#include "audioengine.h" #include "data/project.h" using namespace Xybrid::Audio; using namespace Xybrid::Data; #include "data/graph.h" #include "data/porttypes.h" #include "mainwindow.h" #include "uisocket.h" #include "util/strings.h" #include #include #include #include #include #include #include #ifdef Q_OS_MAC #define FFMPEG "/usr/local/bin/ffmpeg" #else #define FFMPEG "ffmpeg" #endif // zero-initialize AudioEngine* Xybrid::Audio::audioEngine = nullptr; void AudioEngine::init() { if (audioEngine) return; // already set up // instantiate singleton QThread* thread = new QThread; audioEngine = new AudioEngine(nullptr); audioEngine->moveToThread(thread); audioEngine->thread = thread; // hook up signals // ... // and off to the races thread->start(); thread->setPriority(QThread::TimeCriticalPriority); QMetaObject::invokeMethod(audioEngine, &AudioEngine::postInit, Qt::QueuedConnection); } void AudioEngine::postInit() { open(QIODevice::ReadOnly); // set up buffer for per-tick allocation tickBuf = std::make_unique(tickBufSize/sizeof(size_t)); // aligned to size_t tickBufPtr = tickBuf.get(); tickBufEnd = tickBufPtr+tickBufSize; buf.reserve(1024); // 1kb isn't much to make sure it's super unlikely to have to reallocate chTrack.reserve(256); noteEndQueue.reserve(256); nameTrack.reserve(64+1); // +1 to make extra sure it doesn't rehash later if (auto cores = QThread::idealThreadCount(); cores > 1) { cores--; // the engine itself is a worker as well workers.reserve(static_cast(cores)); for (int i = 0; i < cores; i++) { auto wk = new AudioWorker(); wk->thread = new QThread(this); wk->moveToThread(wk->thread); connect(wk->thread, &QThread::started, wk, &AudioWorker::processLoop); wk->thread->start(); wk->thread->setPriority(QThread::TimeCriticalPriority); workers.push_back(wk); } } } void* AudioEngine::tickAlloc(size_t size) { if (auto r = size % sizeof(size_t); r != 0) size += sizeof(size_t) - r; // pad auto n = tickBufPtr.fetch_add(static_cast(size)); if (n + size > tickBufEnd) qWarning() << "Tick buffer overrun!"; return n; } AudioEngine::AudioEngine(QObject *parent) : QIODevice(parent) { } void AudioEngine::initAudio(bool startNow) { if (!output) { const QAudioDeviceInfo& deviceInfo = QAudioDeviceInfo::defaultOutputDevice(); QAudioFormat format; format.setSampleRate(48000); format.setChannelCount(2); format.setSampleSize(16); format.setCodec("audio/pcm"); format.setByteOrder(QAudioFormat::LittleEndian); format.setSampleType(QAudioFormat::SignedInt); if (!deviceInfo.isFormatSupported(format)) { qWarning() << "Default format not supported - trying to use nearest"; format = deviceInfo.nearestFormat(format); } sampleRate = format.sampleRate(); output.reset(new QAudioOutput(deviceInfo, format)); output->setCategory("Xybrid"); output->setObjectName("Xybrid"); // if Qt ever implements naming the stream this way, WE'LL BE READY output->setBufferSize(static_cast(sampleRate*4*( 64.0 )/1000.0)); // 64ms seems to be a sweet spot now } if (startNow) output->start(); } void AudioEngine::deinitAudio() { if (output) { QTimer::singleShot(20, [this] { // delay to flush buffers with silence, else we get leftovers on next playback if (output && mode == Stopped) { output->stop(); output.reset(); } }); } } void AudioEngine::play(std::shared_ptr p, int fromPos) { QMetaObject::invokeMethod(this, [this, p, fromPos] { if (!p) return; // nope project = p; // stop and reset, then init playback queueValid = false; queue.clear(); portLastNoteId.fill(0); project->rootGraph->reset(); initAudio(); for (auto& b : buffer) { b.clear(); b.reserve(static_cast(sampleRate/4)); } seqPos = fromPos; curRow = -1; curTick = -2; tempo = project->tempo; tickAcc = 0; output->start(this); mode = Playing; emit this->playbackModeChanged(); }, Qt::QueuedConnection); } void AudioEngine::stop() { if (mode == Rendering) { mode = Stopped; return; } QMetaObject::invokeMethod(this, [this] { if (project) project->rootGraph->release(); project = nullptr; queueValid = false; queue.clear(); for (auto& b : buffer) b.clear(); deinitAudio(); mode = Stopped; emit this->playbackModeChanged(); }, Qt::QueuedConnection); } uint16_t AudioEngine::preview(std::shared_ptr p, int16_t port, int16_t note, uint16_t nId, Data::Node* node) { if (note > -1) nId = previewNote_++; QMetaObject::invokeMethod(this, [this, p, port, note, nId, node] { if (!p) return; if (mode == Playing || mode == Rendering || mode == PlaybackMode::Paused) return; if (project != p || mode != Previewing) { deinitAudio(); project = p; queueValid = false; queue.clear(); buf.clear(); project->rootGraph->reset(); initAudio(); for (auto& b : buffer) { b.clear(); b.reserve(static_cast(sampleRate/4)); b.resize(static_cast(output->periodSize())); } tempo = project->tempo; output->start(this); mode = Previewing; emit this->playbackModeChanged(); } if (port >= 0 && port <= 255 && (note > -1 || note < -3)) previewPort_ = static_cast(port); // assign port if valid (and note on) if (note < -3) return; // invalid note (port is set before it so that setting the port can be a separate action) if (node && node->project == p.get()) previewNode = node; else previewNode = nullptr; // assemble message size_t bi = buf.size(); buf.resize(bi+5); reinterpret_cast(buf[bi]) = nId; reinterpret_cast(buf[bi+2]) = note; }, Qt::QueuedConnection); return nId; } void AudioEngine::render(std::shared_ptr p, QString filename) { if (!p) return; // yeah, no if (mode != Stopped) { stop(); while (output) QThread::yieldCurrentThread(); } project = p; queueValid = false; queue.clear(); portLastNoteId.fill(0); project->rootGraph->reset(); for (auto& b : buffer) { b.clear(); b.reserve(static_cast(sampleRate/4)); } seqPos = -1; curRow = -1; curTick = -2; tempo = project->tempo; tickAcc = 0; initAudio(); // we actually need the period size. whoops. QProcess enc; QStringList param; param << "-y" << "-f" << "s16le" << "-ac" << "2" << "-ar" << QString::number(sampleRate) << "-i" << "pipe:"; if (!project->title.isEmpty()) param << "-metadata" << qs("title=%1").arg(project->title); if (!project->artist.isEmpty()) param << "-metadata" << qs("artist=%1").arg(project->artist); param << "-f" << "mp3" << "-codec:a" << "libmp3lame"<< "-q:a" << "0"; // specify mp3, vbr v0 param << filename; enc.start(FFMPEG, param); enc.waitForStarted(); std::vector dat; dat.resize(1024); mode = Rendering; while (mode == Rendering) { enc.write(&dat[0], readData(&dat[0], 1024)); //enc.write(read(1024)); } enc.closeWriteChannel(); enc.waitForFinished(); stop(); //qDebug() << enc.readAllStandardOutput(); //qDebug() << enc.readAllStandardError(); } void AudioEngine::buildQueue() { // keep track of what was there before std::unordered_set prev; prev.reserve(queue.size() + 1); for (auto n : queue) prev.insert(n.get()); queue.clear(); // stuff std::deque> q1, q2, qf; auto* qCurrent = &q1; auto* qNext = &q2; if (auto p = project->rootGraph->port(Port::Output, Port::Audio, 0); p) if (auto pt = p->passthroughTo.lock(); pt) if (auto ptn = pt->owner.lock(); ptn) qCurrent->push_back(ptn); while (!qCurrent->empty()) { // ... this could be made more efficient with some redundancy checking, but whatever for (auto n : *qCurrent) { qf.push_front(n); // add to actual queue for (auto p1 : n->inputs) { // data types... for (auto p2 : p1.second) { // ports... for (auto p3 : p2.second->connections) { // connected ports! auto pc = p3.lock(); if (!pc) continue; auto pcn = pc->owner.lock(); if (!pcn) continue; qNext->push_back(pcn); if (auto pp = pc->passthroughTo.lock(); pp) { // if it has a passthrough, also place passthrough's owner after (before) if (auto ppp = pp->owner.lock(); ppp) qNext->push_back(ppp); } } } } } qCurrent->clear(); std::swap(qCurrent, qNext); } // assemble final deduplicated queue std::unordered_set dd; for (auto n : qf) { if (dd.find(n.get()) == dd.end()) { queue.push_back(n); dd.insert(n.get()); } } // reset any newcomers for (auto n : queue) if (prev.find(n.get()) == prev.end()) n->reset(); /*/{ auto dbg = qDebug() << "Queue:"; for (auto n : queue) { dbg << QString::fromStdString(n->pluginName()); if (n->name.empty()) dbg << n.get(); else dbg << QString::fromStdString(n->name); } }//*/ queueValid = true; } qint64 AudioEngine::readData(char *data, qint64 maxlen) { const constexpr qint64 smp = 2; const constexpr qint64 stride = smp*2; qint64 sr = maxlen; qint64 srp = sr - output->periodSize(); bool brk = false; while (sr >= stride) { if (bufPos >= buffer[0].size()) { nextTick(); // process next tick when end of buffer reached brk = true; // signal loop to yield as soon as the period has been fulfilled so previewing works } if (brk && sr < srp) break; // convert non-interleaved floating point into interleaved int16 int16_t* l = reinterpret_cast(data); int16_t* r = reinterpret_cast(data+smp); *l = static_cast(std::clamp(buffer[0][bufPos], -1.0f, 1.0f) * 32767); *r = static_cast(std::clamp(buffer[1][bufPos], -1.0f, 1.0f) * 32767); bufPos++; data += stride; sr -= stride; } return maxlen - sr; } Pattern* AudioEngine::findPattern(int adv) { seqPos += adv; for (int tr = 0; tr < 25; tr++) { if (seqPos < 0) seqPos = 0; SequenceEntry* s = nullptr; if (auto sp = static_cast(seqPos); sp < project->sequence.size()) s = &project->sequence[sp]; if (mode == Rendering && !s) return nullptr; // stop else if (mode == Rendering && s->type == SequenceEntry::LoopTrigger) { seqPos++; continue; } else if (!s || (s->type == SequenceEntry::LoopTrigger && seqPos > 0)) { // off end or explicit loop, find loop point for (seqPos = std::min(seqPos, static_cast(project->sequence.size()) - 1); seqPos >= 0; --seqPos) if (project->sequence[static_cast(seqPos)].type == SequenceEntry::LoopStart) break; continue; } else if (s->type == SequenceEntry::Pattern) return s->pattern().get(); else { seqPos++; continue; } } return nullptr; // out of tries } void AudioEngine::nextTick() { bufPos = 0; if (mode == Paused || mode == Stopped) { // simplest case, just give a 100ms empty buffer buffer[0].clear(); buffer[1].clear(); buffer[0].resize(static_cast(sampleRate/10)); buffer[1].resize(static_cast(sampleRate/10)); } else if (mode == Previewing) { // reset raw buffer tickBufPtr = tickBuf.get(); tickId++; double tickSize = 0.005 * sampleRate; // 5ms fixed tick size for preview tickSize += tickAcc; // add sample remainder from last tick double tickSf = std::floor(tickSize); tickAcc = tickSize - tickSf; size_t ts = static_cast(tickSf); buffer[0].resize(ts); buffer[1].resize(ts); if (!queueValid) buildQueue(); // send previewing commands auto pnode = previewNode; uint8_t pport = 0; if (!pnode) { pnode = project->rootGraph.get(); pport = previewPort_; } if (auto p = std::static_pointer_cast(pnode->port(Port::Input, Port::Command, pport)); p) { p->push(buf); } buf.clear(); processNodes(); if (auto p = std::static_pointer_cast(project->rootGraph->port(Port::Output, Port::Audio, 0)); p) { p->pull(); size_t bufs = ts * sizeof(float); memcpy(buffer[0].data(), p->bufL, bufs); memcpy(buffer[1].data(), p->bufR, bufs); } } else if (mode == Playing || mode == Rendering) { // reset raw buffer tickBufPtr = tickBuf.get(); tickId++; // empty out last tick buffer[0].clear(); buffer[1].clear(); if (!queueValid) buildQueue(); Pattern* p = nullptr; Pattern* pOld = nullptr; p = findPattern(); bool newRow = false; bool newPattern = false; auto advanceSeq = [&] { pOld = p; p = findPattern(1); if (!p) { stop(); return; } curRow = 0; // set pattern things if (p->tempo > 0) tempo = p->tempo; newPattern = true; newRow = true; }; auto advanceRow = [&] { curTick = 0; curRow++; if (!p || curRow >= p->rows) advanceSeq(); if (!p) return; // stopping MainWindow* w = project->socket->window; QMetaObject::invokeMethod(w, [this, w]{ w->playbackPosition(seqPos, curRow); }, Qt::QueuedConnection); // process global commands first for (int c = 0; c < static_cast(p->numChannels()); c++) { if (auto& row = p->rowAt(c, curRow); row.port == -2 && row.params) { for (auto p : *row.params) { if (p[0] == 't' && p[1] > 0) tempo = p[1]; } } } newRow = true; // assemble command buffers noteEndQueue.clear(); if (newPattern) { // notes on named channels carry over to their matching channel on the new pattern (if present); everything else is note-offed if (pOld) { size_t cs = pOld->channels.size(); for (size_t c = 0; c < cs; c++) { auto& ch = pOld->channels[c]; if (!chTrack[c].valid) continue; // skip notes that aren't actually playing if (ch.name.isEmpty()) noteEndQueue.push_back(chTrack[c]); // end notes in unnamed channels right away else nameTrack[&ch.name] = chTrack[c]; // otherwise keep track for later } } chTrack.clear(); // clear and prepare channel note tracking chTrack.resize(p->channels.size()); if (nameTrack.size() > 0) { // if there were any size_t cs = p->channels.size(); for (size_t c = 0; c < cs; c++) { auto& ch = p->channels[c]; if (ch.name.isEmpty()) continue; if (auto nt = nameTrack.find(&ch.name); nt != nameTrack.end() && nt->second.valid) { chTrack[c] = nt->second; // carry over nt->second.valid = false; // and invalidate } } // dump remainder into note end for (auto nt : nameTrack) if (nt.second.valid) noteEndQueue.push_back(nt.second); } nameTrack.clear(); } int chs = static_cast(p->channels.size()); for (int c = 0; c < chs; c++) { auto& ct = chTrack[static_cast(c)]; if (!ct.valid) continue; // no saved note auto& r = p->rowAt(c, curRow); if (r.note != -1 && r.port >= 0 && r.port != ct.port) { // if explicitly specified for a different port... noteEndQueue.push_back(ct); // old note overwritten ct.valid = false; } } auto& cpm = project->rootGraph->inputs[Port::Command]; for (auto p_ : cpm) { auto* pt = static_cast(p_.second.get()); //if (pt->passthroughTo.lock()->connections.empty()) continue; // port isn't hooked up to anything uint8_t idx = pt->index; buf.clear(); for (auto& ne : noteEndQueue) { if (ne.valid && ne.port == idx) { size_t bi = buf.size(); buf.resize(bi+5, 0); reinterpret_cast(buf[bi]) = ne.noteId; // trigger on note id... reinterpret_cast(buf[bi+2]) = -2; // note off } } for (int c = 0; c < chs; c++) { auto& r = p->rowAt(c, curRow); auto& ct = chTrack[static_cast(c)]; int16_t port = r.port; if (port < 0 && ct.valid) port = ct.port; // assume last port used on channel if not specified if (port != idx) continue; NoteInfo rpl; // default initialization, invalid if (r.note >= 0 && r.port != -3) { if (ct.valid) rpl = ct; // replace ct = NoteInfo(idx, portLastNoteId[idx]++); } else if (r.note <= -2 && ct.valid) { ct.valid = false; // invalidate it here but leave note id intact // this condition will allow you to note-off the same note id multiple times but anything // that takes offense to that is a bug anyway } size_t bi = buf.size(); buf.resize(bi+5, 0); reinterpret_cast(buf[bi]) = ct.noteId; // either new note, or note-off on old one reinterpret_cast(buf[bi+2]) = r.note; // shove note into vector auto& np = buf[bi+4]; // number of params if (r.params) { for (auto& p : *r.params) { if (p[0] == ' ') continue; // ignore struts buf.push_back(p[0]); buf.push_back(p[1]); np++; } } if (rpl.valid) { // replacing old note on the same port and channel bi = buf.size(); buf.resize(bi+5, 0); reinterpret_cast(buf[bi]) = rpl.noteId; // trigger on note id... reinterpret_cast(buf[bi+2]) = -2; // note off } } //qDebug() << "port" << idx << "data of size" << buf.size(); pt->push(buf); } }; curTick++; if (p && curTick >= p->time.ticksPerRow) advanceRow(); if (!p) return; // no patterns to be found, abort // (sample rate / seconds per beat) / ticks per beat double tickSize = (1.0 * sampleRate / (static_cast(tempo)/60.0)) / (p->time.rowsPerBeat * p->time.ticksPerRow); tickSize += tickAcc; // add sample remainder from last tick double tickSf = std::floor(tickSize); tickAcc = tickSize - tickSf; size_t ts = static_cast(tickSf); buffer[0].resize(ts); buffer[1].resize(ts); processNodes(); if (auto p = std::static_pointer_cast(project->rootGraph->port(Port::Output, Port::Audio, 0)); p) { p->pull(); size_t bufs = ts * sizeof(float); memcpy(buffer[0].data(), p->bufL, bufs); memcpy(buffer[1].data(), p->bufR, bufs); } // } } void AudioWorkerCore::processQueue() { auto& queue = audioEngine->queue; auto sz = queue.size(); forever { auto idx = audioEngine->queueIndex.fetch_add(1); if (idx >= sz) break; auto n = queue[idx]; while (!n->try_process(true)) QThread::yieldCurrentThread(); } } void AudioEngine::processNodes() { if (workers.empty()) { for (auto n : queue) if (!n->try_process()) qWarning() << "Dependency check failed in single threaded mode!"; } else { queueIndex.store(0); // reset sync index auto wc = static_cast(workers.size()); wsem.release(wc); for (auto w : workers) w->invoke(); processQueue(); // act as the final worker ourselves wsem.acquire(wc); } } AudioWorker::AudioWorker(QObject* parent) : QObject(parent) { } void AudioWorker::invoke() { if (!audioEngine->wsem.tryAcquire(1)) return; signal.release(1); } void AudioWorker::processLoop() { thread->setPriority(QThread::TimeCriticalPriority); QMutex m; auto& s = audioEngine->wsem; forever { signal.acquire(1); processQueue(); s.release(1); //thread->setPriority(QThread::IdlePriority); } }