Mercurial > hg > svapp
diff audioio/AudioCallbackPlaySource.cpp @ 91:9fc4b256c283
* PortAudio driver: do not specify frames per buffer, let PA decide
* Remove old non-RubberBand time stretcher -- it doesn't work with varying
buffer sizes such as the PA driver may now be using
* Rewrite getCurrentPlayingFrame for greater correctness when using long
buffer sizes (interpolating according to audio stream timestamp)
* Several changes to make the timestretch management RT safe(r)
author | Chris Cannam |
---|---|
date | Fri, 08 Feb 2008 17:51:15 +0000 |
parents | ae2627ac7db2 |
children | 792bca285459 |
line wrap: on
line diff
--- a/audioio/AudioCallbackPlaySource.cpp Thu Feb 07 12:35:08 2008 +0000 +++ b/audioio/AudioCallbackPlaySource.cpp Fri Feb 08 17:51:15 2008 +0000 @@ -26,12 +26,10 @@ #include "data/model/SparseOneDimensionalModel.h" #include "plugin/RealTimePluginInstance.h" -#ifdef HAVE_RUBBERBAND +#include "AudioCallbackPlayTarget.h" + #include <rubberband/RubberBandStretcher.h> using namespace RubberBand; -#else -#include "PhaseVocoderTimeStretcher.h" -#endif #include <iostream> #include <cassert> @@ -56,6 +54,9 @@ m_sourceSampleRate(0), m_targetSampleRate(0), m_playLatency(0), + m_target(0), + m_lastRetrievalTimestamp(0.0), + m_lastRetrievedBlockSize(0), m_playing(false), m_exiting(false), m_lastModelEndFrame(0), @@ -64,6 +65,10 @@ m_auditioningPlugin(0), m_auditioningPluginBypassed(false), m_timeStretcher(0), + m_stretchRatio(1.0), + m_stretcherInputCount(0), + m_stretcherInputs(0), + m_stretcherInputSizes(0), m_fillThread(0), m_converter(0), m_crapConverter(0), @@ -107,11 +112,14 @@ delete m_audioGenerator; + for (size_t i = 0; i < m_stretcherInputCount; ++i) { + delete[] m_stretcherInputs[i]; + } + delete[] m_stretcherInputSizes; + delete[] m_stretcherInputs; + m_bufferScavenger.scavenge(true); m_pluginScavenger.scavenge(true); -#ifndef HAVE_RUBBERBAND - m_timeStretcherScavenger.scavenge(true); -#endif } void @@ -371,6 +379,9 @@ // we're just re-seeking. m_mutex.lock(); + if (m_timeStretcher) { + m_timeStretcher->reset(); + } if (m_playing) { m_readBufferFill = m_writeBufferFill = startFrame; if (m_readBuffers) { @@ -391,6 +402,7 @@ m_audioGenerator->reset(); bool changed = !m_playing; + m_lastRetrievalTimestamp = 0; m_playing = true; m_condition.wakeAll(); if (changed) emit playStatusChanged(m_playing); @@ -402,6 +414,7 @@ bool changed = m_playing; m_playing = false; m_condition.wakeAll(); + m_lastRetrievalTimestamp = 0; if (changed) emit playStatusChanged(m_playing); } @@ -452,8 +465,9 @@ } void -AudioCallbackPlaySource::setTargetBlockSize(size_t size) +AudioCallbackPlaySource::setTarget(AudioCallbackPlayTarget *target, size_t size) { + m_target = target; // std::cout << "AudioCallbackPlaySource::setTargetBlockSize() -> " << size << std::endl; assert(size < m_ringBufferSize); m_blockSize = size; @@ -481,134 +495,190 @@ size_t AudioCallbackPlaySource::getCurrentPlayingFrame() { + // This method attempts to estimate which audio sample frame is + // "currently coming through the speakers". + bool resample = false; - double ratio = 1.0; + double resampleRatio = 1.0; - if (getSourceSampleRate() != getTargetSampleRate()) { - resample = true; - ratio = double(getSourceSampleRate()) / double(getTargetSampleRate()); - } + // We resample when filling the ring buffer, and time-stretch when + // draining it. The buffer contains data at the "target rate" and + // the latency provided by the target is also at the target rate. + // Because of the multiple rates involved, we do the actual + // calculation using RealTime instead. - size_t readSpace = 0; + size_t sourceRate = getSourceSampleRate(); + size_t targetRate = getTargetSampleRate(); + + if (sourceRate == 0 || targetRate == 0) return 0; + + size_t inbuffer = 0; // at target rate + for (size_t c = 0; c < getTargetChannelCount(); ++c) { RingBuffer<float> *rb = getReadRingBuffer(c); if (rb) { - size_t spaceHere = rb->getReadSpace(); - if (c == 0 || spaceHere < readSpace) readSpace = spaceHere; + size_t here = rb->getReadSpace(); + if (c == 0 || here < inbuffer) inbuffer = here; } } - if (resample) { - readSpace = size_t(readSpace * ratio + 0.1); + size_t readBufferFill = m_readBufferFill; + size_t lastRetrievedBlockSize = m_lastRetrievedBlockSize; + double lastRetrievalTimestamp = m_lastRetrievalTimestamp; + double currentTime = 0.0; + if (m_target) currentTime = m_target->getCurrentTime(); + + RealTime inbuffer_t = RealTime::frame2RealTime(inbuffer, targetRate); + + size_t latency = m_playLatency; // at target rate + RealTime latency_t = RealTime::frame2RealTime(latency, targetRate); + + size_t stretchlat = 0; + double timeRatio = 1.0; + + if (m_timeStretcher) { + stretchlat = m_timeStretcher->getLatency(); + timeRatio = m_timeStretcher->getTimeRatio(); } - size_t latency = m_playLatency; - if (resample) latency = size_t(m_playLatency * ratio + 0.1); + RealTime stretchlat_t = RealTime::frame2RealTime(stretchlat, targetRate); -#ifdef HAVE_RUBBERBAND - if (m_timeStretcher) { - latency += m_timeStretcher->getLatency(); + // When the target has just requested a block from us, the last + // sample it obtained was our buffer fill frame count minus the + // amount of read space (converted back to source sample rate) + // remaining now. That sample is not expected to be played until + // the target's play latency has elapsed. By the time the + // following block is requested, that sample will be at the + // target's play latency minus the last requested block size away + // from being played. + + RealTime sincerequest_t = RealTime::zeroTime; + RealTime lastretrieved_t = RealTime::zeroTime; + + if (m_target && lastRetrievalTimestamp != 0.0) { + + lastretrieved_t = RealTime::frame2RealTime + (lastRetrievedBlockSize, targetRate); + + // calculate number of frames at target rate that have elapsed + // since the end of the last call to getSourceSamples + + double elapsed = currentTime - lastRetrievalTimestamp; + + if (elapsed > 0.0) { + sincerequest_t = RealTime::fromSeconds(elapsed); + } + + } else { + + lastretrieved_t = RealTime::frame2RealTime + (getTargetBlockSize(), targetRate); } -#else - PhaseVocoderTimeStretcher *timeStretcher = m_timeStretcher; - if (timeStretcher) { - latency += timeStretcher->getProcessingLatency(); + + RealTime bufferedto_t = RealTime::frame2RealTime(readBufferFill, sourceRate); + + if (timeRatio != 1.0) { + lastretrieved_t = lastretrieved_t / timeRatio; + sincerequest_t = sincerequest_t / timeRatio; } -#endif - - latency += readSpace; - size_t bufferedFrame = m_readBufferFill; bool looping = m_viewManager->getPlayLoopMode(); bool constrained = (m_viewManager->getPlaySelectionMode() && !m_viewManager->getSelections().empty()); - size_t framePlaying = bufferedFrame; +#ifdef DEBUG_AUDIO_PLAY_SOURCE_PLAYING + std::cerr << "\nbuffered to: " << bufferedto_t << ", in buffer: " << inbuffer_t << ", time ratio " << timeRatio << "\n stretcher latency: " << stretchlat_t << ", device latency: " << latency_t << "\n since request: " << sincerequest_t << ", last retrieved: " << lastretrieved_t << std::endl; +#endif - if (looping && !constrained) { - while (framePlaying < latency) framePlaying += m_lastModelEndFrame; - } - - if (framePlaying > latency) framePlaying -= latency; - else framePlaying = 0; - -// std::cerr << "framePlaying = " << framePlaying << " -> reference "; - - framePlaying = m_viewManager->alignPlaybackFrameToReference(framePlaying); - -// std::cerr << framePlaying << std::endl; - - if (!constrained) { - if (!looping && framePlaying > m_lastModelEndFrame) { - framePlaying = m_lastModelEndFrame; - stop(); - } - return framePlaying; - } - - bufferedFrame = m_viewManager->alignPlaybackFrameToReference(bufferedFrame); + RealTime end = RealTime::frame2RealTime(m_lastModelEndFrame, sourceRate); MultiSelection::SelectionList selections = m_viewManager->getSelections(); MultiSelection::SelectionList::const_iterator i; -// i = selections.begin(); -// size_t rangeStart = i->getStartFrame(); + // these could be cached from one call to the next, if the + // selection has not changed... but some of the work would still + // need to be done because the playback model may have changed - i = selections.end(); - --i; - size_t rangeEnd = i->getEndFrame(); + std::vector<RealTime> rangeStarts; + std::vector<RealTime> rangeDurations; - for (i = selections.begin(); i != selections.end(); ++i) { - if (i->contains(bufferedFrame)) break; + int inRange = 0; + int index = 0; + + if (constrained) { + + for (i = selections.begin(); i != selections.end(); ++i) { + + RealTime start = + (RealTime::frame2RealTime + (m_viewManager->alignReferenceToPlaybackFrame(i->getStartFrame()), + sourceRate)); + RealTime duration = + (RealTime::frame2RealTime + (m_viewManager->alignReferenceToPlaybackFrame(i->getEndFrame()) - + m_viewManager->alignReferenceToPlaybackFrame(i->getStartFrame()), + sourceRate)); + + rangeStarts.push_back(start); + rangeDurations.push_back(duration); + + if (bufferedto_t >= start) { + inRange = index; + } + + ++index; + } } - size_t f = bufferedFrame; - -// std::cout << "getCurrentPlayingFrame: f=" << f << ", latency=" << latency << ", rangeEnd=" << rangeEnd << std::endl; - - if (i == selections.end()) { - --i; - if (i->getEndFrame() + latency < f) { -// std::cout << "framePlaying = " << framePlaying << ", rangeEnd = " << rangeEnd << std::endl; - - if (!looping && (framePlaying > rangeEnd)) { -// std::cout << "STOPPING" << std::endl; - stop(); - return rangeEnd; - } else { - return framePlaying; - } - } else { -// std::cout << "latency <- " << latency << "-(" << f << "-" << i->getEndFrame() << ")" << std::endl; - latency -= (f - i->getEndFrame()); - f = i->getEndFrame(); - } + if (rangeStarts.empty()) { + rangeStarts.push_back(RealTime::zeroTime); + rangeDurations.push_back(end); } -// std::cout << "i=(" << i->getStartFrame() << "," << i->getEndFrame() << ") f=" << f << ", latency=" << latency << std::endl; + if (inRange >= rangeStarts.size()) inRange = rangeStarts.size()-1; - while (latency > 0) { - size_t offset = f - i->getStartFrame(); - if (offset >= latency) { - if (f > latency) { - framePlaying = f - latency; - } else { - framePlaying = 0; - } - break; - } else { - if (i == selections.begin()) { - if (looping) { - i = selections.end(); - } - } - latency -= offset; - --i; - f = i->getEndFrame(); - } + RealTime playing_t = bufferedto_t - rangeStarts[inRange]; + + playing_t = playing_t + - latency_t - stretchlat_t - lastretrieved_t - inbuffer_t + + sincerequest_t; + +#ifdef DEBUG_AUDIO_PLAY_SOURCE_PLAYING + std::cerr << "playing_t as offset into range " << inRange << " (with start = " << rangeStarts[inRange] << ") = " << playing_t << std::endl; +#endif + + while (playing_t < RealTime::zeroTime) { + + if (inRange == 0) { + if (looping) { + inRange = rangeStarts.size() - 1; + } else { + break; + } + } else { + --inRange; + } + + playing_t = playing_t + rangeDurations[inRange]; } - return framePlaying; + playing_t = playing_t + rangeStarts[inRange]; + +#ifdef DEBUG_AUDIO_PLAY_SOURCE_PLAYING + std::cerr << " playing time: " << playing_t << std::endl; +#endif + + if (!looping) { + if (inRange == rangeStarts.size()-1 && + playing_t >= rangeStarts[inRange] + rangeDurations[inRange]) { + stop(); + } + } + + if (playing_t < RealTime::zeroTime) playing_t = RealTime::zeroTime; + + size_t frame = RealTime::realTime2Frame(playing_t, sourceRate); + return m_viewManager->alignPlaybackFrameToReference(frame); } void @@ -758,65 +828,29 @@ } void -AudioCallbackPlaySource::setTimeStretch(float factor, bool sharpen, bool mono) +AudioCallbackPlaySource::setTimeStretch(float factor) { -#ifdef HAVE_RUBBERBAND - if (m_timeStretcher) { - m_timeStretchRatioMutex.lock(); - m_timeStretcher->setTimeRatio(factor); - m_timeStretchRatioMutex.unlock(); + m_stretchRatio = factor; + + if (m_timeStretcher || (factor == 1.f)) { + // stretch ratio will be set in next process call if appropriate return; } else { + m_stretcherInputCount = getTargetChannelCount(); RubberBandStretcher *stretcher = new RubberBandStretcher (getTargetSampleRate(), - getTargetChannelCount(), + m_stretcherInputCount, RubberBandStretcher::OptionProcessRealTime, factor); + m_stretcherInputs = new float *[m_stretcherInputCount]; + m_stretcherInputSizes = new size_t[m_stretcherInputCount]; + for (size_t c = 0; c < m_stretcherInputCount; ++c) { + m_stretcherInputSizes[c] = 16384; + m_stretcherInputs[c] = new float[m_stretcherInputSizes[c]]; + } m_timeStretcher = stretcher; return; } -#else - // Avoid locks -- create, assign, mark old one for scavenging - // later (as a call to getSourceSamples may still be using it) - - PhaseVocoderTimeStretcher *existingStretcher = m_timeStretcher; - - size_t channels = getTargetChannelCount(); - if (mono) channels = 1; - - if (existingStretcher && - existingStretcher->getRatio() == factor && - existingStretcher->getSharpening() == sharpen && - existingStretcher->getChannelCount() == channels) { - return; - } - - if (factor != 1) { - - if (existingStretcher && - existingStretcher->getSharpening() == sharpen && - existingStretcher->getChannelCount() == channels) { - existingStretcher->setRatio(factor); - return; - } - - PhaseVocoderTimeStretcher *newStretcher = new PhaseVocoderTimeStretcher - (getTargetSampleRate(), - channels, - factor, - sharpen, - getTargetBlockSize()); - - m_timeStretcher = newStretcher; - - } else { - m_timeStretcher = 0; - } - - if (existingStretcher) { - m_timeStretcherScavenger.claim(existingStretcher); - } -#endif } size_t @@ -860,13 +894,22 @@ if (count == 0) return 0; -#ifdef HAVE_RUBBERBAND RubberBandStretcher *ts = m_timeStretcher; float ratio = ts ? ts->getTimeRatio() : 1.f; -#else - PhaseVocoderTimeStretcher *ts = m_timeStretcher; - float ratio = ts ? ts->getRatio() : 1.f; -#endif + + if (ratio != m_stretchRatio) { + if (!ts) { + std::cerr << "WARNING: AudioCallbackPlaySource::getSourceSamples: Time ratio change to " << m_stretchRatio << " is pending, but no stretcher is set" << std::endl; + m_stretchRatio = 1.f; + } else { + ts->setTimeRatio(m_stretchRatio); + } + } + + if (m_target) { + m_lastRetrievedBlockSize = count; + m_lastRetrievalTimestamp = m_target->getCurrentTime(); + } if (!ts || ratio == 1.f) { @@ -900,68 +943,58 @@ applyAuditioningEffect(count, buffer); m_condition.wakeAll(); + return got; } size_t channels = getTargetChannelCount(); + size_t available; + int warned = 0; + size_t fedToStretcher = 0; -#ifdef HAVE_RUBBERBAND - bool mix = false; -#else - bool mix = (channels > 1 && ts->getChannelCount() == 1); + // The input block for a given output is approx output / ratio, + // but we can't predict it exactly, for an adaptive timestretcher. + + while ((available = ts->available()) < count) { + + size_t reqd = lrintf((count - available) / ratio); + reqd = std::max(reqd, ts->getSamplesRequired()); + if (reqd == 0) reqd = 1; + + size_t got = reqd; + +#ifdef DEBUG_AUDIO_PLAY_SOURCE_PLAYING + std::cerr << "reqd = " <<reqd << ", channels = " << channels << ", ic = " << m_stretcherInputCount << std::endl; #endif - size_t available; + for (size_t c = 0; c < channels; ++c) { + if (c >= m_stretcherInputCount) continue; + if (reqd > m_stretcherInputSizes[c]) { + if (c == 0) { + std::cerr << "WARNING: resizing stretcher input buffer from " << m_stretcherInputSizes[c] << " to " << (reqd * 2) << std::endl; + } + delete[] m_stretcherInputs[c]; + m_stretcherInputSizes[c] = reqd * 2; + m_stretcherInputs[c] = new float[m_stretcherInputSizes[c]]; + } + } - int warned = 0; - - // We want output blocks of e.g. 1024 (probably fixed, certainly - // bounded). We can provide input blocks of any size (unbounded) - // at the timestretcher's request. The input block for a given - // output is approx output / ratio, but we can't predict it - // exactly, for an adaptive timestretcher. The stretcher will - // need some additional buffer space. See the time stretcher code - // and comments. - -#ifdef HAVE_RUBBERBAND - m_timeStretchRatioMutex.lock(); - while ((available = ts->available()) < count) { -#else - while ((available = ts->getAvailableOutputSamples()) < count) { + for (size_t c = 0; c < channels; ++c) { + if (c >= m_stretcherInputCount) continue; + RingBuffer<float> *rb = getReadRingBuffer(c); + if (rb) { + size_t gotHere = rb->read(m_stretcherInputs[c], got); + if (gotHere < got) got = gotHere; + +#ifdef DEBUG_AUDIO_PLAY_SOURCE_PLAYING + if (c == 0) { + std::cerr << "feeding stretcher: got " << gotHere + << ", " << rb->getReadSpace() << " remain" << std::endl; + } #endif - - size_t reqd = lrintf((count - available) / ratio); -#ifdef HAVE_RUBBERBAND - reqd = std::max(reqd, ts->getSamplesRequired()); -#else - reqd = std::max(reqd, ts->getRequiredInputSamples()); -#endif - if (reqd == 0) reqd = 1; - float *ib[channels]; - - size_t got = reqd; - - if (mix) { - for (size_t c = 0; c < channels; ++c) { - if (c == 0) ib[c] = new float[reqd]; //!!! fix -- this is a rt function - else ib[c] = 0; - RingBuffer<float> *rb = getReadRingBuffer(c); - if (rb) { - size_t gotHere; - if (c > 0) gotHere = rb->readAdding(ib[0], got); - else gotHere = rb->read(ib[0], got); - if (gotHere < got) got = gotHere; - } - } - } else { - for (size_t c = 0; c < channels; ++c) { - ib[c] = new float[reqd]; //!!! fix -- this is a rt function - RingBuffer<float> *rb = getReadRingBuffer(c); - if (rb) { - size_t gotHere = rb->read(ib[c], got); - if (gotHere < got) got = gotHere; - } + } else { + std::cerr << "WARNING: No ring buffer available for channel " << c << " in stretcher input block" << std::endl; } } @@ -969,46 +1002,20 @@ std::cerr << "WARNING: Read underrun in playback (" << got << " < " << reqd << ")" << std::endl; } - -#ifdef HAVE_RUBBERBAND - ts->process(ib, got, false); -#else - ts->putInput(ib, got); -#endif - for (size_t c = 0; c < channels; ++c) { - delete[] ib[c]; - } + ts->process(m_stretcherInputs, got, false); + + fedToStretcher += got; if (got == 0) break; -#ifdef HAVE_RUBBERBAND if (ts->available() == available) { -#else - if (ts->getAvailableOutputSamples() == available) { -#endif std::cerr << "WARNING: AudioCallbackPlaySource::getSamples: Added " << got << " samples to time stretcher, created no new available output samples (warned = " << warned << ")" << std::endl; if (++warned == 5) break; } } -#ifdef HAVE_RUBBERBAND ts->retrieve(buffer, count); - m_timeStretchRatioMutex.unlock(); -#else - ts->getOutput(buffer, count); -#endif - - if (mix) { - for (size_t c = 1; c < channels; ++c) { - for (size_t i = 0; i < count; ++i) { - buffer[c][i] = buffer[0][i] / channels; - } - } - for (size_t i = 0; i < count; ++i) { - buffer[0][i] /= channels; - } - } applyAuditioningEffect(count, buffer); @@ -1184,11 +1191,7 @@ int err = 0; -#ifdef HAVE_RUBBERBAND if (m_timeStretcher && m_timeStretcher->getTimeRatio() < 0.4) { -#else - if (m_timeStretcher && m_timeStretcher->getRatio() < 0.4) { -#endif #ifdef DEBUG_AUDIO_PLAY_SOURCE std::cout << "Using crappy converter" << std::endl; #endif @@ -1227,7 +1230,13 @@ // space must be a multiple of generatorBlockSize space = (space / generatorBlockSize) * generatorBlockSize; - if (space == 0) return false; + if (space == 0) { +#ifdef DEBUG_AUDIO_PLAY_SOURCE + std::cout << "requested fill is less than generator block size of " + << generatorBlockSize << ", leaving it" << std::endl; +#endif + return false; + } if (tmpSize < channels * space) { delete[] tmp; @@ -1513,9 +1522,6 @@ s.unifyRingBuffers(); s.m_bufferScavenger.scavenge(); s.m_pluginScavenger.scavenge(); -#ifndef HAVE_RUBBERBAND - s.m_timeStretcherScavenger.scavenge(); -#endif if (work && s.m_playing && s.getSourceSampleRate()) {