yading@10: /* yading@10: * Copyright (c) 2012 Pavel Koshevoy yading@10: * yading@10: * This file is part of FFmpeg. yading@10: * yading@10: * FFmpeg is free software; you can redistribute it and/or yading@10: * modify it under the terms of the GNU Lesser General Public yading@10: * License as published by the Free Software Foundation; either yading@10: * version 2.1 of the License, or (at your option) any later version. yading@10: * yading@10: * FFmpeg is distributed in the hope that it will be useful, yading@10: * but WITHOUT ANY WARRANTY; without even the implied warranty of yading@10: * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU yading@10: * Lesser General Public License for more details. yading@10: * yading@10: * You should have received a copy of the GNU Lesser General Public yading@10: * License along with FFmpeg; if not, write to the Free Software yading@10: * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA yading@10: */ yading@10: yading@10: /** yading@10: * @file yading@10: * tempo scaling audio filter -- an implementation of WSOLA algorithm yading@10: * yading@10: * Based on MIT licensed yaeAudioTempoFilter.h and yaeAudioFragment.h yading@10: * from Apprentice Video player by Pavel Koshevoy. yading@10: * https://sourceforge.net/projects/apprenticevideo/ yading@10: * yading@10: * An explanation of SOLA algorithm is available at yading@10: * http://www.surina.net/article/time-and-pitch-scaling.html yading@10: * yading@10: * WSOLA is very similar to SOLA, only one major difference exists between yading@10: * these algorithms. SOLA shifts audio fragments along the output stream, yading@10: * where as WSOLA shifts audio fragments along the input stream. yading@10: * yading@10: * The advantage of WSOLA algorithm is that the overlap region size is yading@10: * always the same, therefore the blending function is constant and yading@10: * can be precomputed. yading@10: */ yading@10: yading@10: #include yading@10: #include "libavcodec/avfft.h" yading@10: #include "libavutil/avassert.h" yading@10: #include "libavutil/avstring.h" yading@10: #include "libavutil/channel_layout.h" yading@10: #include "libavutil/eval.h" yading@10: #include "libavutil/opt.h" yading@10: #include "libavutil/samplefmt.h" yading@10: #include "avfilter.h" yading@10: #include "audio.h" yading@10: #include "internal.h" yading@10: yading@10: /** yading@10: * A fragment of audio waveform yading@10: */ yading@10: typedef struct { yading@10: // index of the first sample of this fragment in the overall waveform; yading@10: // 0: input sample position yading@10: // 1: output sample position yading@10: int64_t position[2]; yading@10: yading@10: // original packed multi-channel samples: yading@10: uint8_t *data; yading@10: yading@10: // number of samples in this fragment: yading@10: int nsamples; yading@10: yading@10: // rDFT transform of the down-mixed mono fragment, used for yading@10: // fast waveform alignment via correlation in frequency domain: yading@10: FFTSample *xdat; yading@10: } AudioFragment; yading@10: yading@10: /** yading@10: * Filter state machine states yading@10: */ yading@10: typedef enum { yading@10: YAE_LOAD_FRAGMENT, yading@10: YAE_ADJUST_POSITION, yading@10: YAE_RELOAD_FRAGMENT, yading@10: YAE_OUTPUT_OVERLAP_ADD, yading@10: YAE_FLUSH_OUTPUT, yading@10: } FilterState; yading@10: yading@10: /** yading@10: * Filter state machine yading@10: */ yading@10: typedef struct { yading@10: const AVClass *class; yading@10: yading@10: // ring-buffer of input samples, necessary because some times yading@10: // input fragment position may be adjusted backwards: yading@10: uint8_t *buffer; yading@10: yading@10: // ring-buffer maximum capacity, expressed in sample rate time base: yading@10: int ring; yading@10: yading@10: // ring-buffer house keeping: yading@10: int size; yading@10: int head; yading@10: int tail; yading@10: yading@10: // 0: input sample position corresponding to the ring buffer tail yading@10: // 1: output sample position yading@10: int64_t position[2]; yading@10: yading@10: // sample format: yading@10: enum AVSampleFormat format; yading@10: yading@10: // number of channels: yading@10: int channels; yading@10: yading@10: // row of bytes to skip from one sample to next, across multple channels; yading@10: // stride = (number-of-channels * bits-per-sample-per-channel) / 8 yading@10: int stride; yading@10: yading@10: // fragment window size, power-of-two integer: yading@10: int window; yading@10: yading@10: // Hann window coefficients, for feathering yading@10: // (blending) the overlapping fragment region: yading@10: float *hann; yading@10: yading@10: // tempo scaling factor: yading@10: double tempo; yading@10: yading@10: // cumulative alignment drift: yading@10: int drift; yading@10: yading@10: // current/previous fragment ring-buffer: yading@10: AudioFragment frag[2]; yading@10: yading@10: // current fragment index: yading@10: uint64_t nfrag; yading@10: yading@10: // current state: yading@10: FilterState state; yading@10: yading@10: // for fast correlation calculation in frequency domain: yading@10: RDFTContext *real_to_complex; yading@10: RDFTContext *complex_to_real; yading@10: FFTSample *correlation; yading@10: yading@10: // for managing AVFilterPad.request_frame and AVFilterPad.filter_frame yading@10: AVFrame *dst_buffer; yading@10: uint8_t *dst; yading@10: uint8_t *dst_end; yading@10: uint64_t nsamples_in; yading@10: uint64_t nsamples_out; yading@10: } ATempoContext; yading@10: yading@10: #define OFFSET(x) offsetof(ATempoContext, x) yading@10: yading@10: static const AVOption atempo_options[] = { yading@10: { "tempo", "set tempo scale factor", yading@10: OFFSET(tempo), AV_OPT_TYPE_DOUBLE, { .dbl = 1.0 }, 0.5, 2.0, yading@10: AV_OPT_FLAG_AUDIO_PARAM | AV_OPT_FLAG_FILTERING_PARAM }, yading@10: { NULL } yading@10: }; yading@10: yading@10: AVFILTER_DEFINE_CLASS(atempo); yading@10: yading@10: /** yading@10: * Reset filter to initial state, do not deallocate existing local buffers. yading@10: */ yading@10: static void yae_clear(ATempoContext *atempo) yading@10: { yading@10: atempo->size = 0; yading@10: atempo->head = 0; yading@10: atempo->tail = 0; yading@10: yading@10: atempo->drift = 0; yading@10: atempo->nfrag = 0; yading@10: atempo->state = YAE_LOAD_FRAGMENT; yading@10: yading@10: atempo->position[0] = 0; yading@10: atempo->position[1] = 0; yading@10: yading@10: atempo->frag[0].position[0] = 0; yading@10: atempo->frag[0].position[1] = 0; yading@10: atempo->frag[0].nsamples = 0; yading@10: yading@10: atempo->frag[1].position[0] = 0; yading@10: atempo->frag[1].position[1] = 0; yading@10: atempo->frag[1].nsamples = 0; yading@10: yading@10: // shift left position of 1st fragment by half a window yading@10: // so that no re-normalization would be required for yading@10: // the left half of the 1st fragment: yading@10: atempo->frag[0].position[0] = -(int64_t)(atempo->window / 2); yading@10: atempo->frag[0].position[1] = -(int64_t)(atempo->window / 2); yading@10: yading@10: av_frame_free(&atempo->dst_buffer); yading@10: atempo->dst = NULL; yading@10: atempo->dst_end = NULL; yading@10: yading@10: atempo->nsamples_in = 0; yading@10: atempo->nsamples_out = 0; yading@10: } yading@10: yading@10: /** yading@10: * Reset filter to initial state and deallocate all buffers. yading@10: */ yading@10: static void yae_release_buffers(ATempoContext *atempo) yading@10: { yading@10: yae_clear(atempo); yading@10: yading@10: av_freep(&atempo->frag[0].data); yading@10: av_freep(&atempo->frag[1].data); yading@10: av_freep(&atempo->frag[0].xdat); yading@10: av_freep(&atempo->frag[1].xdat); yading@10: yading@10: av_freep(&atempo->buffer); yading@10: av_freep(&atempo->hann); yading@10: av_freep(&atempo->correlation); yading@10: yading@10: av_rdft_end(atempo->real_to_complex); yading@10: atempo->real_to_complex = NULL; yading@10: yading@10: av_rdft_end(atempo->complex_to_real); yading@10: atempo->complex_to_real = NULL; yading@10: } yading@10: yading@10: /* av_realloc is not aligned enough; fortunately, the data does not need to yading@10: * be preserved */ yading@10: #define RE_MALLOC_OR_FAIL(field, field_size) \ yading@10: do { \ yading@10: av_freep(&field); \ yading@10: field = av_malloc(field_size); \ yading@10: if (!field) { \ yading@10: yae_release_buffers(atempo); \ yading@10: return AVERROR(ENOMEM); \ yading@10: } \ yading@10: } while (0) yading@10: yading@10: /** yading@10: * Prepare filter for processing audio data of given format, yading@10: * sample rate and number of channels. yading@10: */ yading@10: static int yae_reset(ATempoContext *atempo, yading@10: enum AVSampleFormat format, yading@10: int sample_rate, yading@10: int channels) yading@10: { yading@10: const int sample_size = av_get_bytes_per_sample(format); yading@10: uint32_t nlevels = 0; yading@10: uint32_t pot; yading@10: int i; yading@10: yading@10: atempo->format = format; yading@10: atempo->channels = channels; yading@10: atempo->stride = sample_size * channels; yading@10: yading@10: // pick a segment window size: yading@10: atempo->window = sample_rate / 24; yading@10: yading@10: // adjust window size to be a power-of-two integer: yading@10: nlevels = av_log2(atempo->window); yading@10: pot = 1 << nlevels; yading@10: av_assert0(pot <= atempo->window); yading@10: yading@10: if (pot < atempo->window) { yading@10: atempo->window = pot * 2; yading@10: nlevels++; yading@10: } yading@10: yading@10: // initialize audio fragment buffers: yading@10: RE_MALLOC_OR_FAIL(atempo->frag[0].data, atempo->window * atempo->stride); yading@10: RE_MALLOC_OR_FAIL(atempo->frag[1].data, atempo->window * atempo->stride); yading@10: RE_MALLOC_OR_FAIL(atempo->frag[0].xdat, atempo->window * sizeof(FFTComplex)); yading@10: RE_MALLOC_OR_FAIL(atempo->frag[1].xdat, atempo->window * sizeof(FFTComplex)); yading@10: yading@10: // initialize rDFT contexts: yading@10: av_rdft_end(atempo->real_to_complex); yading@10: atempo->real_to_complex = NULL; yading@10: yading@10: av_rdft_end(atempo->complex_to_real); yading@10: atempo->complex_to_real = NULL; yading@10: yading@10: atempo->real_to_complex = av_rdft_init(nlevels + 1, DFT_R2C); yading@10: if (!atempo->real_to_complex) { yading@10: yae_release_buffers(atempo); yading@10: return AVERROR(ENOMEM); yading@10: } yading@10: yading@10: atempo->complex_to_real = av_rdft_init(nlevels + 1, IDFT_C2R); yading@10: if (!atempo->complex_to_real) { yading@10: yae_release_buffers(atempo); yading@10: return AVERROR(ENOMEM); yading@10: } yading@10: yading@10: RE_MALLOC_OR_FAIL(atempo->correlation, atempo->window * sizeof(FFTComplex)); yading@10: yading@10: atempo->ring = atempo->window * 3; yading@10: RE_MALLOC_OR_FAIL(atempo->buffer, atempo->ring * atempo->stride); yading@10: yading@10: // initialize the Hann window function: yading@10: RE_MALLOC_OR_FAIL(atempo->hann, atempo->window * sizeof(float)); yading@10: yading@10: for (i = 0; i < atempo->window; i++) { yading@10: double t = (double)i / (double)(atempo->window - 1); yading@10: double h = 0.5 * (1.0 - cos(2.0 * M_PI * t)); yading@10: atempo->hann[i] = (float)h; yading@10: } yading@10: yading@10: yae_clear(atempo); yading@10: return 0; yading@10: } yading@10: yading@10: static int yae_set_tempo(AVFilterContext *ctx, const char *arg_tempo) yading@10: { yading@10: ATempoContext *atempo = ctx->priv; yading@10: char *tail = NULL; yading@10: double tempo = av_strtod(arg_tempo, &tail); yading@10: yading@10: if (tail && *tail) { yading@10: av_log(ctx, AV_LOG_ERROR, "Invalid tempo value '%s'\n", arg_tempo); yading@10: return AVERROR(EINVAL); yading@10: } yading@10: yading@10: if (tempo < 0.5 || tempo > 2.0) { yading@10: av_log(ctx, AV_LOG_ERROR, "Tempo value %f exceeds [0.5, 2.0] range\n", yading@10: tempo); yading@10: return AVERROR(EINVAL); yading@10: } yading@10: yading@10: atempo->tempo = tempo; yading@10: return 0; yading@10: } yading@10: yading@10: inline static AudioFragment *yae_curr_frag(ATempoContext *atempo) yading@10: { yading@10: return &atempo->frag[atempo->nfrag % 2]; yading@10: } yading@10: yading@10: inline static AudioFragment *yae_prev_frag(ATempoContext *atempo) yading@10: { yading@10: return &atempo->frag[(atempo->nfrag + 1) % 2]; yading@10: } yading@10: yading@10: /** yading@10: * A helper macro for initializing complex data buffer with scalar data yading@10: * of a given type. yading@10: */ yading@10: #define yae_init_xdat(scalar_type, scalar_max) \ yading@10: do { \ yading@10: const uint8_t *src_end = src + \ yading@10: frag->nsamples * atempo->channels * sizeof(scalar_type); \ yading@10: \ yading@10: FFTSample *xdat = frag->xdat; \ yading@10: scalar_type tmp; \ yading@10: \ yading@10: if (atempo->channels == 1) { \ yading@10: for (; src < src_end; xdat++) { \ yading@10: tmp = *(const scalar_type *)src; \ yading@10: src += sizeof(scalar_type); \ yading@10: \ yading@10: *xdat = (FFTSample)tmp; \ yading@10: } \ yading@10: } else { \ yading@10: FFTSample s, max, ti, si; \ yading@10: int i; \ yading@10: \ yading@10: for (; src < src_end; xdat++) { \ yading@10: tmp = *(const scalar_type *)src; \ yading@10: src += sizeof(scalar_type); \ yading@10: \ yading@10: max = (FFTSample)tmp; \ yading@10: s = FFMIN((FFTSample)scalar_max, \ yading@10: (FFTSample)fabsf(max)); \ yading@10: \ yading@10: for (i = 1; i < atempo->channels; i++) { \ yading@10: tmp = *(const scalar_type *)src; \ yading@10: src += sizeof(scalar_type); \ yading@10: \ yading@10: ti = (FFTSample)tmp; \ yading@10: si = FFMIN((FFTSample)scalar_max, \ yading@10: (FFTSample)fabsf(ti)); \ yading@10: \ yading@10: if (s < si) { \ yading@10: s = si; \ yading@10: max = ti; \ yading@10: } \ yading@10: } \ yading@10: \ yading@10: *xdat = max; \ yading@10: } \ yading@10: } \ yading@10: } while (0) yading@10: yading@10: /** yading@10: * Initialize complex data buffer of a given audio fragment yading@10: * with down-mixed mono data of appropriate scalar type. yading@10: */ yading@10: static void yae_downmix(ATempoContext *atempo, AudioFragment *frag) yading@10: { yading@10: // shortcuts: yading@10: const uint8_t *src = frag->data; yading@10: yading@10: // init complex data buffer used for FFT and Correlation: yading@10: memset(frag->xdat, 0, sizeof(FFTComplex) * atempo->window); yading@10: yading@10: if (atempo->format == AV_SAMPLE_FMT_U8) { yading@10: yae_init_xdat(uint8_t, 127); yading@10: } else if (atempo->format == AV_SAMPLE_FMT_S16) { yading@10: yae_init_xdat(int16_t, 32767); yading@10: } else if (atempo->format == AV_SAMPLE_FMT_S32) { yading@10: yae_init_xdat(int, 2147483647); yading@10: } else if (atempo->format == AV_SAMPLE_FMT_FLT) { yading@10: yae_init_xdat(float, 1); yading@10: } else if (atempo->format == AV_SAMPLE_FMT_DBL) { yading@10: yae_init_xdat(double, 1); yading@10: } yading@10: } yading@10: yading@10: /** yading@10: * Populate the internal data buffer on as-needed basis. yading@10: * yading@10: * @return yading@10: * 0 if requested data was already available or was successfully loaded, yading@10: * AVERROR(EAGAIN) if more input data is required. yading@10: */ yading@10: static int yae_load_data(ATempoContext *atempo, yading@10: const uint8_t **src_ref, yading@10: const uint8_t *src_end, yading@10: int64_t stop_here) yading@10: { yading@10: // shortcut: yading@10: const uint8_t *src = *src_ref; yading@10: const int read_size = stop_here - atempo->position[0]; yading@10: yading@10: if (stop_here <= atempo->position[0]) { yading@10: return 0; yading@10: } yading@10: yading@10: // samples are not expected to be skipped: yading@10: av_assert0(read_size <= atempo->ring); yading@10: yading@10: while (atempo->position[0] < stop_here && src < src_end) { yading@10: int src_samples = (src_end - src) / atempo->stride; yading@10: yading@10: // load data piece-wise, in order to avoid complicating the logic: yading@10: int nsamples = FFMIN(read_size, src_samples); yading@10: int na; yading@10: int nb; yading@10: yading@10: nsamples = FFMIN(nsamples, atempo->ring); yading@10: na = FFMIN(nsamples, atempo->ring - atempo->tail); yading@10: nb = FFMIN(nsamples - na, atempo->ring); yading@10: yading@10: if (na) { yading@10: uint8_t *a = atempo->buffer + atempo->tail * atempo->stride; yading@10: memcpy(a, src, na * atempo->stride); yading@10: yading@10: src += na * atempo->stride; yading@10: atempo->position[0] += na; yading@10: yading@10: atempo->size = FFMIN(atempo->size + na, atempo->ring); yading@10: atempo->tail = (atempo->tail + na) % atempo->ring; yading@10: atempo->head = yading@10: atempo->size < atempo->ring ? yading@10: atempo->tail - atempo->size : yading@10: atempo->tail; yading@10: } yading@10: yading@10: if (nb) { yading@10: uint8_t *b = atempo->buffer; yading@10: memcpy(b, src, nb * atempo->stride); yading@10: yading@10: src += nb * atempo->stride; yading@10: atempo->position[0] += nb; yading@10: yading@10: atempo->size = FFMIN(atempo->size + nb, atempo->ring); yading@10: atempo->tail = (atempo->tail + nb) % atempo->ring; yading@10: atempo->head = yading@10: atempo->size < atempo->ring ? yading@10: atempo->tail - atempo->size : yading@10: atempo->tail; yading@10: } yading@10: } yading@10: yading@10: // pass back the updated source buffer pointer: yading@10: *src_ref = src; yading@10: yading@10: // sanity check: yading@10: av_assert0(atempo->position[0] <= stop_here); yading@10: yading@10: return atempo->position[0] == stop_here ? 0 : AVERROR(EAGAIN); yading@10: } yading@10: yading@10: /** yading@10: * Populate current audio fragment data buffer. yading@10: * yading@10: * @return yading@10: * 0 when the fragment is ready, yading@10: * AVERROR(EAGAIN) if more input data is required. yading@10: */ yading@10: static int yae_load_frag(ATempoContext *atempo, yading@10: const uint8_t **src_ref, yading@10: const uint8_t *src_end) yading@10: { yading@10: // shortcuts: yading@10: AudioFragment *frag = yae_curr_frag(atempo); yading@10: uint8_t *dst; yading@10: int64_t missing, start, zeros; yading@10: uint32_t nsamples; yading@10: const uint8_t *a, *b; yading@10: int i0, i1, n0, n1, na, nb; yading@10: yading@10: int64_t stop_here = frag->position[0] + atempo->window; yading@10: if (src_ref && yae_load_data(atempo, src_ref, src_end, stop_here) != 0) { yading@10: return AVERROR(EAGAIN); yading@10: } yading@10: yading@10: // calculate the number of samples we don't have: yading@10: missing = yading@10: stop_here > atempo->position[0] ? yading@10: stop_here - atempo->position[0] : 0; yading@10: yading@10: nsamples = yading@10: missing < (int64_t)atempo->window ? yading@10: (uint32_t)(atempo->window - missing) : 0; yading@10: yading@10: // setup the output buffer: yading@10: frag->nsamples = nsamples; yading@10: dst = frag->data; yading@10: yading@10: start = atempo->position[0] - atempo->size; yading@10: zeros = 0; yading@10: yading@10: if (frag->position[0] < start) { yading@10: // what we don't have we substitute with zeros: yading@10: zeros = FFMIN(start - frag->position[0], (int64_t)nsamples); yading@10: av_assert0(zeros != nsamples); yading@10: yading@10: memset(dst, 0, zeros * atempo->stride); yading@10: dst += zeros * atempo->stride; yading@10: } yading@10: yading@10: if (zeros == nsamples) { yading@10: return 0; yading@10: } yading@10: yading@10: // get the remaining data from the ring buffer: yading@10: na = (atempo->head < atempo->tail ? yading@10: atempo->tail - atempo->head : yading@10: atempo->ring - atempo->head); yading@10: yading@10: nb = atempo->head < atempo->tail ? 0 : atempo->tail; yading@10: yading@10: // sanity check: yading@10: av_assert0(nsamples <= zeros + na + nb); yading@10: yading@10: a = atempo->buffer + atempo->head * atempo->stride; yading@10: b = atempo->buffer; yading@10: yading@10: i0 = frag->position[0] + zeros - start; yading@10: i1 = i0 < na ? 0 : i0 - na; yading@10: yading@10: n0 = i0 < na ? FFMIN(na - i0, (int)(nsamples - zeros)) : 0; yading@10: n1 = nsamples - zeros - n0; yading@10: yading@10: if (n0) { yading@10: memcpy(dst, a + i0 * atempo->stride, n0 * atempo->stride); yading@10: dst += n0 * atempo->stride; yading@10: } yading@10: yading@10: if (n1) { yading@10: memcpy(dst, b + i1 * atempo->stride, n1 * atempo->stride); yading@10: } yading@10: yading@10: return 0; yading@10: } yading@10: yading@10: /** yading@10: * Prepare for loading next audio fragment. yading@10: */ yading@10: static void yae_advance_to_next_frag(ATempoContext *atempo) yading@10: { yading@10: const double fragment_step = atempo->tempo * (double)(atempo->window / 2); yading@10: yading@10: const AudioFragment *prev; yading@10: AudioFragment *frag; yading@10: yading@10: atempo->nfrag++; yading@10: prev = yae_prev_frag(atempo); yading@10: frag = yae_curr_frag(atempo); yading@10: yading@10: frag->position[0] = prev->position[0] + (int64_t)fragment_step; yading@10: frag->position[1] = prev->position[1] + atempo->window / 2; yading@10: frag->nsamples = 0; yading@10: } yading@10: yading@10: /** yading@10: * Calculate cross-correlation via rDFT. yading@10: * yading@10: * Multiply two vectors of complex numbers (result of real_to_complex rDFT) yading@10: * and transform back via complex_to_real rDFT. yading@10: */ yading@10: static void yae_xcorr_via_rdft(FFTSample *xcorr, yading@10: RDFTContext *complex_to_real, yading@10: const FFTComplex *xa, yading@10: const FFTComplex *xb, yading@10: const int window) yading@10: { yading@10: FFTComplex *xc = (FFTComplex *)xcorr; yading@10: int i; yading@10: yading@10: // NOTE: first element requires special care -- Given Y = rDFT(X), yading@10: // Im(Y[0]) and Im(Y[N/2]) are always zero, therefore av_rdft_calc yading@10: // stores Re(Y[N/2]) in place of Im(Y[0]). yading@10: yading@10: xc->re = xa->re * xb->re; yading@10: xc->im = xa->im * xb->im; yading@10: xa++; yading@10: xb++; yading@10: xc++; yading@10: yading@10: for (i = 1; i < window; i++, xa++, xb++, xc++) { yading@10: xc->re = (xa->re * xb->re + xa->im * xb->im); yading@10: xc->im = (xa->im * xb->re - xa->re * xb->im); yading@10: } yading@10: yading@10: // apply inverse rDFT: yading@10: av_rdft_calc(complex_to_real, xcorr); yading@10: } yading@10: yading@10: /** yading@10: * Calculate alignment offset for given fragment yading@10: * relative to the previous fragment. yading@10: * yading@10: * @return alignment offset of current fragment relative to previous. yading@10: */ yading@10: static int yae_align(AudioFragment *frag, yading@10: const AudioFragment *prev, yading@10: const int window, yading@10: const int delta_max, yading@10: const int drift, yading@10: FFTSample *correlation, yading@10: RDFTContext *complex_to_real) yading@10: { yading@10: int best_offset = -drift; yading@10: FFTSample best_metric = -FLT_MAX; yading@10: FFTSample *xcorr; yading@10: yading@10: int i0; yading@10: int i1; yading@10: int i; yading@10: yading@10: yae_xcorr_via_rdft(correlation, yading@10: complex_to_real, yading@10: (const FFTComplex *)prev->xdat, yading@10: (const FFTComplex *)frag->xdat, yading@10: window); yading@10: yading@10: // identify search window boundaries: yading@10: i0 = FFMAX(window / 2 - delta_max - drift, 0); yading@10: i0 = FFMIN(i0, window); yading@10: yading@10: i1 = FFMIN(window / 2 + delta_max - drift, window - window / 16); yading@10: i1 = FFMAX(i1, 0); yading@10: yading@10: // identify cross-correlation peaks within search window: yading@10: xcorr = correlation + i0; yading@10: yading@10: for (i = i0; i < i1; i++, xcorr++) { yading@10: FFTSample metric = *xcorr; yading@10: yading@10: // normalize: yading@10: FFTSample drifti = (FFTSample)(drift + i); yading@10: metric *= drifti * (FFTSample)(i - i0) * (FFTSample)(i1 - i); yading@10: yading@10: if (metric > best_metric) { yading@10: best_metric = metric; yading@10: best_offset = i - window / 2; yading@10: } yading@10: } yading@10: yading@10: return best_offset; yading@10: } yading@10: yading@10: /** yading@10: * Adjust current fragment position for better alignment yading@10: * with previous fragment. yading@10: * yading@10: * @return alignment correction. yading@10: */ yading@10: static int yae_adjust_position(ATempoContext *atempo) yading@10: { yading@10: const AudioFragment *prev = yae_prev_frag(atempo); yading@10: AudioFragment *frag = yae_curr_frag(atempo); yading@10: yading@10: const int delta_max = atempo->window / 2; yading@10: const int correction = yae_align(frag, yading@10: prev, yading@10: atempo->window, yading@10: delta_max, yading@10: atempo->drift, yading@10: atempo->correlation, yading@10: atempo->complex_to_real); yading@10: yading@10: if (correction) { yading@10: // adjust fragment position: yading@10: frag->position[0] -= correction; yading@10: yading@10: // clear so that the fragment can be reloaded: yading@10: frag->nsamples = 0; yading@10: yading@10: // update cumulative correction drift counter: yading@10: atempo->drift += correction; yading@10: } yading@10: yading@10: return correction; yading@10: } yading@10: yading@10: /** yading@10: * A helper macro for blending the overlap region of previous yading@10: * and current audio fragment. yading@10: */ yading@10: #define yae_blend(scalar_type) \ yading@10: do { \ yading@10: const scalar_type *aaa = (const scalar_type *)a; \ yading@10: const scalar_type *bbb = (const scalar_type *)b; \ yading@10: \ yading@10: scalar_type *out = (scalar_type *)dst; \ yading@10: scalar_type *out_end = (scalar_type *)dst_end; \ yading@10: int64_t i; \ yading@10: \ yading@10: for (i = 0; i < overlap && out < out_end; \ yading@10: i++, atempo->position[1]++, wa++, wb++) { \ yading@10: float w0 = *wa; \ yading@10: float w1 = *wb; \ yading@10: int j; \ yading@10: \ yading@10: for (j = 0; j < atempo->channels; \ yading@10: j++, aaa++, bbb++, out++) { \ yading@10: float t0 = (float)*aaa; \ yading@10: float t1 = (float)*bbb; \ yading@10: \ yading@10: *out = \ yading@10: frag->position[0] + i < 0 ? \ yading@10: *aaa : \ yading@10: (scalar_type)(t0 * w0 + t1 * w1); \ yading@10: } \ yading@10: } \ yading@10: dst = (uint8_t *)out; \ yading@10: } while (0) yading@10: yading@10: /** yading@10: * Blend the overlap region of previous and current audio fragment yading@10: * and output the results to the given destination buffer. yading@10: * yading@10: * @return yading@10: * 0 if the overlap region was completely stored in the dst buffer, yading@10: * AVERROR(EAGAIN) if more destination buffer space is required. yading@10: */ yading@10: static int yae_overlap_add(ATempoContext *atempo, yading@10: uint8_t **dst_ref, yading@10: uint8_t *dst_end) yading@10: { yading@10: // shortcuts: yading@10: const AudioFragment *prev = yae_prev_frag(atempo); yading@10: const AudioFragment *frag = yae_curr_frag(atempo); yading@10: yading@10: const int64_t start_here = FFMAX(atempo->position[1], yading@10: frag->position[1]); yading@10: yading@10: const int64_t stop_here = FFMIN(prev->position[1] + prev->nsamples, yading@10: frag->position[1] + frag->nsamples); yading@10: yading@10: const int64_t overlap = stop_here - start_here; yading@10: yading@10: const int64_t ia = start_here - prev->position[1]; yading@10: const int64_t ib = start_here - frag->position[1]; yading@10: yading@10: const float *wa = atempo->hann + ia; yading@10: const float *wb = atempo->hann + ib; yading@10: yading@10: const uint8_t *a = prev->data + ia * atempo->stride; yading@10: const uint8_t *b = frag->data + ib * atempo->stride; yading@10: yading@10: uint8_t *dst = *dst_ref; yading@10: yading@10: av_assert0(start_here <= stop_here && yading@10: frag->position[1] <= start_here && yading@10: overlap <= frag->nsamples); yading@10: yading@10: if (atempo->format == AV_SAMPLE_FMT_U8) { yading@10: yae_blend(uint8_t); yading@10: } else if (atempo->format == AV_SAMPLE_FMT_S16) { yading@10: yae_blend(int16_t); yading@10: } else if (atempo->format == AV_SAMPLE_FMT_S32) { yading@10: yae_blend(int); yading@10: } else if (atempo->format == AV_SAMPLE_FMT_FLT) { yading@10: yae_blend(float); yading@10: } else if (atempo->format == AV_SAMPLE_FMT_DBL) { yading@10: yae_blend(double); yading@10: } yading@10: yading@10: // pass-back the updated destination buffer pointer: yading@10: *dst_ref = dst; yading@10: yading@10: return atempo->position[1] == stop_here ? 0 : AVERROR(EAGAIN); yading@10: } yading@10: yading@10: /** yading@10: * Feed as much data to the filter as it is able to consume yading@10: * and receive as much processed data in the destination buffer yading@10: * as it is able to produce or store. yading@10: */ yading@10: static void yading@10: yae_apply(ATempoContext *atempo, yading@10: const uint8_t **src_ref, yading@10: const uint8_t *src_end, yading@10: uint8_t **dst_ref, yading@10: uint8_t *dst_end) yading@10: { yading@10: while (1) { yading@10: if (atempo->state == YAE_LOAD_FRAGMENT) { yading@10: // load additional data for the current fragment: yading@10: if (yae_load_frag(atempo, src_ref, src_end) != 0) { yading@10: break; yading@10: } yading@10: yading@10: // down-mix to mono: yading@10: yae_downmix(atempo, yae_curr_frag(atempo)); yading@10: yading@10: // apply rDFT: yading@10: av_rdft_calc(atempo->real_to_complex, yae_curr_frag(atempo)->xdat); yading@10: yading@10: // must load the second fragment before alignment can start: yading@10: if (!atempo->nfrag) { yading@10: yae_advance_to_next_frag(atempo); yading@10: continue; yading@10: } yading@10: yading@10: atempo->state = YAE_ADJUST_POSITION; yading@10: } yading@10: yading@10: if (atempo->state == YAE_ADJUST_POSITION) { yading@10: // adjust position for better alignment: yading@10: if (yae_adjust_position(atempo)) { yading@10: // reload the fragment at the corrected position, so that the yading@10: // Hann window blending would not require normalization: yading@10: atempo->state = YAE_RELOAD_FRAGMENT; yading@10: } else { yading@10: atempo->state = YAE_OUTPUT_OVERLAP_ADD; yading@10: } yading@10: } yading@10: yading@10: if (atempo->state == YAE_RELOAD_FRAGMENT) { yading@10: // load additional data if necessary due to position adjustment: yading@10: if (yae_load_frag(atempo, src_ref, src_end) != 0) { yading@10: break; yading@10: } yading@10: yading@10: // down-mix to mono: yading@10: yae_downmix(atempo, yae_curr_frag(atempo)); yading@10: yading@10: // apply rDFT: yading@10: av_rdft_calc(atempo->real_to_complex, yae_curr_frag(atempo)->xdat); yading@10: yading@10: atempo->state = YAE_OUTPUT_OVERLAP_ADD; yading@10: } yading@10: yading@10: if (atempo->state == YAE_OUTPUT_OVERLAP_ADD) { yading@10: // overlap-add and output the result: yading@10: if (yae_overlap_add(atempo, dst_ref, dst_end) != 0) { yading@10: break; yading@10: } yading@10: yading@10: // advance to the next fragment, repeat: yading@10: yae_advance_to_next_frag(atempo); yading@10: atempo->state = YAE_LOAD_FRAGMENT; yading@10: } yading@10: } yading@10: } yading@10: yading@10: /** yading@10: * Flush any buffered data from the filter. yading@10: * yading@10: * @return yading@10: * 0 if all data was completely stored in the dst buffer, yading@10: * AVERROR(EAGAIN) if more destination buffer space is required. yading@10: */ yading@10: static int yae_flush(ATempoContext *atempo, yading@10: uint8_t **dst_ref, yading@10: uint8_t *dst_end) yading@10: { yading@10: AudioFragment *frag = yae_curr_frag(atempo); yading@10: int64_t overlap_end; yading@10: int64_t start_here; yading@10: int64_t stop_here; yading@10: int64_t offset; yading@10: yading@10: const uint8_t *src; yading@10: uint8_t *dst; yading@10: yading@10: int src_size; yading@10: int dst_size; yading@10: int nbytes; yading@10: yading@10: atempo->state = YAE_FLUSH_OUTPUT; yading@10: yading@10: if (atempo->position[0] == frag->position[0] + frag->nsamples && yading@10: atempo->position[1] == frag->position[1] + frag->nsamples) { yading@10: // the current fragment is already flushed: yading@10: return 0; yading@10: } yading@10: yading@10: if (frag->position[0] + frag->nsamples < atempo->position[0]) { yading@10: // finish loading the current (possibly partial) fragment: yading@10: yae_load_frag(atempo, NULL, NULL); yading@10: yading@10: if (atempo->nfrag) { yading@10: // down-mix to mono: yading@10: yae_downmix(atempo, frag); yading@10: yading@10: // apply rDFT: yading@10: av_rdft_calc(atempo->real_to_complex, frag->xdat); yading@10: yading@10: // align current fragment to previous fragment: yading@10: if (yae_adjust_position(atempo)) { yading@10: // reload the current fragment due to adjusted position: yading@10: yae_load_frag(atempo, NULL, NULL); yading@10: } yading@10: } yading@10: } yading@10: yading@10: // flush the overlap region: yading@10: overlap_end = frag->position[1] + FFMIN(atempo->window / 2, yading@10: frag->nsamples); yading@10: yading@10: while (atempo->position[1] < overlap_end) { yading@10: if (yae_overlap_add(atempo, dst_ref, dst_end) != 0) { yading@10: return AVERROR(EAGAIN); yading@10: } yading@10: } yading@10: yading@10: // flush the remaininder of the current fragment: yading@10: start_here = FFMAX(atempo->position[1], overlap_end); yading@10: stop_here = frag->position[1] + frag->nsamples; yading@10: offset = start_here - frag->position[1]; yading@10: av_assert0(start_here <= stop_here && frag->position[1] <= start_here); yading@10: yading@10: src = frag->data + offset * atempo->stride; yading@10: dst = (uint8_t *)*dst_ref; yading@10: yading@10: src_size = (int)(stop_here - start_here) * atempo->stride; yading@10: dst_size = dst_end - dst; yading@10: nbytes = FFMIN(src_size, dst_size); yading@10: yading@10: memcpy(dst, src, nbytes); yading@10: dst += nbytes; yading@10: yading@10: atempo->position[1] += (nbytes / atempo->stride); yading@10: yading@10: // pass-back the updated destination buffer pointer: yading@10: *dst_ref = (uint8_t *)dst; yading@10: yading@10: return atempo->position[1] == stop_here ? 0 : AVERROR(EAGAIN); yading@10: } yading@10: yading@10: static av_cold int init(AVFilterContext *ctx) yading@10: { yading@10: ATempoContext *atempo = ctx->priv; yading@10: atempo->format = AV_SAMPLE_FMT_NONE; yading@10: atempo->state = YAE_LOAD_FRAGMENT; yading@10: return 0; yading@10: } yading@10: yading@10: static av_cold void uninit(AVFilterContext *ctx) yading@10: { yading@10: ATempoContext *atempo = ctx->priv; yading@10: yae_release_buffers(atempo); yading@10: } yading@10: yading@10: static int query_formats(AVFilterContext *ctx) yading@10: { yading@10: AVFilterChannelLayouts *layouts = NULL; yading@10: AVFilterFormats *formats = NULL; yading@10: yading@10: // WSOLA necessitates an internal sliding window ring buffer yading@10: // for incoming audio stream. yading@10: // yading@10: // Planar sample formats are too cumbersome to store in a ring buffer, yading@10: // therefore planar sample formats are not supported. yading@10: // yading@10: static const enum AVSampleFormat sample_fmts[] = { yading@10: AV_SAMPLE_FMT_U8, yading@10: AV_SAMPLE_FMT_S16, yading@10: AV_SAMPLE_FMT_S32, yading@10: AV_SAMPLE_FMT_FLT, yading@10: AV_SAMPLE_FMT_DBL, yading@10: AV_SAMPLE_FMT_NONE yading@10: }; yading@10: yading@10: layouts = ff_all_channel_layouts(); yading@10: if (!layouts) { yading@10: return AVERROR(ENOMEM); yading@10: } yading@10: ff_set_common_channel_layouts(ctx, layouts); yading@10: yading@10: formats = ff_make_format_list(sample_fmts); yading@10: if (!formats) { yading@10: return AVERROR(ENOMEM); yading@10: } yading@10: ff_set_common_formats(ctx, formats); yading@10: yading@10: formats = ff_all_samplerates(); yading@10: if (!formats) { yading@10: return AVERROR(ENOMEM); yading@10: } yading@10: ff_set_common_samplerates(ctx, formats); yading@10: yading@10: return 0; yading@10: } yading@10: yading@10: static int config_props(AVFilterLink *inlink) yading@10: { yading@10: AVFilterContext *ctx = inlink->dst; yading@10: ATempoContext *atempo = ctx->priv; yading@10: yading@10: enum AVSampleFormat format = inlink->format; yading@10: int sample_rate = (int)inlink->sample_rate; yading@10: int channels = av_get_channel_layout_nb_channels(inlink->channel_layout); yading@10: yading@10: ctx->outputs[0]->flags |= FF_LINK_FLAG_REQUEST_LOOP; yading@10: yading@10: return yae_reset(atempo, format, sample_rate, channels); yading@10: } yading@10: yading@10: static int push_samples(ATempoContext *atempo, yading@10: AVFilterLink *outlink, yading@10: int n_out) yading@10: { yading@10: int ret; yading@10: yading@10: atempo->dst_buffer->sample_rate = outlink->sample_rate; yading@10: atempo->dst_buffer->nb_samples = n_out; yading@10: yading@10: // adjust the PTS: yading@10: atempo->dst_buffer->pts = yading@10: av_rescale_q(atempo->nsamples_out, yading@10: (AVRational){ 1, outlink->sample_rate }, yading@10: outlink->time_base); yading@10: yading@10: ret = ff_filter_frame(outlink, atempo->dst_buffer); yading@10: if (ret < 0) yading@10: return ret; yading@10: atempo->dst_buffer = NULL; yading@10: atempo->dst = NULL; yading@10: atempo->dst_end = NULL; yading@10: yading@10: atempo->nsamples_out += n_out; yading@10: return 0; yading@10: } yading@10: yading@10: static int filter_frame(AVFilterLink *inlink, AVFrame *src_buffer) yading@10: { yading@10: AVFilterContext *ctx = inlink->dst; yading@10: ATempoContext *atempo = ctx->priv; yading@10: AVFilterLink *outlink = ctx->outputs[0]; yading@10: yading@10: int ret = 0; yading@10: int n_in = src_buffer->nb_samples; yading@10: int n_out = (int)(0.5 + ((double)n_in) / atempo->tempo); yading@10: yading@10: const uint8_t *src = src_buffer->data[0]; yading@10: const uint8_t *src_end = src + n_in * atempo->stride; yading@10: yading@10: while (src < src_end) { yading@10: if (!atempo->dst_buffer) { yading@10: atempo->dst_buffer = ff_get_audio_buffer(outlink, n_out); yading@10: if (!atempo->dst_buffer) yading@10: return AVERROR(ENOMEM); yading@10: av_frame_copy_props(atempo->dst_buffer, src_buffer); yading@10: yading@10: atempo->dst = atempo->dst_buffer->data[0]; yading@10: atempo->dst_end = atempo->dst + n_out * atempo->stride; yading@10: } yading@10: yading@10: yae_apply(atempo, &src, src_end, &atempo->dst, atempo->dst_end); yading@10: yading@10: if (atempo->dst == atempo->dst_end) { yading@10: ret = push_samples(atempo, outlink, n_out); yading@10: if (ret < 0) yading@10: goto end; yading@10: } yading@10: } yading@10: yading@10: atempo->nsamples_in += n_in; yading@10: end: yading@10: av_frame_free(&src_buffer); yading@10: return ret; yading@10: } yading@10: yading@10: static int request_frame(AVFilterLink *outlink) yading@10: { yading@10: AVFilterContext *ctx = outlink->src; yading@10: ATempoContext *atempo = ctx->priv; yading@10: int ret; yading@10: yading@10: ret = ff_request_frame(ctx->inputs[0]); yading@10: yading@10: if (ret == AVERROR_EOF) { yading@10: // flush the filter: yading@10: int n_max = atempo->ring; yading@10: int n_out; yading@10: int err = AVERROR(EAGAIN); yading@10: yading@10: while (err == AVERROR(EAGAIN)) { yading@10: if (!atempo->dst_buffer) { yading@10: atempo->dst_buffer = ff_get_audio_buffer(outlink, n_max); yading@10: if (!atempo->dst_buffer) yading@10: return AVERROR(ENOMEM); yading@10: yading@10: atempo->dst = atempo->dst_buffer->data[0]; yading@10: atempo->dst_end = atempo->dst + n_max * atempo->stride; yading@10: } yading@10: yading@10: err = yae_flush(atempo, &atempo->dst, atempo->dst_end); yading@10: yading@10: n_out = ((atempo->dst - atempo->dst_buffer->data[0]) / yading@10: atempo->stride); yading@10: yading@10: if (n_out) { yading@10: ret = push_samples(atempo, outlink, n_out); yading@10: } yading@10: } yading@10: yading@10: av_frame_free(&atempo->dst_buffer); yading@10: atempo->dst = NULL; yading@10: atempo->dst_end = NULL; yading@10: yading@10: return AVERROR_EOF; yading@10: } yading@10: yading@10: return ret; yading@10: } yading@10: yading@10: static int process_command(AVFilterContext *ctx, yading@10: const char *cmd, yading@10: const char *arg, yading@10: char *res, yading@10: int res_len, yading@10: int flags) yading@10: { yading@10: return !strcmp(cmd, "tempo") ? yae_set_tempo(ctx, arg) : AVERROR(ENOSYS); yading@10: } yading@10: yading@10: static const AVFilterPad atempo_inputs[] = { yading@10: { yading@10: .name = "default", yading@10: .type = AVMEDIA_TYPE_AUDIO, yading@10: .filter_frame = filter_frame, yading@10: .config_props = config_props, yading@10: }, yading@10: { NULL } yading@10: }; yading@10: yading@10: static const AVFilterPad atempo_outputs[] = { yading@10: { yading@10: .name = "default", yading@10: .request_frame = request_frame, yading@10: .type = AVMEDIA_TYPE_AUDIO, yading@10: }, yading@10: { NULL } yading@10: }; yading@10: yading@10: AVFilter avfilter_af_atempo = { yading@10: .name = "atempo", yading@10: .description = NULL_IF_CONFIG_SMALL("Adjust audio tempo."), yading@10: .init = init, yading@10: .uninit = uninit, yading@10: .query_formats = query_formats, yading@10: .process_command = process_command, yading@10: .priv_size = sizeof(ATempoContext), yading@10: .priv_class = &atempo_class, yading@10: .inputs = atempo_inputs, yading@10: .outputs = atempo_outputs, yading@10: };