c@41
|
1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
|
c@41
|
2
|
c@41
|
3 /*
|
c@41
|
4 * SegmenterPlugin.cpp
|
c@41
|
5 *
|
c@41
|
6 * Copyright 2008 Centre for Digital Music, Queen Mary, University of London.
|
c@41
|
7 * All rights reserved.
|
c@41
|
8 */
|
c@41
|
9
|
c@41
|
10 #include <iostream>
|
c@44
|
11 #include <cstdio>
|
c@41
|
12
|
c@41
|
13 #include "SimilarityPlugin.h"
|
c@42
|
14 #include "base/Pitch.h"
|
c@41
|
15 #include "dsp/mfcc/MFCC.h"
|
c@42
|
16 #include "dsp/chromagram/Chromagram.h"
|
c@41
|
17 #include "dsp/rateconversion/Decimator.h"
|
c@47
|
18 #include "dsp/rhythm/BeatSpectrum.h"
|
c@47
|
19 #include "maths/KLDivergence.h"
|
c@47
|
20 #include "maths/CosineDistance.h"
|
c@41
|
21
|
c@41
|
22 using std::string;
|
c@41
|
23 using std::vector;
|
c@41
|
24 using std::cerr;
|
c@41
|
25 using std::endl;
|
c@41
|
26 using std::ostringstream;
|
c@41
|
27
|
c@47
|
28 const float
|
c@47
|
29 SimilarityPlugin::m_noRhythm = 0.009;
|
c@47
|
30
|
c@47
|
31 const float
|
c@47
|
32 SimilarityPlugin::m_allRhythm = 0.991;
|
c@47
|
33
|
c@41
|
34 SimilarityPlugin::SimilarityPlugin(float inputSampleRate) :
|
c@41
|
35 Plugin(inputSampleRate),
|
c@42
|
36 m_type(TypeMFCC),
|
c@41
|
37 m_mfcc(0),
|
c@47
|
38 m_rhythmfcc(0),
|
c@42
|
39 m_chromagram(0),
|
c@41
|
40 m_decimator(0),
|
c@42
|
41 m_featureColumnSize(20),
|
c@47
|
42 m_rhythmWeighting(0.f),
|
c@47
|
43 m_rhythmClipDuration(4.f), // seconds
|
c@47
|
44 m_rhythmClipOrigin(40.f), // seconds
|
c@47
|
45 m_rhythmClipFrameSize(0),
|
c@47
|
46 m_rhythmClipFrames(0),
|
c@47
|
47 m_rhythmColumnSize(20),
|
c@41
|
48 m_blockSize(0),
|
c@47
|
49 m_channels(0),
|
c@47
|
50 m_processRate(0),
|
c@47
|
51 m_frameNo(0),
|
c@47
|
52 m_done(false)
|
c@41
|
53 {
|
c@47
|
54 int rate = lrintf(m_inputSampleRate);
|
c@47
|
55 int internalRate = 22050;
|
c@47
|
56 int decimationFactor = rate / internalRate;
|
c@47
|
57 if (decimationFactor < 1) decimationFactor = 1;
|
c@47
|
58
|
c@47
|
59 // must be a power of two
|
c@47
|
60 while (decimationFactor & (decimationFactor - 1)) ++decimationFactor;
|
c@47
|
61
|
c@47
|
62 m_processRate = rate / decimationFactor; // may be 22050, 24000 etc
|
c@41
|
63 }
|
c@41
|
64
|
c@41
|
65 SimilarityPlugin::~SimilarityPlugin()
|
c@41
|
66 {
|
c@41
|
67 delete m_mfcc;
|
c@47
|
68 delete m_rhythmfcc;
|
c@42
|
69 delete m_chromagram;
|
c@41
|
70 delete m_decimator;
|
c@41
|
71 }
|
c@41
|
72
|
c@41
|
73 string
|
c@41
|
74 SimilarityPlugin::getIdentifier() const
|
c@41
|
75 {
|
c@41
|
76 return "qm-similarity";
|
c@41
|
77 }
|
c@41
|
78
|
c@41
|
79 string
|
c@41
|
80 SimilarityPlugin::getName() const
|
c@41
|
81 {
|
c@41
|
82 return "Similarity";
|
c@41
|
83 }
|
c@41
|
84
|
c@41
|
85 string
|
c@41
|
86 SimilarityPlugin::getDescription() const
|
c@41
|
87 {
|
c@42
|
88 return "Return a distance matrix for similarity between the input audio channels";
|
c@41
|
89 }
|
c@41
|
90
|
c@41
|
91 string
|
c@41
|
92 SimilarityPlugin::getMaker() const
|
c@41
|
93 {
|
c@47
|
94 return "Mark Levy, Kurt Jacobson and Chris Cannam, Queen Mary, University of London";
|
c@41
|
95 }
|
c@41
|
96
|
c@41
|
97 int
|
c@41
|
98 SimilarityPlugin::getPluginVersion() const
|
c@41
|
99 {
|
c@41
|
100 return 1;
|
c@41
|
101 }
|
c@41
|
102
|
c@41
|
103 string
|
c@41
|
104 SimilarityPlugin::getCopyright() const
|
c@41
|
105 {
|
c@41
|
106 return "Copyright (c) 2008 - All Rights Reserved";
|
c@41
|
107 }
|
c@41
|
108
|
c@41
|
109 size_t
|
c@41
|
110 SimilarityPlugin::getMinChannelCount() const
|
c@41
|
111 {
|
c@43
|
112 return 1;
|
c@41
|
113 }
|
c@41
|
114
|
c@41
|
115 size_t
|
c@41
|
116 SimilarityPlugin::getMaxChannelCount() const
|
c@41
|
117 {
|
c@41
|
118 return 1024;
|
c@41
|
119 }
|
c@41
|
120
|
c@41
|
121 bool
|
c@41
|
122 SimilarityPlugin::initialise(size_t channels, size_t stepSize, size_t blockSize)
|
c@41
|
123 {
|
c@41
|
124 if (channels < getMinChannelCount() ||
|
c@41
|
125 channels > getMaxChannelCount()) return false;
|
c@41
|
126
|
c@41
|
127 if (stepSize != getPreferredStepSize()) {
|
c@43
|
128 //!!! actually this perhaps shouldn't be an error... similarly
|
c@43
|
129 //using more than getMaxChannelCount channels
|
c@41
|
130 std::cerr << "SimilarityPlugin::initialise: supplied step size "
|
c@41
|
131 << stepSize << " differs from required step size "
|
c@41
|
132 << getPreferredStepSize() << std::endl;
|
c@41
|
133 return false;
|
c@41
|
134 }
|
c@41
|
135
|
c@41
|
136 if (blockSize != getPreferredBlockSize()) {
|
c@41
|
137 std::cerr << "SimilarityPlugin::initialise: supplied block size "
|
c@41
|
138 << blockSize << " differs from required block size "
|
c@41
|
139 << getPreferredBlockSize() << std::endl;
|
c@41
|
140 return false;
|
c@41
|
141 }
|
c@41
|
142
|
c@41
|
143 m_blockSize = blockSize;
|
c@41
|
144 m_channels = channels;
|
c@41
|
145
|
c@44
|
146 m_lastNonEmptyFrame = std::vector<int>(m_channels);
|
c@44
|
147 for (int i = 0; i < m_channels; ++i) m_lastNonEmptyFrame[i] = -1;
|
c@44
|
148 m_frameNo = 0;
|
c@44
|
149
|
c@41
|
150 int decimationFactor = getDecimationFactor();
|
c@41
|
151 if (decimationFactor > 1) {
|
c@42
|
152 m_decimator = new Decimator(m_blockSize, decimationFactor);
|
c@41
|
153 }
|
c@41
|
154
|
c@42
|
155 if (m_type == TypeMFCC) {
|
c@42
|
156
|
c@42
|
157 m_featureColumnSize = 20;
|
c@42
|
158
|
c@47
|
159 MFCCConfig config(m_processRate);
|
c@42
|
160 config.fftsize = 2048;
|
c@42
|
161 config.nceps = m_featureColumnSize - 1;
|
c@42
|
162 config.want_c0 = true;
|
c@45
|
163 config.logpower = 1;
|
c@42
|
164 m_mfcc = new MFCC(config);
|
c@42
|
165 m_fftSize = m_mfcc->getfftlength();
|
c@47
|
166 m_rhythmClipFrameSize = m_fftSize / 4;
|
c@42
|
167
|
c@43
|
168 std::cerr << "MFCC FS = " << config.FS << ", FFT size = " << m_fftSize<< std::endl;
|
c@43
|
169
|
c@42
|
170 } else if (m_type == TypeChroma) {
|
c@42
|
171
|
c@42
|
172 m_featureColumnSize = 12;
|
c@42
|
173
|
c@42
|
174 ChromaConfig config;
|
c@47
|
175 config.FS = m_processRate;
|
c@42
|
176 config.min = Pitch::getFrequencyForPitch(24, 0, 440);
|
c@42
|
177 config.max = Pitch::getFrequencyForPitch(96, 0, 440);
|
c@42
|
178 config.BPO = 12;
|
c@42
|
179 config.CQThresh = 0.0054;
|
c@42
|
180 config.isNormalised = true;
|
c@42
|
181 m_chromagram = new Chromagram(config);
|
c@42
|
182 m_fftSize = m_chromagram->getFrameSize();
|
c@42
|
183
|
c@47
|
184 std::cerr << "fftsize = " << m_fftSize << std::endl;
|
c@47
|
185
|
c@47
|
186 m_rhythmClipFrameSize = m_fftSize / 16;
|
c@47
|
187 while (m_rhythmClipFrameSize < 512) m_rhythmClipFrameSize *= 2;
|
c@47
|
188 std::cerr << "m_rhythmClipFrameSize = " << m_rhythmClipFrameSize << std::endl;
|
c@47
|
189
|
c@42
|
190 std::cerr << "min = "<< config.min << ", max = " << config.max << std::endl;
|
c@42
|
191
|
c@42
|
192 } else {
|
c@42
|
193
|
c@42
|
194 std::cerr << "SimilarityPlugin::initialise: internal error: unknown type " << m_type << std::endl;
|
c@42
|
195 return false;
|
c@42
|
196 }
|
c@41
|
197
|
c@47
|
198 if (needRhythm()) {
|
c@47
|
199 m_rhythmClipFrames =
|
c@47
|
200 int(ceil((m_rhythmClipDuration * m_processRate)
|
c@47
|
201 / m_rhythmClipFrameSize));
|
c@47
|
202 std::cerr << "SimilarityPlugin::initialise: rhythm clip is "
|
c@47
|
203 << m_rhythmClipFrames << " frames of size "
|
c@47
|
204 << m_rhythmClipFrameSize << " at process rate "
|
c@47
|
205 << m_processRate << " ( = "
|
c@47
|
206 << (float(m_rhythmClipFrames * m_rhythmClipFrameSize) / m_processRate) << " sec )"
|
c@47
|
207 << std::endl;
|
c@47
|
208
|
c@47
|
209 MFCCConfig config(m_processRate);
|
c@47
|
210 config.fftsize = m_rhythmClipFrameSize;
|
c@47
|
211 config.nceps = m_featureColumnSize - 1;
|
c@47
|
212 config.want_c0 = true;
|
c@47
|
213 config.logpower = 1;
|
c@47
|
214 config.window = RectangularWindow; // because no overlap
|
c@47
|
215 m_rhythmfcc = new MFCC(config);
|
c@47
|
216 }
|
c@47
|
217
|
c@41
|
218 for (int i = 0; i < m_channels; ++i) {
|
c@47
|
219
|
c@42
|
220 m_values.push_back(FeatureMatrix());
|
c@47
|
221
|
c@47
|
222 if (needRhythm()) {
|
c@47
|
223 m_rhythmValues.push_back(FeatureColumnQueue());
|
c@47
|
224 }
|
c@41
|
225 }
|
c@41
|
226
|
c@47
|
227 m_done = false;
|
c@47
|
228
|
c@41
|
229 return true;
|
c@41
|
230 }
|
c@41
|
231
|
c@41
|
232 void
|
c@41
|
233 SimilarityPlugin::reset()
|
c@41
|
234 {
|
c@41
|
235 //!!!
|
c@47
|
236 m_done = false;
|
c@41
|
237 }
|
c@41
|
238
|
c@41
|
239 int
|
c@41
|
240 SimilarityPlugin::getDecimationFactor() const
|
c@41
|
241 {
|
c@41
|
242 int rate = lrintf(m_inputSampleRate);
|
c@47
|
243 return rate / m_processRate;
|
c@41
|
244 }
|
c@41
|
245
|
c@41
|
246 size_t
|
c@41
|
247 SimilarityPlugin::getPreferredStepSize() const
|
c@41
|
248 {
|
c@42
|
249 if (m_blockSize == 0) calculateBlockSize();
|
c@47
|
250
|
c@47
|
251 // there is also an assumption to this effect in process()
|
c@47
|
252 // (referring to m_fftSize/2 instead of a literal post-decimation
|
c@47
|
253 // step size):
|
c@45
|
254 return m_blockSize/2;
|
c@41
|
255 }
|
c@41
|
256
|
c@41
|
257 size_t
|
c@41
|
258 SimilarityPlugin::getPreferredBlockSize() const
|
c@41
|
259 {
|
c@42
|
260 if (m_blockSize == 0) calculateBlockSize();
|
c@42
|
261 return m_blockSize;
|
c@42
|
262 }
|
c@42
|
263
|
c@42
|
264 void
|
c@42
|
265 SimilarityPlugin::calculateBlockSize() const
|
c@42
|
266 {
|
c@42
|
267 if (m_blockSize != 0) return;
|
c@42
|
268 int decimationFactor = getDecimationFactor();
|
c@42
|
269 if (m_type == TypeChroma) {
|
c@42
|
270 ChromaConfig config;
|
c@47
|
271 config.FS = m_processRate;
|
c@42
|
272 config.min = Pitch::getFrequencyForPitch(24, 0, 440);
|
c@42
|
273 config.max = Pitch::getFrequencyForPitch(96, 0, 440);
|
c@42
|
274 config.BPO = 12;
|
c@42
|
275 config.CQThresh = 0.0054;
|
c@42
|
276 config.isNormalised = false;
|
c@42
|
277 Chromagram *c = new Chromagram(config);
|
c@42
|
278 size_t sz = c->getFrameSize();
|
c@42
|
279 delete c;
|
c@42
|
280 m_blockSize = sz * decimationFactor;
|
c@42
|
281 } else {
|
c@42
|
282 m_blockSize = 2048 * decimationFactor;
|
c@42
|
283 }
|
c@41
|
284 }
|
c@41
|
285
|
c@41
|
286 SimilarityPlugin::ParameterList SimilarityPlugin::getParameterDescriptors() const
|
c@41
|
287 {
|
c@41
|
288 ParameterList list;
|
c@42
|
289
|
c@42
|
290 ParameterDescriptor desc;
|
c@42
|
291 desc.identifier = "featureType";
|
c@42
|
292 desc.name = "Feature Type";
|
c@45
|
293 desc.description = "Audio feature used for similarity measure. Timbral: use the first 20 MFCCs (19 plus C0). Chromatic: use 12 bin-per-octave chroma.";
|
c@42
|
294 desc.unit = "";
|
c@42
|
295 desc.minValue = 0;
|
c@42
|
296 desc.maxValue = 1;
|
c@42
|
297 desc.defaultValue = 0;
|
c@42
|
298 desc.isQuantized = true;
|
c@42
|
299 desc.quantizeStep = 1;
|
c@42
|
300 desc.valueNames.push_back("Timbral (MFCC)");
|
c@42
|
301 desc.valueNames.push_back("Chromatic (Chroma)");
|
c@42
|
302 list.push_back(desc);
|
c@42
|
303
|
c@47
|
304 desc.identifier = "rhythmWeighting";
|
c@47
|
305 desc.name = "Influence of Rhythm";
|
c@47
|
306 desc.description = "Proportion of similarity measure made up from rhythmic similarity component, from 0 (entirely timbral or chromatic) to 100 (entirely rhythmic).";
|
c@47
|
307 desc.unit = "%";
|
c@47
|
308 desc.minValue = 0;
|
c@47
|
309 desc.maxValue = 100;
|
c@47
|
310 desc.defaultValue = 0;
|
c@47
|
311 desc.isQuantized = true;
|
c@47
|
312 desc.quantizeStep = 1;
|
c@47
|
313 desc.valueNames.clear();
|
c@47
|
314 list.push_back(desc);
|
c@47
|
315
|
c@41
|
316 return list;
|
c@41
|
317 }
|
c@41
|
318
|
c@41
|
319 float
|
c@41
|
320 SimilarityPlugin::getParameter(std::string param) const
|
c@41
|
321 {
|
c@42
|
322 if (param == "featureType") {
|
c@42
|
323 if (m_type == TypeMFCC) return 0;
|
c@42
|
324 else if (m_type == TypeChroma) return 1;
|
c@42
|
325 else return 0;
|
c@47
|
326 } else if (param == "rhythmWeighting") {
|
c@47
|
327 return nearbyint(m_rhythmWeighting * 100.0);
|
c@42
|
328 }
|
c@42
|
329
|
c@41
|
330 std::cerr << "WARNING: SimilarityPlugin::getParameter: unknown parameter \""
|
c@41
|
331 << param << "\"" << std::endl;
|
c@41
|
332 return 0.0;
|
c@41
|
333 }
|
c@41
|
334
|
c@41
|
335 void
|
c@41
|
336 SimilarityPlugin::setParameter(std::string param, float value)
|
c@41
|
337 {
|
c@42
|
338 if (param == "featureType") {
|
c@42
|
339 int v = int(value + 0.1);
|
c@42
|
340 Type prevType = m_type;
|
c@42
|
341 if (v == 0) m_type = TypeMFCC;
|
c@42
|
342 else if (v == 1) m_type = TypeChroma;
|
c@42
|
343 if (m_type != prevType) m_blockSize = 0;
|
c@42
|
344 return;
|
c@47
|
345 } else if (param == "rhythmWeighting") {
|
c@47
|
346 m_rhythmWeighting = value / 100;
|
c@47
|
347 return;
|
c@42
|
348 }
|
c@42
|
349
|
c@41
|
350 std::cerr << "WARNING: SimilarityPlugin::setParameter: unknown parameter \""
|
c@41
|
351 << param << "\"" << std::endl;
|
c@41
|
352 }
|
c@41
|
353
|
c@41
|
354 SimilarityPlugin::OutputList
|
c@41
|
355 SimilarityPlugin::getOutputDescriptors() const
|
c@41
|
356 {
|
c@41
|
357 OutputList list;
|
c@41
|
358
|
c@41
|
359 OutputDescriptor similarity;
|
c@43
|
360 similarity.identifier = "distancematrix";
|
c@43
|
361 similarity.name = "Distance Matrix";
|
c@43
|
362 similarity.description = "Distance matrix for similarity metric. Smaller = more similar. Should be symmetrical.";
|
c@41
|
363 similarity.unit = "";
|
c@41
|
364 similarity.hasFixedBinCount = true;
|
c@41
|
365 similarity.binCount = m_channels;
|
c@41
|
366 similarity.hasKnownExtents = false;
|
c@41
|
367 similarity.isQuantized = false;
|
c@41
|
368 similarity.sampleType = OutputDescriptor::FixedSampleRate;
|
c@41
|
369 similarity.sampleRate = 1;
|
c@41
|
370
|
c@43
|
371 m_distanceMatrixOutput = list.size();
|
c@41
|
372 list.push_back(similarity);
|
c@41
|
373
|
c@43
|
374 OutputDescriptor simvec;
|
c@43
|
375 simvec.identifier = "distancevector";
|
c@43
|
376 simvec.name = "Distance from First Channel";
|
c@43
|
377 simvec.description = "Distance vector for similarity of each channel to the first channel. Smaller = more similar.";
|
c@43
|
378 simvec.unit = "";
|
c@43
|
379 simvec.hasFixedBinCount = true;
|
c@43
|
380 simvec.binCount = m_channels;
|
c@43
|
381 simvec.hasKnownExtents = false;
|
c@43
|
382 simvec.isQuantized = false;
|
c@43
|
383 simvec.sampleType = OutputDescriptor::FixedSampleRate;
|
c@43
|
384 simvec.sampleRate = 1;
|
c@43
|
385
|
c@43
|
386 m_distanceVectorOutput = list.size();
|
c@43
|
387 list.push_back(simvec);
|
c@43
|
388
|
c@44
|
389 OutputDescriptor sortvec;
|
c@44
|
390 sortvec.identifier = "sorteddistancevector";
|
c@44
|
391 sortvec.name = "Ordered Distances from First Channel";
|
c@44
|
392 sortvec.description = "Vector of the order of other channels in similarity to the first, followed by distance vector for similarity of each to the first. Smaller = more similar.";
|
c@44
|
393 sortvec.unit = "";
|
c@44
|
394 sortvec.hasFixedBinCount = true;
|
c@44
|
395 sortvec.binCount = m_channels;
|
c@44
|
396 sortvec.hasKnownExtents = false;
|
c@44
|
397 sortvec.isQuantized = false;
|
c@44
|
398 sortvec.sampleType = OutputDescriptor::FixedSampleRate;
|
c@44
|
399 sortvec.sampleRate = 1;
|
c@44
|
400
|
c@44
|
401 m_sortedVectorOutput = list.size();
|
c@44
|
402 list.push_back(sortvec);
|
c@44
|
403
|
c@41
|
404 OutputDescriptor means;
|
c@41
|
405 means.identifier = "means";
|
c@42
|
406 means.name = "Feature Means";
|
c@43
|
407 means.description = "Means of the feature bins. Feature time (sec) corresponds to input channel. Number of bins depends on selected feature type.";
|
c@41
|
408 means.unit = "";
|
c@41
|
409 means.hasFixedBinCount = true;
|
c@43
|
410 means.binCount = m_featureColumnSize;
|
c@41
|
411 means.hasKnownExtents = false;
|
c@41
|
412 means.isQuantized = false;
|
c@43
|
413 means.sampleType = OutputDescriptor::FixedSampleRate;
|
c@43
|
414 means.sampleRate = 1;
|
c@41
|
415
|
c@43
|
416 m_meansOutput = list.size();
|
c@41
|
417 list.push_back(means);
|
c@41
|
418
|
c@41
|
419 OutputDescriptor variances;
|
c@41
|
420 variances.identifier = "variances";
|
c@42
|
421 variances.name = "Feature Variances";
|
c@43
|
422 variances.description = "Variances of the feature bins. Feature time (sec) corresponds to input channel. Number of bins depends on selected feature type.";
|
c@41
|
423 variances.unit = "";
|
c@41
|
424 variances.hasFixedBinCount = true;
|
c@43
|
425 variances.binCount = m_featureColumnSize;
|
c@41
|
426 variances.hasKnownExtents = false;
|
c@41
|
427 variances.isQuantized = false;
|
c@43
|
428 variances.sampleType = OutputDescriptor::FixedSampleRate;
|
c@43
|
429 variances.sampleRate = 1;
|
c@41
|
430
|
c@43
|
431 m_variancesOutput = list.size();
|
c@41
|
432 list.push_back(variances);
|
c@41
|
433
|
c@47
|
434 OutputDescriptor beatspectrum;
|
c@47
|
435 beatspectrum.identifier = "beatspectrum";
|
c@47
|
436 beatspectrum.name = "Beat Spectra";
|
c@47
|
437 beatspectrum.description = "Rhythmic self-similarity vectors (beat spectra) for the input channels. Feature time (sec) corresponds to input channel. Not returned if rhythm weighting is zero.";
|
c@47
|
438 beatspectrum.unit = "";
|
c@47
|
439 if (m_rhythmClipFrames > 0) {
|
c@47
|
440 beatspectrum.hasFixedBinCount = true;
|
c@47
|
441 beatspectrum.binCount = m_rhythmClipFrames / 2;
|
c@47
|
442 } else {
|
c@47
|
443 beatspectrum.hasFixedBinCount = false;
|
c@47
|
444 }
|
c@47
|
445 beatspectrum.hasKnownExtents = false;
|
c@47
|
446 beatspectrum.isQuantized = false;
|
c@47
|
447 beatspectrum.sampleType = OutputDescriptor::FixedSampleRate;
|
c@47
|
448 beatspectrum.sampleRate = 1;
|
c@47
|
449
|
c@47
|
450 m_beatSpectraOutput = list.size();
|
c@47
|
451 list.push_back(beatspectrum);
|
c@47
|
452
|
c@41
|
453 return list;
|
c@41
|
454 }
|
c@41
|
455
|
c@41
|
456 SimilarityPlugin::FeatureSet
|
c@41
|
457 SimilarityPlugin::process(const float *const *inputBuffers, Vamp::RealTime /* timestamp */)
|
c@41
|
458 {
|
c@47
|
459 if (m_done) {
|
c@47
|
460 return FeatureSet();
|
c@47
|
461 }
|
c@47
|
462
|
c@41
|
463 double *dblbuf = new double[m_blockSize];
|
c@41
|
464 double *decbuf = dblbuf;
|
c@42
|
465 if (m_decimator) decbuf = new double[m_fftSize];
|
c@42
|
466
|
c@47
|
467 double *raw = new double[std::max(m_featureColumnSize,
|
c@47
|
468 m_rhythmColumnSize)];
|
c@41
|
469
|
c@43
|
470 float threshold = 1e-10;
|
c@43
|
471
|
c@47
|
472 bool someRhythmFrameNeeded = false;
|
c@47
|
473
|
c@41
|
474 for (size_t c = 0; c < m_channels; ++c) {
|
c@41
|
475
|
c@43
|
476 bool empty = true;
|
c@43
|
477
|
c@41
|
478 for (int i = 0; i < m_blockSize; ++i) {
|
c@43
|
479 float val = inputBuffers[c][i];
|
c@43
|
480 if (fabs(val) > threshold) empty = false;
|
c@43
|
481 dblbuf[i] = val;
|
c@41
|
482 }
|
c@41
|
483
|
c@47
|
484 if (empty) {
|
c@47
|
485 if (needRhythm() && ((m_frameNo % 2) == 0)) {
|
c@47
|
486 for (int i = 0; i < m_fftSize / m_rhythmClipFrameSize; ++i) {
|
c@47
|
487 if (m_rhythmValues[c].size() < m_rhythmClipFrames) {
|
c@47
|
488 FeatureColumn mf(m_rhythmColumnSize);
|
c@47
|
489 for (int i = 0; i < m_rhythmColumnSize; ++i) {
|
c@47
|
490 mf[i] = 0.0;
|
c@47
|
491 }
|
c@47
|
492 m_rhythmValues[c].push_back(mf);
|
c@47
|
493 }
|
c@47
|
494 }
|
c@47
|
495 }
|
c@47
|
496 continue;
|
c@47
|
497 }
|
c@47
|
498
|
c@44
|
499 m_lastNonEmptyFrame[c] = m_frameNo;
|
c@43
|
500
|
c@41
|
501 if (m_decimator) {
|
c@41
|
502 m_decimator->process(dblbuf, decbuf);
|
c@41
|
503 }
|
c@42
|
504
|
c@47
|
505 if (needTimbre()) {
|
c@47
|
506
|
c@47
|
507 if (m_type == TypeMFCC) {
|
c@47
|
508 m_mfcc->process(decbuf, raw);
|
c@47
|
509 } else if (m_type == TypeChroma) {
|
c@47
|
510 raw = m_chromagram->process(decbuf);
|
c@47
|
511 }
|
c@41
|
512
|
c@47
|
513 FeatureColumn mf(m_featureColumnSize);
|
c@47
|
514 for (int i = 0; i < m_featureColumnSize; ++i) {
|
c@47
|
515 mf[i] = raw[i];
|
c@47
|
516 }
|
c@47
|
517
|
c@47
|
518 m_values[c].push_back(mf);
|
c@44
|
519 }
|
c@41
|
520
|
c@47
|
521 // std::cerr << "needRhythm = " << needRhythm() << ", frame = " << m_frameNo << std::endl;
|
c@47
|
522
|
c@47
|
523 if (needRhythm() && ((m_frameNo % 2) == 0)) {
|
c@47
|
524
|
c@47
|
525 // The incoming frames are overlapping; we only use every
|
c@47
|
526 // other one, because we don't want the overlap (it would
|
c@47
|
527 // screw up the rhythm)
|
c@47
|
528
|
c@47
|
529 int frameOffset = 0;
|
c@47
|
530
|
c@47
|
531 while (frameOffset + m_rhythmClipFrameSize <= m_fftSize) {
|
c@47
|
532
|
c@47
|
533 bool needRhythmFrame = true;
|
c@47
|
534
|
c@47
|
535 if (m_rhythmValues[c].size() >= m_rhythmClipFrames) {
|
c@47
|
536
|
c@47
|
537 needRhythmFrame = false;
|
c@47
|
538
|
c@47
|
539 // assumes hopsize = framesize/2
|
c@47
|
540 float current = m_frameNo * (m_fftSize/2) + frameOffset;
|
c@47
|
541 current = current / m_processRate;
|
c@47
|
542 if (current - m_rhythmClipDuration < m_rhythmClipOrigin) {
|
c@47
|
543 needRhythmFrame = true;
|
c@47
|
544 m_rhythmValues[c].pop_front();
|
c@47
|
545 }
|
c@47
|
546
|
c@47
|
547 if (needRhythmFrame) {
|
c@47
|
548 std::cerr << "at current = " <<current << " (frame = " << m_frameNo << "), have " << m_rhythmValues[c].size() << ", need rhythm = " << needRhythmFrame << std::endl;
|
c@47
|
549 }
|
c@47
|
550
|
c@47
|
551 }
|
c@47
|
552
|
c@47
|
553 if (needRhythmFrame) {
|
c@47
|
554
|
c@47
|
555 someRhythmFrameNeeded = true;
|
c@47
|
556
|
c@47
|
557 m_rhythmfcc->process(decbuf + frameOffset, raw);
|
c@47
|
558
|
c@47
|
559 FeatureColumn mf(m_rhythmColumnSize);
|
c@47
|
560 for (int i = 0; i < m_rhythmColumnSize; ++i) {
|
c@47
|
561 mf[i] = raw[i];
|
c@47
|
562 }
|
c@47
|
563
|
c@47
|
564 m_rhythmValues[c].push_back(mf);
|
c@47
|
565 }
|
c@47
|
566
|
c@47
|
567 frameOffset += m_rhythmClipFrameSize;
|
c@47
|
568 }
|
c@47
|
569 }
|
c@47
|
570 }
|
c@47
|
571
|
c@47
|
572 if (!needTimbre() && !someRhythmFrameNeeded && ((m_frameNo % 2) == 0)) {
|
c@47
|
573 std::cerr << "done!" << std::endl;
|
c@47
|
574 m_done = true;
|
c@41
|
575 }
|
c@41
|
576
|
c@41
|
577 if (m_decimator) delete[] decbuf;
|
c@41
|
578 delete[] dblbuf;
|
c@47
|
579 delete[] raw;
|
c@41
|
580
|
c@44
|
581 ++m_frameNo;
|
c@44
|
582
|
c@41
|
583 return FeatureSet();
|
c@41
|
584 }
|
c@41
|
585
|
c@47
|
586 SimilarityPlugin::FeatureMatrix
|
c@47
|
587 SimilarityPlugin::calculateTimbral(FeatureSet &returnFeatures)
|
c@41
|
588 {
|
c@47
|
589 FeatureMatrix m(m_channels); // means
|
c@47
|
590 FeatureMatrix v(m_channels); // variances
|
c@41
|
591
|
c@41
|
592 for (int i = 0; i < m_channels; ++i) {
|
c@41
|
593
|
c@42
|
594 FeatureColumn mean(m_featureColumnSize), variance(m_featureColumnSize);
|
c@41
|
595
|
c@42
|
596 for (int j = 0; j < m_featureColumnSize; ++j) {
|
c@41
|
597
|
c@43
|
598 mean[j] = 0.0;
|
c@43
|
599 variance[j] = 0.0;
|
c@41
|
600 int count;
|
c@41
|
601
|
c@44
|
602 // We want to take values up to, but not including, the
|
c@44
|
603 // last non-empty frame (which may be partial)
|
c@43
|
604
|
c@44
|
605 int sz = m_lastNonEmptyFrame[i];
|
c@44
|
606 if (sz < 0) sz = 0;
|
c@43
|
607
|
c@41
|
608 count = 0;
|
c@43
|
609 for (int k = 0; k < sz; ++k) {
|
c@42
|
610 double val = m_values[i][k][j];
|
c@41
|
611 if (isnan(val) || isinf(val)) continue;
|
c@41
|
612 mean[j] += val;
|
c@41
|
613 ++count;
|
c@41
|
614 }
|
c@41
|
615 if (count > 0) mean[j] /= count;
|
c@41
|
616
|
c@41
|
617 count = 0;
|
c@43
|
618 for (int k = 0; k < sz; ++k) {
|
c@42
|
619 double val = ((m_values[i][k][j] - mean[j]) *
|
c@42
|
620 (m_values[i][k][j] - mean[j]));
|
c@41
|
621 if (isnan(val) || isinf(val)) continue;
|
c@41
|
622 variance[j] += val;
|
c@41
|
623 ++count;
|
c@41
|
624 }
|
c@41
|
625 if (count > 0) variance[j] /= count;
|
c@41
|
626 }
|
c@41
|
627
|
c@41
|
628 m[i] = mean;
|
c@41
|
629 v[i] = variance;
|
c@41
|
630 }
|
c@41
|
631
|
c@42
|
632 // "Despite the fact that MFCCs extracted from music are clearly
|
c@42
|
633 // not Gaussian, [14] showed, somewhat surprisingly, that a
|
c@42
|
634 // similarity function comparing single Gaussians modelling MFCCs
|
c@42
|
635 // for each track can perform as well as mixture models. A great
|
c@42
|
636 // advantage of using single Gaussians is that a simple closed
|
c@42
|
637 // form exists for the KL divergence." -- Mark Levy, "Lightweight
|
c@42
|
638 // measures for timbral similarity of musical audio"
|
c@42
|
639 // (http://www.elec.qmul.ac.uk/easaier/papers/mlevytimbralsimilarity.pdf)
|
c@46
|
640
|
c@46
|
641 KLDivergence kld;
|
c@47
|
642 FeatureMatrix distances(m_channels);
|
c@42
|
643
|
c@41
|
644 for (int i = 0; i < m_channels; ++i) {
|
c@41
|
645 for (int j = 0; j < m_channels; ++j) {
|
c@46
|
646 double d = kld.distance(m[i], v[i], m[j], v[j]);
|
c@41
|
647 distances[i].push_back(d);
|
c@41
|
648 }
|
c@41
|
649 }
|
c@47
|
650
|
c@44
|
651 Feature feature;
|
c@44
|
652 feature.hasTimestamp = true;
|
c@44
|
653
|
c@44
|
654 char labelBuffer[100];
|
c@43
|
655
|
c@41
|
656 for (int i = 0; i < m_channels; ++i) {
|
c@41
|
657
|
c@41
|
658 feature.timestamp = Vamp::RealTime(i, 0);
|
c@41
|
659
|
c@44
|
660 sprintf(labelBuffer, "Means for channel %d", i+1);
|
c@44
|
661 feature.label = labelBuffer;
|
c@44
|
662
|
c@41
|
663 feature.values.clear();
|
c@42
|
664 for (int k = 0; k < m_featureColumnSize; ++k) {
|
c@41
|
665 feature.values.push_back(m[i][k]);
|
c@41
|
666 }
|
c@41
|
667
|
c@43
|
668 returnFeatures[m_meansOutput].push_back(feature);
|
c@41
|
669
|
c@44
|
670 sprintf(labelBuffer, "Variances for channel %d", i+1);
|
c@44
|
671 feature.label = labelBuffer;
|
c@44
|
672
|
c@41
|
673 feature.values.clear();
|
c@42
|
674 for (int k = 0; k < m_featureColumnSize; ++k) {
|
c@41
|
675 feature.values.push_back(v[i][k]);
|
c@41
|
676 }
|
c@41
|
677
|
c@43
|
678 returnFeatures[m_variancesOutput].push_back(feature);
|
c@47
|
679 }
|
c@47
|
680
|
c@47
|
681 return distances;
|
c@47
|
682 }
|
c@47
|
683
|
c@47
|
684 SimilarityPlugin::FeatureMatrix
|
c@47
|
685 SimilarityPlugin::calculateRhythmic(FeatureSet &returnFeatures)
|
c@47
|
686 {
|
c@47
|
687 if (!needRhythm()) return FeatureMatrix();
|
c@47
|
688
|
c@47
|
689 BeatSpectrum bscalc;
|
c@47
|
690 CosineDistance cd;
|
c@47
|
691
|
c@47
|
692 // Our rhythm feature matrix is a deque of vectors for practical
|
c@47
|
693 // reasons, but BeatSpectrum::process wants a vector of vectors
|
c@47
|
694 // (which is what FeatureMatrix happens to be).
|
c@47
|
695
|
c@47
|
696 FeatureMatrixSet bsinput(m_channels);
|
c@47
|
697 for (int i = 0; i < m_channels; ++i) {
|
c@47
|
698 for (int j = 0; j < m_rhythmValues[i].size(); ++j) {
|
c@47
|
699 bsinput[i].push_back(m_rhythmValues[i][j]);
|
c@47
|
700 }
|
c@47
|
701 }
|
c@47
|
702
|
c@47
|
703 FeatureMatrix bs(m_channels);
|
c@47
|
704 for (int i = 0; i < m_channels; ++i) {
|
c@47
|
705 bs[i] = bscalc.process(bsinput[i]);
|
c@47
|
706 }
|
c@47
|
707
|
c@47
|
708 FeatureMatrix distances(m_channels);
|
c@47
|
709 for (int i = 0; i < m_channels; ++i) {
|
c@47
|
710 for (int j = 0; j < m_channels; ++j) {
|
c@47
|
711 double d = cd.distance(bs[i], bs[j]);
|
c@47
|
712 distances[i].push_back(d);
|
c@47
|
713 }
|
c@47
|
714 }
|
c@47
|
715
|
c@47
|
716 Feature feature;
|
c@47
|
717 feature.hasTimestamp = true;
|
c@47
|
718
|
c@47
|
719 char labelBuffer[100];
|
c@47
|
720
|
c@47
|
721 for (int i = 0; i < m_channels; ++i) {
|
c@47
|
722
|
c@47
|
723 feature.timestamp = Vamp::RealTime(i, 0);
|
c@47
|
724
|
c@47
|
725 sprintf(labelBuffer, "Beat spectrum for channel %d", i+1);
|
c@47
|
726 feature.label = labelBuffer;
|
c@47
|
727
|
c@47
|
728 feature.values.clear();
|
c@47
|
729 for (int j = 0; j < bs[i].size(); ++j) {
|
c@47
|
730 feature.values.push_back(bs[i][j]);
|
c@47
|
731 }
|
c@47
|
732
|
c@47
|
733 returnFeatures[m_beatSpectraOutput].push_back(feature);
|
c@47
|
734 }
|
c@47
|
735
|
c@47
|
736 return distances;
|
c@47
|
737 }
|
c@47
|
738
|
c@47
|
739 double
|
c@47
|
740 SimilarityPlugin::getDistance(const FeatureMatrix &timbral,
|
c@47
|
741 const FeatureMatrix &rhythmic,
|
c@47
|
742 int i, int j)
|
c@47
|
743 {
|
c@47
|
744 double distance = 1.0;
|
c@47
|
745 if (needTimbre()) distance *= timbral[i][j];
|
c@47
|
746 if (needRhythm()) distance *= rhythmic[i][j];
|
c@47
|
747 return distance;
|
c@47
|
748 }
|
c@47
|
749
|
c@47
|
750 SimilarityPlugin::FeatureSet
|
c@47
|
751 SimilarityPlugin::getRemainingFeatures()
|
c@47
|
752 {
|
c@47
|
753 FeatureSet returnFeatures;
|
c@47
|
754
|
c@47
|
755 // We want to return a matrix of the distances between channels,
|
c@47
|
756 // but Vamp doesn't have a matrix return type so we will actually
|
c@47
|
757 // return a series of vectors
|
c@47
|
758
|
c@47
|
759 FeatureMatrix timbralDistances, rhythmicDistances;
|
c@47
|
760
|
c@47
|
761 if (needTimbre()) {
|
c@47
|
762 timbralDistances = calculateTimbral(returnFeatures);
|
c@47
|
763 }
|
c@47
|
764
|
c@47
|
765 if (needRhythm()) {
|
c@47
|
766 rhythmicDistances = calculateRhythmic(returnFeatures);
|
c@47
|
767 }
|
c@47
|
768
|
c@47
|
769 // We give all features a timestamp, otherwise hosts will tend to
|
c@47
|
770 // stamp them at the end of the file, which is annoying
|
c@47
|
771
|
c@47
|
772 Feature feature;
|
c@47
|
773 feature.hasTimestamp = true;
|
c@47
|
774
|
c@47
|
775 Feature distanceVectorFeature;
|
c@47
|
776 distanceVectorFeature.label = "Distance from first channel";
|
c@47
|
777 distanceVectorFeature.hasTimestamp = true;
|
c@47
|
778 distanceVectorFeature.timestamp = Vamp::RealTime::zeroTime;
|
c@47
|
779
|
c@47
|
780 std::map<double, int> sorted;
|
c@47
|
781
|
c@47
|
782 char labelBuffer[100];
|
c@47
|
783
|
c@47
|
784 for (int i = 0; i < m_channels; ++i) {
|
c@47
|
785
|
c@47
|
786 feature.timestamp = Vamp::RealTime(i, 0);
|
c@41
|
787
|
c@41
|
788 feature.values.clear();
|
c@41
|
789 for (int j = 0; j < m_channels; ++j) {
|
c@47
|
790 double dist = getDistance(timbralDistances, rhythmicDistances, i, j);
|
c@47
|
791 feature.values.push_back(dist);
|
c@41
|
792 }
|
c@43
|
793
|
c@44
|
794 sprintf(labelBuffer, "Distances from channel %d", i+1);
|
c@44
|
795 feature.label = labelBuffer;
|
c@41
|
796
|
c@43
|
797 returnFeatures[m_distanceMatrixOutput].push_back(feature);
|
c@43
|
798
|
c@47
|
799 double fromFirst =
|
c@47
|
800 getDistance(timbralDistances, rhythmicDistances, 0, i);
|
c@44
|
801
|
c@47
|
802 distanceVectorFeature.values.push_back(fromFirst);
|
c@47
|
803 sorted[fromFirst] = i;
|
c@41
|
804 }
|
c@41
|
805
|
c@43
|
806 returnFeatures[m_distanceVectorOutput].push_back(distanceVectorFeature);
|
c@43
|
807
|
c@44
|
808 feature.label = "Order of channels by similarity to first channel";
|
c@44
|
809 feature.values.clear();
|
c@44
|
810 feature.timestamp = Vamp::RealTime(0, 0);
|
c@44
|
811
|
c@44
|
812 for (std::map<double, int>::iterator i = sorted.begin();
|
c@44
|
813 i != sorted.end(); ++i) {
|
c@45
|
814 feature.values.push_back(i->second + 1);
|
c@44
|
815 }
|
c@44
|
816
|
c@44
|
817 returnFeatures[m_sortedVectorOutput].push_back(feature);
|
c@44
|
818
|
c@44
|
819 feature.label = "Ordered distances of channels from first channel";
|
c@44
|
820 feature.values.clear();
|
c@44
|
821 feature.timestamp = Vamp::RealTime(1, 0);
|
c@44
|
822
|
c@44
|
823 for (std::map<double, int>::iterator i = sorted.begin();
|
c@44
|
824 i != sorted.end(); ++i) {
|
c@44
|
825 feature.values.push_back(i->first);
|
c@44
|
826 }
|
c@44
|
827
|
c@44
|
828 returnFeatures[m_sortedVectorOutput].push_back(feature);
|
c@44
|
829
|
c@41
|
830 return returnFeatures;
|
c@41
|
831 }
|