annotate plugins/SimilarityPlugin.cpp @ 47:f8c5f11e60a6

* Add rhythmic similarity to similarity plugin. Needs testing, and the current weighting parameter (not properly used) needs revision.
author Chris Cannam <c.cannam@qmul.ac.uk>
date Fri, 18 Jan 2008 18:11:01 +0000
parents 26a2e341d358
children 3b4572153ce3
rev   line source
c@41 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
c@41 2
c@41 3 /*
c@41 4 * SegmenterPlugin.cpp
c@41 5 *
c@41 6 * Copyright 2008 Centre for Digital Music, Queen Mary, University of London.
c@41 7 * All rights reserved.
c@41 8 */
c@41 9
c@41 10 #include <iostream>
c@44 11 #include <cstdio>
c@41 12
c@41 13 #include "SimilarityPlugin.h"
c@42 14 #include "base/Pitch.h"
c@41 15 #include "dsp/mfcc/MFCC.h"
c@42 16 #include "dsp/chromagram/Chromagram.h"
c@41 17 #include "dsp/rateconversion/Decimator.h"
c@47 18 #include "dsp/rhythm/BeatSpectrum.h"
c@47 19 #include "maths/KLDivergence.h"
c@47 20 #include "maths/CosineDistance.h"
c@41 21
c@41 22 using std::string;
c@41 23 using std::vector;
c@41 24 using std::cerr;
c@41 25 using std::endl;
c@41 26 using std::ostringstream;
c@41 27
c@47 28 const float
c@47 29 SimilarityPlugin::m_noRhythm = 0.009;
c@47 30
c@47 31 const float
c@47 32 SimilarityPlugin::m_allRhythm = 0.991;
c@47 33
c@41 34 SimilarityPlugin::SimilarityPlugin(float inputSampleRate) :
c@41 35 Plugin(inputSampleRate),
c@42 36 m_type(TypeMFCC),
c@41 37 m_mfcc(0),
c@47 38 m_rhythmfcc(0),
c@42 39 m_chromagram(0),
c@41 40 m_decimator(0),
c@42 41 m_featureColumnSize(20),
c@47 42 m_rhythmWeighting(0.f),
c@47 43 m_rhythmClipDuration(4.f), // seconds
c@47 44 m_rhythmClipOrigin(40.f), // seconds
c@47 45 m_rhythmClipFrameSize(0),
c@47 46 m_rhythmClipFrames(0),
c@47 47 m_rhythmColumnSize(20),
c@41 48 m_blockSize(0),
c@47 49 m_channels(0),
c@47 50 m_processRate(0),
c@47 51 m_frameNo(0),
c@47 52 m_done(false)
c@41 53 {
c@47 54 int rate = lrintf(m_inputSampleRate);
c@47 55 int internalRate = 22050;
c@47 56 int decimationFactor = rate / internalRate;
c@47 57 if (decimationFactor < 1) decimationFactor = 1;
c@47 58
c@47 59 // must be a power of two
c@47 60 while (decimationFactor & (decimationFactor - 1)) ++decimationFactor;
c@47 61
c@47 62 m_processRate = rate / decimationFactor; // may be 22050, 24000 etc
c@41 63 }
c@41 64
c@41 65 SimilarityPlugin::~SimilarityPlugin()
c@41 66 {
c@41 67 delete m_mfcc;
c@47 68 delete m_rhythmfcc;
c@42 69 delete m_chromagram;
c@41 70 delete m_decimator;
c@41 71 }
c@41 72
c@41 73 string
c@41 74 SimilarityPlugin::getIdentifier() const
c@41 75 {
c@41 76 return "qm-similarity";
c@41 77 }
c@41 78
c@41 79 string
c@41 80 SimilarityPlugin::getName() const
c@41 81 {
c@41 82 return "Similarity";
c@41 83 }
c@41 84
c@41 85 string
c@41 86 SimilarityPlugin::getDescription() const
c@41 87 {
c@42 88 return "Return a distance matrix for similarity between the input audio channels";
c@41 89 }
c@41 90
c@41 91 string
c@41 92 SimilarityPlugin::getMaker() const
c@41 93 {
c@47 94 return "Mark Levy, Kurt Jacobson and Chris Cannam, Queen Mary, University of London";
c@41 95 }
c@41 96
c@41 97 int
c@41 98 SimilarityPlugin::getPluginVersion() const
c@41 99 {
c@41 100 return 1;
c@41 101 }
c@41 102
c@41 103 string
c@41 104 SimilarityPlugin::getCopyright() const
c@41 105 {
c@41 106 return "Copyright (c) 2008 - All Rights Reserved";
c@41 107 }
c@41 108
c@41 109 size_t
c@41 110 SimilarityPlugin::getMinChannelCount() const
c@41 111 {
c@43 112 return 1;
c@41 113 }
c@41 114
c@41 115 size_t
c@41 116 SimilarityPlugin::getMaxChannelCount() const
c@41 117 {
c@41 118 return 1024;
c@41 119 }
c@41 120
c@41 121 bool
c@41 122 SimilarityPlugin::initialise(size_t channels, size_t stepSize, size_t blockSize)
c@41 123 {
c@41 124 if (channels < getMinChannelCount() ||
c@41 125 channels > getMaxChannelCount()) return false;
c@41 126
c@41 127 if (stepSize != getPreferredStepSize()) {
c@43 128 //!!! actually this perhaps shouldn't be an error... similarly
c@43 129 //using more than getMaxChannelCount channels
c@41 130 std::cerr << "SimilarityPlugin::initialise: supplied step size "
c@41 131 << stepSize << " differs from required step size "
c@41 132 << getPreferredStepSize() << std::endl;
c@41 133 return false;
c@41 134 }
c@41 135
c@41 136 if (blockSize != getPreferredBlockSize()) {
c@41 137 std::cerr << "SimilarityPlugin::initialise: supplied block size "
c@41 138 << blockSize << " differs from required block size "
c@41 139 << getPreferredBlockSize() << std::endl;
c@41 140 return false;
c@41 141 }
c@41 142
c@41 143 m_blockSize = blockSize;
c@41 144 m_channels = channels;
c@41 145
c@44 146 m_lastNonEmptyFrame = std::vector<int>(m_channels);
c@44 147 for (int i = 0; i < m_channels; ++i) m_lastNonEmptyFrame[i] = -1;
c@44 148 m_frameNo = 0;
c@44 149
c@41 150 int decimationFactor = getDecimationFactor();
c@41 151 if (decimationFactor > 1) {
c@42 152 m_decimator = new Decimator(m_blockSize, decimationFactor);
c@41 153 }
c@41 154
c@42 155 if (m_type == TypeMFCC) {
c@42 156
c@42 157 m_featureColumnSize = 20;
c@42 158
c@47 159 MFCCConfig config(m_processRate);
c@42 160 config.fftsize = 2048;
c@42 161 config.nceps = m_featureColumnSize - 1;
c@42 162 config.want_c0 = true;
c@45 163 config.logpower = 1;
c@42 164 m_mfcc = new MFCC(config);
c@42 165 m_fftSize = m_mfcc->getfftlength();
c@47 166 m_rhythmClipFrameSize = m_fftSize / 4;
c@42 167
c@43 168 std::cerr << "MFCC FS = " << config.FS << ", FFT size = " << m_fftSize<< std::endl;
c@43 169
c@42 170 } else if (m_type == TypeChroma) {
c@42 171
c@42 172 m_featureColumnSize = 12;
c@42 173
c@42 174 ChromaConfig config;
c@47 175 config.FS = m_processRate;
c@42 176 config.min = Pitch::getFrequencyForPitch(24, 0, 440);
c@42 177 config.max = Pitch::getFrequencyForPitch(96, 0, 440);
c@42 178 config.BPO = 12;
c@42 179 config.CQThresh = 0.0054;
c@42 180 config.isNormalised = true;
c@42 181 m_chromagram = new Chromagram(config);
c@42 182 m_fftSize = m_chromagram->getFrameSize();
c@42 183
c@47 184 std::cerr << "fftsize = " << m_fftSize << std::endl;
c@47 185
c@47 186 m_rhythmClipFrameSize = m_fftSize / 16;
c@47 187 while (m_rhythmClipFrameSize < 512) m_rhythmClipFrameSize *= 2;
c@47 188 std::cerr << "m_rhythmClipFrameSize = " << m_rhythmClipFrameSize << std::endl;
c@47 189
c@42 190 std::cerr << "min = "<< config.min << ", max = " << config.max << std::endl;
c@42 191
c@42 192 } else {
c@42 193
c@42 194 std::cerr << "SimilarityPlugin::initialise: internal error: unknown type " << m_type << std::endl;
c@42 195 return false;
c@42 196 }
c@41 197
c@47 198 if (needRhythm()) {
c@47 199 m_rhythmClipFrames =
c@47 200 int(ceil((m_rhythmClipDuration * m_processRate)
c@47 201 / m_rhythmClipFrameSize));
c@47 202 std::cerr << "SimilarityPlugin::initialise: rhythm clip is "
c@47 203 << m_rhythmClipFrames << " frames of size "
c@47 204 << m_rhythmClipFrameSize << " at process rate "
c@47 205 << m_processRate << " ( = "
c@47 206 << (float(m_rhythmClipFrames * m_rhythmClipFrameSize) / m_processRate) << " sec )"
c@47 207 << std::endl;
c@47 208
c@47 209 MFCCConfig config(m_processRate);
c@47 210 config.fftsize = m_rhythmClipFrameSize;
c@47 211 config.nceps = m_featureColumnSize - 1;
c@47 212 config.want_c0 = true;
c@47 213 config.logpower = 1;
c@47 214 config.window = RectangularWindow; // because no overlap
c@47 215 m_rhythmfcc = new MFCC(config);
c@47 216 }
c@47 217
c@41 218 for (int i = 0; i < m_channels; ++i) {
c@47 219
c@42 220 m_values.push_back(FeatureMatrix());
c@47 221
c@47 222 if (needRhythm()) {
c@47 223 m_rhythmValues.push_back(FeatureColumnQueue());
c@47 224 }
c@41 225 }
c@41 226
c@47 227 m_done = false;
c@47 228
c@41 229 return true;
c@41 230 }
c@41 231
c@41 232 void
c@41 233 SimilarityPlugin::reset()
c@41 234 {
c@41 235 //!!!
c@47 236 m_done = false;
c@41 237 }
c@41 238
c@41 239 int
c@41 240 SimilarityPlugin::getDecimationFactor() const
c@41 241 {
c@41 242 int rate = lrintf(m_inputSampleRate);
c@47 243 return rate / m_processRate;
c@41 244 }
c@41 245
c@41 246 size_t
c@41 247 SimilarityPlugin::getPreferredStepSize() const
c@41 248 {
c@42 249 if (m_blockSize == 0) calculateBlockSize();
c@47 250
c@47 251 // there is also an assumption to this effect in process()
c@47 252 // (referring to m_fftSize/2 instead of a literal post-decimation
c@47 253 // step size):
c@45 254 return m_blockSize/2;
c@41 255 }
c@41 256
c@41 257 size_t
c@41 258 SimilarityPlugin::getPreferredBlockSize() const
c@41 259 {
c@42 260 if (m_blockSize == 0) calculateBlockSize();
c@42 261 return m_blockSize;
c@42 262 }
c@42 263
c@42 264 void
c@42 265 SimilarityPlugin::calculateBlockSize() const
c@42 266 {
c@42 267 if (m_blockSize != 0) return;
c@42 268 int decimationFactor = getDecimationFactor();
c@42 269 if (m_type == TypeChroma) {
c@42 270 ChromaConfig config;
c@47 271 config.FS = m_processRate;
c@42 272 config.min = Pitch::getFrequencyForPitch(24, 0, 440);
c@42 273 config.max = Pitch::getFrequencyForPitch(96, 0, 440);
c@42 274 config.BPO = 12;
c@42 275 config.CQThresh = 0.0054;
c@42 276 config.isNormalised = false;
c@42 277 Chromagram *c = new Chromagram(config);
c@42 278 size_t sz = c->getFrameSize();
c@42 279 delete c;
c@42 280 m_blockSize = sz * decimationFactor;
c@42 281 } else {
c@42 282 m_blockSize = 2048 * decimationFactor;
c@42 283 }
c@41 284 }
c@41 285
c@41 286 SimilarityPlugin::ParameterList SimilarityPlugin::getParameterDescriptors() const
c@41 287 {
c@41 288 ParameterList list;
c@42 289
c@42 290 ParameterDescriptor desc;
c@42 291 desc.identifier = "featureType";
c@42 292 desc.name = "Feature Type";
c@45 293 desc.description = "Audio feature used for similarity measure. Timbral: use the first 20 MFCCs (19 plus C0). Chromatic: use 12 bin-per-octave chroma.";
c@42 294 desc.unit = "";
c@42 295 desc.minValue = 0;
c@42 296 desc.maxValue = 1;
c@42 297 desc.defaultValue = 0;
c@42 298 desc.isQuantized = true;
c@42 299 desc.quantizeStep = 1;
c@42 300 desc.valueNames.push_back("Timbral (MFCC)");
c@42 301 desc.valueNames.push_back("Chromatic (Chroma)");
c@42 302 list.push_back(desc);
c@42 303
c@47 304 desc.identifier = "rhythmWeighting";
c@47 305 desc.name = "Influence of Rhythm";
c@47 306 desc.description = "Proportion of similarity measure made up from rhythmic similarity component, from 0 (entirely timbral or chromatic) to 100 (entirely rhythmic).";
c@47 307 desc.unit = "%";
c@47 308 desc.minValue = 0;
c@47 309 desc.maxValue = 100;
c@47 310 desc.defaultValue = 0;
c@47 311 desc.isQuantized = true;
c@47 312 desc.quantizeStep = 1;
c@47 313 desc.valueNames.clear();
c@47 314 list.push_back(desc);
c@47 315
c@41 316 return list;
c@41 317 }
c@41 318
c@41 319 float
c@41 320 SimilarityPlugin::getParameter(std::string param) const
c@41 321 {
c@42 322 if (param == "featureType") {
c@42 323 if (m_type == TypeMFCC) return 0;
c@42 324 else if (m_type == TypeChroma) return 1;
c@42 325 else return 0;
c@47 326 } else if (param == "rhythmWeighting") {
c@47 327 return nearbyint(m_rhythmWeighting * 100.0);
c@42 328 }
c@42 329
c@41 330 std::cerr << "WARNING: SimilarityPlugin::getParameter: unknown parameter \""
c@41 331 << param << "\"" << std::endl;
c@41 332 return 0.0;
c@41 333 }
c@41 334
c@41 335 void
c@41 336 SimilarityPlugin::setParameter(std::string param, float value)
c@41 337 {
c@42 338 if (param == "featureType") {
c@42 339 int v = int(value + 0.1);
c@42 340 Type prevType = m_type;
c@42 341 if (v == 0) m_type = TypeMFCC;
c@42 342 else if (v == 1) m_type = TypeChroma;
c@42 343 if (m_type != prevType) m_blockSize = 0;
c@42 344 return;
c@47 345 } else if (param == "rhythmWeighting") {
c@47 346 m_rhythmWeighting = value / 100;
c@47 347 return;
c@42 348 }
c@42 349
c@41 350 std::cerr << "WARNING: SimilarityPlugin::setParameter: unknown parameter \""
c@41 351 << param << "\"" << std::endl;
c@41 352 }
c@41 353
c@41 354 SimilarityPlugin::OutputList
c@41 355 SimilarityPlugin::getOutputDescriptors() const
c@41 356 {
c@41 357 OutputList list;
c@41 358
c@41 359 OutputDescriptor similarity;
c@43 360 similarity.identifier = "distancematrix";
c@43 361 similarity.name = "Distance Matrix";
c@43 362 similarity.description = "Distance matrix for similarity metric. Smaller = more similar. Should be symmetrical.";
c@41 363 similarity.unit = "";
c@41 364 similarity.hasFixedBinCount = true;
c@41 365 similarity.binCount = m_channels;
c@41 366 similarity.hasKnownExtents = false;
c@41 367 similarity.isQuantized = false;
c@41 368 similarity.sampleType = OutputDescriptor::FixedSampleRate;
c@41 369 similarity.sampleRate = 1;
c@41 370
c@43 371 m_distanceMatrixOutput = list.size();
c@41 372 list.push_back(similarity);
c@41 373
c@43 374 OutputDescriptor simvec;
c@43 375 simvec.identifier = "distancevector";
c@43 376 simvec.name = "Distance from First Channel";
c@43 377 simvec.description = "Distance vector for similarity of each channel to the first channel. Smaller = more similar.";
c@43 378 simvec.unit = "";
c@43 379 simvec.hasFixedBinCount = true;
c@43 380 simvec.binCount = m_channels;
c@43 381 simvec.hasKnownExtents = false;
c@43 382 simvec.isQuantized = false;
c@43 383 simvec.sampleType = OutputDescriptor::FixedSampleRate;
c@43 384 simvec.sampleRate = 1;
c@43 385
c@43 386 m_distanceVectorOutput = list.size();
c@43 387 list.push_back(simvec);
c@43 388
c@44 389 OutputDescriptor sortvec;
c@44 390 sortvec.identifier = "sorteddistancevector";
c@44 391 sortvec.name = "Ordered Distances from First Channel";
c@44 392 sortvec.description = "Vector of the order of other channels in similarity to the first, followed by distance vector for similarity of each to the first. Smaller = more similar.";
c@44 393 sortvec.unit = "";
c@44 394 sortvec.hasFixedBinCount = true;
c@44 395 sortvec.binCount = m_channels;
c@44 396 sortvec.hasKnownExtents = false;
c@44 397 sortvec.isQuantized = false;
c@44 398 sortvec.sampleType = OutputDescriptor::FixedSampleRate;
c@44 399 sortvec.sampleRate = 1;
c@44 400
c@44 401 m_sortedVectorOutput = list.size();
c@44 402 list.push_back(sortvec);
c@44 403
c@41 404 OutputDescriptor means;
c@41 405 means.identifier = "means";
c@42 406 means.name = "Feature Means";
c@43 407 means.description = "Means of the feature bins. Feature time (sec) corresponds to input channel. Number of bins depends on selected feature type.";
c@41 408 means.unit = "";
c@41 409 means.hasFixedBinCount = true;
c@43 410 means.binCount = m_featureColumnSize;
c@41 411 means.hasKnownExtents = false;
c@41 412 means.isQuantized = false;
c@43 413 means.sampleType = OutputDescriptor::FixedSampleRate;
c@43 414 means.sampleRate = 1;
c@41 415
c@43 416 m_meansOutput = list.size();
c@41 417 list.push_back(means);
c@41 418
c@41 419 OutputDescriptor variances;
c@41 420 variances.identifier = "variances";
c@42 421 variances.name = "Feature Variances";
c@43 422 variances.description = "Variances of the feature bins. Feature time (sec) corresponds to input channel. Number of bins depends on selected feature type.";
c@41 423 variances.unit = "";
c@41 424 variances.hasFixedBinCount = true;
c@43 425 variances.binCount = m_featureColumnSize;
c@41 426 variances.hasKnownExtents = false;
c@41 427 variances.isQuantized = false;
c@43 428 variances.sampleType = OutputDescriptor::FixedSampleRate;
c@43 429 variances.sampleRate = 1;
c@41 430
c@43 431 m_variancesOutput = list.size();
c@41 432 list.push_back(variances);
c@41 433
c@47 434 OutputDescriptor beatspectrum;
c@47 435 beatspectrum.identifier = "beatspectrum";
c@47 436 beatspectrum.name = "Beat Spectra";
c@47 437 beatspectrum.description = "Rhythmic self-similarity vectors (beat spectra) for the input channels. Feature time (sec) corresponds to input channel. Not returned if rhythm weighting is zero.";
c@47 438 beatspectrum.unit = "";
c@47 439 if (m_rhythmClipFrames > 0) {
c@47 440 beatspectrum.hasFixedBinCount = true;
c@47 441 beatspectrum.binCount = m_rhythmClipFrames / 2;
c@47 442 } else {
c@47 443 beatspectrum.hasFixedBinCount = false;
c@47 444 }
c@47 445 beatspectrum.hasKnownExtents = false;
c@47 446 beatspectrum.isQuantized = false;
c@47 447 beatspectrum.sampleType = OutputDescriptor::FixedSampleRate;
c@47 448 beatspectrum.sampleRate = 1;
c@47 449
c@47 450 m_beatSpectraOutput = list.size();
c@47 451 list.push_back(beatspectrum);
c@47 452
c@41 453 return list;
c@41 454 }
c@41 455
c@41 456 SimilarityPlugin::FeatureSet
c@41 457 SimilarityPlugin::process(const float *const *inputBuffers, Vamp::RealTime /* timestamp */)
c@41 458 {
c@47 459 if (m_done) {
c@47 460 return FeatureSet();
c@47 461 }
c@47 462
c@41 463 double *dblbuf = new double[m_blockSize];
c@41 464 double *decbuf = dblbuf;
c@42 465 if (m_decimator) decbuf = new double[m_fftSize];
c@42 466
c@47 467 double *raw = new double[std::max(m_featureColumnSize,
c@47 468 m_rhythmColumnSize)];
c@41 469
c@43 470 float threshold = 1e-10;
c@43 471
c@47 472 bool someRhythmFrameNeeded = false;
c@47 473
c@41 474 for (size_t c = 0; c < m_channels; ++c) {
c@41 475
c@43 476 bool empty = true;
c@43 477
c@41 478 for (int i = 0; i < m_blockSize; ++i) {
c@43 479 float val = inputBuffers[c][i];
c@43 480 if (fabs(val) > threshold) empty = false;
c@43 481 dblbuf[i] = val;
c@41 482 }
c@41 483
c@47 484 if (empty) {
c@47 485 if (needRhythm() && ((m_frameNo % 2) == 0)) {
c@47 486 for (int i = 0; i < m_fftSize / m_rhythmClipFrameSize; ++i) {
c@47 487 if (m_rhythmValues[c].size() < m_rhythmClipFrames) {
c@47 488 FeatureColumn mf(m_rhythmColumnSize);
c@47 489 for (int i = 0; i < m_rhythmColumnSize; ++i) {
c@47 490 mf[i] = 0.0;
c@47 491 }
c@47 492 m_rhythmValues[c].push_back(mf);
c@47 493 }
c@47 494 }
c@47 495 }
c@47 496 continue;
c@47 497 }
c@47 498
c@44 499 m_lastNonEmptyFrame[c] = m_frameNo;
c@43 500
c@41 501 if (m_decimator) {
c@41 502 m_decimator->process(dblbuf, decbuf);
c@41 503 }
c@42 504
c@47 505 if (needTimbre()) {
c@47 506
c@47 507 if (m_type == TypeMFCC) {
c@47 508 m_mfcc->process(decbuf, raw);
c@47 509 } else if (m_type == TypeChroma) {
c@47 510 raw = m_chromagram->process(decbuf);
c@47 511 }
c@41 512
c@47 513 FeatureColumn mf(m_featureColumnSize);
c@47 514 for (int i = 0; i < m_featureColumnSize; ++i) {
c@47 515 mf[i] = raw[i];
c@47 516 }
c@47 517
c@47 518 m_values[c].push_back(mf);
c@44 519 }
c@41 520
c@47 521 // std::cerr << "needRhythm = " << needRhythm() << ", frame = " << m_frameNo << std::endl;
c@47 522
c@47 523 if (needRhythm() && ((m_frameNo % 2) == 0)) {
c@47 524
c@47 525 // The incoming frames are overlapping; we only use every
c@47 526 // other one, because we don't want the overlap (it would
c@47 527 // screw up the rhythm)
c@47 528
c@47 529 int frameOffset = 0;
c@47 530
c@47 531 while (frameOffset + m_rhythmClipFrameSize <= m_fftSize) {
c@47 532
c@47 533 bool needRhythmFrame = true;
c@47 534
c@47 535 if (m_rhythmValues[c].size() >= m_rhythmClipFrames) {
c@47 536
c@47 537 needRhythmFrame = false;
c@47 538
c@47 539 // assumes hopsize = framesize/2
c@47 540 float current = m_frameNo * (m_fftSize/2) + frameOffset;
c@47 541 current = current / m_processRate;
c@47 542 if (current - m_rhythmClipDuration < m_rhythmClipOrigin) {
c@47 543 needRhythmFrame = true;
c@47 544 m_rhythmValues[c].pop_front();
c@47 545 }
c@47 546
c@47 547 if (needRhythmFrame) {
c@47 548 std::cerr << "at current = " <<current << " (frame = " << m_frameNo << "), have " << m_rhythmValues[c].size() << ", need rhythm = " << needRhythmFrame << std::endl;
c@47 549 }
c@47 550
c@47 551 }
c@47 552
c@47 553 if (needRhythmFrame) {
c@47 554
c@47 555 someRhythmFrameNeeded = true;
c@47 556
c@47 557 m_rhythmfcc->process(decbuf + frameOffset, raw);
c@47 558
c@47 559 FeatureColumn mf(m_rhythmColumnSize);
c@47 560 for (int i = 0; i < m_rhythmColumnSize; ++i) {
c@47 561 mf[i] = raw[i];
c@47 562 }
c@47 563
c@47 564 m_rhythmValues[c].push_back(mf);
c@47 565 }
c@47 566
c@47 567 frameOffset += m_rhythmClipFrameSize;
c@47 568 }
c@47 569 }
c@47 570 }
c@47 571
c@47 572 if (!needTimbre() && !someRhythmFrameNeeded && ((m_frameNo % 2) == 0)) {
c@47 573 std::cerr << "done!" << std::endl;
c@47 574 m_done = true;
c@41 575 }
c@41 576
c@41 577 if (m_decimator) delete[] decbuf;
c@41 578 delete[] dblbuf;
c@47 579 delete[] raw;
c@41 580
c@44 581 ++m_frameNo;
c@44 582
c@41 583 return FeatureSet();
c@41 584 }
c@41 585
c@47 586 SimilarityPlugin::FeatureMatrix
c@47 587 SimilarityPlugin::calculateTimbral(FeatureSet &returnFeatures)
c@41 588 {
c@47 589 FeatureMatrix m(m_channels); // means
c@47 590 FeatureMatrix v(m_channels); // variances
c@41 591
c@41 592 for (int i = 0; i < m_channels; ++i) {
c@41 593
c@42 594 FeatureColumn mean(m_featureColumnSize), variance(m_featureColumnSize);
c@41 595
c@42 596 for (int j = 0; j < m_featureColumnSize; ++j) {
c@41 597
c@43 598 mean[j] = 0.0;
c@43 599 variance[j] = 0.0;
c@41 600 int count;
c@41 601
c@44 602 // We want to take values up to, but not including, the
c@44 603 // last non-empty frame (which may be partial)
c@43 604
c@44 605 int sz = m_lastNonEmptyFrame[i];
c@44 606 if (sz < 0) sz = 0;
c@43 607
c@41 608 count = 0;
c@43 609 for (int k = 0; k < sz; ++k) {
c@42 610 double val = m_values[i][k][j];
c@41 611 if (isnan(val) || isinf(val)) continue;
c@41 612 mean[j] += val;
c@41 613 ++count;
c@41 614 }
c@41 615 if (count > 0) mean[j] /= count;
c@41 616
c@41 617 count = 0;
c@43 618 for (int k = 0; k < sz; ++k) {
c@42 619 double val = ((m_values[i][k][j] - mean[j]) *
c@42 620 (m_values[i][k][j] - mean[j]));
c@41 621 if (isnan(val) || isinf(val)) continue;
c@41 622 variance[j] += val;
c@41 623 ++count;
c@41 624 }
c@41 625 if (count > 0) variance[j] /= count;
c@41 626 }
c@41 627
c@41 628 m[i] = mean;
c@41 629 v[i] = variance;
c@41 630 }
c@41 631
c@42 632 // "Despite the fact that MFCCs extracted from music are clearly
c@42 633 // not Gaussian, [14] showed, somewhat surprisingly, that a
c@42 634 // similarity function comparing single Gaussians modelling MFCCs
c@42 635 // for each track can perform as well as mixture models. A great
c@42 636 // advantage of using single Gaussians is that a simple closed
c@42 637 // form exists for the KL divergence." -- Mark Levy, "Lightweight
c@42 638 // measures for timbral similarity of musical audio"
c@42 639 // (http://www.elec.qmul.ac.uk/easaier/papers/mlevytimbralsimilarity.pdf)
c@46 640
c@46 641 KLDivergence kld;
c@47 642 FeatureMatrix distances(m_channels);
c@42 643
c@41 644 for (int i = 0; i < m_channels; ++i) {
c@41 645 for (int j = 0; j < m_channels; ++j) {
c@46 646 double d = kld.distance(m[i], v[i], m[j], v[j]);
c@41 647 distances[i].push_back(d);
c@41 648 }
c@41 649 }
c@47 650
c@44 651 Feature feature;
c@44 652 feature.hasTimestamp = true;
c@44 653
c@44 654 char labelBuffer[100];
c@43 655
c@41 656 for (int i = 0; i < m_channels; ++i) {
c@41 657
c@41 658 feature.timestamp = Vamp::RealTime(i, 0);
c@41 659
c@44 660 sprintf(labelBuffer, "Means for channel %d", i+1);
c@44 661 feature.label = labelBuffer;
c@44 662
c@41 663 feature.values.clear();
c@42 664 for (int k = 0; k < m_featureColumnSize; ++k) {
c@41 665 feature.values.push_back(m[i][k]);
c@41 666 }
c@41 667
c@43 668 returnFeatures[m_meansOutput].push_back(feature);
c@41 669
c@44 670 sprintf(labelBuffer, "Variances for channel %d", i+1);
c@44 671 feature.label = labelBuffer;
c@44 672
c@41 673 feature.values.clear();
c@42 674 for (int k = 0; k < m_featureColumnSize; ++k) {
c@41 675 feature.values.push_back(v[i][k]);
c@41 676 }
c@41 677
c@43 678 returnFeatures[m_variancesOutput].push_back(feature);
c@47 679 }
c@47 680
c@47 681 return distances;
c@47 682 }
c@47 683
c@47 684 SimilarityPlugin::FeatureMatrix
c@47 685 SimilarityPlugin::calculateRhythmic(FeatureSet &returnFeatures)
c@47 686 {
c@47 687 if (!needRhythm()) return FeatureMatrix();
c@47 688
c@47 689 BeatSpectrum bscalc;
c@47 690 CosineDistance cd;
c@47 691
c@47 692 // Our rhythm feature matrix is a deque of vectors for practical
c@47 693 // reasons, but BeatSpectrum::process wants a vector of vectors
c@47 694 // (which is what FeatureMatrix happens to be).
c@47 695
c@47 696 FeatureMatrixSet bsinput(m_channels);
c@47 697 for (int i = 0; i < m_channels; ++i) {
c@47 698 for (int j = 0; j < m_rhythmValues[i].size(); ++j) {
c@47 699 bsinput[i].push_back(m_rhythmValues[i][j]);
c@47 700 }
c@47 701 }
c@47 702
c@47 703 FeatureMatrix bs(m_channels);
c@47 704 for (int i = 0; i < m_channels; ++i) {
c@47 705 bs[i] = bscalc.process(bsinput[i]);
c@47 706 }
c@47 707
c@47 708 FeatureMatrix distances(m_channels);
c@47 709 for (int i = 0; i < m_channels; ++i) {
c@47 710 for (int j = 0; j < m_channels; ++j) {
c@47 711 double d = cd.distance(bs[i], bs[j]);
c@47 712 distances[i].push_back(d);
c@47 713 }
c@47 714 }
c@47 715
c@47 716 Feature feature;
c@47 717 feature.hasTimestamp = true;
c@47 718
c@47 719 char labelBuffer[100];
c@47 720
c@47 721 for (int i = 0; i < m_channels; ++i) {
c@47 722
c@47 723 feature.timestamp = Vamp::RealTime(i, 0);
c@47 724
c@47 725 sprintf(labelBuffer, "Beat spectrum for channel %d", i+1);
c@47 726 feature.label = labelBuffer;
c@47 727
c@47 728 feature.values.clear();
c@47 729 for (int j = 0; j < bs[i].size(); ++j) {
c@47 730 feature.values.push_back(bs[i][j]);
c@47 731 }
c@47 732
c@47 733 returnFeatures[m_beatSpectraOutput].push_back(feature);
c@47 734 }
c@47 735
c@47 736 return distances;
c@47 737 }
c@47 738
c@47 739 double
c@47 740 SimilarityPlugin::getDistance(const FeatureMatrix &timbral,
c@47 741 const FeatureMatrix &rhythmic,
c@47 742 int i, int j)
c@47 743 {
c@47 744 double distance = 1.0;
c@47 745 if (needTimbre()) distance *= timbral[i][j];
c@47 746 if (needRhythm()) distance *= rhythmic[i][j];
c@47 747 return distance;
c@47 748 }
c@47 749
c@47 750 SimilarityPlugin::FeatureSet
c@47 751 SimilarityPlugin::getRemainingFeatures()
c@47 752 {
c@47 753 FeatureSet returnFeatures;
c@47 754
c@47 755 // We want to return a matrix of the distances between channels,
c@47 756 // but Vamp doesn't have a matrix return type so we will actually
c@47 757 // return a series of vectors
c@47 758
c@47 759 FeatureMatrix timbralDistances, rhythmicDistances;
c@47 760
c@47 761 if (needTimbre()) {
c@47 762 timbralDistances = calculateTimbral(returnFeatures);
c@47 763 }
c@47 764
c@47 765 if (needRhythm()) {
c@47 766 rhythmicDistances = calculateRhythmic(returnFeatures);
c@47 767 }
c@47 768
c@47 769 // We give all features a timestamp, otherwise hosts will tend to
c@47 770 // stamp them at the end of the file, which is annoying
c@47 771
c@47 772 Feature feature;
c@47 773 feature.hasTimestamp = true;
c@47 774
c@47 775 Feature distanceVectorFeature;
c@47 776 distanceVectorFeature.label = "Distance from first channel";
c@47 777 distanceVectorFeature.hasTimestamp = true;
c@47 778 distanceVectorFeature.timestamp = Vamp::RealTime::zeroTime;
c@47 779
c@47 780 std::map<double, int> sorted;
c@47 781
c@47 782 char labelBuffer[100];
c@47 783
c@47 784 for (int i = 0; i < m_channels; ++i) {
c@47 785
c@47 786 feature.timestamp = Vamp::RealTime(i, 0);
c@41 787
c@41 788 feature.values.clear();
c@41 789 for (int j = 0; j < m_channels; ++j) {
c@47 790 double dist = getDistance(timbralDistances, rhythmicDistances, i, j);
c@47 791 feature.values.push_back(dist);
c@41 792 }
c@43 793
c@44 794 sprintf(labelBuffer, "Distances from channel %d", i+1);
c@44 795 feature.label = labelBuffer;
c@41 796
c@43 797 returnFeatures[m_distanceMatrixOutput].push_back(feature);
c@43 798
c@47 799 double fromFirst =
c@47 800 getDistance(timbralDistances, rhythmicDistances, 0, i);
c@44 801
c@47 802 distanceVectorFeature.values.push_back(fromFirst);
c@47 803 sorted[fromFirst] = i;
c@41 804 }
c@41 805
c@43 806 returnFeatures[m_distanceVectorOutput].push_back(distanceVectorFeature);
c@43 807
c@44 808 feature.label = "Order of channels by similarity to first channel";
c@44 809 feature.values.clear();
c@44 810 feature.timestamp = Vamp::RealTime(0, 0);
c@44 811
c@44 812 for (std::map<double, int>::iterator i = sorted.begin();
c@44 813 i != sorted.end(); ++i) {
c@45 814 feature.values.push_back(i->second + 1);
c@44 815 }
c@44 816
c@44 817 returnFeatures[m_sortedVectorOutput].push_back(feature);
c@44 818
c@44 819 feature.label = "Ordered distances of channels from first channel";
c@44 820 feature.values.clear();
c@44 821 feature.timestamp = Vamp::RealTime(1, 0);
c@44 822
c@44 823 for (std::map<double, int>::iterator i = sorted.begin();
c@44 824 i != sorted.end(); ++i) {
c@44 825 feature.values.push_back(i->first);
c@44 826 }
c@44 827
c@44 828 returnFeatures[m_sortedVectorOutput].push_back(feature);
c@44 829
c@41 830 return returnFeatures;
c@41 831 }