annotate data/fileio/CSVFileReader.cpp @ 1886:f803d3c33f76 tip

Switch off copious debug in soft synth driving
author Chris Cannam
date Fri, 14 Aug 2020 10:44:44 +0100
parents 566476eeeb80
children
rev   line source
Chris@148 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@148 2
Chris@148 3 /*
Chris@148 4 Sonic Visualiser
Chris@148 5 An audio file viewer and annotation editor.
Chris@148 6 Centre for Digital Music, Queen Mary, University of London.
Chris@148 7 This file copyright 2006 Chris Cannam.
Chris@148 8
Chris@148 9 This program is free software; you can redistribute it and/or
Chris@148 10 modify it under the terms of the GNU General Public License as
Chris@148 11 published by the Free Software Foundation; either version 2 of the
Chris@148 12 License, or (at your option) any later version. See the file
Chris@148 13 COPYING included with this distribution for more information.
Chris@148 14 */
Chris@148 15
Chris@148 16 #include "CSVFileReader.h"
Chris@148 17
Chris@150 18 #include "model/Model.h"
Chris@148 19 #include "base/RealTime.h"
Chris@631 20 #include "base/StringBits.h"
Chris@1491 21 #include "base/ProgressReporter.h"
Chris@1519 22 #include "base/RecordDirectory.h"
Chris@148 23 #include "model/SparseOneDimensionalModel.h"
Chris@148 24 #include "model/SparseTimeValueModel.h"
Chris@152 25 #include "model/EditableDenseThreeDimensionalModel.h"
Chris@628 26 #include "model/RegionModel.h"
Chris@897 27 #include "model/NoteModel.h"
Chris@1793 28 #include "model/BoxModel.h"
Chris@1488 29 #include "model/WritableWaveFileModel.h"
Chris@308 30 #include "DataFileReaderFactory.h"
Chris@148 31
Chris@148 32 #include <QFile>
Chris@1519 33 #include <QDir>
Chris@1030 34 #include <QFileInfo>
Chris@148 35 #include <QString>
Chris@148 36 #include <QRegExp>
Chris@148 37 #include <QStringList>
Chris@148 38 #include <QTextStream>
Chris@1519 39 #include <QDateTime>
Chris@148 40
Chris@148 41 #include <iostream>
Chris@628 42 #include <map>
Chris@1428 43 #include <string>
Chris@148 44
Chris@1113 45 using namespace std;
Chris@1113 46
Chris@392 47 CSVFileReader::CSVFileReader(QString path, CSVFormat format,
Chris@1491 48 sv_samplerate_t mainModelSampleRate,
Chris@1491 49 ProgressReporter *reporter) :
Chris@392 50 m_format(format),
Chris@1582 51 m_device(nullptr),
Chris@1009 52 m_ownDevice(true),
Chris@631 53 m_warnings(0),
Chris@1491 54 m_mainModelSampleRate(mainModelSampleRate),
Chris@1491 55 m_fileSize(0),
Chris@1491 56 m_readCount(0),
Chris@1492 57 m_progress(-1),
Chris@1491 58 m_reporter(reporter)
Chris@148 59 {
Chris@1009 60 QFile *file = new QFile(path);
Chris@148 61 bool good = false;
Chris@148 62
Chris@1009 63 if (!file->exists()) {
Chris@1429 64 m_error = QFile::tr("File \"%1\" does not exist").arg(path);
Chris@1009 65 } else if (!file->open(QIODevice::ReadOnly | QIODevice::Text)) {
Chris@1429 66 m_error = QFile::tr("Failed to open file \"%1\"").arg(path);
Chris@148 67 } else {
Chris@1429 68 good = true;
Chris@148 69 }
Chris@148 70
Chris@1009 71 if (good) {
Chris@1009 72 m_device = file;
Chris@1030 73 m_filename = QFileInfo(path).fileName();
Chris@1491 74 m_fileSize = file->size();
Chris@1491 75 if (m_reporter) m_reporter->setDefinite(true);
Chris@1009 76 } else {
Chris@1429 77 delete file;
Chris@148 78 }
Chris@148 79 }
Chris@148 80
Chris@1009 81 CSVFileReader::CSVFileReader(QIODevice *device, CSVFormat format,
Chris@1491 82 sv_samplerate_t mainModelSampleRate,
Chris@1491 83 ProgressReporter *reporter) :
Chris@1009 84 m_format(format),
Chris@1009 85 m_device(device),
Chris@1009 86 m_ownDevice(false),
Chris@1009 87 m_warnings(0),
Chris@1491 88 m_mainModelSampleRate(mainModelSampleRate),
Chris@1491 89 m_fileSize(0),
Chris@1491 90 m_readCount(0),
Chris@1492 91 m_progress(-1),
Chris@1491 92 m_reporter(reporter)
Chris@1009 93 {
Chris@1491 94 if (m_reporter) m_reporter->setDefinite(false);
Chris@1009 95 }
Chris@1009 96
Chris@148 97 CSVFileReader::~CSVFileReader()
Chris@148 98 {
Chris@1009 99 SVDEBUG << "CSVFileReader::~CSVFileReader: device is " << m_device << endl;
Chris@148 100
Chris@1009 101 if (m_device && m_ownDevice) {
Chris@1009 102 SVDEBUG << "CSVFileReader::CSVFileReader: Closing device" << endl;
Chris@1009 103 m_device->close();
Chris@1009 104 delete m_device;
Chris@148 105 }
Chris@148 106 }
Chris@148 107
Chris@148 108 bool
Chris@148 109 CSVFileReader::isOK() const
Chris@148 110 {
Chris@1582 111 return (m_device != nullptr);
Chris@148 112 }
Chris@148 113
Chris@148 114 QString
Chris@148 115 CSVFileReader::getError() const
Chris@148 116 {
Chris@148 117 return m_error;
Chris@148 118 }
Chris@148 119
Chris@1867 120 bool
Chris@1047 121 CSVFileReader::convertTimeValue(QString s, int lineno,
Chris@1047 122 sv_samplerate_t sampleRate,
Chris@1867 123 int windowSize,
Chris@1867 124 sv_frame_t &calculatedFrame) const
Chris@631 125 {
Chris@631 126 QRegExp nonNumericRx("[^0-9eE.,+-]");
Chris@897 127 int warnLimit = 10;
Chris@631 128
Chris@631 129 CSVFormat::TimeUnits timeUnits = m_format.getTimeUnits();
Chris@631 130
Chris@1867 131 calculatedFrame = 0;
Chris@631 132
Chris@631 133 bool ok = false;
Chris@631 134 QString numeric = s;
Chris@631 135 numeric.remove(nonNumericRx);
Chris@631 136
Chris@631 137 if (timeUnits == CSVFormat::TimeSeconds) {
Chris@631 138
Chris@631 139 double time = numeric.toDouble(&ok);
Chris@631 140 if (!ok) time = StringBits::stringToDoubleLocaleFree(numeric, &ok);
Chris@1038 141 calculatedFrame = sv_frame_t(time * sampleRate + 0.5);
Chris@990 142
Chris@990 143 } else if (timeUnits == CSVFormat::TimeMilliseconds) {
Chris@990 144
Chris@990 145 double time = numeric.toDouble(&ok);
Chris@990 146 if (!ok) time = StringBits::stringToDoubleLocaleFree(numeric, &ok);
Chris@1038 147 calculatedFrame = sv_frame_t((time / 1000.0) * sampleRate + 0.5);
Chris@631 148
Chris@631 149 } else {
Chris@631 150
Chris@631 151 long n = numeric.toLong(&ok);
Chris@631 152 if (n >= 0) calculatedFrame = n;
Chris@631 153
Chris@631 154 if (timeUnits == CSVFormat::TimeWindows) {
Chris@631 155 calculatedFrame *= windowSize;
Chris@631 156 }
Chris@631 157 }
Chris@631 158
Chris@631 159 if (!ok) {
Chris@631 160 if (m_warnings < warnLimit) {
Chris@1428 161 SVCERR << "WARNING: CSVFileReader::load: "
Chris@844 162 << "Bad time format (\"" << s
Chris@631 163 << "\") in data line "
Chris@843 164 << lineno+1 << endl;
Chris@631 165 } else if (m_warnings == warnLimit) {
Chris@1428 166 SVCERR << "WARNING: Too many warnings" << endl;
Chris@631 167 }
Chris@631 168 ++m_warnings;
Chris@631 169 }
Chris@631 170
Chris@631 171 return calculatedFrame;
Chris@631 172 }
Chris@631 173
Chris@148 174 Model *
Chris@148 175 CSVFileReader::load() const
Chris@148 176 {
Chris@1582 177 if (!m_device) return nullptr;
Chris@148 178
Chris@628 179 CSVFormat::ModelType modelType = m_format.getModelType();
Chris@392 180 CSVFormat::TimingType timingType = m_format.getTimingType();
Chris@628 181 CSVFormat::TimeUnits timeUnits = m_format.getTimeUnits();
Chris@1047 182 sv_samplerate_t sampleRate = m_format.getSampleRate();
Chris@929 183 int windowSize = m_format.getWindowSize();
Chris@631 184 QChar separator = m_format.getSeparator();
Chris@631 185 bool allowQuoting = m_format.getAllowQuoting();
Chris@148 186
Chris@392 187 if (timingType == CSVFormat::ExplicitTiming) {
Chris@611 188 if (modelType == CSVFormat::ThreeDimensionalModel) {
Chris@611 189 // This will be overridden later if more than one line
Chris@611 190 // appears in our file, but we want to choose a default
Chris@611 191 // that's likely to be visible
Chris@611 192 windowSize = 1024;
Chris@611 193 } else {
Chris@611 194 windowSize = 1;
Chris@611 195 }
Chris@1429 196 if (timeUnits == CSVFormat::TimeSeconds ||
Chris@990 197 timeUnits == CSVFormat::TimeMilliseconds) {
Chris@1429 198 sampleRate = m_mainModelSampleRate;
Chris@1429 199 }
Chris@148 200 }
Chris@148 201
Chris@1582 202 SparseOneDimensionalModel *model1 = nullptr;
Chris@1582 203 SparseTimeValueModel *model2 = nullptr;
Chris@1582 204 RegionModel *model2a = nullptr;
Chris@1582 205 NoteModel *model2b = nullptr;
Chris@1793 206 BoxModel *model2c = nullptr;
Chris@1582 207 EditableDenseThreeDimensionalModel *model3 = nullptr;
Chris@1582 208 WritableWaveFileModel *modelW = nullptr;
Chris@1582 209 Model *model = nullptr;
Chris@148 210
Chris@1009 211 QTextStream in(m_device);
Chris@148 212
Chris@148 213 unsigned int warnings = 0, warnLimit = 10;
Chris@148 214 unsigned int lineno = 0;
Chris@148 215
Chris@148 216 float min = 0.0, max = 0.0;
Chris@148 217
Chris@1038 218 sv_frame_t frameNo = 0;
Chris@1038 219 sv_frame_t duration = 0;
Chris@1038 220 sv_frame_t endFrame = 0;
Chris@631 221
Chris@631 222 bool haveAnyValue = false;
Chris@631 223 bool haveEndTime = false;
Chris@897 224 bool pitchLooksLikeMIDI = true;
Chris@631 225
Chris@1038 226 sv_frame_t startFrame = 0; // for calculation of dense model resolution
Chris@631 227 bool firstEverValue = true;
Chris@631 228
Chris@676 229 int valueColumns = 0;
Chris@676 230 for (int i = 0; i < m_format.getColumnCount(); ++i) {
Chris@676 231 if (m_format.getColumnPurpose(i) == CSVFormat::ColumnValue) {
Chris@676 232 ++valueColumns;
Chris@676 233 }
Chris@676 234 }
Chris@676 235
Chris@1518 236 int audioChannels = 0;
Chris@1582 237 float **audioSamples = nullptr;
Chris@1518 238 float sampleShift = 0.f;
Chris@1518 239 float sampleScale = 1.f;
Chris@1518 240
Chris@1518 241 if (modelType == CSVFormat::WaveFileModel) {
Chris@1518 242
Chris@1518 243 audioChannels = valueColumns;
Chris@1518 244
Chris@1518 245 audioSamples =
Chris@1518 246 breakfastquay::allocate_and_zero_channels<float>
Chris@1518 247 (audioChannels, 1);
Chris@1518 248
Chris@1518 249 switch (m_format.getAudioSampleRange()) {
Chris@1518 250 case CSVFormat::SampleRangeSigned1:
Chris@1518 251 case CSVFormat::SampleRangeOther:
Chris@1518 252 sampleShift = 0.f;
Chris@1518 253 sampleScale = 1.f;
Chris@1518 254 break;
Chris@1518 255 case CSVFormat::SampleRangeUnsigned255:
Chris@1518 256 sampleShift = -128.f;
Chris@1518 257 sampleScale = 1.f / 128.f;
Chris@1518 258 break;
Chris@1518 259 case CSVFormat::SampleRangeSigned32767:
Chris@1518 260 sampleShift = 0.f;
Chris@1518 261 sampleScale = 1.f / 32768.f;
Chris@1518 262 break;
Chris@1518 263 }
Chris@1518 264 }
Chris@1518 265
Chris@1518 266 map<QString, int> labelCountMap;
Chris@1518 267
Chris@1872 268 bool atStart = true;
Chris@1509 269 bool abandoned = false;
Chris@1509 270
Chris@1509 271 while (!in.atEnd() && !abandoned) {
Chris@148 272
Chris@283 273 // QTextStream's readLine doesn't cope with old-style Mac
Chris@283 274 // CR-only line endings. Why did they bother making the class
Chris@283 275 // cope with more than one sort of line ending, if it still
Chris@283 276 // can't be configured to cope with all the common sorts?
Chris@148 277
Chris@283 278 // For the time being we'll deal with this case (which is
Chris@283 279 // relatively uncommon for us, but still necessary to handle)
Chris@283 280 // by reading the entire file using a single readLine, and
Chris@283 281 // splitting it. For CR and CR/LF line endings this will just
Chris@283 282 // read a line at a time, and that's obviously OK.
Chris@148 283
Chris@283 284 QString chunk = in.readLine();
Chris@283 285 QStringList lines = chunk.split('\r', QString::SkipEmptyParts);
Chris@1491 286
Chris@1491 287 m_readCount += chunk.size() + 1;
Chris@1491 288
Chris@1491 289 if (m_reporter) {
Chris@1491 290 if (m_reporter->wasCancelled()) {
Chris@1509 291 abandoned = true;
Chris@1491 292 break;
Chris@1491 293 }
Chris@1491 294 int progress;
Chris@1491 295 if (m_fileSize > 0) {
Chris@1491 296 progress = int((double(m_readCount) / double(m_fileSize))
Chris@1491 297 * 100.0);
Chris@1491 298 } else {
Chris@1491 299 progress = int(m_readCount / 10000);
Chris@1491 300 }
Chris@1491 301 if (progress != m_progress) {
Chris@1491 302 m_reporter->setProgress(progress);
Chris@1491 303 m_progress = progress;
Chris@1491 304 }
Chris@1491 305 }
Chris@283 306
Chris@897 307 for (int li = 0; li < lines.size(); ++li) {
Chris@1872 308
Chris@1872 309 QString line = lines[li];
Chris@1872 310 if (line.startsWith("#")) continue;
Chris@148 311
Chris@1872 312 if (atStart) {
Chris@1872 313 atStart = false;
Chris@1872 314 if (m_format.getHeaderStatus() == CSVFormat::HeaderPresent) {
Chris@1872 315 continue;
Chris@1872 316 }
Chris@1872 317 }
Chris@283 318
Chris@631 319 QStringList list = StringBits::split(line, separator, allowQuoting);
Chris@283 320 if (!model) {
Chris@283 321
Chris@1519 322 QString modelName = m_filename;
Chris@1519 323
Chris@283 324 switch (modelType) {
Chris@283 325
Chris@392 326 case CSVFormat::OneDimensionalModel:
Chris@1867 327 SVDEBUG << "CSVFileReader: Creating sparse one-dimensional model" << endl;
Chris@283 328 model1 = new SparseOneDimensionalModel(sampleRate, windowSize);
Chris@283 329 model = model1;
Chris@283 330 break;
Chris@1429 331
Chris@392 332 case CSVFormat::TwoDimensionalModel:
Chris@1867 333 SVDEBUG << "CSVFileReader: Creating sparse time-value model" << endl;
Chris@283 334 model2 = new SparseTimeValueModel(sampleRate, windowSize, false);
Chris@283 335 model = model2;
Chris@283 336 break;
Chris@1429 337
Chris@628 338 case CSVFormat::TwoDimensionalModelWithDuration:
Chris@1867 339 SVDEBUG << "CSVFileReader: Creating region model" << endl;
Chris@628 340 model2a = new RegionModel(sampleRate, windowSize, false);
Chris@628 341 model = model2a;
Chris@628 342 break;
Chris@1429 343
Chris@897 344 case CSVFormat::TwoDimensionalModelWithDurationAndPitch:
Chris@1867 345 SVDEBUG << "CSVFileReader: Creating note model" << endl;
Chris@897 346 model2b = new NoteModel(sampleRate, windowSize, false);
Chris@897 347 model = model2b;
Chris@897 348 break;
Chris@1429 349
Chris@1793 350 case CSVFormat::TwoDimensionalModelWithDurationAndExtent:
Chris@1867 351 SVDEBUG << "CSVFileReader: Creating box model" << endl;
Chris@1793 352 model2c = new BoxModel(sampleRate, windowSize, false);
Chris@1793 353 model = model2c;
Chris@1793 354 break;
Chris@1793 355
Chris@392 356 case CSVFormat::ThreeDimensionalModel:
Chris@1867 357 SVDEBUG << "CSVFileReader: Creating editable dense three-dimensional model" << endl;
Chris@535 358 model3 = new EditableDenseThreeDimensionalModel
Chris@1777 359 (sampleRate, windowSize, valueColumns);
Chris@283 360 model = model3;
Chris@283 361 break;
Chris@1488 362
Chris@1488 363 case CSVFormat::WaveFileModel:
Chris@1517 364 {
Chris@1867 365 SVDEBUG << "CSVFileReader: Creating writable wave-file model" << endl;
Chris@1517 366 bool normalise = (m_format.getAudioSampleRange()
Chris@1517 367 == CSVFormat::SampleRangeOther);
Chris@1519 368 QString path = getConvertedAudioFilePath();
Chris@1488 369 modelW = new WritableWaveFileModel
Chris@1520 370 (path, sampleRate, valueColumns,
Chris@1520 371 normalise ?
Chris@1520 372 WritableWaveFileModel::Normalisation::Peak :
Chris@1520 373 WritableWaveFileModel::Normalisation::None);
Chris@1519 374 modelName = QFileInfo(path).fileName();
Chris@1488 375 model = modelW;
Chris@1488 376 break;
Chris@283 377 }
Chris@1517 378 }
Chris@1030 379
Chris@1508 380 if (model && model->isOK()) {
Chris@1519 381 if (modelName != "") {
Chris@1519 382 model->setObjectName(modelName);
Chris@1030 383 }
Chris@1030 384 }
Chris@283 385 }
Chris@148 386
Chris@1508 387 if (!model || !model->isOK()) {
Chris@1508 388 SVCERR << "Failed to create model to load CSV file into"
Chris@1508 389 << endl;
Chris@1508 390 if (model) {
Chris@1508 391 delete model;
Chris@1582 392 model = nullptr;
Chris@1793 393 model1 = nullptr; model2 = nullptr;
Chris@1793 394 model2a = nullptr; model2b = nullptr; model2c = nullptr;
Chris@1582 395 model3 = nullptr; modelW = nullptr;
Chris@1508 396 }
Chris@1509 397 abandoned = true;
Chris@1508 398 break;
Chris@1508 399 }
Chris@1508 400
Chris@631 401 float value = 0.f;
Chris@1793 402 float otherValue = 0.f;
Chris@897 403 float pitch = 0.f;
Chris@631 404 QString label = "";
Chris@1867 405 bool ok = true;
Chris@148 406
Chris@631 407 duration = 0.f;
Chris@631 408 haveEndTime = false;
Chris@1518 409
Chris@283 410 for (int i = 0; i < list.size(); ++i) {
Chris@148 411
Chris@631 412 QString s = list[i];
Chris@631 413
Chris@631 414 CSVFormat::ColumnPurpose purpose = m_format.getColumnPurpose(i);
Chris@631 415
Chris@631 416 switch (purpose) {
Chris@631 417
Chris@631 418 case CSVFormat::ColumnUnknown:
Chris@631 419 break;
Chris@631 420
Chris@631 421 case CSVFormat::ColumnStartTime:
Chris@1867 422 if (!convertTimeValue(s, lineno, sampleRate, windowSize, frameNo)) {
Chris@1867 423 ok = false;
Chris@1867 424 }
Chris@631 425 break;
Chris@631 426
Chris@631 427 case CSVFormat::ColumnEndTime:
Chris@1867 428 if (convertTimeValue(s, lineno, sampleRate, windowSize, endFrame)) {
Chris@1867 429 haveEndTime = true;
Chris@1867 430 }
Chris@631 431 break;
Chris@631 432
Chris@631 433 case CSVFormat::ColumnDuration:
Chris@1867 434 if (!convertTimeValue(s, lineno, sampleRate, windowSize, duration)) {
Chris@1867 435 ok = false;
Chris@1867 436 }
Chris@631 437 break;
Chris@631 438
Chris@631 439 case CSVFormat::ColumnValue:
Chris@1793 440 if (haveAnyValue) {
Chris@1793 441 otherValue = value;
Chris@1793 442 }
Chris@631 443 value = s.toFloat();
Chris@631 444 haveAnyValue = true;
Chris@631 445 break;
Chris@631 446
Chris@897 447 case CSVFormat::ColumnPitch:
Chris@897 448 pitch = s.toFloat();
Chris@897 449 if (pitch < 0.f || pitch > 127.f) {
Chris@897 450 pitchLooksLikeMIDI = false;
Chris@897 451 }
Chris@897 452 break;
Chris@897 453
Chris@631 454 case CSVFormat::ColumnLabel:
Chris@631 455 label = s;
Chris@631 456 break;
Chris@283 457 }
Chris@631 458 }
Chris@148 459
Chris@1867 460 if (!ok) {
Chris@1867 461 continue;
Chris@1867 462 }
Chris@1867 463
Chris@1113 464 ++labelCountMap[label];
Chris@1113 465
Chris@631 466 if (haveEndTime) { // ... calculate duration now all cols read
Chris@631 467 if (endFrame > frameNo) {
Chris@631 468 duration = endFrame - frameNo;
Chris@628 469 }
Chris@283 470 }
Chris@148 471
Chris@392 472 if (modelType == CSVFormat::OneDimensionalModel) {
Chris@1429 473
Chris@1658 474 Event point(frameNo, label);
Chris@1658 475 model1->add(point);
Chris@148 476
Chris@392 477 } else if (modelType == CSVFormat::TwoDimensionalModel) {
Chris@148 478
Chris@1651 479 Event point(frameNo, value, label);
Chris@1651 480 model2->add(point);
Chris@148 481
Chris@628 482 } else if (modelType == CSVFormat::TwoDimensionalModelWithDuration) {
Chris@628 483
Chris@1649 484 Event region(frameNo, value, duration, label);
Chris@1649 485 model2a->add(region);
Chris@628 486
Chris@897 487 } else if (modelType == CSVFormat::TwoDimensionalModelWithDurationAndPitch) {
Chris@897 488
Chris@897 489 float level = ((value >= 0.f && value <= 1.f) ? value : 1.f);
Chris@1643 490 Event note(frameNo, pitch, duration, level, label);
Chris@1644 491 model2b->add(note);
Chris@897 492
Chris@1793 493 } else if (modelType == CSVFormat::TwoDimensionalModelWithDurationAndExtent) {
Chris@1793 494
Chris@1793 495 float level = 0.f;
Chris@1793 496 if (value > otherValue) {
Chris@1793 497 level = value - otherValue;
Chris@1793 498 value = otherValue;
Chris@1793 499 } else {
Chris@1793 500 level = otherValue - value;
Chris@1793 501 }
Chris@1793 502 Event box(frameNo, value, duration, level, label);
Chris@1793 503 model2c->add(box);
Chris@1793 504
Chris@392 505 } else if (modelType == CSVFormat::ThreeDimensionalModel) {
Chris@148 506
Chris@283 507 DenseThreeDimensionalModel::Column values;
Chris@148 508
Chris@631 509 for (int i = 0; i < list.size(); ++i) {
Chris@148 510
Chris@676 511 if (m_format.getColumnPurpose(i) != CSVFormat::ColumnValue) {
Chris@676 512 continue;
Chris@676 513 }
Chris@676 514
Chris@283 515 bool ok = false;
Chris@283 516 float value = list[i].toFloat(&ok);
Chris@611 517
Chris@676 518 values.push_back(value);
Chris@1429 519
Chris@631 520 if (firstEverValue || value < min) min = value;
Chris@631 521 if (firstEverValue || value > max) max = value;
Chris@676 522
Chris@631 523 if (firstEverValue) {
Chris@611 524 startFrame = frameNo;
Chris@611 525 model3->setStartFrame(startFrame);
Chris@611 526 } else if (lineno == 1 &&
Chris@611 527 timingType == CSVFormat::ExplicitTiming) {
Chris@1038 528 model3->setResolution(int(frameNo - startFrame));
Chris@611 529 }
Chris@631 530
Chris@631 531 firstEverValue = false;
Chris@148 532
Chris@283 533 if (!ok) {
Chris@283 534 if (warnings < warnLimit) {
Chris@1428 535 SVCERR << "WARNING: CSVFileReader::load: "
Chris@390 536 << "Non-numeric value \""
Chris@844 537 << list[i]
Chris@491 538 << "\" in data line " << lineno+1
Chris@843 539 << ":" << endl;
Chris@1428 540 SVCERR << line << endl;
Chris@283 541 ++warnings;
Chris@283 542 } else if (warnings == warnLimit) {
Chris@1428 543 // SVCERR << "WARNING: Too many warnings" << endl;
Chris@283 544 }
Chris@283 545 }
Chris@283 546 }
Chris@1429 547
Chris@690 548 // SVDEBUG << "Setting bin values for count " << lineno << ", frame "
Chris@687 549 // << frameNo << ", time " << RealTime::frame2RealTime(frameNo, sampleRate) << endl;
Chris@148 550
Chris@611 551 model3->setColumn(lineno, values);
Chris@1488 552
Chris@1488 553 } else if (modelType == CSVFormat::WaveFileModel) {
Chris@1488 554
Chris@1518 555 int channel = 0;
Chris@1490 556
Chris@1518 557 for (int i = 0;
Chris@1518 558 i < list.size() && channel < audioChannels;
Chris@1518 559 ++i) {
Chris@1488 560
Chris@1490 561 if (m_format.getColumnPurpose(i) !=
Chris@1490 562 CSVFormat::ColumnValue) {
Chris@1488 563 continue;
Chris@1488 564 }
Chris@1488 565
Chris@1488 566 bool ok = false;
Chris@1488 567 float value = list[i].toFloat(&ok);
Chris@1518 568 if (!ok) {
Chris@1518 569 value = 0.f;
Chris@1518 570 }
Chris@1517 571
Chris@1518 572 value += sampleShift;
Chris@1518 573 value *= sampleScale;
Chris@1488 574
Chris@1518 575 audioSamples[channel][0] = value;
Chris@1490 576
Chris@1490 577 ++channel;
Chris@1488 578 }
Chris@1488 579
Chris@1518 580 while (channel < audioChannels) {
Chris@1518 581 audioSamples[channel][0] = 0.f;
Chris@1518 582 ++channel;
Chris@1518 583 }
Chris@1488 584
Chris@1518 585 bool ok = modelW->addSamples(audioSamples, 1);
Chris@1488 586
Chris@1488 587 if (!ok) {
Chris@1488 588 if (warnings < warnLimit) {
Chris@1488 589 SVCERR << "WARNING: CSVFileReader::load: "
Chris@1488 590 << "Unable to add sample to wave-file model"
Chris@1488 591 << endl;
Chris@1488 592 SVCERR << line << endl;
Chris@1488 593 ++warnings;
Chris@1488 594 }
Chris@1488 595 }
Chris@283 596 }
Chris@1488 597
Chris@283 598 ++lineno;
Chris@392 599 if (timingType == CSVFormat::ImplicitTiming ||
Chris@283 600 list.size() == 0) {
Chris@283 601 frameNo += windowSize;
Chris@283 602 }
Chris@283 603 }
Chris@148 604 }
Chris@148 605
Chris@631 606 if (!haveAnyValue) {
Chris@631 607 if (model2a) {
Chris@631 608 // assign values for regions based on label frequency; we
Chris@631 609 // have this in our labelCountMap, sort of
Chris@631 610
Chris@1113 611 map<int, map<QString, float> > countLabelValueMap;
Chris@1113 612 for (map<QString, int>::iterator i = labelCountMap.begin();
Chris@631 613 i != labelCountMap.end(); ++i) {
Chris@1113 614 countLabelValueMap[i->second][i->first] = -1.f;
Chris@631 615 }
Chris@631 616
Chris@631 617 float v = 0.f;
Chris@1113 618 for (map<int, map<QString, float> >::iterator i =
Chris@631 619 countLabelValueMap.end(); i != countLabelValueMap.begin(); ) {
Chris@631 620 --i;
Chris@1428 621 SVCERR << "count -> " << i->first << endl;
Chris@1113 622 for (map<QString, float>::iterator j = i->second.begin();
Chris@631 623 j != i->second.end(); ++j) {
Chris@631 624 j->second = v;
Chris@1428 625 SVCERR << "label -> " << j->first << ", value " << v << endl;
Chris@631 626 v = v + 1.f;
Chris@631 627 }
Chris@631 628 }
Chris@631 629
Chris@1649 630 map<Event, Event> eventMap;
Chris@1649 631
Chris@1649 632 EventVector allEvents = model2a->getAllEvents();
Chris@1649 633 for (const Event &e: allEvents) {
Chris@1649 634 int count = labelCountMap[e.getLabel()];
Chris@1649 635 v = countLabelValueMap[count][e.getLabel()];
Chris@1649 636 // SVCERR << "mapping from label \"" << p.label
Chris@1649 637 // << "\" (count " << count
Chris@1649 638 // << ") to value " << v << endl;
Chris@1649 639 eventMap[e] = Event(e.getFrame(), v,
Chris@1649 640 e.getDuration(), e.getLabel());
Chris@631 641 }
Chris@631 642
Chris@1649 643 for (const auto &i: eventMap) {
Chris@1113 644 // There could be duplicate regions; if so replace
Chris@1113 645 // them all -- but we need to check we're not
Chris@1113 646 // replacing a region by itself (or else this will
Chris@1113 647 // never terminate)
Chris@1649 648 if (i.first.getValue() == i.second.getValue()) {
Chris@1113 649 continue;
Chris@1113 650 }
Chris@1649 651 while (model2a->containsEvent(i.first)) {
Chris@1649 652 model2a->remove(i.first);
Chris@1649 653 model2a->add(i.second);
Chris@1113 654 }
Chris@631 655 }
Chris@631 656 }
Chris@631 657 }
Chris@631 658
Chris@897 659 if (model2b) {
Chris@897 660 if (pitchLooksLikeMIDI) {
Chris@897 661 model2b->setScaleUnits("MIDI Pitch");
Chris@897 662 } else {
Chris@897 663 model2b->setScaleUnits("Hz");
Chris@897 664 }
Chris@897 665 }
Chris@897 666
Chris@961 667 if (model3) {
Chris@1429 668 model3->setMinimumLevel(min);
Chris@1429 669 model3->setMaximumLevel(max);
Chris@148 670 }
Chris@148 671
Chris@1489 672 if (modelW) {
Chris@1518 673 breakfastquay::deallocate_channels(audioSamples, audioChannels);
Chris@1493 674 modelW->updateModel();
Chris@1489 675 modelW->writeComplete();
Chris@1489 676 }
Chris@1489 677
Chris@148 678 return model;
Chris@148 679 }
Chris@148 680
Chris@1519 681 QString
Chris@1519 682 CSVFileReader::getConvertedAudioFilePath() const
Chris@1519 683 {
Chris@1519 684 QString base = m_filename;
Chris@1519 685 base.replace(QRegExp("[/\\,.:;~<>\"'|?%*]+"), "_");
Chris@1519 686
Chris@1519 687 QString convertedFileDir = RecordDirectory::getConvertedAudioDirectory();
Chris@1519 688 if (convertedFileDir == "") {
Chris@1519 689 SVCERR << "WARNING: CSVFileReader::getConvertedAudioFilePath: Failed to retrieve converted audio directory" << endl;
Chris@1519 690 return "";
Chris@1519 691 }
Chris@1519 692
Chris@1519 693 auto ms = QDateTime::currentDateTime().toMSecsSinceEpoch();
Chris@1519 694 auto s = ms / 1000; // there is a toSecsSinceEpoch in Qt 5.8 but
Chris@1519 695 // we currently want to support older versions
Chris@1519 696
Chris@1519 697 return QDir(convertedFileDir).filePath
Chris@1519 698 (QString("%1-%2.wav").arg(base).arg(s));
Chris@1519 699 }
Chris@1519 700