annotate data/fileio/CSVFileReader.cpp @ 1700:c1208b211d8c single-point

Ensure test fails rather than crashing if this reader doesn't get created
author Chris Cannam <cannam@all-day-breakfast.com>
date Fri, 03 May 2019 15:02:09 +0100
parents 5b7b01da430a
children d484490cdf69
rev   line source
Chris@148 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@148 2
Chris@148 3 /*
Chris@148 4 Sonic Visualiser
Chris@148 5 An audio file viewer and annotation editor.
Chris@148 6 Centre for Digital Music, Queen Mary, University of London.
Chris@148 7 This file copyright 2006 Chris Cannam.
Chris@148 8
Chris@148 9 This program is free software; you can redistribute it and/or
Chris@148 10 modify it under the terms of the GNU General Public License as
Chris@148 11 published by the Free Software Foundation; either version 2 of the
Chris@148 12 License, or (at your option) any later version. See the file
Chris@148 13 COPYING included with this distribution for more information.
Chris@148 14 */
Chris@148 15
Chris@148 16 #include "CSVFileReader.h"
Chris@148 17
Chris@150 18 #include "model/Model.h"
Chris@148 19 #include "base/RealTime.h"
Chris@631 20 #include "base/StringBits.h"
Chris@1491 21 #include "base/ProgressReporter.h"
Chris@1519 22 #include "base/RecordDirectory.h"
Chris@148 23 #include "model/SparseOneDimensionalModel.h"
Chris@148 24 #include "model/SparseTimeValueModel.h"
Chris@152 25 #include "model/EditableDenseThreeDimensionalModel.h"
Chris@628 26 #include "model/RegionModel.h"
Chris@897 27 #include "model/NoteModel.h"
Chris@1488 28 #include "model/WritableWaveFileModel.h"
Chris@308 29 #include "DataFileReaderFactory.h"
Chris@148 30
Chris@148 31 #include <QFile>
Chris@1519 32 #include <QDir>
Chris@1030 33 #include <QFileInfo>
Chris@148 34 #include <QString>
Chris@148 35 #include <QRegExp>
Chris@148 36 #include <QStringList>
Chris@148 37 #include <QTextStream>
Chris@1519 38 #include <QDateTime>
Chris@148 39
Chris@148 40 #include <iostream>
Chris@628 41 #include <map>
Chris@1428 42 #include <string>
Chris@148 43
Chris@1113 44 using namespace std;
Chris@1113 45
Chris@392 46 CSVFileReader::CSVFileReader(QString path, CSVFormat format,
Chris@1491 47 sv_samplerate_t mainModelSampleRate,
Chris@1491 48 ProgressReporter *reporter) :
Chris@392 49 m_format(format),
Chris@1582 50 m_device(nullptr),
Chris@1009 51 m_ownDevice(true),
Chris@631 52 m_warnings(0),
Chris@1491 53 m_mainModelSampleRate(mainModelSampleRate),
Chris@1491 54 m_fileSize(0),
Chris@1491 55 m_readCount(0),
Chris@1492 56 m_progress(-1),
Chris@1491 57 m_reporter(reporter)
Chris@148 58 {
Chris@1009 59 QFile *file = new QFile(path);
Chris@148 60 bool good = false;
Chris@148 61
Chris@1009 62 if (!file->exists()) {
Chris@1429 63 m_error = QFile::tr("File \"%1\" does not exist").arg(path);
Chris@1009 64 } else if (!file->open(QIODevice::ReadOnly | QIODevice::Text)) {
Chris@1429 65 m_error = QFile::tr("Failed to open file \"%1\"").arg(path);
Chris@148 66 } else {
Chris@1429 67 good = true;
Chris@148 68 }
Chris@148 69
Chris@1009 70 if (good) {
Chris@1009 71 m_device = file;
Chris@1030 72 m_filename = QFileInfo(path).fileName();
Chris@1491 73 m_fileSize = file->size();
Chris@1491 74 if (m_reporter) m_reporter->setDefinite(true);
Chris@1009 75 } else {
Chris@1429 76 delete file;
Chris@148 77 }
Chris@148 78 }
Chris@148 79
Chris@1009 80 CSVFileReader::CSVFileReader(QIODevice *device, CSVFormat format,
Chris@1491 81 sv_samplerate_t mainModelSampleRate,
Chris@1491 82 ProgressReporter *reporter) :
Chris@1009 83 m_format(format),
Chris@1009 84 m_device(device),
Chris@1009 85 m_ownDevice(false),
Chris@1009 86 m_warnings(0),
Chris@1491 87 m_mainModelSampleRate(mainModelSampleRate),
Chris@1491 88 m_fileSize(0),
Chris@1491 89 m_readCount(0),
Chris@1492 90 m_progress(-1),
Chris@1491 91 m_reporter(reporter)
Chris@1009 92 {
Chris@1491 93 if (m_reporter) m_reporter->setDefinite(false);
Chris@1009 94 }
Chris@1009 95
Chris@148 96 CSVFileReader::~CSVFileReader()
Chris@148 97 {
Chris@1009 98 SVDEBUG << "CSVFileReader::~CSVFileReader: device is " << m_device << endl;
Chris@148 99
Chris@1009 100 if (m_device && m_ownDevice) {
Chris@1009 101 SVDEBUG << "CSVFileReader::CSVFileReader: Closing device" << endl;
Chris@1009 102 m_device->close();
Chris@1009 103 delete m_device;
Chris@148 104 }
Chris@148 105 }
Chris@148 106
Chris@148 107 bool
Chris@148 108 CSVFileReader::isOK() const
Chris@148 109 {
Chris@1582 110 return (m_device != nullptr);
Chris@148 111 }
Chris@148 112
Chris@148 113 QString
Chris@148 114 CSVFileReader::getError() const
Chris@148 115 {
Chris@148 116 return m_error;
Chris@148 117 }
Chris@148 118
Chris@1038 119 sv_frame_t
Chris@1047 120 CSVFileReader::convertTimeValue(QString s, int lineno,
Chris@1047 121 sv_samplerate_t sampleRate,
Chris@929 122 int windowSize) const
Chris@631 123 {
Chris@631 124 QRegExp nonNumericRx("[^0-9eE.,+-]");
Chris@897 125 int warnLimit = 10;
Chris@631 126
Chris@631 127 CSVFormat::TimeUnits timeUnits = m_format.getTimeUnits();
Chris@631 128
Chris@1038 129 sv_frame_t calculatedFrame = 0;
Chris@631 130
Chris@631 131 bool ok = false;
Chris@631 132 QString numeric = s;
Chris@631 133 numeric.remove(nonNumericRx);
Chris@631 134
Chris@631 135 if (timeUnits == CSVFormat::TimeSeconds) {
Chris@631 136
Chris@631 137 double time = numeric.toDouble(&ok);
Chris@631 138 if (!ok) time = StringBits::stringToDoubleLocaleFree(numeric, &ok);
Chris@1038 139 calculatedFrame = sv_frame_t(time * sampleRate + 0.5);
Chris@990 140
Chris@990 141 } else if (timeUnits == CSVFormat::TimeMilliseconds) {
Chris@990 142
Chris@990 143 double time = numeric.toDouble(&ok);
Chris@990 144 if (!ok) time = StringBits::stringToDoubleLocaleFree(numeric, &ok);
Chris@1038 145 calculatedFrame = sv_frame_t((time / 1000.0) * sampleRate + 0.5);
Chris@631 146
Chris@631 147 } else {
Chris@631 148
Chris@631 149 long n = numeric.toLong(&ok);
Chris@631 150 if (n >= 0) calculatedFrame = n;
Chris@631 151
Chris@631 152 if (timeUnits == CSVFormat::TimeWindows) {
Chris@631 153 calculatedFrame *= windowSize;
Chris@631 154 }
Chris@631 155 }
Chris@631 156
Chris@631 157 if (!ok) {
Chris@631 158 if (m_warnings < warnLimit) {
Chris@1428 159 SVCERR << "WARNING: CSVFileReader::load: "
Chris@844 160 << "Bad time format (\"" << s
Chris@631 161 << "\") in data line "
Chris@843 162 << lineno+1 << endl;
Chris@631 163 } else if (m_warnings == warnLimit) {
Chris@1428 164 SVCERR << "WARNING: Too many warnings" << endl;
Chris@631 165 }
Chris@631 166 ++m_warnings;
Chris@631 167 }
Chris@631 168
Chris@631 169 return calculatedFrame;
Chris@631 170 }
Chris@631 171
Chris@148 172 Model *
Chris@148 173 CSVFileReader::load() const
Chris@148 174 {
Chris@1582 175 if (!m_device) return nullptr;
Chris@148 176
Chris@628 177 CSVFormat::ModelType modelType = m_format.getModelType();
Chris@392 178 CSVFormat::TimingType timingType = m_format.getTimingType();
Chris@628 179 CSVFormat::TimeUnits timeUnits = m_format.getTimeUnits();
Chris@1047 180 sv_samplerate_t sampleRate = m_format.getSampleRate();
Chris@929 181 int windowSize = m_format.getWindowSize();
Chris@631 182 QChar separator = m_format.getSeparator();
Chris@631 183 bool allowQuoting = m_format.getAllowQuoting();
Chris@148 184
Chris@392 185 if (timingType == CSVFormat::ExplicitTiming) {
Chris@611 186 if (modelType == CSVFormat::ThreeDimensionalModel) {
Chris@611 187 // This will be overridden later if more than one line
Chris@611 188 // appears in our file, but we want to choose a default
Chris@611 189 // that's likely to be visible
Chris@611 190 windowSize = 1024;
Chris@611 191 } else {
Chris@611 192 windowSize = 1;
Chris@611 193 }
Chris@1429 194 if (timeUnits == CSVFormat::TimeSeconds ||
Chris@990 195 timeUnits == CSVFormat::TimeMilliseconds) {
Chris@1429 196 sampleRate = m_mainModelSampleRate;
Chris@1429 197 }
Chris@148 198 }
Chris@148 199
Chris@1582 200 SparseOneDimensionalModel *model1 = nullptr;
Chris@1582 201 SparseTimeValueModel *model2 = nullptr;
Chris@1582 202 RegionModel *model2a = nullptr;
Chris@1582 203 NoteModel *model2b = nullptr;
Chris@1582 204 EditableDenseThreeDimensionalModel *model3 = nullptr;
Chris@1582 205 WritableWaveFileModel *modelW = nullptr;
Chris@1582 206 Model *model = nullptr;
Chris@148 207
Chris@1009 208 QTextStream in(m_device);
Chris@148 209
Chris@148 210 unsigned int warnings = 0, warnLimit = 10;
Chris@148 211 unsigned int lineno = 0;
Chris@148 212
Chris@148 213 float min = 0.0, max = 0.0;
Chris@148 214
Chris@1038 215 sv_frame_t frameNo = 0;
Chris@1038 216 sv_frame_t duration = 0;
Chris@1038 217 sv_frame_t endFrame = 0;
Chris@631 218
Chris@631 219 bool haveAnyValue = false;
Chris@631 220 bool haveEndTime = false;
Chris@897 221 bool pitchLooksLikeMIDI = true;
Chris@631 222
Chris@1038 223 sv_frame_t startFrame = 0; // for calculation of dense model resolution
Chris@631 224 bool firstEverValue = true;
Chris@631 225
Chris@676 226 int valueColumns = 0;
Chris@676 227 for (int i = 0; i < m_format.getColumnCount(); ++i) {
Chris@676 228 if (m_format.getColumnPurpose(i) == CSVFormat::ColumnValue) {
Chris@676 229 ++valueColumns;
Chris@676 230 }
Chris@676 231 }
Chris@676 232
Chris@1518 233 int audioChannels = 0;
Chris@1582 234 float **audioSamples = nullptr;
Chris@1518 235 float sampleShift = 0.f;
Chris@1518 236 float sampleScale = 1.f;
Chris@1518 237
Chris@1518 238 if (modelType == CSVFormat::WaveFileModel) {
Chris@1518 239
Chris@1518 240 audioChannels = valueColumns;
Chris@1518 241
Chris@1518 242 audioSamples =
Chris@1518 243 breakfastquay::allocate_and_zero_channels<float>
Chris@1518 244 (audioChannels, 1);
Chris@1518 245
Chris@1518 246 switch (m_format.getAudioSampleRange()) {
Chris@1518 247 case CSVFormat::SampleRangeSigned1:
Chris@1518 248 case CSVFormat::SampleRangeOther:
Chris@1518 249 sampleShift = 0.f;
Chris@1518 250 sampleScale = 1.f;
Chris@1518 251 break;
Chris@1518 252 case CSVFormat::SampleRangeUnsigned255:
Chris@1518 253 sampleShift = -128.f;
Chris@1518 254 sampleScale = 1.f / 128.f;
Chris@1518 255 break;
Chris@1518 256 case CSVFormat::SampleRangeSigned32767:
Chris@1518 257 sampleShift = 0.f;
Chris@1518 258 sampleScale = 1.f / 32768.f;
Chris@1518 259 break;
Chris@1518 260 }
Chris@1518 261 }
Chris@1518 262
Chris@1518 263 map<QString, int> labelCountMap;
Chris@1518 264
Chris@1509 265 bool abandoned = false;
Chris@1509 266
Chris@1509 267 while (!in.atEnd() && !abandoned) {
Chris@148 268
Chris@283 269 // QTextStream's readLine doesn't cope with old-style Mac
Chris@283 270 // CR-only line endings. Why did they bother making the class
Chris@283 271 // cope with more than one sort of line ending, if it still
Chris@283 272 // can't be configured to cope with all the common sorts?
Chris@148 273
Chris@283 274 // For the time being we'll deal with this case (which is
Chris@283 275 // relatively uncommon for us, but still necessary to handle)
Chris@283 276 // by reading the entire file using a single readLine, and
Chris@283 277 // splitting it. For CR and CR/LF line endings this will just
Chris@283 278 // read a line at a time, and that's obviously OK.
Chris@148 279
Chris@283 280 QString chunk = in.readLine();
Chris@283 281 QStringList lines = chunk.split('\r', QString::SkipEmptyParts);
Chris@1491 282
Chris@1491 283 m_readCount += chunk.size() + 1;
Chris@1491 284
Chris@1491 285 if (m_reporter) {
Chris@1491 286 if (m_reporter->wasCancelled()) {
Chris@1509 287 abandoned = true;
Chris@1491 288 break;
Chris@1491 289 }
Chris@1491 290 int progress;
Chris@1491 291 if (m_fileSize > 0) {
Chris@1491 292 progress = int((double(m_readCount) / double(m_fileSize))
Chris@1491 293 * 100.0);
Chris@1491 294 } else {
Chris@1491 295 progress = int(m_readCount / 10000);
Chris@1491 296 }
Chris@1491 297 if (progress != m_progress) {
Chris@1491 298 m_reporter->setProgress(progress);
Chris@1491 299 m_progress = progress;
Chris@1491 300 }
Chris@1491 301 }
Chris@283 302
Chris@897 303 for (int li = 0; li < lines.size(); ++li) {
Chris@148 304
Chris@283 305 QString line = lines[li];
Chris@1009 306
Chris@283 307 if (line.startsWith("#")) continue;
Chris@283 308
Chris@631 309 QStringList list = StringBits::split(line, separator, allowQuoting);
Chris@283 310 if (!model) {
Chris@283 311
Chris@1519 312 QString modelName = m_filename;
Chris@1519 313
Chris@283 314 switch (modelType) {
Chris@283 315
Chris@392 316 case CSVFormat::OneDimensionalModel:
Chris@283 317 model1 = new SparseOneDimensionalModel(sampleRate, windowSize);
Chris@283 318 model = model1;
Chris@283 319 break;
Chris@1429 320
Chris@392 321 case CSVFormat::TwoDimensionalModel:
Chris@283 322 model2 = new SparseTimeValueModel(sampleRate, windowSize, false);
Chris@283 323 model = model2;
Chris@283 324 break;
Chris@1429 325
Chris@628 326 case CSVFormat::TwoDimensionalModelWithDuration:
Chris@628 327 model2a = new RegionModel(sampleRate, windowSize, false);
Chris@628 328 model = model2a;
Chris@628 329 break;
Chris@1429 330
Chris@897 331 case CSVFormat::TwoDimensionalModelWithDurationAndPitch:
Chris@897 332 model2b = new NoteModel(sampleRate, windowSize, false);
Chris@897 333 model = model2b;
Chris@897 334 break;
Chris@1429 335
Chris@392 336 case CSVFormat::ThreeDimensionalModel:
Chris@535 337 model3 = new EditableDenseThreeDimensionalModel
Chris@535 338 (sampleRate,
Chris@535 339 windowSize,
Chris@676 340 valueColumns,
Chris@535 341 EditableDenseThreeDimensionalModel::NoCompression);
Chris@283 342 model = model3;
Chris@283 343 break;
Chris@1488 344
Chris@1488 345 case CSVFormat::WaveFileModel:
Chris@1517 346 {
Chris@1517 347 bool normalise = (m_format.getAudioSampleRange()
Chris@1517 348 == CSVFormat::SampleRangeOther);
Chris@1519 349 QString path = getConvertedAudioFilePath();
Chris@1488 350 modelW = new WritableWaveFileModel
Chris@1520 351 (path, sampleRate, valueColumns,
Chris@1520 352 normalise ?
Chris@1520 353 WritableWaveFileModel::Normalisation::Peak :
Chris@1520 354 WritableWaveFileModel::Normalisation::None);
Chris@1519 355 modelName = QFileInfo(path).fileName();
Chris@1488 356 model = modelW;
Chris@1488 357 break;
Chris@283 358 }
Chris@1517 359 }
Chris@1030 360
Chris@1508 361 if (model && model->isOK()) {
Chris@1519 362 if (modelName != "") {
Chris@1519 363 model->setObjectName(modelName);
Chris@1030 364 }
Chris@1030 365 }
Chris@283 366 }
Chris@148 367
Chris@1508 368 if (!model || !model->isOK()) {
Chris@1508 369 SVCERR << "Failed to create model to load CSV file into"
Chris@1508 370 << endl;
Chris@1508 371 if (model) {
Chris@1508 372 delete model;
Chris@1582 373 model = nullptr;
Chris@1582 374 model1 = nullptr; model2 = nullptr; model2a = nullptr; model2b = nullptr;
Chris@1582 375 model3 = nullptr; modelW = nullptr;
Chris@1508 376 }
Chris@1509 377 abandoned = true;
Chris@1508 378 break;
Chris@1508 379 }
Chris@1508 380
Chris@631 381 float value = 0.f;
Chris@897 382 float pitch = 0.f;
Chris@631 383 QString label = "";
Chris@148 384
Chris@631 385 duration = 0.f;
Chris@631 386 haveEndTime = false;
Chris@1518 387
Chris@283 388 for (int i = 0; i < list.size(); ++i) {
Chris@148 389
Chris@631 390 QString s = list[i];
Chris@631 391
Chris@631 392 CSVFormat::ColumnPurpose purpose = m_format.getColumnPurpose(i);
Chris@631 393
Chris@631 394 switch (purpose) {
Chris@631 395
Chris@631 396 case CSVFormat::ColumnUnknown:
Chris@631 397 break;
Chris@631 398
Chris@631 399 case CSVFormat::ColumnStartTime:
Chris@631 400 frameNo = convertTimeValue(s, lineno, sampleRate, windowSize);
Chris@631 401 break;
Chris@631 402
Chris@631 403 case CSVFormat::ColumnEndTime:
Chris@631 404 endFrame = convertTimeValue(s, lineno, sampleRate, windowSize);
Chris@631 405 haveEndTime = true;
Chris@631 406 break;
Chris@631 407
Chris@631 408 case CSVFormat::ColumnDuration:
Chris@631 409 duration = convertTimeValue(s, lineno, sampleRate, windowSize);
Chris@631 410 break;
Chris@631 411
Chris@631 412 case CSVFormat::ColumnValue:
Chris@631 413 value = s.toFloat();
Chris@631 414 haveAnyValue = true;
Chris@631 415 break;
Chris@631 416
Chris@897 417 case CSVFormat::ColumnPitch:
Chris@897 418 pitch = s.toFloat();
Chris@897 419 if (pitch < 0.f || pitch > 127.f) {
Chris@897 420 pitchLooksLikeMIDI = false;
Chris@897 421 }
Chris@897 422 break;
Chris@897 423
Chris@631 424 case CSVFormat::ColumnLabel:
Chris@631 425 label = s;
Chris@631 426 break;
Chris@283 427 }
Chris@631 428 }
Chris@148 429
Chris@1113 430 ++labelCountMap[label];
Chris@1113 431
Chris@631 432 if (haveEndTime) { // ... calculate duration now all cols read
Chris@631 433 if (endFrame > frameNo) {
Chris@631 434 duration = endFrame - frameNo;
Chris@628 435 }
Chris@283 436 }
Chris@148 437
Chris@392 438 if (modelType == CSVFormat::OneDimensionalModel) {
Chris@1429 439
Chris@1658 440 Event point(frameNo, label);
Chris@1658 441 model1->add(point);
Chris@148 442
Chris@392 443 } else if (modelType == CSVFormat::TwoDimensionalModel) {
Chris@148 444
Chris@1651 445 Event point(frameNo, value, label);
Chris@1651 446 model2->add(point);
Chris@148 447
Chris@628 448 } else if (modelType == CSVFormat::TwoDimensionalModelWithDuration) {
Chris@628 449
Chris@1649 450 Event region(frameNo, value, duration, label);
Chris@1649 451 model2a->add(region);
Chris@628 452
Chris@897 453 } else if (modelType == CSVFormat::TwoDimensionalModelWithDurationAndPitch) {
Chris@897 454
Chris@897 455 float level = ((value >= 0.f && value <= 1.f) ? value : 1.f);
Chris@1643 456 Event note(frameNo, pitch, duration, level, label);
Chris@1644 457 model2b->add(note);
Chris@897 458
Chris@392 459 } else if (modelType == CSVFormat::ThreeDimensionalModel) {
Chris@148 460
Chris@283 461 DenseThreeDimensionalModel::Column values;
Chris@148 462
Chris@631 463 for (int i = 0; i < list.size(); ++i) {
Chris@148 464
Chris@676 465 if (m_format.getColumnPurpose(i) != CSVFormat::ColumnValue) {
Chris@676 466 continue;
Chris@676 467 }
Chris@676 468
Chris@283 469 bool ok = false;
Chris@283 470 float value = list[i].toFloat(&ok);
Chris@611 471
Chris@676 472 values.push_back(value);
Chris@1429 473
Chris@631 474 if (firstEverValue || value < min) min = value;
Chris@631 475 if (firstEverValue || value > max) max = value;
Chris@676 476
Chris@631 477 if (firstEverValue) {
Chris@611 478 startFrame = frameNo;
Chris@611 479 model3->setStartFrame(startFrame);
Chris@611 480 } else if (lineno == 1 &&
Chris@611 481 timingType == CSVFormat::ExplicitTiming) {
Chris@1038 482 model3->setResolution(int(frameNo - startFrame));
Chris@611 483 }
Chris@631 484
Chris@631 485 firstEverValue = false;
Chris@148 486
Chris@283 487 if (!ok) {
Chris@283 488 if (warnings < warnLimit) {
Chris@1428 489 SVCERR << "WARNING: CSVFileReader::load: "
Chris@390 490 << "Non-numeric value \""
Chris@844 491 << list[i]
Chris@491 492 << "\" in data line " << lineno+1
Chris@843 493 << ":" << endl;
Chris@1428 494 SVCERR << line << endl;
Chris@283 495 ++warnings;
Chris@283 496 } else if (warnings == warnLimit) {
Chris@1428 497 // SVCERR << "WARNING: Too many warnings" << endl;
Chris@283 498 }
Chris@283 499 }
Chris@283 500 }
Chris@1429 501
Chris@690 502 // SVDEBUG << "Setting bin values for count " << lineno << ", frame "
Chris@687 503 // << frameNo << ", time " << RealTime::frame2RealTime(frameNo, sampleRate) << endl;
Chris@148 504
Chris@611 505 model3->setColumn(lineno, values);
Chris@1488 506
Chris@1488 507 } else if (modelType == CSVFormat::WaveFileModel) {
Chris@1488 508
Chris@1518 509 int channel = 0;
Chris@1490 510
Chris@1518 511 for (int i = 0;
Chris@1518 512 i < list.size() && channel < audioChannels;
Chris@1518 513 ++i) {
Chris@1488 514
Chris@1490 515 if (m_format.getColumnPurpose(i) !=
Chris@1490 516 CSVFormat::ColumnValue) {
Chris@1488 517 continue;
Chris@1488 518 }
Chris@1488 519
Chris@1488 520 bool ok = false;
Chris@1488 521 float value = list[i].toFloat(&ok);
Chris@1518 522 if (!ok) {
Chris@1518 523 value = 0.f;
Chris@1518 524 }
Chris@1517 525
Chris@1518 526 value += sampleShift;
Chris@1518 527 value *= sampleScale;
Chris@1488 528
Chris@1518 529 audioSamples[channel][0] = value;
Chris@1490 530
Chris@1490 531 ++channel;
Chris@1488 532 }
Chris@1488 533
Chris@1518 534 while (channel < audioChannels) {
Chris@1518 535 audioSamples[channel][0] = 0.f;
Chris@1518 536 ++channel;
Chris@1518 537 }
Chris@1488 538
Chris@1518 539 bool ok = modelW->addSamples(audioSamples, 1);
Chris@1488 540
Chris@1488 541 if (!ok) {
Chris@1488 542 if (warnings < warnLimit) {
Chris@1488 543 SVCERR << "WARNING: CSVFileReader::load: "
Chris@1488 544 << "Unable to add sample to wave-file model"
Chris@1488 545 << endl;
Chris@1488 546 SVCERR << line << endl;
Chris@1488 547 ++warnings;
Chris@1488 548 }
Chris@1488 549 }
Chris@283 550 }
Chris@1488 551
Chris@283 552 ++lineno;
Chris@392 553 if (timingType == CSVFormat::ImplicitTiming ||
Chris@283 554 list.size() == 0) {
Chris@283 555 frameNo += windowSize;
Chris@283 556 }
Chris@283 557 }
Chris@148 558 }
Chris@148 559
Chris@631 560 if (!haveAnyValue) {
Chris@631 561 if (model2a) {
Chris@631 562 // assign values for regions based on label frequency; we
Chris@631 563 // have this in our labelCountMap, sort of
Chris@631 564
Chris@1113 565 map<int, map<QString, float> > countLabelValueMap;
Chris@1113 566 for (map<QString, int>::iterator i = labelCountMap.begin();
Chris@631 567 i != labelCountMap.end(); ++i) {
Chris@1113 568 countLabelValueMap[i->second][i->first] = -1.f;
Chris@631 569 }
Chris@631 570
Chris@631 571 float v = 0.f;
Chris@1113 572 for (map<int, map<QString, float> >::iterator i =
Chris@631 573 countLabelValueMap.end(); i != countLabelValueMap.begin(); ) {
Chris@631 574 --i;
Chris@1428 575 SVCERR << "count -> " << i->first << endl;
Chris@1113 576 for (map<QString, float>::iterator j = i->second.begin();
Chris@631 577 j != i->second.end(); ++j) {
Chris@631 578 j->second = v;
Chris@1428 579 SVCERR << "label -> " << j->first << ", value " << v << endl;
Chris@631 580 v = v + 1.f;
Chris@631 581 }
Chris@631 582 }
Chris@631 583
Chris@1649 584 map<Event, Event> eventMap;
Chris@1649 585
Chris@1649 586 EventVector allEvents = model2a->getAllEvents();
Chris@1649 587 for (const Event &e: allEvents) {
Chris@1649 588 int count = labelCountMap[e.getLabel()];
Chris@1649 589 v = countLabelValueMap[count][e.getLabel()];
Chris@1649 590 // SVCERR << "mapping from label \"" << p.label
Chris@1649 591 // << "\" (count " << count
Chris@1649 592 // << ") to value " << v << endl;
Chris@1649 593 eventMap[e] = Event(e.getFrame(), v,
Chris@1649 594 e.getDuration(), e.getLabel());
Chris@631 595 }
Chris@631 596
Chris@1649 597 for (const auto &i: eventMap) {
Chris@1113 598 // There could be duplicate regions; if so replace
Chris@1113 599 // them all -- but we need to check we're not
Chris@1113 600 // replacing a region by itself (or else this will
Chris@1113 601 // never terminate)
Chris@1649 602 if (i.first.getValue() == i.second.getValue()) {
Chris@1113 603 continue;
Chris@1113 604 }
Chris@1649 605 while (model2a->containsEvent(i.first)) {
Chris@1649 606 model2a->remove(i.first);
Chris@1649 607 model2a->add(i.second);
Chris@1113 608 }
Chris@631 609 }
Chris@631 610 }
Chris@631 611 }
Chris@631 612
Chris@897 613 if (model2b) {
Chris@897 614 if (pitchLooksLikeMIDI) {
Chris@897 615 model2b->setScaleUnits("MIDI Pitch");
Chris@897 616 } else {
Chris@897 617 model2b->setScaleUnits("Hz");
Chris@897 618 }
Chris@897 619 }
Chris@897 620
Chris@961 621 if (model3) {
Chris@1429 622 model3->setMinimumLevel(min);
Chris@1429 623 model3->setMaximumLevel(max);
Chris@148 624 }
Chris@148 625
Chris@1489 626 if (modelW) {
Chris@1518 627 breakfastquay::deallocate_channels(audioSamples, audioChannels);
Chris@1493 628 modelW->updateModel();
Chris@1489 629 modelW->writeComplete();
Chris@1489 630 }
Chris@1489 631
Chris@148 632 return model;
Chris@148 633 }
Chris@148 634
Chris@1519 635 QString
Chris@1519 636 CSVFileReader::getConvertedAudioFilePath() const
Chris@1519 637 {
Chris@1519 638 QString base = m_filename;
Chris@1519 639 base.replace(QRegExp("[/\\,.:;~<>\"'|?%*]+"), "_");
Chris@1519 640
Chris@1519 641 QString convertedFileDir = RecordDirectory::getConvertedAudioDirectory();
Chris@1519 642 if (convertedFileDir == "") {
Chris@1519 643 SVCERR << "WARNING: CSVFileReader::getConvertedAudioFilePath: Failed to retrieve converted audio directory" << endl;
Chris@1519 644 return "";
Chris@1519 645 }
Chris@1519 646
Chris@1519 647 auto ms = QDateTime::currentDateTime().toMSecsSinceEpoch();
Chris@1519 648 auto s = ms / 1000; // there is a toSecsSinceEpoch in Qt 5.8 but
Chris@1519 649 // we currently want to support older versions
Chris@1519 650
Chris@1519 651 return QDir(convertedFileDir).filePath
Chris@1519 652 (QString("%1-%2.wav").arg(base).arg(s));
Chris@1519 653 }
Chris@1519 654