comparison data/fileio/CSVFileReader.cpp @ 1867:2654bf447a84

CSV reader tests and fixes - avoid creating null events for lines in which the timings could not be read
author Chris Cannam
date Thu, 11 Jun 2020 14:09:59 +0100
parents f0ffc88a36b3
children 566476eeeb80
comparison
equal deleted inserted replaced
1866:b4b11af915f4 1867:2654bf447a84
115 CSVFileReader::getError() const 115 CSVFileReader::getError() const
116 { 116 {
117 return m_error; 117 return m_error;
118 } 118 }
119 119
120 sv_frame_t 120 bool
121 CSVFileReader::convertTimeValue(QString s, int lineno, 121 CSVFileReader::convertTimeValue(QString s, int lineno,
122 sv_samplerate_t sampleRate, 122 sv_samplerate_t sampleRate,
123 int windowSize) const 123 int windowSize,
124 sv_frame_t &calculatedFrame) const
124 { 125 {
125 QRegExp nonNumericRx("[^0-9eE.,+-]"); 126 QRegExp nonNumericRx("[^0-9eE.,+-]");
126 int warnLimit = 10; 127 int warnLimit = 10;
127 128
128 CSVFormat::TimeUnits timeUnits = m_format.getTimeUnits(); 129 CSVFormat::TimeUnits timeUnits = m_format.getTimeUnits();
129 130
130 sv_frame_t calculatedFrame = 0; 131 calculatedFrame = 0;
131 132
132 bool ok = false; 133 bool ok = false;
133 QString numeric = s; 134 QString numeric = s;
134 numeric.remove(nonNumericRx); 135 numeric.remove(nonNumericRx);
135 136
314 QString modelName = m_filename; 315 QString modelName = m_filename;
315 316
316 switch (modelType) { 317 switch (modelType) {
317 318
318 case CSVFormat::OneDimensionalModel: 319 case CSVFormat::OneDimensionalModel:
320 SVDEBUG << "CSVFileReader: Creating sparse one-dimensional model" << endl;
319 model1 = new SparseOneDimensionalModel(sampleRate, windowSize); 321 model1 = new SparseOneDimensionalModel(sampleRate, windowSize);
320 model = model1; 322 model = model1;
321 break; 323 break;
322 324
323 case CSVFormat::TwoDimensionalModel: 325 case CSVFormat::TwoDimensionalModel:
326 SVDEBUG << "CSVFileReader: Creating sparse time-value model" << endl;
324 model2 = new SparseTimeValueModel(sampleRate, windowSize, false); 327 model2 = new SparseTimeValueModel(sampleRate, windowSize, false);
325 model = model2; 328 model = model2;
326 break; 329 break;
327 330
328 case CSVFormat::TwoDimensionalModelWithDuration: 331 case CSVFormat::TwoDimensionalModelWithDuration:
332 SVDEBUG << "CSVFileReader: Creating region model" << endl;
329 model2a = new RegionModel(sampleRate, windowSize, false); 333 model2a = new RegionModel(sampleRate, windowSize, false);
330 model = model2a; 334 model = model2a;
331 break; 335 break;
332 336
333 case CSVFormat::TwoDimensionalModelWithDurationAndPitch: 337 case CSVFormat::TwoDimensionalModelWithDurationAndPitch:
338 SVDEBUG << "CSVFileReader: Creating note model" << endl;
334 model2b = new NoteModel(sampleRate, windowSize, false); 339 model2b = new NoteModel(sampleRate, windowSize, false);
335 model = model2b; 340 model = model2b;
336 break; 341 break;
337 342
338 case CSVFormat::TwoDimensionalModelWithDurationAndExtent: 343 case CSVFormat::TwoDimensionalModelWithDurationAndExtent:
344 SVDEBUG << "CSVFileReader: Creating box model" << endl;
339 model2c = new BoxModel(sampleRate, windowSize, false); 345 model2c = new BoxModel(sampleRate, windowSize, false);
340 model = model2c; 346 model = model2c;
341 break; 347 break;
342 348
343 case CSVFormat::ThreeDimensionalModel: 349 case CSVFormat::ThreeDimensionalModel:
350 SVDEBUG << "CSVFileReader: Creating editable dense three-dimensional model" << endl;
344 model3 = new EditableDenseThreeDimensionalModel 351 model3 = new EditableDenseThreeDimensionalModel
345 (sampleRate, windowSize, valueColumns); 352 (sampleRate, windowSize, valueColumns);
346 model = model3; 353 model = model3;
347 break; 354 break;
348 355
349 case CSVFormat::WaveFileModel: 356 case CSVFormat::WaveFileModel:
350 { 357 {
358 SVDEBUG << "CSVFileReader: Creating writable wave-file model" << endl;
351 bool normalise = (m_format.getAudioSampleRange() 359 bool normalise = (m_format.getAudioSampleRange()
352 == CSVFormat::SampleRangeOther); 360 == CSVFormat::SampleRangeOther);
353 QString path = getConvertedAudioFilePath(); 361 QString path = getConvertedAudioFilePath();
354 modelW = new WritableWaveFileModel 362 modelW = new WritableWaveFileModel
355 (path, sampleRate, valueColumns, 363 (path, sampleRate, valueColumns,
385 393
386 float value = 0.f; 394 float value = 0.f;
387 float otherValue = 0.f; 395 float otherValue = 0.f;
388 float pitch = 0.f; 396 float pitch = 0.f;
389 QString label = ""; 397 QString label = "";
398 bool ok = true;
390 399
391 duration = 0.f; 400 duration = 0.f;
392 haveEndTime = false; 401 haveEndTime = false;
393 402
394 for (int i = 0; i < list.size(); ++i) { 403 for (int i = 0; i < list.size(); ++i) {
401 410
402 case CSVFormat::ColumnUnknown: 411 case CSVFormat::ColumnUnknown:
403 break; 412 break;
404 413
405 case CSVFormat::ColumnStartTime: 414 case CSVFormat::ColumnStartTime:
406 frameNo = convertTimeValue(s, lineno, sampleRate, windowSize); 415 if (!convertTimeValue(s, lineno, sampleRate, windowSize, frameNo)) {
416 ok = false;
417 }
407 break; 418 break;
408 419
409 case CSVFormat::ColumnEndTime: 420 case CSVFormat::ColumnEndTime:
410 endFrame = convertTimeValue(s, lineno, sampleRate, windowSize); 421 if (convertTimeValue(s, lineno, sampleRate, windowSize, endFrame)) {
411 haveEndTime = true; 422 haveEndTime = true;
423 }
412 break; 424 break;
413 425
414 case CSVFormat::ColumnDuration: 426 case CSVFormat::ColumnDuration:
415 duration = convertTimeValue(s, lineno, sampleRate, windowSize); 427 if (!convertTimeValue(s, lineno, sampleRate, windowSize, duration)) {
428 ok = false;
429 }
416 break; 430 break;
417 431
418 case CSVFormat::ColumnValue: 432 case CSVFormat::ColumnValue:
419 if (haveAnyValue) { 433 if (haveAnyValue) {
420 otherValue = value; 434 otherValue = value;
434 label = s; 448 label = s;
435 break; 449 break;
436 } 450 }
437 } 451 }
438 452
453 if (!ok) {
454 continue;
455 }
456
439 ++labelCountMap[label]; 457 ++labelCountMap[label];
440 458
441 if (haveEndTime) { // ... calculate duration now all cols read 459 if (haveEndTime) { // ... calculate duration now all cols read
442 if (endFrame > frameNo) { 460 if (endFrame > frameNo) {
443 duration = endFrame - frameNo; 461 duration = endFrame - frameNo;