comparison data/fileio/CSVFileReader.cpp @ 1450:a12fd0456f0c streaming-csv-writer

Merge from default branch
author Chris Cannam
date Tue, 17 Apr 2018 10:35:42 +0100
parents 48e9f538e6e9
children 53fa8d57b728
comparison
equal deleted inserted replaced
1449:deabf9fd3d28 1450:a12fd0456f0c
32 #include <QStringList> 32 #include <QStringList>
33 #include <QTextStream> 33 #include <QTextStream>
34 34
35 #include <iostream> 35 #include <iostream>
36 #include <map> 36 #include <map>
37 #include <string>
37 38
38 using namespace std; 39 using namespace std;
39 40
40 CSVFileReader::CSVFileReader(QString path, CSVFormat format, 41 CSVFileReader::CSVFileReader(QString path, CSVFormat format,
41 sv_samplerate_t mainModelSampleRate) : 42 sv_samplerate_t mainModelSampleRate) :
47 { 48 {
48 QFile *file = new QFile(path); 49 QFile *file = new QFile(path);
49 bool good = false; 50 bool good = false;
50 51
51 if (!file->exists()) { 52 if (!file->exists()) {
52 m_error = QFile::tr("File \"%1\" does not exist").arg(path); 53 m_error = QFile::tr("File \"%1\" does not exist").arg(path);
53 } else if (!file->open(QIODevice::ReadOnly | QIODevice::Text)) { 54 } else if (!file->open(QIODevice::ReadOnly | QIODevice::Text)) {
54 m_error = QFile::tr("Failed to open file \"%1\"").arg(path); 55 m_error = QFile::tr("Failed to open file \"%1\"").arg(path);
55 } else { 56 } else {
56 good = true; 57 good = true;
57 } 58 }
58 59
59 if (good) { 60 if (good) {
60 m_device = file; 61 m_device = file;
61 m_filename = QFileInfo(path).fileName(); 62 m_filename = QFileInfo(path).fileName();
62 } else { 63 } else {
63 delete file; 64 delete file;
64 } 65 }
65 } 66 }
66 67
67 CSVFileReader::CSVFileReader(QIODevice *device, CSVFormat format, 68 CSVFileReader::CSVFileReader(QIODevice *device, CSVFormat format,
68 sv_samplerate_t mainModelSampleRate) : 69 sv_samplerate_t mainModelSampleRate) :
135 } 136 }
136 } 137 }
137 138
138 if (!ok) { 139 if (!ok) {
139 if (m_warnings < warnLimit) { 140 if (m_warnings < warnLimit) {
140 cerr << "WARNING: CSVFileReader::load: " 141 SVCERR << "WARNING: CSVFileReader::load: "
141 << "Bad time format (\"" << s 142 << "Bad time format (\"" << s
142 << "\") in data line " 143 << "\") in data line "
143 << lineno+1 << endl; 144 << lineno+1 << endl;
144 } else if (m_warnings == warnLimit) { 145 } else if (m_warnings == warnLimit) {
145 cerr << "WARNING: Too many warnings" << endl; 146 SVCERR << "WARNING: Too many warnings" << endl;
146 } 147 }
147 ++m_warnings; 148 ++m_warnings;
148 } 149 }
149 150
150 return calculatedFrame; 151 return calculatedFrame;
170 // that's likely to be visible 171 // that's likely to be visible
171 windowSize = 1024; 172 windowSize = 1024;
172 } else { 173 } else {
173 windowSize = 1; 174 windowSize = 1;
174 } 175 }
175 if (timeUnits == CSVFormat::TimeSeconds || 176 if (timeUnits == CSVFormat::TimeSeconds ||
176 timeUnits == CSVFormat::TimeMilliseconds) { 177 timeUnits == CSVFormat::TimeMilliseconds) {
177 sampleRate = m_mainModelSampleRate; 178 sampleRate = m_mainModelSampleRate;
178 } 179 }
179 } 180 }
180 181
181 SparseOneDimensionalModel *model1 = 0; 182 SparseOneDimensionalModel *model1 = 0;
182 SparseTimeValueModel *model2 = 0; 183 SparseTimeValueModel *model2 = 0;
183 RegionModel *model2a = 0; 184 RegionModel *model2a = 0;
241 242
242 case CSVFormat::OneDimensionalModel: 243 case CSVFormat::OneDimensionalModel:
243 model1 = new SparseOneDimensionalModel(sampleRate, windowSize); 244 model1 = new SparseOneDimensionalModel(sampleRate, windowSize);
244 model = model1; 245 model = model1;
245 break; 246 break;
246 247
247 case CSVFormat::TwoDimensionalModel: 248 case CSVFormat::TwoDimensionalModel:
248 model2 = new SparseTimeValueModel(sampleRate, windowSize, false); 249 model2 = new SparseTimeValueModel(sampleRate, windowSize, false);
249 model = model2; 250 model = model2;
250 break; 251 break;
251 252
252 case CSVFormat::TwoDimensionalModelWithDuration: 253 case CSVFormat::TwoDimensionalModelWithDuration:
253 model2a = new RegionModel(sampleRate, windowSize, false); 254 model2a = new RegionModel(sampleRate, windowSize, false);
254 model = model2a; 255 model = model2a;
255 break; 256 break;
256 257
257 case CSVFormat::TwoDimensionalModelWithDurationAndPitch: 258 case CSVFormat::TwoDimensionalModelWithDurationAndPitch:
258 model2b = new NoteModel(sampleRate, windowSize, false); 259 model2b = new NoteModel(sampleRate, windowSize, false);
259 model = model2b; 260 model = model2b;
260 break; 261 break;
261 262
262 case CSVFormat::ThreeDimensionalModel: 263 case CSVFormat::ThreeDimensionalModel:
263 model3 = new EditableDenseThreeDimensionalModel 264 model3 = new EditableDenseThreeDimensionalModel
264 (sampleRate, 265 (sampleRate,
265 windowSize, 266 windowSize,
266 valueColumns, 267 valueColumns,
332 duration = endFrame - frameNo; 333 duration = endFrame - frameNo;
333 } 334 }
334 } 335 }
335 336
336 if (modelType == CSVFormat::OneDimensionalModel) { 337 if (modelType == CSVFormat::OneDimensionalModel) {
337 338
338 SparseOneDimensionalModel::Point point(frameNo, label); 339 SparseOneDimensionalModel::Point point(frameNo, label);
339 model1->addPoint(point); 340 model1->addPoint(point);
340 341
341 } else if (modelType == CSVFormat::TwoDimensionalModel) { 342 } else if (modelType == CSVFormat::TwoDimensionalModel) {
342 343
366 367
367 bool ok = false; 368 bool ok = false;
368 float value = list[i].toFloat(&ok); 369 float value = list[i].toFloat(&ok);
369 370
370 values.push_back(value); 371 values.push_back(value);
371 372
372 if (firstEverValue || value < min) min = value; 373 if (firstEverValue || value < min) min = value;
373 if (firstEverValue || value > max) max = value; 374 if (firstEverValue || value > max) max = value;
374 375
375 if (firstEverValue) { 376 if (firstEverValue) {
376 startFrame = frameNo; 377 startFrame = frameNo;
382 383
383 firstEverValue = false; 384 firstEverValue = false;
384 385
385 if (!ok) { 386 if (!ok) {
386 if (warnings < warnLimit) { 387 if (warnings < warnLimit) {
387 cerr << "WARNING: CSVFileReader::load: " 388 SVCERR << "WARNING: CSVFileReader::load: "
388 << "Non-numeric value \"" 389 << "Non-numeric value \""
389 << list[i] 390 << list[i]
390 << "\" in data line " << lineno+1 391 << "\" in data line " << lineno+1
391 << ":" << endl; 392 << ":" << endl;
392 cerr << line << endl; 393 SVCERR << line << endl;
393 ++warnings; 394 ++warnings;
394 } else if (warnings == warnLimit) { 395 } else if (warnings == warnLimit) {
395 // cerr << "WARNING: Too many warnings" << endl; 396 // SVCERR << "WARNING: Too many warnings" << endl;
396 } 397 }
397 } 398 }
398 } 399 }
399 400
400 // SVDEBUG << "Setting bin values for count " << lineno << ", frame " 401 // SVDEBUG << "Setting bin values for count " << lineno << ", frame "
401 // << frameNo << ", time " << RealTime::frame2RealTime(frameNo, sampleRate) << endl; 402 // << frameNo << ", time " << RealTime::frame2RealTime(frameNo, sampleRate) << endl;
402 403
403 model3->setColumn(lineno, values); 404 model3->setColumn(lineno, values);
404 } 405 }
424 425
425 float v = 0.f; 426 float v = 0.f;
426 for (map<int, map<QString, float> >::iterator i = 427 for (map<int, map<QString, float> >::iterator i =
427 countLabelValueMap.end(); i != countLabelValueMap.begin(); ) { 428 countLabelValueMap.end(); i != countLabelValueMap.begin(); ) {
428 --i; 429 --i;
429 cerr << "count -> " << i->first << endl; 430 SVCERR << "count -> " << i->first << endl;
430 for (map<QString, float>::iterator j = i->second.begin(); 431 for (map<QString, float>::iterator j = i->second.begin();
431 j != i->second.end(); ++j) { 432 j != i->second.end(); ++j) {
432 j->second = v; 433 j->second = v;
433 cerr << "label -> " << j->first << ", value " << v << endl; 434 SVCERR << "label -> " << j->first << ", value " << v << endl;
434 v = v + 1.f; 435 v = v + 1.f;
435 } 436 }
436 } 437 }
437 438
438 map<RegionModel::Point, RegionModel::Point, 439 map<RegionModel::Point, RegionModel::Point,
441 model2a->getPoints().begin(); 442 model2a->getPoints().begin();
442 i != model2a->getPoints().end(); ++i) { 443 i != model2a->getPoints().end(); ++i) {
443 RegionModel::Point p(*i); 444 RegionModel::Point p(*i);
444 int count = labelCountMap[p.label]; 445 int count = labelCountMap[p.label];
445 v = countLabelValueMap[count][p.label]; 446 v = countLabelValueMap[count][p.label];
446 cerr << "mapping from label \"" << p.label << "\" (count " << count << ") to value " << v << endl; 447 // SVCERR << "mapping from label \"" << p.label << "\" (count " << count << ") to value " << v << endl;
447 RegionModel::Point pp(p.frame, v, p.duration, p.label); 448 RegionModel::Point pp(p.frame, v, p.duration, p.label);
448 pointMap[p] = pp; 449 pointMap[p] = pp;
449 } 450 }
450 451
451 for (map<RegionModel::Point, RegionModel::Point>::iterator i = 452 for (map<RegionModel::Point, RegionModel::Point>::iterator i =
472 model2b->setScaleUnits("Hz"); 473 model2b->setScaleUnits("Hz");
473 } 474 }
474 } 475 }
475 476
476 if (model3) { 477 if (model3) {
477 model3->setMinimumLevel(min); 478 model3->setMinimumLevel(min);
478 model3->setMaximumLevel(max); 479 model3->setMaximumLevel(max);
479 } 480 }
480 481
481 return model; 482 return model;
482 } 483 }
483 484