annotate data/fileio/CSVFileReader.cpp @ 1119:e22bfe8ca248 tonioni

fixes the problem that the last note was not exported to csv
author Matthias Mauch <mail@matthiasmauch.net>
date Fri, 14 Aug 2015 18:16:14 +0100
parents 26cf6d5251ec
children 39019ce29178
rev   line source
Chris@148 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@148 2
Chris@148 3 /*
Chris@148 4 Sonic Visualiser
Chris@148 5 An audio file viewer and annotation editor.
Chris@148 6 Centre for Digital Music, Queen Mary, University of London.
Chris@148 7 This file copyright 2006 Chris Cannam.
Chris@148 8
Chris@148 9 This program is free software; you can redistribute it and/or
Chris@148 10 modify it under the terms of the GNU General Public License as
Chris@148 11 published by the Free Software Foundation; either version 2 of the
Chris@148 12 License, or (at your option) any later version. See the file
Chris@148 13 COPYING included with this distribution for more information.
Chris@148 14 */
Chris@148 15
Chris@148 16 #include "CSVFileReader.h"
Chris@148 17
Chris@150 18 #include "model/Model.h"
Chris@148 19 #include "base/RealTime.h"
Chris@631 20 #include "base/StringBits.h"
Chris@148 21 #include "model/SparseOneDimensionalModel.h"
Chris@148 22 #include "model/SparseTimeValueModel.h"
Chris@152 23 #include "model/EditableDenseThreeDimensionalModel.h"
Chris@628 24 #include "model/RegionModel.h"
Chris@897 25 #include "model/NoteModel.h"
Chris@308 26 #include "DataFileReaderFactory.h"
Chris@148 27
Chris@148 28 #include <QFile>
Chris@1030 29 #include <QFileInfo>
Chris@148 30 #include <QString>
Chris@148 31 #include <QRegExp>
Chris@148 32 #include <QStringList>
Chris@148 33 #include <QTextStream>
mail@1119 34 #include <QIODevice>
Chris@148 35
Chris@148 36 #include <iostream>
Chris@628 37 #include <map>
Chris@148 38
Chris@392 39 CSVFileReader::CSVFileReader(QString path, CSVFormat format,
Chris@1047 40 sv_samplerate_t mainModelSampleRate) :
Chris@392 41 m_format(format),
Chris@1009 42 m_device(0),
Chris@1009 43 m_ownDevice(true),
Chris@631 44 m_warnings(0),
Chris@148 45 m_mainModelSampleRate(mainModelSampleRate)
Chris@148 46 {
Chris@1009 47 QFile *file = new QFile(path);
Chris@148 48 bool good = false;
Chris@148 49
Chris@1009 50 if (!file->exists()) {
Chris@148 51 m_error = QFile::tr("File \"%1\" does not exist").arg(path);
Chris@1009 52 } else if (!file->open(QIODevice::ReadOnly | QIODevice::Text)) {
Chris@148 53 m_error = QFile::tr("Failed to open file \"%1\"").arg(path);
Chris@148 54 } else {
Chris@148 55 good = true;
Chris@148 56 }
Chris@148 57
Chris@1009 58 if (good) {
Chris@1009 59 m_device = file;
Chris@1030 60 m_filename = QFileInfo(path).fileName();
Chris@1009 61 } else {
Chris@1009 62 delete file;
Chris@148 63 }
Chris@148 64 }
Chris@148 65
Chris@1009 66 CSVFileReader::CSVFileReader(QIODevice *device, CSVFormat format,
Chris@1047 67 sv_samplerate_t mainModelSampleRate) :
Chris@1009 68 m_format(format),
Chris@1009 69 m_device(device),
Chris@1009 70 m_ownDevice(false),
Chris@1009 71 m_warnings(0),
Chris@1009 72 m_mainModelSampleRate(mainModelSampleRate)
Chris@1009 73 {
Chris@1009 74 }
Chris@1009 75
Chris@148 76 CSVFileReader::~CSVFileReader()
Chris@148 77 {
Chris@1009 78 SVDEBUG << "CSVFileReader::~CSVFileReader: device is " << m_device << endl;
Chris@148 79
Chris@1009 80 if (m_device && m_ownDevice) {
Chris@1009 81 SVDEBUG << "CSVFileReader::CSVFileReader: Closing device" << endl;
Chris@1009 82 m_device->close();
Chris@1009 83 delete m_device;
Chris@148 84 }
Chris@148 85 }
Chris@148 86
Chris@148 87 bool
Chris@148 88 CSVFileReader::isOK() const
Chris@148 89 {
Chris@1009 90 return (m_device != 0);
Chris@148 91 }
Chris@148 92
Chris@148 93 QString
Chris@148 94 CSVFileReader::getError() const
Chris@148 95 {
Chris@148 96 return m_error;
Chris@148 97 }
Chris@148 98
Chris@1038 99 sv_frame_t
Chris@1047 100 CSVFileReader::convertTimeValue(QString s, int lineno,
Chris@1047 101 sv_samplerate_t sampleRate,
Chris@929 102 int windowSize) const
Chris@631 103 {
Chris@631 104 QRegExp nonNumericRx("[^0-9eE.,+-]");
Chris@897 105 int warnLimit = 10;
Chris@631 106
Chris@631 107 CSVFormat::TimeUnits timeUnits = m_format.getTimeUnits();
Chris@631 108
Chris@1038 109 sv_frame_t calculatedFrame = 0;
Chris@631 110
Chris@631 111 bool ok = false;
Chris@631 112 QString numeric = s;
Chris@631 113 numeric.remove(nonNumericRx);
Chris@631 114
Chris@631 115 if (timeUnits == CSVFormat::TimeSeconds) {
Chris@631 116
Chris@631 117 double time = numeric.toDouble(&ok);
Chris@631 118 if (!ok) time = StringBits::stringToDoubleLocaleFree(numeric, &ok);
Chris@1038 119 calculatedFrame = sv_frame_t(time * sampleRate + 0.5);
Chris@990 120
Chris@990 121 } else if (timeUnits == CSVFormat::TimeMilliseconds) {
Chris@990 122
Chris@990 123 double time = numeric.toDouble(&ok);
Chris@990 124 if (!ok) time = StringBits::stringToDoubleLocaleFree(numeric, &ok);
Chris@1038 125 calculatedFrame = sv_frame_t((time / 1000.0) * sampleRate + 0.5);
Chris@631 126
Chris@631 127 } else {
Chris@631 128
Chris@631 129 long n = numeric.toLong(&ok);
Chris@631 130 if (n >= 0) calculatedFrame = n;
Chris@631 131
Chris@631 132 if (timeUnits == CSVFormat::TimeWindows) {
Chris@631 133 calculatedFrame *= windowSize;
Chris@631 134 }
Chris@631 135 }
Chris@631 136
Chris@631 137 if (!ok) {
Chris@631 138 if (m_warnings < warnLimit) {
Chris@843 139 cerr << "WARNING: CSVFileReader::load: "
Chris@844 140 << "Bad time format (\"" << s
Chris@631 141 << "\") in data line "
Chris@843 142 << lineno+1 << endl;
Chris@631 143 } else if (m_warnings == warnLimit) {
Chris@843 144 cerr << "WARNING: Too many warnings" << endl;
Chris@631 145 }
Chris@631 146 ++m_warnings;
Chris@631 147 }
Chris@631 148
Chris@631 149 return calculatedFrame;
Chris@631 150 }
Chris@631 151
Chris@148 152 Model *
Chris@148 153 CSVFileReader::load() const
Chris@148 154 {
Chris@1009 155 if (!m_device) return 0;
Chris@148 156
Chris@628 157 CSVFormat::ModelType modelType = m_format.getModelType();
Chris@392 158 CSVFormat::TimingType timingType = m_format.getTimingType();
Chris@628 159 CSVFormat::TimeUnits timeUnits = m_format.getTimeUnits();
Chris@1047 160 sv_samplerate_t sampleRate = m_format.getSampleRate();
Chris@929 161 int windowSize = m_format.getWindowSize();
Chris@631 162 QChar separator = m_format.getSeparator();
Chris@631 163 bool allowQuoting = m_format.getAllowQuoting();
Chris@148 164
Chris@392 165 if (timingType == CSVFormat::ExplicitTiming) {
Chris@611 166 if (modelType == CSVFormat::ThreeDimensionalModel) {
Chris@611 167 // This will be overridden later if more than one line
Chris@611 168 // appears in our file, but we want to choose a default
Chris@611 169 // that's likely to be visible
Chris@611 170 windowSize = 1024;
Chris@611 171 } else {
Chris@611 172 windowSize = 1;
Chris@611 173 }
Chris@990 174 if (timeUnits == CSVFormat::TimeSeconds ||
Chris@990 175 timeUnits == CSVFormat::TimeMilliseconds) {
Chris@148 176 sampleRate = m_mainModelSampleRate;
Chris@148 177 }
Chris@148 178 }
Chris@148 179
Chris@148 180 SparseOneDimensionalModel *model1 = 0;
Chris@148 181 SparseTimeValueModel *model2 = 0;
Chris@628 182 RegionModel *model2a = 0;
Chris@897 183 NoteModel *model2b = 0;
Chris@152 184 EditableDenseThreeDimensionalModel *model3 = 0;
Chris@148 185 Model *model = 0;
Chris@148 186
Chris@1009 187 QTextStream in(m_device);
Chris@148 188
Chris@148 189 unsigned int warnings = 0, warnLimit = 10;
Chris@148 190 unsigned int lineno = 0;
Chris@148 191
Chris@148 192 float min = 0.0, max = 0.0;
Chris@148 193
Chris@1038 194 sv_frame_t frameNo = 0;
Chris@1038 195 sv_frame_t duration = 0;
Chris@1038 196 sv_frame_t endFrame = 0;
Chris@631 197
Chris@631 198 bool haveAnyValue = false;
Chris@631 199 bool haveEndTime = false;
Chris@897 200 bool pitchLooksLikeMIDI = true;
Chris@631 201
Chris@1038 202 sv_frame_t startFrame = 0; // for calculation of dense model resolution
Chris@631 203 bool firstEverValue = true;
Chris@148 204
Chris@631 205 std::map<QString, int> labelCountMap;
Chris@631 206
Chris@676 207 int valueColumns = 0;
Chris@676 208 for (int i = 0; i < m_format.getColumnCount(); ++i) {
Chris@676 209 if (m_format.getColumnPurpose(i) == CSVFormat::ColumnValue) {
Chris@676 210 ++valueColumns;
Chris@676 211 }
Chris@676 212 }
Chris@676 213
Chris@148 214 while (!in.atEnd()) {
Chris@148 215
Chris@283 216 // QTextStream's readLine doesn't cope with old-style Mac
Chris@283 217 // CR-only line endings. Why did they bother making the class
Chris@283 218 // cope with more than one sort of line ending, if it still
Chris@283 219 // can't be configured to cope with all the common sorts?
Chris@148 220
Chris@283 221 // For the time being we'll deal with this case (which is
Chris@283 222 // relatively uncommon for us, but still necessary to handle)
Chris@283 223 // by reading the entire file using a single readLine, and
Chris@283 224 // splitting it. For CR and CR/LF line endings this will just
Chris@283 225 // read a line at a time, and that's obviously OK.
Chris@148 226
Chris@283 227 QString chunk = in.readLine();
Chris@283 228 QStringList lines = chunk.split('\r', QString::SkipEmptyParts);
Chris@283 229
Chris@897 230 for (int li = 0; li < lines.size(); ++li) {
Chris@148 231
Chris@283 232 QString line = lines[li];
Chris@1009 233
Chris@283 234 if (line.startsWith("#")) continue;
Chris@283 235
Chris@631 236 QStringList list = StringBits::split(line, separator, allowQuoting);
Chris@283 237 if (!model) {
Chris@283 238
Chris@283 239 switch (modelType) {
Chris@283 240
Chris@392 241 case CSVFormat::OneDimensionalModel:
Chris@283 242 model1 = new SparseOneDimensionalModel(sampleRate, windowSize);
Chris@283 243 model = model1;
Chris@283 244 break;
Chris@148 245
Chris@392 246 case CSVFormat::TwoDimensionalModel:
Chris@283 247 model2 = new SparseTimeValueModel(sampleRate, windowSize, false);
Chris@283 248 model = model2;
Chris@283 249 break;
Chris@148 250
Chris@628 251 case CSVFormat::TwoDimensionalModelWithDuration:
Chris@628 252 model2a = new RegionModel(sampleRate, windowSize, false);
Chris@628 253 model = model2a;
Chris@628 254 break;
Chris@628 255
Chris@897 256 case CSVFormat::TwoDimensionalModelWithDurationAndPitch:
Chris@897 257 model2b = new NoteModel(sampleRate, windowSize, false);
Chris@897 258 model = model2b;
Chris@897 259 break;
Chris@897 260
Chris@392 261 case CSVFormat::ThreeDimensionalModel:
Chris@535 262 model3 = new EditableDenseThreeDimensionalModel
Chris@535 263 (sampleRate,
Chris@535 264 windowSize,
Chris@676 265 valueColumns,
Chris@535 266 EditableDenseThreeDimensionalModel::NoCompression);
Chris@283 267 model = model3;
Chris@283 268 break;
Chris@283 269 }
Chris@1030 270
Chris@1030 271 if (model) {
Chris@1030 272 if (m_filename != "") {
Chris@1030 273 model->setObjectName(m_filename);
Chris@1030 274 }
Chris@1030 275 }
Chris@283 276 }
Chris@148 277
Chris@631 278 float value = 0.f;
Chris@897 279 float pitch = 0.f;
Chris@631 280 QString label = "";
Chris@148 281
Chris@631 282 duration = 0.f;
Chris@631 283 haveEndTime = false;
Chris@628 284
Chris@283 285 for (int i = 0; i < list.size(); ++i) {
Chris@148 286
Chris@631 287 QString s = list[i];
Chris@631 288
Chris@631 289 CSVFormat::ColumnPurpose purpose = m_format.getColumnPurpose(i);
Chris@631 290
Chris@631 291 switch (purpose) {
Chris@631 292
Chris@631 293 case CSVFormat::ColumnUnknown:
Chris@631 294 break;
Chris@631 295
Chris@631 296 case CSVFormat::ColumnStartTime:
Chris@631 297 frameNo = convertTimeValue(s, lineno, sampleRate, windowSize);
Chris@631 298 break;
Chris@631 299
Chris@631 300 case CSVFormat::ColumnEndTime:
Chris@631 301 endFrame = convertTimeValue(s, lineno, sampleRate, windowSize);
Chris@631 302 haveEndTime = true;
Chris@631 303 break;
Chris@631 304
Chris@631 305 case CSVFormat::ColumnDuration:
Chris@631 306 duration = convertTimeValue(s, lineno, sampleRate, windowSize);
Chris@631 307 break;
Chris@631 308
Chris@631 309 case CSVFormat::ColumnValue:
Chris@631 310 value = s.toFloat();
Chris@631 311 haveAnyValue = true;
Chris@631 312 break;
Chris@631 313
Chris@897 314 case CSVFormat::ColumnPitch:
Chris@897 315 pitch = s.toFloat();
Chris@897 316 if (pitch < 0.f || pitch > 127.f) {
Chris@897 317 pitchLooksLikeMIDI = false;
Chris@897 318 }
Chris@897 319 break;
Chris@897 320
Chris@631 321 case CSVFormat::ColumnLabel:
Chris@631 322 label = s;
Chris@631 323 ++labelCountMap[label];
Chris@631 324 break;
Chris@283 325 }
Chris@631 326 }
Chris@148 327
Chris@631 328 if (haveEndTime) { // ... calculate duration now all cols read
Chris@631 329 if (endFrame > frameNo) {
Chris@631 330 duration = endFrame - frameNo;
Chris@628 331 }
Chris@283 332 }
Chris@148 333
Chris@392 334 if (modelType == CSVFormat::OneDimensionalModel) {
Chris@148 335
Chris@631 336 SparseOneDimensionalModel::Point point(frameNo, label);
Chris@283 337 model1->addPoint(point);
Chris@148 338
Chris@392 339 } else if (modelType == CSVFormat::TwoDimensionalModel) {
Chris@148 340
Chris@631 341 SparseTimeValueModel::Point point(frameNo, value, label);
Chris@283 342 model2->addPoint(point);
Chris@148 343
Chris@628 344 } else if (modelType == CSVFormat::TwoDimensionalModelWithDuration) {
Chris@628 345
Chris@631 346 RegionModel::Point point(frameNo, value, duration, label);
Chris@628 347 model2a->addPoint(point);
Chris@628 348
Chris@897 349 } else if (modelType == CSVFormat::TwoDimensionalModelWithDurationAndPitch) {
Chris@897 350
Chris@897 351 float level = ((value >= 0.f && value <= 1.f) ? value : 1.f);
Chris@897 352 NoteModel::Point point(frameNo, pitch, duration, level, label);
Chris@897 353 model2b->addPoint(point);
Chris@897 354
Chris@392 355 } else if (modelType == CSVFormat::ThreeDimensionalModel) {
Chris@148 356
Chris@283 357 DenseThreeDimensionalModel::Column values;
Chris@148 358
Chris@631 359 for (int i = 0; i < list.size(); ++i) {
Chris@148 360
Chris@676 361 if (m_format.getColumnPurpose(i) != CSVFormat::ColumnValue) {
Chris@676 362 continue;
Chris@676 363 }
Chris@676 364
Chris@283 365 bool ok = false;
Chris@283 366 float value = list[i].toFloat(&ok);
Chris@611 367
Chris@676 368 values.push_back(value);
Chris@148 369
Chris@631 370 if (firstEverValue || value < min) min = value;
Chris@631 371 if (firstEverValue || value > max) max = value;
Chris@676 372
Chris@631 373 if (firstEverValue) {
Chris@611 374 startFrame = frameNo;
Chris@611 375 model3->setStartFrame(startFrame);
Chris@611 376 } else if (lineno == 1 &&
Chris@611 377 timingType == CSVFormat::ExplicitTiming) {
Chris@1038 378 model3->setResolution(int(frameNo - startFrame));
Chris@611 379 }
Chris@631 380
Chris@631 381 firstEverValue = false;
Chris@148 382
Chris@283 383 if (!ok) {
Chris@283 384 if (warnings < warnLimit) {
Chris@843 385 cerr << "WARNING: CSVFileReader::load: "
Chris@390 386 << "Non-numeric value \""
Chris@844 387 << list[i]
Chris@491 388 << "\" in data line " << lineno+1
Chris@843 389 << ":" << endl;
Chris@843 390 cerr << line << endl;
Chris@283 391 ++warnings;
Chris@283 392 } else if (warnings == warnLimit) {
Chris@843 393 // cerr << "WARNING: Too many warnings" << endl;
Chris@283 394 }
Chris@283 395 }
Chris@283 396 }
Chris@148 397
Chris@690 398 // SVDEBUG << "Setting bin values for count " << lineno << ", frame "
Chris@687 399 // << frameNo << ", time " << RealTime::frame2RealTime(frameNo, sampleRate) << endl;
Chris@148 400
Chris@611 401 model3->setColumn(lineno, values);
Chris@283 402 }
Chris@148 403
Chris@283 404 ++lineno;
Chris@392 405 if (timingType == CSVFormat::ImplicitTiming ||
Chris@283 406 list.size() == 0) {
Chris@283 407 frameNo += windowSize;
Chris@283 408 }
Chris@283 409 }
Chris@148 410 }
Chris@148 411
Chris@631 412 if (!haveAnyValue) {
Chris@631 413 if (model2a) {
Chris@631 414 // assign values for regions based on label frequency; we
Chris@631 415 // have this in our labelCountMap, sort of
Chris@631 416
Chris@631 417 std::map<int, std::map<QString, float> > countLabelValueMap;
Chris@631 418 for (std::map<QString, int>::iterator i = labelCountMap.begin();
Chris@631 419 i != labelCountMap.end(); ++i) {
Chris@631 420 countLabelValueMap[i->second][i->first] = 0.f;
Chris@631 421 }
Chris@631 422
Chris@631 423 float v = 0.f;
Chris@631 424 for (std::map<int, std::map<QString, float> >::iterator i =
Chris@631 425 countLabelValueMap.end(); i != countLabelValueMap.begin(); ) {
Chris@631 426 --i;
Chris@631 427 for (std::map<QString, float>::iterator j = i->second.begin();
Chris@631 428 j != i->second.end(); ++j) {
Chris@631 429 j->second = v;
Chris@631 430 v = v + 1.f;
Chris@631 431 }
Chris@631 432 }
Chris@631 433
Chris@631 434 std::map<RegionModel::Point, RegionModel::Point,
Chris@631 435 RegionModel::Point::Comparator> pointMap;
Chris@631 436 for (RegionModel::PointList::const_iterator i =
Chris@631 437 model2a->getPoints().begin();
Chris@631 438 i != model2a->getPoints().end(); ++i) {
Chris@631 439 RegionModel::Point p(*i);
Chris@631 440 v = countLabelValueMap[labelCountMap[p.label]][p.label];
Chris@631 441 RegionModel::Point pp(p.frame, v, p.duration, p.label);
Chris@631 442 pointMap[p] = pp;
Chris@631 443 }
Chris@631 444
Chris@631 445 for (std::map<RegionModel::Point, RegionModel::Point>::iterator i =
Chris@631 446 pointMap.begin(); i != pointMap.end(); ++i) {
Chris@631 447 model2a->deletePoint(i->first);
Chris@631 448 model2a->addPoint(i->second);
Chris@631 449 }
Chris@631 450 }
Chris@631 451 }
Chris@631 452
Chris@897 453 if (model2b) {
Chris@897 454 if (pitchLooksLikeMIDI) {
Chris@897 455 model2b->setScaleUnits("MIDI Pitch");
Chris@897 456 } else {
Chris@897 457 model2b->setScaleUnits("Hz");
Chris@897 458 }
Chris@897 459 }
Chris@897 460
Chris@961 461 if (model3) {
Chris@148 462 model3->setMinimumLevel(min);
Chris@148 463 model3->setMaximumLevel(max);
Chris@148 464 }
Chris@148 465
Chris@148 466 return model;
Chris@148 467 }
Chris@148 468