Chris@148: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
Chris@148: 
Chris@148: /*
Chris@148:     Sonic Visualiser
Chris@148:     An audio file viewer and annotation editor.
Chris@148:     Centre for Digital Music, Queen Mary, University of London.
Chris@148:     This file copyright 2006 Chris Cannam.
Chris@148:     
Chris@148:     This program is free software; you can redistribute it and/or
Chris@148:     modify it under the terms of the GNU General Public License as
Chris@148:     published by the Free Software Foundation; either version 2 of the
Chris@148:     License, or (at your option) any later version.  See the file
Chris@148:     COPYING included with this distribution for more information.
Chris@148: */
Chris@148: 
Chris@148: #include "CSVFileReader.h"
Chris@148: 
Chris@150: #include "model/Model.h"
Chris@148: #include "base/RealTime.h"
Chris@631: #include "base/StringBits.h"
Chris@1491: #include "base/ProgressReporter.h"
Chris@1519: #include "base/RecordDirectory.h"
Chris@148: #include "model/SparseOneDimensionalModel.h"
Chris@148: #include "model/SparseTimeValueModel.h"
Chris@152: #include "model/EditableDenseThreeDimensionalModel.h"
Chris@628: #include "model/RegionModel.h"
Chris@897: #include "model/NoteModel.h"
Chris@1793: #include "model/BoxModel.h"
Chris@1488: #include "model/WritableWaveFileModel.h"
Chris@308: #include "DataFileReaderFactory.h"
Chris@148: 
Chris@148: #include <QFile>
Chris@1519: #include <QDir>
Chris@1030: #include <QFileInfo>
Chris@148: #include <QString>
Chris@148: #include <QRegExp>
Chris@148: #include <QStringList>
Chris@148: #include <QTextStream>
Chris@1519: #include <QDateTime>
Chris@148: 
Chris@148: #include <iostream>
Chris@628: #include <map>
Chris@1428: #include <string>
Chris@148: 
Chris@1113: using namespace std;
Chris@1113: 
Chris@392: CSVFileReader::CSVFileReader(QString path, CSVFormat format,
Chris@1491:                              sv_samplerate_t mainModelSampleRate,
Chris@1491:                              ProgressReporter *reporter) :
Chris@392:     m_format(format),
Chris@1582:     m_device(nullptr),
Chris@1009:     m_ownDevice(true),
Chris@631:     m_warnings(0),
Chris@1491:     m_mainModelSampleRate(mainModelSampleRate),
Chris@1491:     m_fileSize(0),
Chris@1491:     m_readCount(0),
Chris@1492:     m_progress(-1),
Chris@1491:     m_reporter(reporter)
Chris@148: {
Chris@1009:     QFile *file = new QFile(path);
Chris@148:     bool good = false;
Chris@148:     
Chris@1009:     if (!file->exists()) {
Chris@1429:         m_error = QFile::tr("File \"%1\" does not exist").arg(path);
Chris@1009:     } else if (!file->open(QIODevice::ReadOnly | QIODevice::Text)) {
Chris@1429:         m_error = QFile::tr("Failed to open file \"%1\"").arg(path);
Chris@148:     } else {
Chris@1429:         good = true;
Chris@148:     }
Chris@148: 
Chris@1009:     if (good) {
Chris@1009:         m_device = file;
Chris@1030:         m_filename = QFileInfo(path).fileName();
Chris@1491:         m_fileSize = file->size();
Chris@1491:         if (m_reporter) m_reporter->setDefinite(true);
Chris@1009:     } else {
Chris@1429:         delete file;
Chris@148:     }
Chris@148: }
Chris@148: 
Chris@1009: CSVFileReader::CSVFileReader(QIODevice *device, CSVFormat format,
Chris@1491:                              sv_samplerate_t mainModelSampleRate,
Chris@1491:                              ProgressReporter *reporter) :
Chris@1009:     m_format(format),
Chris@1009:     m_device(device),
Chris@1009:     m_ownDevice(false),
Chris@1009:     m_warnings(0),
Chris@1491:     m_mainModelSampleRate(mainModelSampleRate),
Chris@1491:     m_fileSize(0),
Chris@1491:     m_readCount(0),
Chris@1492:     m_progress(-1),
Chris@1491:     m_reporter(reporter)
Chris@1009: {
Chris@1491:     if (m_reporter) m_reporter->setDefinite(false);
Chris@1009: }
Chris@1009: 
Chris@148: CSVFileReader::~CSVFileReader()
Chris@148: {
Chris@1009:     SVDEBUG << "CSVFileReader::~CSVFileReader: device is " << m_device << endl;
Chris@148: 
Chris@1009:     if (m_device && m_ownDevice) {
Chris@1009:         SVDEBUG << "CSVFileReader::CSVFileReader: Closing device" << endl;
Chris@1009:         m_device->close();
Chris@1009:         delete m_device;
Chris@148:     }
Chris@148: }
Chris@148: 
Chris@148: bool
Chris@148: CSVFileReader::isOK() const
Chris@148: {
Chris@1582:     return (m_device != nullptr);
Chris@148: }
Chris@148: 
Chris@148: QString
Chris@148: CSVFileReader::getError() const
Chris@148: {
Chris@148:     return m_error;
Chris@148: }
Chris@148: 
Chris@1867: bool
Chris@1047: CSVFileReader::convertTimeValue(QString s, int lineno,
Chris@1047:                                 sv_samplerate_t sampleRate,
Chris@1867:                                 int windowSize,
Chris@1867:                                 sv_frame_t &calculatedFrame) const
Chris@631: {
Chris@631:     QRegExp nonNumericRx("[^0-9eE.,+-]");
Chris@897:     int warnLimit = 10;
Chris@631: 
Chris@631:     CSVFormat::TimeUnits timeUnits = m_format.getTimeUnits();
Chris@631: 
Chris@1867:     calculatedFrame = 0;
Chris@631: 
Chris@631:     bool ok = false;
Chris@631:     QString numeric = s;
Chris@631:     numeric.remove(nonNumericRx);
Chris@631:     
Chris@631:     if (timeUnits == CSVFormat::TimeSeconds) {
Chris@631: 
Chris@631:         double time = numeric.toDouble(&ok);
Chris@631:         if (!ok) time = StringBits::stringToDoubleLocaleFree(numeric, &ok);
Chris@1038:         calculatedFrame = sv_frame_t(time * sampleRate + 0.5);
Chris@990:     
Chris@990:     } else if (timeUnits == CSVFormat::TimeMilliseconds) {
Chris@990: 
Chris@990:         double time = numeric.toDouble(&ok);
Chris@990:         if (!ok) time = StringBits::stringToDoubleLocaleFree(numeric, &ok);
Chris@1038:         calculatedFrame = sv_frame_t((time / 1000.0) * sampleRate + 0.5);
Chris@631:         
Chris@631:     } else {
Chris@631:         
Chris@631:         long n = numeric.toLong(&ok);
Chris@631:         if (n >= 0) calculatedFrame = n;
Chris@631:         
Chris@631:         if (timeUnits == CSVFormat::TimeWindows) {
Chris@631:             calculatedFrame *= windowSize;
Chris@631:         }
Chris@631:     }
Chris@631:     
Chris@631:     if (!ok) {
Chris@631:         if (m_warnings < warnLimit) {
Chris@1428:             SVCERR << "WARNING: CSVFileReader::load: "
Chris@844:                       << "Bad time format (\"" << s
Chris@631:                       << "\") in data line "
Chris@843:                       << lineno+1 << endl;
Chris@631:         } else if (m_warnings == warnLimit) {
Chris@1428:             SVCERR << "WARNING: Too many warnings" << endl;
Chris@631:         }
Chris@631:         ++m_warnings;
Chris@631:     }
Chris@631: 
Chris@631:     return calculatedFrame;
Chris@631: }
Chris@631: 
Chris@148: Model *
Chris@148: CSVFileReader::load() const
Chris@148: {
Chris@1582:     if (!m_device) return nullptr;
Chris@148: 
Chris@628:     CSVFormat::ModelType modelType = m_format.getModelType();
Chris@392:     CSVFormat::TimingType timingType = m_format.getTimingType();
Chris@628:     CSVFormat::TimeUnits timeUnits = m_format.getTimeUnits();
Chris@1047:     sv_samplerate_t sampleRate = m_format.getSampleRate();
Chris@929:     int windowSize = m_format.getWindowSize();
Chris@631:     QChar separator = m_format.getSeparator();
Chris@631:     bool allowQuoting = m_format.getAllowQuoting();
Chris@148: 
Chris@392:     if (timingType == CSVFormat::ExplicitTiming) {
Chris@611:         if (modelType == CSVFormat::ThreeDimensionalModel) {
Chris@611:             // This will be overridden later if more than one line
Chris@611:             // appears in our file, but we want to choose a default
Chris@611:             // that's likely to be visible
Chris@611:             windowSize = 1024;
Chris@611:         } else {
Chris@611:             windowSize = 1;
Chris@611:         }
Chris@1429:         if (timeUnits == CSVFormat::TimeSeconds ||
Chris@990:             timeUnits == CSVFormat::TimeMilliseconds) {
Chris@1429:             sampleRate = m_mainModelSampleRate;
Chris@1429:         }
Chris@148:     }
Chris@148: 
Chris@1582:     SparseOneDimensionalModel *model1 = nullptr;
Chris@1582:     SparseTimeValueModel *model2 = nullptr;
Chris@1582:     RegionModel *model2a = nullptr;
Chris@1582:     NoteModel *model2b = nullptr;
Chris@1793:     BoxModel *model2c = nullptr;
Chris@1582:     EditableDenseThreeDimensionalModel *model3 = nullptr;
Chris@1582:     WritableWaveFileModel *modelW = nullptr;
Chris@1582:     Model *model = nullptr;
Chris@148: 
Chris@1009:     QTextStream in(m_device);
Chris@148: 
Chris@148:     unsigned int warnings = 0, warnLimit = 10;
Chris@148:     unsigned int lineno = 0;
Chris@148: 
Chris@148:     float min = 0.0, max = 0.0;
Chris@148: 
Chris@1038:     sv_frame_t frameNo = 0;
Chris@1038:     sv_frame_t duration = 0;
Chris@1038:     sv_frame_t endFrame = 0;
Chris@631: 
Chris@631:     bool haveAnyValue = false;
Chris@631:     bool haveEndTime = false;
Chris@897:     bool pitchLooksLikeMIDI = true;
Chris@631: 
Chris@1038:     sv_frame_t startFrame = 0; // for calculation of dense model resolution
Chris@631:     bool firstEverValue = true;
Chris@631:     
Chris@676:     int valueColumns = 0;
Chris@676:     for (int i = 0; i < m_format.getColumnCount(); ++i) {
Chris@676:         if (m_format.getColumnPurpose(i) == CSVFormat::ColumnValue) {
Chris@676:             ++valueColumns;
Chris@676:         }
Chris@676:     }
Chris@676: 
Chris@1518:     int audioChannels = 0;
Chris@1582:     float **audioSamples = nullptr;
Chris@1518:     float sampleShift = 0.f;
Chris@1518:     float sampleScale = 1.f;
Chris@1518: 
Chris@1518:     if (modelType == CSVFormat::WaveFileModel) {
Chris@1518: 
Chris@1518:         audioChannels = valueColumns;
Chris@1518:                 
Chris@1518:         audioSamples =
Chris@1518:             breakfastquay::allocate_and_zero_channels<float>
Chris@1518:             (audioChannels, 1);
Chris@1518: 
Chris@1518:         switch (m_format.getAudioSampleRange()) {
Chris@1518:         case CSVFormat::SampleRangeSigned1:
Chris@1518:         case CSVFormat::SampleRangeOther:
Chris@1518:             sampleShift = 0.f;
Chris@1518:             sampleScale = 1.f;
Chris@1518:             break;
Chris@1518:         case CSVFormat::SampleRangeUnsigned255:
Chris@1518:             sampleShift = -128.f;
Chris@1518:             sampleScale = 1.f / 128.f;
Chris@1518:             break;
Chris@1518:         case CSVFormat::SampleRangeSigned32767:
Chris@1518:             sampleShift = 0.f;
Chris@1518:             sampleScale = 1.f / 32768.f;
Chris@1518:             break;
Chris@1518:         }
Chris@1518:     }
Chris@1518: 
Chris@1518:     map<QString, int> labelCountMap;
Chris@1518: 
Chris@1872:     bool atStart = true;
Chris@1509:     bool abandoned = false;
Chris@1509:     
Chris@1509:     while (!in.atEnd() && !abandoned) {
Chris@148: 
Chris@283:         // QTextStream's readLine doesn't cope with old-style Mac
Chris@283:         // CR-only line endings.  Why did they bother making the class
Chris@283:         // cope with more than one sort of line ending, if it still
Chris@283:         // can't be configured to cope with all the common sorts?
Chris@148: 
Chris@283:         // For the time being we'll deal with this case (which is
Chris@283:         // relatively uncommon for us, but still necessary to handle)
Chris@283:         // by reading the entire file using a single readLine, and
Chris@283:         // splitting it.  For CR and CR/LF line endings this will just
Chris@283:         // read a line at a time, and that's obviously OK.
Chris@148: 
Chris@283:         QString chunk = in.readLine();
Chris@283:         QStringList lines = chunk.split('\r', QString::SkipEmptyParts);
Chris@1491: 
Chris@1491:         m_readCount += chunk.size() + 1;
Chris@1491: 
Chris@1491:         if (m_reporter) {
Chris@1491:             if (m_reporter->wasCancelled()) {
Chris@1509:                 abandoned = true;
Chris@1491:                 break;
Chris@1491:             }
Chris@1491:             int progress;
Chris@1491:             if (m_fileSize > 0) {
Chris@1491:                 progress = int((double(m_readCount) / double(m_fileSize))
Chris@1491:                                * 100.0);
Chris@1491:             } else {
Chris@1491:                 progress = int(m_readCount / 10000);
Chris@1491:             }
Chris@1491:             if (progress != m_progress) {
Chris@1491:                 m_reporter->setProgress(progress);
Chris@1491:                 m_progress = progress;
Chris@1491:             }
Chris@1491:         }
Chris@283:         
Chris@897:         for (int li = 0; li < lines.size(); ++li) {
Chris@1872:             
Chris@1872:             QString line = lines[li];
Chris@1872:             if (line.startsWith("#")) continue;
Chris@148: 
Chris@1872:             if (atStart) {
Chris@1872:                 atStart = false;
Chris@1872:                 if (m_format.getHeaderStatus() == CSVFormat::HeaderPresent) {
Chris@1872:                     continue;
Chris@1872:                 }
Chris@1872:             }
Chris@283: 
Chris@631:             QStringList list = StringBits::split(line, separator, allowQuoting);
Chris@283:             if (!model) {
Chris@283: 
Chris@1519:                 QString modelName = m_filename;
Chris@1519:                 
Chris@283:                 switch (modelType) {
Chris@283: 
Chris@392:                 case CSVFormat::OneDimensionalModel:
Chris@1867:                     SVDEBUG << "CSVFileReader: Creating sparse one-dimensional model" << endl;
Chris@283:                     model1 = new SparseOneDimensionalModel(sampleRate, windowSize);
Chris@283:                     model = model1;
Chris@283:                     break;
Chris@1429:                 
Chris@392:                 case CSVFormat::TwoDimensionalModel:
Chris@1867:                     SVDEBUG << "CSVFileReader: Creating sparse time-value model" << endl;
Chris@283:                     model2 = new SparseTimeValueModel(sampleRate, windowSize, false);
Chris@283:                     model = model2;
Chris@283:                     break;
Chris@1429:                 
Chris@628:                 case CSVFormat::TwoDimensionalModelWithDuration:
Chris@1867:                     SVDEBUG << "CSVFileReader: Creating region model" << endl;
Chris@628:                     model2a = new RegionModel(sampleRate, windowSize, false);
Chris@628:                     model = model2a;
Chris@628:                     break;
Chris@1429:                 
Chris@897:                 case CSVFormat::TwoDimensionalModelWithDurationAndPitch:
Chris@1867:                     SVDEBUG << "CSVFileReader: Creating note model" << endl;
Chris@897:                     model2b = new NoteModel(sampleRate, windowSize, false);
Chris@897:                     model = model2b;
Chris@897:                     break;
Chris@1429:                 
Chris@1793:                 case CSVFormat::TwoDimensionalModelWithDurationAndExtent:
Chris@1867:                     SVDEBUG << "CSVFileReader: Creating box model" << endl;
Chris@1793:                     model2c = new BoxModel(sampleRate, windowSize, false);
Chris@1793:                     model = model2c;
Chris@1793:                     break;
Chris@1793:                 
Chris@392:                 case CSVFormat::ThreeDimensionalModel:
Chris@1867:                     SVDEBUG << "CSVFileReader: Creating editable dense three-dimensional model" << endl;
Chris@535:                     model3 = new EditableDenseThreeDimensionalModel
Chris@1777:                         (sampleRate, windowSize, valueColumns);
Chris@283:                     model = model3;
Chris@283:                     break;
Chris@1488: 
Chris@1488:                 case CSVFormat::WaveFileModel:
Chris@1517:                 {
Chris@1867:                     SVDEBUG << "CSVFileReader: Creating writable wave-file model" << endl;
Chris@1517:                     bool normalise = (m_format.getAudioSampleRange()
Chris@1517:                                       == CSVFormat::SampleRangeOther);
Chris@1519:                     QString path = getConvertedAudioFilePath();
Chris@1488:                     modelW = new WritableWaveFileModel
Chris@1520:                         (path, sampleRate, valueColumns,
Chris@1520:                          normalise ?
Chris@1520:                          WritableWaveFileModel::Normalisation::Peak :
Chris@1520:                          WritableWaveFileModel::Normalisation::None);
Chris@1519:                     modelName = QFileInfo(path).fileName();
Chris@1488:                     model = modelW;
Chris@1488:                     break;
Chris@283:                 }
Chris@1517:                 }
Chris@1030: 
Chris@1508:                 if (model && model->isOK()) {
Chris@1519:                     if (modelName != "") {
Chris@1519:                         model->setObjectName(modelName);
Chris@1030:                     }
Chris@1030:                 }
Chris@283:             }
Chris@148: 
Chris@1508:             if (!model || !model->isOK()) {
Chris@1508:                 SVCERR << "Failed to create model to load CSV file into"
Chris@1508:                        << endl;
Chris@1508:                 if (model) {
Chris@1508:                     delete model;
Chris@1582:                     model = nullptr;
Chris@1793:                     model1 = nullptr; model2 = nullptr;
Chris@1793:                     model2a = nullptr; model2b = nullptr; model2c = nullptr;
Chris@1582:                     model3 = nullptr; modelW = nullptr;
Chris@1508:                 }
Chris@1509:                 abandoned = true;
Chris@1508:                 break;
Chris@1508:             }
Chris@1508:             
Chris@631:             float value = 0.f;
Chris@1793:             float otherValue = 0.f;
Chris@897:             float pitch = 0.f;
Chris@631:             QString label = "";
Chris@1867:             bool ok = true;
Chris@148: 
Chris@631:             duration = 0.f;
Chris@631:             haveEndTime = false;
Chris@1518:             
Chris@283:             for (int i = 0; i < list.size(); ++i) {
Chris@148: 
Chris@631:                 QString s = list[i];
Chris@631: 
Chris@631:                 CSVFormat::ColumnPurpose purpose = m_format.getColumnPurpose(i);
Chris@631: 
Chris@631:                 switch (purpose) {
Chris@631: 
Chris@631:                 case CSVFormat::ColumnUnknown:
Chris@631:                     break;
Chris@631: 
Chris@631:                 case CSVFormat::ColumnStartTime:
Chris@1867:                     if (!convertTimeValue(s, lineno, sampleRate, windowSize, frameNo)) {
Chris@1867:                         ok = false;
Chris@1867:                     }
Chris@631:                     break;
Chris@631:                 
Chris@631:                 case CSVFormat::ColumnEndTime:
Chris@1867:                     if (convertTimeValue(s, lineno, sampleRate, windowSize, endFrame)) {
Chris@1867:                         haveEndTime = true;
Chris@1867:                     }
Chris@631:                     break;
Chris@631: 
Chris@631:                 case CSVFormat::ColumnDuration:
Chris@1867:                     if (!convertTimeValue(s, lineno, sampleRate, windowSize, duration)) {
Chris@1867:                         ok = false;
Chris@1867:                     }
Chris@631:                     break;
Chris@631: 
Chris@631:                 case CSVFormat::ColumnValue:
Chris@1793:                     if (haveAnyValue) {
Chris@1793:                         otherValue = value;
Chris@1793:                     }
Chris@631:                     value = s.toFloat();
Chris@631:                     haveAnyValue = true;
Chris@631:                     break;
Chris@631: 
Chris@897:                 case CSVFormat::ColumnPitch:
Chris@897:                     pitch = s.toFloat();
Chris@897:                     if (pitch < 0.f || pitch > 127.f) {
Chris@897:                         pitchLooksLikeMIDI = false;
Chris@897:                     }
Chris@897:                     break;
Chris@897: 
Chris@631:                 case CSVFormat::ColumnLabel:
Chris@631:                     label = s;
Chris@631:                     break;
Chris@283:                 }
Chris@631:             }
Chris@148: 
Chris@1867:             if (!ok) {
Chris@1867:                 continue;
Chris@1867:             }
Chris@1867:             
Chris@1113:             ++labelCountMap[label];
Chris@1113:             
Chris@631:             if (haveEndTime) { // ... calculate duration now all cols read
Chris@631:                 if (endFrame > frameNo) {
Chris@631:                     duration = endFrame - frameNo;
Chris@628:                 }
Chris@283:             }
Chris@148: 
Chris@392:             if (modelType == CSVFormat::OneDimensionalModel) {
Chris@1429:             
Chris@1658:                 Event point(frameNo, label);
Chris@1658:                 model1->add(point);
Chris@148: 
Chris@392:             } else if (modelType == CSVFormat::TwoDimensionalModel) {
Chris@148: 
Chris@1651:                 Event point(frameNo, value, label);
Chris@1651:                 model2->add(point);
Chris@148: 
Chris@628:             } else if (modelType == CSVFormat::TwoDimensionalModelWithDuration) {
Chris@628: 
Chris@1649:                 Event region(frameNo, value, duration, label);
Chris@1649:                 model2a->add(region);
Chris@628: 
Chris@897:             } else if (modelType == CSVFormat::TwoDimensionalModelWithDurationAndPitch) {
Chris@897: 
Chris@897:                 float level = ((value >= 0.f && value <= 1.f) ? value : 1.f);
Chris@1643:                 Event note(frameNo, pitch, duration, level, label);
Chris@1644:                 model2b->add(note);
Chris@897: 
Chris@1793:             } else if (modelType == CSVFormat::TwoDimensionalModelWithDurationAndExtent) {
Chris@1793: 
Chris@1793:                 float level = 0.f;
Chris@1793:                 if (value > otherValue) {
Chris@1793:                     level = value - otherValue;
Chris@1793:                     value = otherValue;
Chris@1793:                 } else {
Chris@1793:                     level = otherValue - value;
Chris@1793:                 }
Chris@1793:                 Event box(frameNo, value, duration, level, label);
Chris@1793:                 model2c->add(box);
Chris@1793: 
Chris@392:             } else if (modelType == CSVFormat::ThreeDimensionalModel) {
Chris@148: 
Chris@283:                 DenseThreeDimensionalModel::Column values;
Chris@148: 
Chris@631:                 for (int i = 0; i < list.size(); ++i) {
Chris@148: 
Chris@676:                     if (m_format.getColumnPurpose(i) != CSVFormat::ColumnValue) {
Chris@676:                         continue;
Chris@676:                     }
Chris@676: 
Chris@283:                     bool ok = false;
Chris@283:                     float value = list[i].toFloat(&ok);
Chris@611: 
Chris@676:                     values.push_back(value);
Chris@1429:             
Chris@631:                     if (firstEverValue || value < min) min = value;
Chris@631:                     if (firstEverValue || value > max) max = value;
Chris@676:                     
Chris@631:                     if (firstEverValue) {
Chris@611:                         startFrame = frameNo;
Chris@611:                         model3->setStartFrame(startFrame);
Chris@611:                     } else if (lineno == 1 &&
Chris@611:                                timingType == CSVFormat::ExplicitTiming) {
Chris@1038:                         model3->setResolution(int(frameNo - startFrame));
Chris@611:                     }
Chris@631:                     
Chris@631:                     firstEverValue = false;
Chris@148: 
Chris@283:                     if (!ok) {
Chris@283:                         if (warnings < warnLimit) {
Chris@1428:                             SVCERR << "WARNING: CSVFileReader::load: "
Chris@390:                                       << "Non-numeric value \""
Chris@844:                                       << list[i]
Chris@491:                                       << "\" in data line " << lineno+1
Chris@843:                                       << ":" << endl;
Chris@1428:                             SVCERR << line << endl;
Chris@283:                             ++warnings;
Chris@283:                         } else if (warnings == warnLimit) {
Chris@1428: //                            SVCERR << "WARNING: Too many warnings" << endl;
Chris@283:                         }
Chris@283:                     }
Chris@283:                 }
Chris@1429:         
Chris@690: //                SVDEBUG << "Setting bin values for count " << lineno << ", frame "
Chris@687: //                          << frameNo << ", time " << RealTime::frame2RealTime(frameNo, sampleRate) << endl;
Chris@148: 
Chris@611:                 model3->setColumn(lineno, values);
Chris@1488: 
Chris@1488:             } else if (modelType == CSVFormat::WaveFileModel) {
Chris@1488: 
Chris@1518:                 int channel = 0;
Chris@1490: 
Chris@1518:                 for (int i = 0;
Chris@1518:                      i < list.size() && channel < audioChannels;
Chris@1518:                      ++i) {
Chris@1488: 
Chris@1490:                     if (m_format.getColumnPurpose(i) !=
Chris@1490:                         CSVFormat::ColumnValue) {
Chris@1488:                         continue;
Chris@1488:                     }
Chris@1488: 
Chris@1488:                     bool ok = false;
Chris@1488:                     float value = list[i].toFloat(&ok);
Chris@1518:                     if (!ok) {
Chris@1518:                         value = 0.f;
Chris@1518:                     }
Chris@1517: 
Chris@1518:                     value += sampleShift;
Chris@1518:                     value *= sampleScale;
Chris@1488:                     
Chris@1518:                     audioSamples[channel][0] = value;
Chris@1490: 
Chris@1490:                     ++channel;
Chris@1488:                 }
Chris@1488: 
Chris@1518:                 while (channel < audioChannels) {
Chris@1518:                     audioSamples[channel][0] = 0.f;
Chris@1518:                     ++channel;
Chris@1518:                 }
Chris@1488: 
Chris@1518:                 bool ok = modelW->addSamples(audioSamples, 1);
Chris@1488:                 
Chris@1488:                 if (!ok) {
Chris@1488:                     if (warnings < warnLimit) {
Chris@1488:                         SVCERR << "WARNING: CSVFileReader::load: "
Chris@1488:                                << "Unable to add sample to wave-file model"
Chris@1488:                                << endl;
Chris@1488:                         SVCERR << line << endl;
Chris@1488:                         ++warnings;
Chris@1488:                     }
Chris@1488:                 }
Chris@283:             }
Chris@1488:             
Chris@283:             ++lineno;
Chris@392:             if (timingType == CSVFormat::ImplicitTiming ||
Chris@283:                 list.size() == 0) {
Chris@283:                 frameNo += windowSize;
Chris@283:             }
Chris@283:         }
Chris@148:     }
Chris@148: 
Chris@631:     if (!haveAnyValue) {
Chris@631:         if (model2a) {
Chris@631:             // assign values for regions based on label frequency; we
Chris@631:             // have this in our labelCountMap, sort of
Chris@631: 
Chris@1113:             map<int, map<QString, float> > countLabelValueMap;
Chris@1113:             for (map<QString, int>::iterator i = labelCountMap.begin();
Chris@631:                  i != labelCountMap.end(); ++i) {
Chris@1113:                 countLabelValueMap[i->second][i->first] = -1.f;
Chris@631:             }
Chris@631: 
Chris@631:             float v = 0.f;
Chris@1113:             for (map<int, map<QString, float> >::iterator i =
Chris@631:                      countLabelValueMap.end(); i != countLabelValueMap.begin(); ) {
Chris@631:                 --i;
Chris@1428:                 SVCERR << "count -> " << i->first << endl;
Chris@1113:                 for (map<QString, float>::iterator j = i->second.begin();
Chris@631:                      j != i->second.end(); ++j) {
Chris@631:                     j->second = v;
Chris@1428:                     SVCERR << "label -> " << j->first << ", value " << v << endl;
Chris@631:                     v = v + 1.f;
Chris@631:                 }
Chris@631:             }
Chris@631: 
Chris@1649:             map<Event, Event> eventMap;
Chris@1649: 
Chris@1649:             EventVector allEvents = model2a->getAllEvents();
Chris@1649:             for (const Event &e: allEvents) {
Chris@1649:                 int count = labelCountMap[e.getLabel()];
Chris@1649:                 v = countLabelValueMap[count][e.getLabel()];
Chris@1649:                 // SVCERR << "mapping from label \"" << p.label
Chris@1649:                 //       << "\" (count " << count
Chris@1649:                 //       << ") to value " << v << endl;
Chris@1649:                 eventMap[e] = Event(e.getFrame(), v,
Chris@1649:                                     e.getDuration(), e.getLabel());
Chris@631:             }
Chris@631: 
Chris@1649:             for (const auto &i: eventMap) {
Chris@1113:                 // There could be duplicate regions; if so replace
Chris@1113:                 // them all -- but we need to check we're not
Chris@1113:                 // replacing a region by itself (or else this will
Chris@1113:                 // never terminate)
Chris@1649:                 if (i.first.getValue() == i.second.getValue()) {
Chris@1113:                     continue;
Chris@1113:                 }
Chris@1649:                 while (model2a->containsEvent(i.first)) {
Chris@1649:                     model2a->remove(i.first);
Chris@1649:                     model2a->add(i.second);
Chris@1113:                 }
Chris@631:             }
Chris@631:         }
Chris@631:     }
Chris@631:                 
Chris@897:     if (model2b) {
Chris@897:         if (pitchLooksLikeMIDI) {
Chris@897:             model2b->setScaleUnits("MIDI Pitch");
Chris@897:         } else {
Chris@897:             model2b->setScaleUnits("Hz");
Chris@897:         }
Chris@897:     }
Chris@897: 
Chris@961:     if (model3) {
Chris@1429:         model3->setMinimumLevel(min);
Chris@1429:         model3->setMaximumLevel(max);
Chris@148:     }
Chris@148: 
Chris@1489:     if (modelW) {
Chris@1518:         breakfastquay::deallocate_channels(audioSamples, audioChannels);
Chris@1493:         modelW->updateModel();
Chris@1489:         modelW->writeComplete();
Chris@1489:     }
Chris@1489: 
Chris@148:     return model;
Chris@148: }
Chris@148: 
Chris@1519: QString
Chris@1519: CSVFileReader::getConvertedAudioFilePath() const
Chris@1519: {
Chris@1519:     QString base = m_filename;
Chris@1519:     base.replace(QRegExp("[/\\,.:;~<>\"'|?%*]+"), "_");
Chris@1519: 
Chris@1519:     QString convertedFileDir = RecordDirectory::getConvertedAudioDirectory();
Chris@1519:     if (convertedFileDir == "") {
Chris@1519:         SVCERR << "WARNING: CSVFileReader::getConvertedAudioFilePath: Failed to retrieve converted audio directory" << endl;
Chris@1519:         return "";
Chris@1519:     }
Chris@1519: 
Chris@1519:     auto ms = QDateTime::currentDateTime().toMSecsSinceEpoch();
Chris@1519:     auto s = ms / 1000; // there is a toSecsSinceEpoch in Qt 5.8 but
Chris@1519:                         // we currently want to support older versions
Chris@1519:     
Chris@1519:     return QDir(convertedFileDir).filePath
Chris@1519:         (QString("%1-%2.wav").arg(base).arg(s));
Chris@1519: }
Chris@1519: