# HG changeset patch # User Chris Cannam # Date 1591880999 -3600 # Node ID 2654bf447a84175ff9d50fcb267a24ce826bbaef # Parent b4b11af915f4a478ab45dcddfe183e4dc8569964 CSV reader tests and fixes - avoid creating null events for lines in which the timings could not be read diff -r b4b11af915f4 -r 2654bf447a84 data/fileio/CSVFileReader.cpp --- a/data/fileio/CSVFileReader.cpp Thu Jun 11 14:07:56 2020 +0100 +++ b/data/fileio/CSVFileReader.cpp Thu Jun 11 14:09:59 2020 +0100 @@ -117,17 +117,18 @@ return m_error; } -sv_frame_t +bool CSVFileReader::convertTimeValue(QString s, int lineno, sv_samplerate_t sampleRate, - int windowSize) const + int windowSize, + sv_frame_t &calculatedFrame) const { QRegExp nonNumericRx("[^0-9eE.,+-]"); int warnLimit = 10; CSVFormat::TimeUnits timeUnits = m_format.getTimeUnits(); - sv_frame_t calculatedFrame = 0; + calculatedFrame = 0; bool ok = false; QString numeric = s; @@ -316,31 +317,37 @@ switch (modelType) { case CSVFormat::OneDimensionalModel: + SVDEBUG << "CSVFileReader: Creating sparse one-dimensional model" << endl; model1 = new SparseOneDimensionalModel(sampleRate, windowSize); model = model1; break; case CSVFormat::TwoDimensionalModel: + SVDEBUG << "CSVFileReader: Creating sparse time-value model" << endl; model2 = new SparseTimeValueModel(sampleRate, windowSize, false); model = model2; break; case CSVFormat::TwoDimensionalModelWithDuration: + SVDEBUG << "CSVFileReader: Creating region model" << endl; model2a = new RegionModel(sampleRate, windowSize, false); model = model2a; break; case CSVFormat::TwoDimensionalModelWithDurationAndPitch: + SVDEBUG << "CSVFileReader: Creating note model" << endl; model2b = new NoteModel(sampleRate, windowSize, false); model = model2b; break; case CSVFormat::TwoDimensionalModelWithDurationAndExtent: + SVDEBUG << "CSVFileReader: Creating box model" << endl; model2c = new BoxModel(sampleRate, windowSize, false); model = model2c; break; case CSVFormat::ThreeDimensionalModel: + SVDEBUG << "CSVFileReader: Creating editable dense three-dimensional model" << endl; model3 = new EditableDenseThreeDimensionalModel (sampleRate, windowSize, valueColumns); model = model3; @@ -348,6 +355,7 @@ case CSVFormat::WaveFileModel: { + SVDEBUG << "CSVFileReader: Creating writable wave-file model" << endl; bool normalise = (m_format.getAudioSampleRange() == CSVFormat::SampleRangeOther); QString path = getConvertedAudioFilePath(); @@ -387,6 +395,7 @@ float otherValue = 0.f; float pitch = 0.f; QString label = ""; + bool ok = true; duration = 0.f; haveEndTime = false; @@ -403,16 +412,21 @@ break; case CSVFormat::ColumnStartTime: - frameNo = convertTimeValue(s, lineno, sampleRate, windowSize); + if (!convertTimeValue(s, lineno, sampleRate, windowSize, frameNo)) { + ok = false; + } break; case CSVFormat::ColumnEndTime: - endFrame = convertTimeValue(s, lineno, sampleRate, windowSize); - haveEndTime = true; + if (convertTimeValue(s, lineno, sampleRate, windowSize, endFrame)) { + haveEndTime = true; + } break; case CSVFormat::ColumnDuration: - duration = convertTimeValue(s, lineno, sampleRate, windowSize); + if (!convertTimeValue(s, lineno, sampleRate, windowSize, duration)) { + ok = false; + } break; case CSVFormat::ColumnValue: @@ -436,6 +450,10 @@ } } + if (!ok) { + continue; + } + ++labelCountMap[label]; if (haveEndTime) { // ... calculate duration now all cols read diff -r b4b11af915f4 -r 2654bf447a84 data/fileio/CSVFileReader.h --- a/data/fileio/CSVFileReader.h Thu Jun 11 14:07:56 2020 +0100 +++ b/data/fileio/CSVFileReader.h Thu Jun 11 14:09:59 2020 +0100 @@ -70,8 +70,8 @@ mutable int m_progress; ProgressReporter *m_reporter; - sv_frame_t convertTimeValue(QString, int lineno, sv_samplerate_t sampleRate, - int windowSize) const; + bool convertTimeValue(QString, int lineno, sv_samplerate_t sampleRate, + int windowSize, sv_frame_t &calculatedFrame) const; QString getConvertedAudioFilePath() const; }; diff -r b4b11af915f4 -r 2654bf447a84 data/fileio/test/CSVReaderTest.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data/fileio/test/CSVReaderTest.h Thu Jun 11 14:09:59 2020 +0100 @@ -0,0 +1,234 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Sonic Visualiser + An audio file viewer and annotation editor. + Centre for Digital Music, Queen Mary, University of London. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef TEST_CSV_READER_H +#define TEST_CSV_READER_H + +#include "../CSVFileReader.h" + +#include "data/model/SparseOneDimensionalModel.h" +#include "data/model/SparseTimeValueModel.h" +#include "data/model/RegionModel.h" +#include "data/model/EditableDenseThreeDimensionalModel.h" + +#include "base/Debug.h" + +#include + +#include +#include +#include + +#include + +using namespace std; + +class CSVReaderTest : public QObject +{ + Q_OBJECT + +private: + QDir csvDir; + sv_samplerate_t mainRate; + +public: + CSVReaderTest(QString base) { + if (base == "") { + base = "svcore/data/fileio/test"; + } + csvDir = QDir(base + "/csv"); + mainRate = 44100; + } + +private: + void loadFrom(QString filename, Model *&model) { + QString path(csvDir.filePath(filename)); + CSVFormat f; + f.guessFormatFor(path); + CSVFileReader reader(path, f, mainRate); + model = reader.load(); + QVERIFY(model); + QVERIFY(reader.isOK()); + QCOMPARE(reader.getError(), QString()); + } + +private slots: + void init() { + if (!csvDir.exists()) { + SVCERR << "ERROR: CSV test file directory \"" << csvDir.absolutePath() << "\" does not exist" << endl; + QVERIFY2(csvDir.exists(), "CSV test file directory not found"); + } + } + + void modelType1DSamples() { + Model *model = nullptr; + loadFrom("model-type-1d-samples.csv", model); + auto actual = qobject_cast(model); + QVERIFY(actual); + QCOMPARE(actual->getAllEvents().size(), 5); + //!!! + the actual contents + delete model; + } + + void modelType1DSeconds() { + Model *model = nullptr; + loadFrom("model-type-1d-seconds.csv", model); + auto actual = qobject_cast(model); + QVERIFY(actual); + QCOMPARE(actual->getAllEvents().size(), 5); + delete model; + } + + void modelType2DDurationSamples() { + Model *model = nullptr; + loadFrom("model-type-2d-duration-samples.csv", model); + auto actual = qobject_cast(model); + QVERIFY(actual); + QCOMPARE(actual->getAllEvents().size(), 5); + delete model; + } + + void modelType2DDurationSeconds() { + Model *model = nullptr; + loadFrom("model-type-2d-duration-seconds.csv", model); + auto actual = qobject_cast(model); + QVERIFY(actual); + QCOMPARE(actual->getAllEvents().size(), 5); + delete model; + } + + void badNegativeDuration() { + Model *model = nullptr; + loadFrom("bad-negative-duration.csv", model); + auto actual = qobject_cast(model); + QVERIFY(actual); + //!!! + check duration has been corrected + QCOMPARE(actual->getAllEvents().size(), 5); + delete model; + } + + void modelType2DEndTimeSamples() { + Model *model = nullptr; + loadFrom("model-type-2d-endtime-samples.csv", model); + auto actual = qobject_cast(model); + QVERIFY(actual); + QCOMPARE(actual->getAllEvents().size(), 5); + delete model; + } + + void modelType2DEndTimeSeconds() { + Model *model = nullptr; + loadFrom("model-type-2d-endtime-seconds.csv", model); + auto actual = qobject_cast(model); + QVERIFY(actual); + QCOMPARE(actual->getAllEvents().size(), 5); + delete model; + } + + void modelType2DImplicit() { + Model *model = nullptr; + loadFrom("model-type-2d-implicit.csv", model); + auto actual = qobject_cast(model); + QVERIFY(actual); + QCOMPARE(actual->getAllEvents().size(), 5); + delete model; + } + + void modelType2DSamples() { + Model *model = nullptr; + loadFrom("model-type-2d-samples.csv", model); + auto actual = qobject_cast(model); + QVERIFY(actual); + QCOMPARE(actual->getAllEvents().size(), 5); + delete model; + } + + void modelType2DSeconds() { + Model *model = nullptr; + loadFrom("model-type-2d-seconds.csv", model); + auto actual = qobject_cast(model); + QVERIFY(actual); + QCOMPARE(actual->getAllEvents().size(), 5); + delete model; + } + + void modelType3DImplicit() { + Model *model = nullptr; + loadFrom("model-type-3d-implicit.csv", model); + auto actual = qobject_cast(model); + QVERIFY(actual); + QCOMPARE(actual->getWidth(), 6); + QCOMPARE(actual->getHeight(), 6); + delete model; + } + + void modelType3DSamples() { + Model *model = nullptr; + loadFrom("model-type-3d-samples.csv", model); + auto actual = qobject_cast(model); + QVERIFY(actual); + QCOMPARE(actual->getWidth(), 6); + QCOMPARE(actual->getHeight(), 6); + delete model; + } + + void modelType3DSeconds() { + Model *model = nullptr; + loadFrom("model-type-3d-seconds.csv", model); + auto actual = qobject_cast(model); + QVERIFY(actual); + QCOMPARE(actual->getWidth(), 6); + QCOMPARE(actual->getHeight(), 6); + delete model; + } + + void withBlankLines1D() { + Model *model = nullptr; + loadFrom("with-blank-lines-1d.csv", model); + auto actual = qobject_cast(model); + QVERIFY(actual); + QCOMPARE(actual->getAllEvents().size(), 5); + delete model; + } + + void withBlankLines2D() { + Model *model = nullptr; + loadFrom("with-blank-lines-2d.csv", model); + auto actual = qobject_cast(model); + QVERIFY(actual); + QCOMPARE(actual->getAllEvents().size(), 5); + delete model; + } + + void withBlankLines3D() { + Model *model = nullptr; + loadFrom("with-blank-lines-3d.csv", model); + auto actual = qobject_cast(model); + QVERIFY(actual); + QCOMPARE(actual->getWidth(), 6); + QCOMPARE(actual->getHeight(), 6); + delete model; + } + + void quoting() { + Model *model = nullptr; + loadFrom("quoting.csv", model); + auto actual = qobject_cast(model); + QVERIFY(actual); + QCOMPARE(actual->getAllEvents().size(), 5); + delete model; + } +}; + +#endif diff -r b4b11af915f4 -r 2654bf447a84 data/fileio/test/csv/bad-negative-duration.csv --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data/fileio/test/csv/bad-negative-duration.csv Thu Jun 11 14:09:59 2020 +0100 @@ -0,0 +1,6 @@ +# As model-type-2d-duration-seconds.csv but with a negative value for a duration +1.1,4,620 +2.2,4.2,880 +3.3,0.4,440 +4.4,3.8,213 +5.5,-2.3,123 diff -r b4b11af915f4 -r 2654bf447a84 data/fileio/test/csv/model-type-2d-duration-seconds.csv --- a/data/fileio/test/csv/model-type-2d-duration-seconds.csv Thu Jun 11 14:07:56 2020 +0100 +++ b/data/fileio/test/csv/model-type-2d-duration-seconds.csv Thu Jun 11 14:09:59 2020 +0100 @@ -5,4 +5,4 @@ 2.2,4.2,880 3.3,0.4,440 4.4,3.8,213 -5.5,-2.3,123 +5.5,2.3,123 diff -r b4b11af915f4 -r 2654bf447a84 data/fileio/test/csv/quoting.csv --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data/fileio/test/csv/quoting.csv Thu Jun 11 14:09:59 2020 +0100 @@ -0,0 +1,5 @@ +1,2,Label 1 +"2",4 +3,"6","Labels 3a, 3b" +4,8.0,"Label \"4\"" +5, 1,"Label ""5""" \ No newline at end of file diff -r b4b11af915f4 -r 2654bf447a84 data/fileio/test/csv/with-blank-lines-1d.csv --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data/fileio/test/csv/with-blank-lines-1d.csv Thu Jun 11 14:09:59 2020 +0100 @@ -0,0 +1,9 @@ + +3.2 First thing + +4.4 Second thing +5.5 Third thing + +6.3 Fourth thing +7.8 Fifth thing + diff -r b4b11af915f4 -r 2654bf447a84 data/fileio/test/csv/with-blank-lines-2d.csv --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data/fileio/test/csv/with-blank-lines-2d.csv Thu Jun 11 14:09:59 2020 +0100 @@ -0,0 +1,12 @@ + +45678,4 + +123239,4.2 + 320130,0.4 + +# Also include some CR/LF variations: +452103,3.8 + + +# And let's not have a newline after this last line: +620301,-2.3 \ No newline at end of file diff -r b4b11af915f4 -r 2654bf447a84 data/fileio/test/csv/with-blank-lines-3d.csv --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data/fileio/test/csv/with-blank-lines-3d.csv Thu Jun 11 14:09:59 2020 +0100 @@ -0,0 +1,10 @@ + +22050,143.0,2.0,-1.3,0.0,0.0,1.0 +44100,0.2,0.1,-3.0,0.0,0.1,0.143 + +66150,0.143,0.2,-3.1,0.0,0.0,0.1 +88200,2.0,1.0,-0.3,0.0,1.0,143.0 + +110250,0.0,0.0,0.1,0.143,0.2,-3.1 +132300,0.0,1.0,143.0,2.0,1.0,-0.3 + diff -r b4b11af915f4 -r 2654bf447a84 data/fileio/test/files.pri --- a/data/fileio/test/files.pri Thu Jun 11 14:07:56 2020 +0100 +++ b/data/fileio/test/files.pri Thu Jun 11 14:09:59 2020 +0100 @@ -9,6 +9,7 @@ EncodingTest.h \ MIDIFileReaderTest.h \ CSVFormatTest.h \ + CSVReaderTest.h \ CSVStreamWriterTest.h TEST_SOURCES += \ diff -r b4b11af915f4 -r 2654bf447a84 data/fileio/test/svcore-data-fileio-test.cpp --- a/data/fileio/test/svcore-data-fileio-test.cpp Thu Jun 11 14:07:56 2020 +0100 +++ b/data/fileio/test/svcore-data-fileio-test.cpp Thu Jun 11 14:09:59 2020 +0100 @@ -18,6 +18,7 @@ #include "EncodingTest.h" #include "MIDIFileReaderTest.h" #include "CSVFormatTest.h" +#include "CSVReaderTest.h" #include "CSVStreamWriterTest.h" #include "system/Init.h" @@ -90,6 +91,12 @@ } { + CSVReaderTest t(testDir); + if (QTest::qExec(&t, argc, argv) == 0) ++good; + else ++bad; + } + + { CSVStreamWriterTest t; if (QTest::qExec(&t, argc, argv) == 0) ++good; else ++bad;