changeset 1867:2654bf447a84

CSV reader tests and fixes - avoid creating null events for lines in which the timings could not be read

--- a/data/fileio/CSVFileReader.cpp	Thu Jun 11 14:07:56 2020 +0100
+++ b/data/fileio/CSVFileReader.cpp	Thu Jun 11 14:09:59 2020 +0100
@@ -117,17 +117,18 @@
     return m_error;
 }
 
-sv_frame_t
+bool
 CSVFileReader::convertTimeValue(QString s, int lineno,
                                 sv_samplerate_t sampleRate,
-                                int windowSize) const
+                                int windowSize,
+                                sv_frame_t &calculatedFrame) const
 {
     QRegExp nonNumericRx("[^0-9eE.,+-]");
     int warnLimit = 10;
 
     CSVFormat::TimeUnits timeUnits = m_format.getTimeUnits();
 
-    sv_frame_t calculatedFrame = 0;
+    calculatedFrame = 0;
 
     bool ok = false;
     QString numeric = s;
@@ -316,31 +317,37 @@
                 switch (modelType) {
 
                 case CSVFormat::OneDimensionalModel:
+                    SVDEBUG << "CSVFileReader: Creating sparse one-dimensional model" << endl;
                     model1 = new SparseOneDimensionalModel(sampleRate, windowSize);
                     model = model1;
                     break;
                 
                 case CSVFormat::TwoDimensionalModel:
+                    SVDEBUG << "CSVFileReader: Creating sparse time-value model" << endl;
                     model2 = new SparseTimeValueModel(sampleRate, windowSize, false);
                     model = model2;
                     break;
                 
                 case CSVFormat::TwoDimensionalModelWithDuration:
+                    SVDEBUG << "CSVFileReader: Creating region model" << endl;
                     model2a = new RegionModel(sampleRate, windowSize, false);
                     model = model2a;
                     break;
                 
                 case CSVFormat::TwoDimensionalModelWithDurationAndPitch:
+                    SVDEBUG << "CSVFileReader: Creating note model" << endl;
                     model2b = new NoteModel(sampleRate, windowSize, false);
                     model = model2b;
                     break;
                 
                 case CSVFormat::TwoDimensionalModelWithDurationAndExtent:
+                    SVDEBUG << "CSVFileReader: Creating box model" << endl;
                     model2c = new BoxModel(sampleRate, windowSize, false);
                     model = model2c;
                     break;
                 
                 case CSVFormat::ThreeDimensionalModel:
+                    SVDEBUG << "CSVFileReader: Creating editable dense three-dimensional model" << endl;
                     model3 = new EditableDenseThreeDimensionalModel
                         (sampleRate, windowSize, valueColumns);
                     model = model3;
@@ -348,6 +355,7 @@
 
                 case CSVFormat::WaveFileModel:
                 {
+                    SVDEBUG << "CSVFileReader: Creating writable wave-file model" << endl;
                     bool normalise = (m_format.getAudioSampleRange()
                                       == CSVFormat::SampleRangeOther);
                     QString path = getConvertedAudioFilePath();
@@ -387,6 +395,7 @@
             float otherValue = 0.f;
             float pitch = 0.f;
             QString label = "";
+            bool ok = true;
 
             duration = 0.f;
             haveEndTime = false;
@@ -403,16 +412,21 @@
                     break;
 
                 case CSVFormat::ColumnStartTime:
-                    frameNo = convertTimeValue(s, lineno, sampleRate, windowSize);
+                    if (!convertTimeValue(s, lineno, sampleRate, windowSize, frameNo)) {
+                        ok = false;
+                    }
                     break;
                 
                 case CSVFormat::ColumnEndTime:
-                    endFrame = convertTimeValue(s, lineno, sampleRate, windowSize);
-                    haveEndTime = true;
+                    if (convertTimeValue(s, lineno, sampleRate, windowSize, endFrame)) {
+                        haveEndTime = true;
+                    }
                     break;
 
                 case CSVFormat::ColumnDuration:
-                    duration = convertTimeValue(s, lineno, sampleRate, windowSize);
+                    if (!convertTimeValue(s, lineno, sampleRate, windowSize, duration)) {
+                        ok = false;
+                    }
                     break;
 
                 case CSVFormat::ColumnValue:
@@ -436,6 +450,10 @@
                 }
             }
 
+            if (!ok) {
+                continue;
+            }
+            
             ++labelCountMap[label];
             
             if (haveEndTime) { // ... calculate duration now all cols read

--- a/data/fileio/CSVFileReader.h	Thu Jun 11 14:07:56 2020 +0100
+++ b/data/fileio/CSVFileReader.h	Thu Jun 11 14:09:59 2020 +0100
@@ -70,8 +70,8 @@
     mutable int m_progress;
     ProgressReporter *m_reporter;
 
-    sv_frame_t convertTimeValue(QString, int lineno, sv_samplerate_t sampleRate,
-                                int windowSize) const;
+    bool convertTimeValue(QString, int lineno, sv_samplerate_t sampleRate,
+                          int windowSize, sv_frame_t &calculatedFrame) const;
 
     QString getConvertedAudioFilePath() const;
 };

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data/fileio/test/CSVReaderTest.h	Thu Jun 11 14:09:59 2020 +0100
@@ -0,0 +1,234 @@
+/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
+
+/*
+    Sonic Visualiser
+    An audio file viewer and annotation editor.
+    Centre for Digital Music, Queen Mary, University of London.
+    
+    This program is free software; you can redistribute it and/or
+    modify it under the terms of the GNU General Public License as
+    published by the Free Software Foundation; either version 2 of the
+    License, or (at your option) any later version.  See the file
+    COPYING included with this distribution for more information.
+*/
+
+#ifndef TEST_CSV_READER_H
+#define TEST_CSV_READER_H
+
+#include "../CSVFileReader.h"
+
+#include "data/model/SparseOneDimensionalModel.h"
+#include "data/model/SparseTimeValueModel.h"
+#include "data/model/RegionModel.h"
+#include "data/model/EditableDenseThreeDimensionalModel.h"
+
+#include "base/Debug.h"
+
+#include <cmath>
+
+#include <QObject>
+#include <QtTest>
+#include <QDir>
+
+#include <iostream>
+
+using namespace std;
+
+class CSVReaderTest : public QObject
+{
+    Q_OBJECT
+
+private:
+    QDir csvDir;
+    sv_samplerate_t mainRate;
+
+public:
+    CSVReaderTest(QString base) {
+        if (base == "") {
+            base = "svcore/data/fileio/test";
+        }
+        csvDir = QDir(base + "/csv");
+        mainRate = 44100;
+    }
+
+private:
+    void loadFrom(QString filename, Model *&model) {
+        QString path(csvDir.filePath(filename));
+        CSVFormat f;
+        f.guessFormatFor(path);
+        CSVFileReader reader(path, f, mainRate);
+        model = reader.load();
+        QVERIFY(model);
+        QVERIFY(reader.isOK());
+        QCOMPARE(reader.getError(), QString());
+    }
+
+private slots:
+    void init() {
+        if (!csvDir.exists()) {
+            SVCERR << "ERROR: CSV test file directory \"" << csvDir.absolutePath() << "\" does not exist" << endl;
+            QVERIFY2(csvDir.exists(), "CSV test file directory not found");
+        }
+    }
+
+    void modelType1DSamples() {
+        Model *model = nullptr;
+        loadFrom("model-type-1d-samples.csv", model);
+        auto actual = qobject_cast<SparseOneDimensionalModel *>(model);
+        QVERIFY(actual);
+        QCOMPARE(actual->getAllEvents().size(), 5);
+        //!!! + the actual contents
+        delete model;
+    }
+
+    void modelType1DSeconds() {
+        Model *model = nullptr;
+        loadFrom("model-type-1d-seconds.csv", model);
+        auto actual = qobject_cast<SparseOneDimensionalModel *>(model);
+        QVERIFY(actual);
+        QCOMPARE(actual->getAllEvents().size(), 5);
+        delete model;
+    }
+
+    void modelType2DDurationSamples() {
+        Model *model = nullptr;
+        loadFrom("model-type-2d-duration-samples.csv", model);
+        auto actual = qobject_cast<RegionModel *>(model);
+        QVERIFY(actual);
+        QCOMPARE(actual->getAllEvents().size(), 5);
+        delete model;
+    }
+    
+    void modelType2DDurationSeconds() {
+        Model *model = nullptr;
+        loadFrom("model-type-2d-duration-seconds.csv", model);
+        auto actual = qobject_cast<RegionModel *>(model);
+        QVERIFY(actual);
+        QCOMPARE(actual->getAllEvents().size(), 5);
+        delete model;
+    }
+    
+    void badNegativeDuration() {
+        Model *model = nullptr;
+        loadFrom("bad-negative-duration.csv", model);
+        auto actual = qobject_cast<RegionModel *>(model);
+        QVERIFY(actual);
+        //!!! + check duration has been corrected
+        QCOMPARE(actual->getAllEvents().size(), 5);
+        delete model;
+    }
+    
+    void modelType2DEndTimeSamples() {
+        Model *model = nullptr;
+        loadFrom("model-type-2d-endtime-samples.csv", model);
+        auto actual = qobject_cast<RegionModel *>(model);
+        QVERIFY(actual);
+        QCOMPARE(actual->getAllEvents().size(), 5);
+        delete model;
+    }
+    
+    void modelType2DEndTimeSeconds() {
+        Model *model = nullptr;
+        loadFrom("model-type-2d-endtime-seconds.csv", model);
+        auto actual = qobject_cast<RegionModel *>(model);
+        QVERIFY(actual);
+        QCOMPARE(actual->getAllEvents().size(), 5);
+        delete model;
+    }
+    
+    void modelType2DImplicit() {
+        Model *model = nullptr;
+        loadFrom("model-type-2d-implicit.csv", model);
+        auto actual = qobject_cast<SparseTimeValueModel *>(model);
+        QVERIFY(actual);
+        QCOMPARE(actual->getAllEvents().size(), 5);
+        delete model;
+    }
+    
+    void modelType2DSamples() {
+        Model *model = nullptr;
+        loadFrom("model-type-2d-samples.csv", model);
+        auto actual = qobject_cast<SparseTimeValueModel *>(model);
+        QVERIFY(actual);
+        QCOMPARE(actual->getAllEvents().size(), 5);
+        delete model;
+    }
+    
+    void modelType2DSeconds() {
+        Model *model = nullptr;
+        loadFrom("model-type-2d-seconds.csv", model);
+        auto actual = qobject_cast<SparseTimeValueModel *>(model);
+        QVERIFY(actual);
+        QCOMPARE(actual->getAllEvents().size(), 5);
+        delete model;
+    }
+    
+    void modelType3DImplicit() {
+        Model *model = nullptr;
+        loadFrom("model-type-3d-implicit.csv", model);
+        auto actual = qobject_cast<EditableDenseThreeDimensionalModel *>(model);
+        QVERIFY(actual);
+        QCOMPARE(actual->getWidth(), 6);
+        QCOMPARE(actual->getHeight(), 6);
+        delete model;
+    }
+    
+    void modelType3DSamples() {
+        Model *model = nullptr;
+        loadFrom("model-type-3d-samples.csv", model);
+        auto actual = qobject_cast<EditableDenseThreeDimensionalModel *>(model);
+        QVERIFY(actual);
+        QCOMPARE(actual->getWidth(), 6);
+        QCOMPARE(actual->getHeight(), 6);
+        delete model;
+    }
+    
+    void modelType3DSeconds() {
+        Model *model = nullptr;
+        loadFrom("model-type-3d-seconds.csv", model);
+        auto actual = qobject_cast<EditableDenseThreeDimensionalModel *>(model);
+        QVERIFY(actual);
+        QCOMPARE(actual->getWidth(), 6);
+        QCOMPARE(actual->getHeight(), 6);
+        delete model;
+    }
+    
+    void withBlankLines1D() {
+        Model *model = nullptr;
+        loadFrom("with-blank-lines-1d.csv", model);
+        auto actual = qobject_cast<SparseOneDimensionalModel *>(model);
+        QVERIFY(actual);
+        QCOMPARE(actual->getAllEvents().size(), 5);
+        delete model;
+    }
+    
+    void withBlankLines2D() {
+        Model *model = nullptr;
+        loadFrom("with-blank-lines-2d.csv", model);
+        auto actual = qobject_cast<SparseTimeValueModel *>(model);
+        QVERIFY(actual);
+        QCOMPARE(actual->getAllEvents().size(), 5);
+        delete model;
+    }
+    
+    void withBlankLines3D() {
+        Model *model = nullptr;
+        loadFrom("with-blank-lines-3d.csv", model);
+        auto actual = qobject_cast<EditableDenseThreeDimensionalModel *>(model);
+        QVERIFY(actual);
+        QCOMPARE(actual->getWidth(), 6);
+        QCOMPARE(actual->getHeight(), 6);
+        delete model;
+    }
+
+    void quoting() {
+        Model *model = nullptr;
+        loadFrom("quoting.csv", model);
+        auto actual = qobject_cast<SparseTimeValueModel *>(model);
+        QVERIFY(actual);
+        QCOMPARE(actual->getAllEvents().size(), 5);
+        delete model;
+    }
+};
+
+#endif

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data/fileio/test/csv/bad-negative-duration.csv	Thu Jun 11 14:09:59 2020 +0100
@@ -0,0 +1,6 @@
+# As model-type-2d-duration-seconds.csv but with a negative value for a duration
+1.1,4,620
+2.2,4.2,880
+3.3,0.4,440
+4.4,3.8,213
+5.5,-2.3,123

--- a/data/fileio/test/csv/model-type-2d-duration-seconds.csv	Thu Jun 11 14:07:56 2020 +0100
+++ b/data/fileio/test/csv/model-type-2d-duration-seconds.csv	Thu Jun 11 14:09:59 2020 +0100
@@ -5,4 +5,4 @@
 2.2,4.2,880
 3.3,0.4,440
 4.4,3.8,213
-5.5,-2.3,123
+5.5,2.3,123

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data/fileio/test/csv/quoting.csv	Thu Jun 11 14:09:59 2020 +0100
@@ -0,0 +1,5 @@
+1,2,Label 1
+"2",4
+3,"6","Labels 3a, 3b"
+4,8.0,"Label \"4\""
+5, 1,"Label ""5"""
\ No newline at end of file

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data/fileio/test/csv/with-blank-lines-1d.csv	Thu Jun 11 14:09:59 2020 +0100
@@ -0,0 +1,9 @@
+
+3.2	First thing
+ 
+4.4	Second thing
+5.5	Third thing
+
+6.3	Fourth thing
+7.8	Fifth thing
+

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data/fileio/test/csv/with-blank-lines-2d.csv	Thu Jun 11 14:09:59 2020 +0100
@@ -0,0 +1,12 @@
+
+45678,4
+
+123239,4.2
+
320130,0.4
+
+# Also include some CR/LF variations:
+452103,3.8
+
+
+# And let's not have a newline after this last line:
+620301,-2.3
\ No newline at end of file

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data/fileio/test/csv/with-blank-lines-3d.csv	Thu Jun 11 14:09:59 2020 +0100
@@ -0,0 +1,10 @@
+
+22050,143.0,2.0,-1.3,0.0,0.0,1.0
+44100,0.2,0.1,-3.0,0.0,0.1,0.143
+
+66150,0.143,0.2,-3.1,0.0,0.0,0.1
+88200,2.0,1.0,-0.3,0.0,1.0,143.0
+ 
+110250,0.0,0.0,0.1,0.143,0.2,-3.1
+132300,0.0,1.0,143.0,2.0,1.0,-0.3
+

--- a/data/fileio/test/files.pri	Thu Jun 11 14:07:56 2020 +0100
+++ b/data/fileio/test/files.pri	Thu Jun 11 14:09:59 2020 +0100
@@ -9,6 +9,7 @@
 	EncodingTest.h \
 	MIDIFileReaderTest.h \
 	CSVFormatTest.h \
+	CSVReaderTest.h \
 	CSVStreamWriterTest.h
      
 TEST_SOURCES += \

--- a/data/fileio/test/svcore-data-fileio-test.cpp	Thu Jun 11 14:07:56 2020 +0100
+++ b/data/fileio/test/svcore-data-fileio-test.cpp	Thu Jun 11 14:09:59 2020 +0100
@@ -18,6 +18,7 @@
 #include "EncodingTest.h"
 #include "MIDIFileReaderTest.h"
 #include "CSVFormatTest.h"
+#include "CSVReaderTest.h"
 #include "CSVStreamWriterTest.h"
 
 #include "system/Init.h"
@@ -90,6 +91,12 @@
     }
 
     {
+        CSVReaderTest t(testDir);
+        if (QTest::qExec(&t, argc, argv) == 0) ++good;
+        else ++bad;
+    }
+
+    {
         CSVStreamWriterTest t;
         if (QTest::qExec(&t, argc, argv) == 0) ++good;
         else ++bad;