# HG changeset patch
# User Chris Cannam
# Date 1456403768 0
# Node ID d582d56620106419f6fe18ecd72d877bd5a818ad
# Parent  f4315a0ade8983e0624b40cb47a3c90b428a023e# Parent  c58e123419fb9c3e8e48b2a903a59a2b5f1d2278
Merge from default branch

diff -r f4315a0ade89 -r d582d5662010 .hgsubstate
--- a/.hgsubstate	Mon Feb 22 14:27:19 2016 +0000
+++ b/.hgsubstate	Thu Feb 25 12:36:08 2016 +0000
@@ -1,4 +1,4 @@
 d16f0fd6db6104d87882bc43788a3bb1b0f8c528 dataquay
 55ece8862b6d3a54aad271a53f9c1615e5d3bcf8 sv-dependency-builds
-d9c766274c8b87a41a577b3688edb2f067105cf4 svcore
+7c4f4701b49fe6d0b3388788866933f03cc7849c svcore
 632d90c185ecc8655f7a85ba58dc568351449dfd vamp-plugin-sdk
diff -r f4315a0ade89 -r d582d5662010 runner.pro
--- a/runner.pro	Mon Feb 22 14:27:19 2016 +0000
+++ b/runner.pro	Thu Feb 25 12:36:08 2016 +0000
@@ -80,7 +80,7 @@
 
 LIBS = $$MY_LIBS $$LIBS
 
-#PRE_TARGETDEPS += svcore/libsvcore.a
+PRE_TARGETDEPS += svcore/libsvcore.a
 
 HEADERS += \
         vamp-plugin-sdk/vamp-hostsdk/PluginBase.h \
diff -r f4315a0ade89 -r d582d5662010 runner/FeatureExtractionManager.cpp
--- a/runner/FeatureExtractionManager.cpp	Mon Feb 22 14:27:19 2016 +0000
+++ b/runner/FeatureExtractionManager.cpp	Thu Feb 25 12:36:08 2016 +0000
@@ -427,11 +427,36 @@
 }
 
 bool FeatureExtractionManager::addFeatureExtractorFromFile
-(QString transformXmlFile, const vector<FeatureWriter*> &writers)
+(QString transformFile, const vector<FeatureWriter*> &writers)
 {
+    // We support two formats for transform description files, XML (in
+    // a format specific to Sonic Annotator) and RDF/Turtle. The RDF
+    // format can describe multiple transforms in a single file, the
+    // XML only one.
+    
+    // Possible errors we should report:
+    //
+    // 1. File does not exist or cannot be opened
+    // 2. File is ostensibly XML, but is not parseable
+    // 3. File is ostensibly Turtle, but is not parseable
+    // 4. File is XML, but contains no valid transform (e.g. is unrelated XML)
+    // 5. File is Turtle, but contains no valid transform(s)
+    // 6. File is Turtle and contains both valid and invalid transform(s)
+
+    {
+        // We don't actually need to open this here yet, we just hoist
+        // it to the top for error reporting purposes
+        QFile file(transformFile);
+        if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
+            // Error case 1. File does not exist or cannot be opened
+            cerr << "ERROR: Failed to open transform file \"" << transformFile
+                 << "\" for reading" << endl;
+            return false;
+        }
+    }
+    
     bool tryRdf = true;
-
-    if (transformXmlFile.endsWith(".xml") || transformXmlFile.endsWith(".XML")) {
+    if (transformFile.endsWith(".xml") || transformFile.endsWith(".XML")) {
         // We don't support RDF-XML (and nor does the underlying
         // parser library) so skip the RDF parse if the filename
         // suggests XML, to avoid puking out a load of errors from
@@ -439,44 +464,90 @@
         tryRdf = false;
     }
 
+    bool tryXml = true;
+    if (transformFile.endsWith(".ttl") || transformFile.endsWith(".TTL") ||
+        transformFile.endsWith(".ntriples") || transformFile.endsWith(".NTRIPLES") ||
+        transformFile.endsWith(".n3") || transformFile.endsWith(".N3")) {
+        tryXml = false;
+    }
+
+    QString rdfError, xmlError;
+    
     if (tryRdf) {
+
         RDFTransformFactory factory
-            (QUrl::fromLocalFile(QFileInfo(transformXmlFile).absoluteFilePath())
+            (QUrl::fromLocalFile(QFileInfo(transformFile).absoluteFilePath())
              .toString());
         ProgressPrinter printer("Parsing transforms RDF file");
         std::vector<Transform> transforms = factory.getTransforms(&printer);
-        if (!factory.isOK()) {
-            cerr << "WARNING: FeatureExtractionManager::addFeatureExtractorFromFile: Failed to parse transforms file: " << factory.getErrorString().toStdString() << endl;
+
+        if (factory.isOK()) {
+            if (transforms.empty()) {
+                cerr << "ERROR: Transform file \"" << transformFile
+                     << "\" is valid RDF but defines no transforms" << endl;
+                return false;
+            } else {
+                bool success = true;
+                for (int i = 0; i < (int)transforms.size(); ++i) {
+                    if (!addFeatureExtractor(transforms[i], writers)) {
+                        success = false;
+                    }
+                }
+                return success;
+            }
+        } else { // !factory.isOK()
             if (factory.isRDF()) {
-                return false; // no point trying it as XML
+                cerr << "ERROR: Invalid transform RDF file \"" << transformFile
+                     << "\": " << factory.getErrorString() << endl;
+                return false;
             }
-        }
-        if (!transforms.empty()) {
-            bool success = true;
-            for (int i = 0; i < (int)transforms.size(); ++i) {
-                if (!addFeatureExtractor(transforms[i], writers)) {
-                    success = false;
-                }
-            }
-            return success;
+
+            // the not-RDF case: fall through without reporting an
+            // error, so we try the file as XML, and if that fails, we
+            // print a general unparseable-file error
+            rdfError = factory.getErrorString();
         }
     }
 
-    QFile file(transformXmlFile);
-    if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
-        cerr << "ERROR: Failed to open transform XML file \""
-             << transformXmlFile.toStdString() << "\" for reading" << endl;
-        return false;
+    if (tryXml) {
+        
+        QFile file(transformFile);
+        if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
+            cerr << "ERROR: Failed to open transform file \""
+                 << transformFile.toStdString() << "\" for reading" << endl;
+            return false;
+        }
+        
+        QTextStream *qts = new QTextStream(&file);
+        QString qs = qts->readAll();
+        delete qts;
+        file.close();
+    
+        Transform transform(qs);
+        xmlError = transform.getErrorString();
+
+        if (xmlError == "") {
+
+            if (transform.getIdentifier() == "") {
+                cerr << "ERROR: Transform file \"" << transformFile
+                     << "\" is valid XML but defines no transform" << endl;
+                return false;
+            }
+
+            return addFeatureExtractor(transform, writers);
+        }
     }
 
-    QTextStream *qts = new QTextStream(&file);
-    QString qs = qts->readAll();
-    delete qts;
-    file.close();
+    cerr << "ERROR: Transform file \"" << transformFile
+         << "\" could not be parsed" << endl;
+    if (rdfError != "") {
+        cerr << "ERROR: RDF parser reported: " << rdfError << endl;
+    }
+    if (xmlError != "") {
+        cerr << "ERROR: XML parser reported: " << xmlError << endl;
+    }
 
-    Transform transform(qs);
-
-    return addFeatureExtractor(transform, writers);
+    return false;
 }
 
 void FeatureExtractionManager::addSource(QString audioSource, bool willMultiplex)
@@ -489,7 +560,7 @@
 
     if (m_channels == 0 || m_defaultSampleRate == 0) {
 
-        ProgressPrinter retrievalProgress("Determining default rate and channel count from first input file...");
+        ProgressPrinter retrievalProgress("Retrieving first input file to determine default rate and channel count...");
 
         FileSource source(audioSource, &retrievalProgress);
         if (!source.isAvailable()) {
@@ -524,14 +595,14 @@
             if (m_channels == 0) {
                 m_channels = reader->getChannelCount();
                 cerr << "Taking default channel count of "
-                     << reader->getChannelCount() << " from file" << endl;
+                     << reader->getChannelCount() << " from audio file" << endl;
             }
         }
 
         if (m_defaultSampleRate == 0) {
             m_defaultSampleRate = reader->getNativeRate();
             cerr << "Taking default sample rate of "
-                 << reader->getNativeRate() << "Hz from file" << endl;
+                 << reader->getNativeRate() << "Hz from audio file" << endl;
             cerr << "(Note: Default may be overridden by transforms)" << endl;
         }
 
diff -r f4315a0ade89 -r d582d5662010 tests/include.sh
--- a/tests/include.sh	Mon Feb 22 14:27:19 2016 +0000
+++ b/tests/include.sh	Thu Feb 25 12:36:08 2016 +0000
@@ -83,7 +83,7 @@
 	echo "--"
 	cat "$3"
 	echo "--"
-	echo "Diff:"
+	echo "Diff (output on left, expected on right):"
 	echo "--"
 	sdiff -w78 "$2" "$3"
 	echo "--"
diff -r f4315a0ade89 -r d582d5662010 tests/test-parse-errors/expected/empty.ttl.txt
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-parse-errors/expected/empty.ttl.txt	Thu Feb 25 12:36:08 2016 +0000
@@ -0,0 +1,1 @@
+RDF parser reported: Failed to import model from URL
diff -r f4315a0ade89 -r d582d5662010 tests/test-parse-errors/expected/empty.xml.txt
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-parse-errors/expected/empty.xml.txt	Thu Feb 25 12:36:08 2016 +0000
@@ -0,0 +1,1 @@
+XML parser reported: unexpected end of file at line 1, column 1
diff -r f4315a0ade89 -r d582d5662010 tests/test-parse-errors/expected/garbage.dat.txt
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-parse-errors/expected/garbage.dat.txt	Thu Feb 25 12:36:08 2016 +0000
@@ -0,0 +1,2 @@
+ERROR: RDF parser reported:
+ERROR: XML parser reported:
diff -r f4315a0ade89 -r d582d5662010 tests/test-parse-errors/expected/invalid-turtle.ttl.txt
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-parse-errors/expected/invalid-turtle.ttl.txt	Thu Feb 25 12:36:08 2016 +0000
@@ -0,0 +1,1 @@
+RDF parser reported: Failed to import model from URL
diff -r f4315a0ade89 -r d582d5662010 tests/test-parse-errors/expected/invalid-xml.xml.txt
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-parse-errors/expected/invalid-xml.xml.txt	Thu Feb 25 12:36:08 2016 +0000
@@ -0,0 +1,1 @@
+XML parser reported: tag mismatch
diff -r f4315a0ade89 -r d582d5662010 tests/test-parse-errors/expected/valid-turtle-no-transform.ttl.txt
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-parse-errors/expected/valid-turtle-no-transform.ttl.txt	Thu Feb 25 12:36:08 2016 +0000
@@ -0,0 +1,1 @@
+valid RDF but defines no transforms
diff -r f4315a0ade89 -r d582d5662010 tests/test-parse-errors/expected/valid-xml-no-transform.xml.txt
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-parse-errors/expected/valid-xml-no-transform.xml.txt	Thu Feb 25 12:36:08 2016 +0000
@@ -0,0 +1,1 @@
+valid XML but defines no transform
diff -r f4315a0ade89 -r d582d5662010 tests/test-parse-errors/inputs/garbage.dat
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-parse-errors/inputs/garbage.dat	Thu Feb 25 12:36:08 2016 +0000
@@ -0,0 +1,1 @@
+ŽºûgÿáZ~zÙN9Ô¬b$äˆ7:2såÍ𤇧0åH2	zêZ4¡¾¦ž�#Â<A¦±n´�#ï|l¬¦TsûÃÍ:o„_äÜBØK°ê¥µ+-%ó
W”Y¸­Ÿ×ü[NÏ-Áµþt×L…àøf7Ò¶¼
\ No newline at end of file
diff -r f4315a0ade89 -r d582d5662010 tests/test-parse-errors/inputs/invalid-turtle.ttl
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-parse-errors/inputs/invalid-turtle.ttl	Thu Feb 25 12:36:08 2016 +0000
@@ -0,0 +1,2 @@
+# Trailing ;
+<http://example/s> <http://example/p> <http://example/o> ;
diff -r f4315a0ade89 -r d582d5662010 tests/test-parse-errors/inputs/invalid-xml.xml
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-parse-errors/inputs/invalid-xml.xml	Thu Feb 25 12:36:08 2016 +0000
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0">
+  <channel>
+    <title>A Title</title>
+    <link>http://example.com</link>
+    <description/>
+    <!-- NB missing end-tag for channel -->
+</rss>
diff -r f4315a0ade89 -r d582d5662010 tests/test-parse-errors/inputs/valid-turtle-no-transform.ttl
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-parse-errors/inputs/valid-turtle-no-transform.ttl	Thu Feb 25 12:36:08 2016 +0000
@@ -0,0 +1,1 @@
+<http://example/s> <http://example/p> <http://example/o> .
diff -r f4315a0ade89 -r d582d5662010 tests/test-parse-errors/inputs/valid-xml-no-transform.xml
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-parse-errors/inputs/valid-xml-no-transform.xml	Thu Feb 25 12:36:08 2016 +0000
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0">
+  <channel>
+    <title>A Title</title>
+    <link>http://example.com</link>
+    <description/>
+  </channel>
+</rss>
diff -r f4315a0ade89 -r d582d5662010 tests/test-parse-errors/test-parse-errors.sh
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-parse-errors/test-parse-errors.sh	Thu Feb 25 12:36:08 2016 +0000
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+. ../include.sh
+
+infile=$audiopath/3clicks8.wav
+tmpfile=$mypath/tmp_$$
+trap "rm -f $tmpfile" 0
+
+for transform in "$mypath"/inputs/* ; do
+
+    base=$(basename "$transform")
+    expected="$mypath"/expected/"$base".txt
+
+    if [ ! -f "$expected" ]; then
+	fail "Internal error: Expected file $expected not found for transform $transform"
+    fi
+
+    if $r -t "$transform" -w csv --csv-one-file /dev/null "$infile" 2>"$tmpfile" ; then
+	fail "Erroneously succeeds in running bogus transform $transform"
+    fi
+
+    cat "$expected" | while read line; do
+	if ! fgrep -q "$line" "$tmpfile" ; then
+	    fail "Expected output text \"$line\" not found in diagnostic output for transform $base"
+	fi
+    done
+    
+done
+
diff -r f4315a0ade89 -r d582d5662010 tests/test.sh
--- a/tests/test.sh	Mon Feb 22 14:27:19 2016 +0000
+++ b/tests/test.sh	Thu Feb 25 12:36:08 2016 +0000
@@ -5,6 +5,7 @@
 for x in \
     supportprogs \
     helpfulflags \
+    parse-errors \
     transforms-basic \
     audioformat \
     vamp-test-plugin \