changeset 138:c0802d1b1fad

Another evaluation script, and a README

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/testdata/evaluation/README	Thu May 08 13:41:13 2014 +0100
@@ -0,0 +1,9 @@
+
+batchProcessingEvaluate.m evaluates a set of .lab output files against
+MIDI "ground truth".
+
+The MIDI files can be found in ../TRIOS-groundtruth, and some examples
+of the .lab files in ../TRIOS-mirex2012-matlab and ../TRIOS-silvet.
+
+evaluate_lab.yeti evaluates one lab file against another, onsets only,
+to a given timing tolerance.

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/testdata/evaluation/evaluate_lab.yeti	Thu May 08 13:41:13 2014 +0100
@@ -0,0 +1,64 @@
+
+// Take two lab files of the form
+//
+// onset offset frequency
+//
+// and report their onset-only note-level accuracy F-measure.
+
+program evaluate_lab;
+
+usage () =
+   (eprintln "\nUsage: evaluate_lab reqd reference.lab transcribed.lab\n";
+    eprintln "where reqd is the number of milliseconds allowed for timing error (+/-)\n");
+
+toMIDIPitch f = 
+    round (12 * (Math#log(f / 220) / Math#log(2)) + 57);
+
+suck f =
+   (str = openInFile f "UTF-8"; // better to use readFile here, oh well
+    d = map do line:
+        case list (strSplit "\t" line) of
+        onset::offset::frequency::_:
+            { onset = number onset, midi = toMIDIPitch (number frequency) };
+        _:
+            failWith "badly formed line: \(line)";
+        esac;
+    done (str.lines ());
+    str.close ();
+    d);
+
+select f = fold do r x: if f x then x::r else r fi done [];
+
+evaluate permitted ref trans =
+   (reference = suck ref;
+    transcribed = suck trans;
+    accurate = 
+        select do here :
+            any do other:
+                here.midi == other.midi and
+                    abs (here.onset - other.onset) < permitted
+            done reference
+        done transcribed;
+    {
+        ntot = length transcribed,
+        nref = length reference,
+        ncorr = length accurate,
+    });
+
+pc n =
+    int (n * 1000) / 10;
+
+report { ntot, nref, ncorr } =
+   (nfp = ntot - ncorr;
+    nfn = nref - ncorr;
+    rec = ncorr / nref;
+    pre = ncorr / ntot;
+    f = 2 * ((pre * rec) / (pre + rec));
+    acc = ncorr / (ncorr + nfp + nfn);
+    println "precision \(pc pre), recall \(pc rec), accuracy \(pc acc), F \(pc f)");
+
+case (list _argv) of
+reqd::ref::trans::[]: report (evaluate (number reqd / 1000) ref trans);
+_: usage ();
+esac;
+