changeset 2:dcaf371d33e3

add a script to iterate over collected outputs and invoke the evaluator

--- a/scene_classification/loadClassificationOutput.m	Sat Apr 13 09:42:08 2013 +0100
+++ b/scene_classification/loadClassificationOutput.m	Tue May 07 13:42:10 2013 +0100
@@ -1,7 +1,7 @@
 function [fileID,classID] = loadClassificationOutput(filename)
 
 % Open raw file
-fid = fopen(filename,'r+');
+fid = fopen(filename,'r');
 
 % Read 1st line
 tline = fgetl(fid);

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/scene_classification/sceneClassificationMetrics_collectevals.m	Tue May 07 13:42:10 2013 +0100
@@ -0,0 +1,20 @@
+% Script to invoke the evaluation on all the output from the submissions, which must previously have been collected.
+% Compiles the accuracies etc into a CSV file
+
+basepath = '/home/dan/aasp_april2013_eval'
+numfolds = 5
+% then using the list of submissions from the folderlisting:
+subs = cellstr(['cooper_tam_system0'; 'cooper_tam_system1'; 'cooper_tam_system2'; 'elizalde'; 'geiger'; 'johnshopkins'; 'krijnders'; 'mtg_roma'; 'nam_hyung_lee'; 'olivetti'; 'pooledmajority'; 'rahman_system1'; 'rahman_system2'; 'rakotomamonjy_gasso'; 'smacpy'])
+
+csvfp = fopen(sprintf('%s/eval_results_sc/eval_results_sc.csv', basepath), 'w+');
+fprintf(csvfp, 'submission,acc,std,confint,accfold1,accfold2,accfold3,accfold4,accfold5\n');
+for which=1:length(subs)
+	f = subs{which};
+	subpath = sprintf('%s/submitter_filelists/%s.txt', basepath, f)
+	[confusionMat, AccFolds, Acc, Std] = sceneClassificationMetrics_eval(numfolds, sprintf('%s/filelist_GTs.txt', basepath), subpath);
+	Acc
+	ConfInt = Std * 1.96 / sqrt(numfolds)
+	fprintf(csvfp, '%s,%g,%g,%g,%g,%g,%g,%g,%g\n', f, Acc, Std, ConfInt, AccFolds(1), AccFolds(2), AccFolds(3), AccFolds(4), AccFolds(5));
+end
+fclose(csvfp);
+

--- a/scene_classification/sceneClassificationMetrics_eval.m	Sat Apr 13 09:42:08 2013 +0100
+++ b/scene_classification/sceneClassificationMetrics_eval.m	Tue May 07 13:42:10 2013 +0100
@@ -13,10 +13,9 @@
 confusionMat = zeros(10,10);
 AccFolds = zeros(1,numfolds);
 
-
 % For each fold
-fid1 = fopen(foldsGTlist,'r+');
-fid2 = fopen(foldstestlist,'r+');
+fid1 = fopen(foldsGTlist,'r');
+fid2 = fopen(foldstestlist,'r');
 for i=1:numfolds
     
     % Load classification output and ground truth