dan@2: % Script to invoke the evaluation on all the output from the submissions, which must previously have been collected.
dan@2: % Compiles the accuracies etc into a CSV file
dan@2: 
dan@2: basepath = '/home/dan/aasp_april2013_eval'
dan@2: numfolds = 5
dan@2: % then using the list of submissions from the folderlisting:
dan@5: subs = cellstr(['cooper_tam_system0_c'; 'cooper_tam_system1_c'; 'cooper_tam_system2_c'; ...
dan@5: 'elizalde'; 'geiger'; 'johnshopkins'; 'krijnders'; ...
dan@5: 'mtg_nogueira_0'; 'mtg_nogueira_1'; 'mtg_nogueira_2'; ...
dan@5: 'mtg_roma'; 'mtg_roma_1'; 
dan@5: 'nam_hyung_lee'; 'olivetti_withdels'; 'pooledmajority'; 'rahman_system1'; 'rahman_system2'; 'rakotomamonjy_gasso'; 'smacpy'])
dan@4: 
dan@4: bigconfusion = 0
dan@2: 
dan@2: csvfp = fopen(sprintf('%s/eval_results_sc/eval_results_sc.csv', basepath), 'w+');
dan@2: fprintf(csvfp, 'submission,acc,std,confint,accfold1,accfold2,accfold3,accfold4,accfold5\n');
dan@2: for which=1:length(subs)
dan@2: 	f = subs{which};
dan@2: 	subpath = sprintf('%s/submitter_filelists/%s.txt', basepath, f)
dan@2: 	[confusionMat, AccFolds, Acc, Std] = sceneClassificationMetrics_eval(numfolds, sprintf('%s/filelist_GTs.txt', basepath), subpath);
dan@2: 	Acc
dan@2: 	ConfInt = Std * 1.96 / sqrt(numfolds)
dan@2: 	fprintf(csvfp, '%s,%g,%g,%g,%g,%g,%g,%g,%g\n', f, Acc, Std, ConfInt, AccFolds(1), AccFolds(2), AccFolds(3), AccFolds(4), AccFolds(5));
dan@3: 	% and write confusion matrices out
dan@3: 	dlmwrite(sprintf('%s/eval_results_sc/confusion_%s.csv', basepath, f), confusionMat);
dan@4: 	if bigconfusion == 0
dan@4: 		bigconfusion = confusionMat;
dan@4: 	else
dan@4: 		bigconfusion += confusionMat;
dan@4: 	end
dan@2: end
dan@5: 
dan@5: rowsum = sum(bigconfusion');
dan@5: if any(rowsum != rowsum(1))
dan@5: 	rowsum
dan@6: 	error('Not all row-sums are equal -- this must be the case for a confusion matrix with balanced classes')
dan@5: end
dan@5: 
dan@5: bigconfusion = round(bigconfusion * 100 / rowsum(1));
dan@5: 
dan@4: dlmwrite(sprintf('%s/eval_results_sc/total_confusion.csv', basepath), bigconfusion);
dan@4: bigconfusion
dan@2: fclose(csvfp);
dan@2: