view scene_classification/sceneClassificationMetrics_collectevals.m @ 4:d58b1f9bd24b

overall confusion matrix too

% Script to invoke the evaluation on all the output from the submissions, which must previously have been collected.
% Compiles the accuracies etc into a CSV file

basepath = '/home/dan/aasp_april2013_eval'
numfolds = 5
% then using the list of submissions from the folderlisting:
subs = cellstr(['cooper_tam_system0'; 'cooper_tam_system1'; 'cooper_tam_system2'; 'cooper_tam_system0_b'; 'cooper_tam_system1_b'; 'cooper_tam_system2_b'; 'elizalde'; 'geiger'; 'johnshopkins'; 'krijnders'; 'mtg_roma'; 'nam_hyung_lee'; 'olivetti'; 'olivetti_secondrun'; 'olivetti_withdels'; 'pooledmajority'; 'rahman_system1'; 'rahman_system2'; 'rakotomamonjy_gasso'; 'smacpy'])

bigconfusion = 0

csvfp = fopen(sprintf('%s/eval_results_sc/eval_results_sc.csv', basepath), 'w+');
fprintf(csvfp, 'submission,acc,std,confint,accfold1,accfold2,accfold3,accfold4,accfold5\n');
for which=1:length(subs)
	f = subs{which};
	subpath = sprintf('%s/submitter_filelists/%s.txt', basepath, f)
	[confusionMat, AccFolds, Acc, Std] = sceneClassificationMetrics_eval(numfolds, sprintf('%s/filelist_GTs.txt', basepath), subpath);
	Acc
	ConfInt = Std * 1.96 / sqrt(numfolds)
	fprintf(csvfp, '%s,%g,%g,%g,%g,%g,%g,%g,%g\n', f, Acc, Std, ConfInt, AccFolds(1), AccFolds(2), AccFolds(3), AccFolds(4), AccFolds(5));
	% and write confusion matrices out
	dlmwrite(sprintf('%s/eval_results_sc/confusion_%s.csv', basepath, f), confusionMat);
	if bigconfusion == 0
		bigconfusion = confusionMat;
	else
		bigconfusion += confusionMat;
	end
end
dlmwrite(sprintf('%s/eval_results_sc/total_confusion.csv', basepath), bigconfusion);
bigconfusion
fclose(csvfp);