view scene_classification/sceneClassificationMetrics_collectevals.m @ 10:507300d2ed66 tip

merge

% Script to invoke the evaluation on all the output from the submissions, which must previously have been collected.
% Compiles the accuracies etc into a CSV file

basepath = '/home/dan/aasp_april2013_eval'
numfolds = 5
% then using the list of submissions from the folderlisting:
subs = cellstr(['cooper_tam_system0_c'; 'cooper_tam_system1_c'; 'cooper_tam_system2_c'; ...
'elizalde'; 'geiger'; 'johnshopkins'; 'krijnders'; ...
'mtg_nogueira_0'; 'mtg_nogueira_1'; 'mtg_nogueira_2'; ...
'mtg_roma'; 'mtg_roma_1'; 
'nam_hyung_lee'; 'olivetti_withdels'; 'pooledmajority'; 'rahman_system1'; 'rahman_system2'; 'rakotomamonjy_gasso'; 'smacpy'])

bigconfusion = 0

csvfp = fopen(sprintf('%s/eval_results_sc/eval_results_sc.csv', basepath), 'w+');
fprintf(csvfp, 'submission,acc,std,confint,accfold1,accfold2,accfold3,accfold4,accfold5\n');
for which=1:length(subs)
	f = subs{which};
	subpath = sprintf('%s/submitter_filelists/%s.txt', basepath, f)
	[confusionMat, AccFolds, Acc, Std] = sceneClassificationMetrics_eval(numfolds, sprintf('%s/filelist_GTs.txt', basepath), subpath);
	Acc
	ConfInt = Std * 1.96 / sqrt(numfolds)
	fprintf(csvfp, '%s,%g,%g,%g,%g,%g,%g,%g,%g\n', f, Acc, Std, ConfInt, AccFolds(1), AccFolds(2), AccFolds(3), AccFolds(4), AccFolds(5));
	% and write confusion matrices out
	dlmwrite(sprintf('%s/eval_results_sc/confusion_%s.csv', basepath, f), confusionMat);
	if bigconfusion == 0
		bigconfusion = confusionMat;
	else
		bigconfusion += confusionMat;
	end
end

rowsum = sum(bigconfusion');
if any(rowsum != rowsum(1))
	rowsum
	error('Not all row-sums are equal -- this must be the case for a confusion matrix with balanced classes')
end

bigconfusion = round(bigconfusion * 100 / rowsum(1));

dlmwrite(sprintf('%s/eval_results_sc/total_confusion.csv', basepath), bigconfusion);
bigconfusion
fclose(csvfp);