changeset 0:2c9915c861d8

First version

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/event_detection/convertEventListToEventRoll.m	Fri Mar 08 11:33:20 2013 +0000
@@ -0,0 +1,17 @@
+function [eventRoll] = convertEventListToEventRoll(onset,offset,classNames)
+
+
+% Initialize
+eventRoll = zeros(ceil(offset(length(offset))*100),16);
+eventID = {'alert','clearthroat','cough','doorslam','drawer','keyboard','keys',...
+    'knock','laughter','mouse','pageturn','pendrop','phone','printer','speech','switch'};
+
+
+% Fill-in eventRoll
+for i=1:length(onset)
+    
+    pos = strmatch(classNames{i}, eventID);
+    
+    eventRoll(floor(onset(i)*100):ceil(offset(i)*100),pos) = 1;
+    
+end;

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/event_detection/eventDetectionMetrics_classWiseEventBased.m	Fri Mar 08 11:33:20 2013 +0000
@@ -0,0 +1,80 @@
+function [results] = eventDetectionMetrics_classWiseEventBased(outputFile,GTFile)
+
+% Class-wise event-based evaluation for event detection task
+% outputFile: the output of the event detection system
+% GTFile: the ground truth list of events
+
+% Initialize
+eventID = {'alert','clearthroat','cough','doorslam','drawer','keyboard','keys',...
+    'knock','laughter','mouse','pageturn','pendrop','phone','printer','speech','switch'};
+
+
+% Load event list from output and ground-truth
+[onset,offset,classNames] = loadEventsList(outputFile);
+[onsetGT,offsetGT,classNamesGT] = loadEventsList(GTFile);
+
+
+% Total number of detected and reference events per class
+Ntot = zeros(16,1);
+for i=1:length(onset)
+    pos = strmatch(classNames{i}, eventID);
+    Ntot(pos) = Ntot(pos)+1;
+end;
+
+Nref = zeros(16,1);
+for i=1:length(onsetGT)
+    pos = strmatch(classNamesGT{i}, eventID);
+    Nref(pos) = Nref(pos)+1;
+end;
+I = find(Nref>0); % index for classes present in ground-truth
+
+
+% Number of correctly transcribed events per class, onset within a +/-100 ms range
+Ncorr = zeros(16,1);
+NcorrOff = zeros(16,1);
+for j=1:length(onsetGT)
+    for i=1:length(onset)
+        
+        if( strcmp(classNames{i},classNamesGT{j}) && (abs(onsetGT(j)-onset(i))<=0.1) )
+            pos = strmatch(classNames{i}, eventID);
+            Ncorr(pos) = Ncorr(pos)+1;
+            
+            % If offset within a +/-100 ms range or within 50% of ground-truth event's duration
+            if abs(offsetGT(j) - offset(i)) <= max(0.1, 0.5 * (offsetGT(j) - onsetGT(j)))
+                pos = strmatch(classNames{i}, eventID);
+                NcorrOff(pos) = NcorrOff(pos) +1;
+            end;
+            
+            break; % In order to not evaluate duplicates
+            
+        end;
+    end;
+end;
+
+
+% Compute onset-only class-wise event-based metrics
+Nfp = Ntot-Ncorr;
+Nfn = Nref-Ncorr;
+Nsubs = min(Nfp,Nfn);
+tempRec = Ncorr(I)./(Nref(I)+eps);
+tempPre = Ncorr(I)./(Ntot(I)+eps);
+results.Rec = mean(tempRec);
+results.Pre = mean(tempPre);
+tempF =  2*((tempPre.*tempRec)./(tempPre+tempRec+eps));
+results.F = mean(tempF);
+tempAEER = (Nfn(I)+Nfp(I)+Nsubs(I))./(Nref(I)+eps);
+results.AEER = mean(tempAEER);
+
+
+% Compute onset-offset class-wise event-based metrics
+NfpOff = Ntot-NcorrOff;
+NfnOff = Nref-NcorrOff;
+NsubsOff = min(NfpOff,NfnOff);
+tempRecOff = NcorrOff(I)./(Nref(I)+eps);
+tempPreOff = NcorrOff(I)./(Ntot(I)+eps);
+results.RecOff = mean(tempRecOff);
+results.PreOff = mean(tempPreOff);
+tempFOff =  2*((tempPreOff.*tempRecOff)./(tempPreOff+tempRecOff+eps));
+results.FOff = mean(tempFOff);
+tempAEEROff = (NfnOff(I)+NfpOff(I)+NsubsOff(I))./(Nref(I)+eps);
+results.AEEROff = mean(tempAEEROff);

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/event_detection/eventDetectionMetrics_eventBased.m	Fri Mar 08 11:33:20 2013 +0000
@@ -0,0 +1,56 @@
+function [results] = eventDetectionMetrics_eventBased(outputFile,GTFile)
+
+% Event-based evaluation for event detection task
+% outputFile: the output of the event detection system
+% GTFile: the ground truth list of events
+
+% Load event list from output and ground-truth
+[onset,offset,classNames] = loadEventsList(outputFile);
+[onsetGT,offsetGT,classNamesGT] = loadEventsList(GTFile);
+
+
+% Total number of detected and reference events
+Ntot = length(onset);
+Nref = length(onsetGT);
+
+
+% Number of correctly transcribed events, onset within a +/-100 ms range
+Ncorr = 0;
+NcorrOff = 0;
+for j=1:length(onsetGT)
+    for i=1:length(onset)
+        
+        if( strcmp(classNames{i},classNamesGT{j}) && (abs(onsetGT(j)-onset(i))<=0.1) )
+            Ncorr = Ncorr+1; 
+            
+            % If offset within a +/-100 ms range or within 50% of ground-truth event's duration
+            if abs(offsetGT(j) - offset(i)) <= max(0.1, 0.5 * (offsetGT(j) - onsetGT(j)))
+                NcorrOff = NcorrOff +1;
+            end;
+            
+            break; % In order to not evaluate duplicates
+            
+        end;
+    end;
+    
+end;
+
+
+% Compute onset-only event-based metrics
+Nfp = Ntot-Ncorr;
+Nfn = Nref-Ncorr;
+Nsubs = min(Nfp,Nfn);
+results.Rec = Ncorr/(Nref+eps);
+results.Pre = Ncorr/(Ntot+eps);
+results.F = 2*((results.Pre*results.Rec)/(results.Pre+results.Rec+eps));
+results.AEER= (Nfn+Nfp+Nsubs)/(Nref+eps);
+
+
+% Compute onset-offset event-based metrics
+NfpOff = Ntot-NcorrOff;
+NfnOff = Nref-NcorrOff;
+NsubsOff = min(NfpOff,NfnOff);
+results.RecOff = NcorrOff/(Nref+eps);
+results.PreOff = NcorrOff/(Ntot+eps);
+results.FOff = 2*((results.PreOff*results.RecOff)/(results.PreOff+results.RecOff+eps));
+results.AEEROff= (NfnOff+NfpOff+NsubsOff)/(Nref+eps);

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/event_detection/eventDetectionMetrics_frameBased.m	Fri Mar 08 11:33:20 2013 +0000
@@ -0,0 +1,34 @@
+function [results] = eventDetectionMetrics_frameBased(outputFile,GTFile)
+
+% Frame-based evaluation for event detection task
+% outputFile: the output of the event detection system
+% GTFile: the ground truth list of events
+
+% Load event list from output and ground-truth
+[onset,offset,classNames] = loadEventsList(outputFile);
+[onsetGT,offsetGT,classNamesGT] = loadEventsList(GTFile);
+
+
+% Convert event list into frame-based representation (10msec resolution)
+[eventRoll] = convertEventListToEventRoll(onset,offset,classNames);
+[eventRollGT] = convertEventListToEventRoll(onsetGT,offsetGT,classNamesGT);
+
+
+% Fix durations of eventRolls
+if (size(eventRollGT,1) > size(eventRoll,1)) eventRoll = [eventRoll; zeros(size(eventRollGT,1)-size(eventRoll,1),16)]; end;
+if (size(eventRoll,1) > size(eventRollGT,1)) eventRollGT = [eventRollGT; zeros(size(eventRoll,1)-size(eventRollGT,1),16)]; end;
+
+
+% Compute frame-based metrics
+Nref = sum(sum(eventRollGT));
+Ntot = sum(sum(eventRoll));
+Ntp = sum(sum(eventRoll+eventRollGT > 1));
+Nfp = sum(sum(eventRoll-eventRollGT > 0));
+Nfn = sum(sum(eventRollGT-eventRoll > 0));
+Nsubs = min(Nfp,Nfn);
+
+
+results.Rec = Ntp/(Nref+eps);
+results.Pre = Ntp/(Ntot+eps);
+results.F = 2*((results.Pre*results.Rec)/(results.Pre+results.Rec+eps));
+results.AEER = (Nfn+Nfp+Nsubs)/(Nref+eps);

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/event_detection/loadEventsList.m	Fri Mar 08 11:33:20 2013 +0000
@@ -0,0 +1,27 @@
+function [onset,offset,classNames] = loadEventsList(filename)
+
+% Open raw file
+fid = fopen(filename,'r+');
+
+% Read 1st line
+tline = fgetl(fid);
+onset_offset(:,1) = sscanf(tline, '%f\t%f\t%*s');
+classNames{1} = char(sscanf(tline, '%*f\t%*f\t%s')');
+
+% Read rest of the lines
+i=1;
+while ischar(tline)
+    i = i+1;
+    tline = fgetl(fid);
+    if (ischar(tline))
+        onset_offset(:,i) = sscanf(tline, '%f\t%f\t%*s');
+        classNames{i} = char(sscanf(tline, '%*f\t%*f\t%s')');
+    end;
+end
+
+% Split onset_offset
+onset = onset_offset(1,:)';
+offset = onset_offset(2,:)';
+
+% Close file
+fclose(fid);
\ No newline at end of file

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/scene_classification/fold1_test_ubhjwk.txt	Fri Mar 08 11:33:20 2013 +0000
@@ -0,0 +1,20 @@
+tmp/1_lsiknx/1_qnsnmm.wav	bus
+tmp/1_lsiknx/6_jvgwsj.wav	restaurant
+tmp/1_lsiknx/1_itqeyo.wav	busystreet
+tmp/1_lsiknx/6_okasnj.wav	busystreet
+tmp/1_lsiknx/1_xctssd.wav	office
+tmp/1_lsiknx/6_hudnps.wav	office
+tmp/1_lsiknx/1_zzdynb.wav	quietstreet
+tmp/1_lsiknx/6_uisfys.wav	quietstreet
+tmp/1_lsiknx/1_qrgvha.wav	restaurant
+tmp/1_lsiknx/6_dxhmur.wav	restaurant
+tmp/1_lsiknx/1_cglqsz.wav	supermarket
+tmp/1_lsiknx/6_nglrop.wav	supermarket
+tmp/1_lsiknx/1_mjugvv.wav	openairmarket
+tmp/1_lsiknx/6_fhtnxb.wav	openairmarket
+tmp/1_lsiknx/1_bmdzly.wav	park
+tmp/1_lsiknx/6_tuhcjv.wav	supermarket
+tmp/1_lsiknx/1_yyzbli.wav	tubestation
+tmp/1_lsiknx/6_qvnesx.wav	tube
+tmp/1_lsiknx/1_bjscrh.wav	tubestation
+tmp/1_lsiknx/6_upmhdh.wav	tubestation

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/scene_classification/foldGTlist.txt	Fri Mar 08 11:33:20 2013 +0000
@@ -0,0 +1,5 @@
+fold1_testgt_ubhjwk.txt
+fold2_testgt_stxbia.txt
+fold3_testgt_bcbkdl.txt
+fold4_testgt_bdbtlr.txt
+fold5_testgt_mdbsir.txt

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/scene_classification/foldtestlist.txt	Fri Mar 08 11:33:20 2013 +0000
@@ -0,0 +1,5 @@
+fold1_test_ubhjwk.txt
+fold2_test_stxbia.txt
+fold3_test_bcbkdl.txt
+fold4_test_bdbtlr.txt
+fold5_test_mdbsir.txt

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/scene_classification/loadClassificationOutput.m	Fri Mar 08 11:33:20 2013 +0000
@@ -0,0 +1,26 @@
+function [fileID,classID] = loadClassificationOutput(filename)
+
+% Open raw file
+fid = fopen(filename,'r+');
+
+% Read 1st line
+tline = fgetl(fid);
+fileID{1} = char(sscanf(tline, '%s\t%*s'));
+classID{1} = char(sscanf(tline, '%*s\t%s'));
+
+% Read rest of the lines
+i=1;
+while ischar(tline)
+    i = i+1;
+    tline = fgetl(fid);
+    if (ischar(tline))
+        
+        fileID{i} = char(sscanf(tline, '%s\t%*s'));
+        classID{i} = char(sscanf(tline, '%*s\t%s'));
+
+    end;
+end
+
+% Close file
+fclose(fid);
+

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/scene_classification/sceneClassificationMetrics_eval.m	Fri Mar 08 11:33:20 2013 +0000
@@ -0,0 +1,48 @@
+function [confusionMat, AccFolds, Acc, Std] = sceneClassificationMetrics_eval(numfolds, foldsGTlist, foldstestlist)
+
+% Function takes as input a number of folds (numfolds), a text file
+% containing a list of filenames with the ground truth per fold (foldsGTlist), 
+% and a text file containing a list of filenames with the classification
+% output (foldstestlist)
+%
+% e.g. [confusionMat, AccFolds, Acc, Std] = sceneClassificationMetrics_eval(5, 'foldGTlist.txt', 'foldtestlist.txt');
+
+
+% Initialize
+classList = {'bus','busystreet','office','openairmarket','park','quietstreet','restaurant','supermarket','tube','tubestation'};
+confusionMat = zeros(10,10);
+AccFolds = zeros(1,numfolds);
+
+
+% For each fold
+fid1 = fopen(foldsGTlist,'r+');
+fid2 = fopen(foldstestlist,'r+');
+for i=1:numfolds
+    
+    % Load classification output and ground truth
+    tline1 = fgetl(fid1);
+    [fileIDGT,classIDGT] = loadClassificationOutput(tline1);
+    tline2 = fgetl(fid2);
+    [fileID,classID] = loadClassificationOutput(tline2);
+    
+    % Compute confusion matrix per fold
+    confusionMatTemp = zeros(10,10);
+    for j=1:length(classIDGT)
+        pos = strmatch(fileIDGT{j}, fileID, 'exact');
+        posClassGT = strmatch(classIDGT{j}, classList, 'exact');
+        posClass = strmatch(classID{pos}, classList, 'exact');
+        confusionMatTemp(posClassGT,posClass) = confusionMatTemp(posClassGT,posClass) + 1;
+    end
+    
+    % Compute accuracy per fold
+    AccFolds(i) = sum(diag(confusionMatTemp))/sum(sum(confusionMatTemp));
+    confusionMat = confusionMat + confusionMatTemp;
+    
+end
+fclose(fid1);
+fclose(fid2);
+
+
+% Compute global accuracy and std
+Acc = mean(AccFolds);
+Std = std(AccFolds);