comparison scripts/evaluation_stats.py @ 1316:279930a008ca

All interfaces support comment boxes. Comment box identification matches presented tag (for instance, AB will be Comment on fragment A, rather than 1). Tighter buffer loading protocol, audioObjects register with the buffer rather than checking for buffer existence (which can be buggy depending on the buffer state). Buffers now have a state to ensure exact location in loading chain (downloading, decoding, LUFS, ready).

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+import xml.etree.ElementTree as ET
+import os       # for getting files from directory
+import operator # for sorting data with multiple keys
+import sys      # for accessing command line arguments
+
+# Command line arguments
+assert len(sys.argv)<3, "evaluation_stats takes at most 1 command line argument\n"+\
+                        "Use: python evaluation_stats.py [results_folder]"
+
+# XML results files location
+if len(sys.argv) == 1:
+    folder_name = "../saves"    # Looks in 'saves/' folder from 'scripts/' folder
+    print "Use: python evaluation_stats.py [results_folder]"
+    print "Using default path: " + folder_name
+elif len(sys.argv) == 2:
+    folder_name = sys.argv[1]   # First command line argument is folder
+
+# Turn number of seconds (int) to '[minutes] min [seconds] s' (string)
+def seconds2timestr(time_in_seconds):
+    time_in_minutes = int(time_in_seconds/60)
+    remaining_seconds = int(time_in_seconds%60)
+    return str(time_in_minutes) + " min " + str(remaining_seconds) + " s"
+
+# stats initialisation
+number_of_XML_files  = 0
+number_of_pages      = 0
+number_of_fragments  = 0
+total_empty_comments = 0
+total_not_played     = 0
+total_not_moved      = 0
+time_per_page_accum  = 0
+
+# arrays initialisation
+page_names       = []
+page_count       = []
+duration_page    = []      # duration of experiment in function of page content
+duration_order   = []      # duration of experiment in function of page number
+fragments_per_page = []    # number of fragments for corresponding page
+
+# get every XML file in folder
+files_list = os.listdir(folder_name)
+for file in files_list: # iterate over all files in files_list
+    if file.endswith(".xml"): # check if XML file
+        number_of_XML_files += 1
+        tree = ET.parse(folder_name + '/' + file)
+        root = tree.getroot()
+        
+        print file # print file name (subject name)
+        
+        # reset for new subject
+        total_duration = 0
+        page_number = 0
+        
+        # get list of all page names
+        for audioholder in root.findall("./audioholder"):   # iterate over pages
+            page_name = audioholder.get('id')               # get page name
+            
+            if page_name is None: # ignore 'empty' audio_holders
+                print "WARNING: " + file + " contains empty audio holder. (evaluation_stats.py)"
+                break # move on to next
+            
+            number_of_comments = 0 # for this page
+            number_of_missing_comments = 0 # for this page
+            not_played = 0 # for this page
+            not_moved = 0 # for this page
+            
+            # 'testTime' keeps total duration: subtract time so far for duration of this audioholder
+            duration = float(audioholder.find("./metric/metricresult[@id='testTime']").text) - total_duration
+            
+            # total duration of test
+            total_duration += duration
+            
+            # number of audio elements
+            audioelements = audioholder.findall("./audioelement") # get audioelements
+            number_of_fragments += len(audioelements) # add length of this list to total
+            
+            # number of comments (interesting if comments not mandatory)
+            for audioelement in audioelements:
+                response = audioelement.find("./comment/response")
+                was_played = audioelement.find("./metric/metricresult/[@name='elementFlagListenedTo']")
+                was_moved = audioelement.find("./metric/metricresult/[@name='elementFlagMoved']")
+                if response.text is not None and len(response.text) > 1: 
+                    number_of_comments += 1
+                else: 
+                    number_of_missing_comments += 1
+                if was_played is not None and was_played.text == 'false': 
+                    not_played += 1
+                if was_moved is not None and was_moved.text == 'false': 
+                    not_moved += 1
+            
+            # update global counters
+            total_empty_comments += number_of_missing_comments
+            total_not_played += not_played
+            total_not_moved += not_moved
+            
+            # print audioholder id and duration
+            print "    " + page_name + ": " + seconds2timestr(duration) + ", "\
+                  + str(number_of_comments)+"/"\
+                  +str(number_of_comments+number_of_missing_comments)+" comments"
+            
+            # number of audio elements not played
+            if not_played > 1:
+                print 'ATTENTION: '+str(not_played)+' fragments were not listened to!'
+            if not_played == 1: 
+                print 'ATTENTION: one fragment was not listened to!'
+            
+            # number of audio element markers not moved
+            if not_moved > 1:
+                print 'ATTENTION: '+str(not_moved)+' markers were not moved!'
+            if not_moved == 1: 
+                print 'ATTENTION: one marker was not moved!'
+            
+            # keep track of duration in function of page index
+            if len(duration_order)>page_number:
+                duration_order[page_number].append(duration)
+            else:
+                duration_order.append([duration])
+                
+            # keep list of audioholder ids and count how many times each audioholder id
+            # was tested, how long it took, and how many fragments there were (if number of 
+            # fragments is different, store as different audioholder id)
+            if page_name in page_names: 
+                page_index = page_names.index(page_name) # get index
+                # check if number of audioelements the same
+                if len(audioelements) == fragments_per_page[page_index]: 
+                    page_count[page_index] += 1
+                    duration_page[page_index].append(duration)
+                else: # make new entry
+                    alt_page_name = page_name+"("+str(len(audioelements))+")"
+                    if alt_page_name in page_names: # if already there
+                        alt_page_index = page_names.index(alt_page_name) # get index
+                        page_count[alt_page_index] += 1
+                        duration_page[alt_page_index].append(duration)
+                    else: 
+                        page_names.append(alt_page_name)
+                        page_count.append(1)
+                        duration_page.append([duration])
+                        fragments_per_page.append(len(audioelements))
+            else: 
+                page_names.append(page_name)
+                page_count.append(1)
+                duration_page.append([duration])
+                fragments_per_page.append(len(audioelements))
+                
+            # bookkeeping
+            page_number += 1 # increase page count for this specific test
+            number_of_pages += 1 # increase total number of pages
+            time_per_page_accum += duration # total duration (for average time spent per page)
+
+        # print total duration of this test
+        print "    TOTAL: " + seconds2timestr(total_duration)
+
+
+# PRINT EVERYTHING
+
+print "Number of XML files: " + str(number_of_XML_files)
+print "Number of pages: " + str(number_of_pages)
+print "Number of fragments: " + str(number_of_fragments)
+print "Number of empty comments: " + str(total_empty_comments) +\
+      " (" + str(round(100.0*total_empty_comments/number_of_fragments,2)) + "%)"
+print "Number of unplayed fragments: " + str(total_not_played) +\
+      " (" + str(round(100.0*total_not_played/number_of_fragments,2)) + "%)"
+print "Number of unmoved markers: " + str(total_not_moved) +\
+      " (" + str(round(100.0*total_not_moved/number_of_fragments,2)) + "%)"
+print "Average time per page: " + seconds2timestr(time_per_page_accum/number_of_pages)
+
+# Pages and number of times tested
+page_count_strings = list(str(x) for x in page_count)
+count_list = page_names + page_count_strings
+count_list[::2] = page_names
+count_list[1::2] = page_count_strings
+print "Pages tested: " + str(count_list)
+
+# Average duration for first, second, ... page
+print "Average duration per page:"
+for page_number in range(len(duration_order)): 
+    print "        page " + str(page_number+1) + ": " +\
+        seconds2timestr(sum(duration_order[page_number])/len(duration_order[page_number])) +\
+            " ("+str(len(duration_order[page_number]))+" subjects)"
+
+
+# Sort pages by number of audioelements, then by duration
+
+# average duration and number of subjects per page
+average_duration_page = []
+number_of_subjects_page = []
+for line in duration_page:
+    number_of_subjects_page.append(len(line))
+    average_duration_page.append(sum(line)/len(line))
+
+# combine and sort in function of number of audioelements and duration
+combined_list = [page_names, average_duration_page, fragments_per_page, number_of_subjects_page]
+combined_list = sorted(zip(*combined_list), key=operator.itemgetter(1, 2)) # sort
+
+# Show average duration for all songs
+print "Average duration per audioholder:"
+for page_index in range(len(page_names)):
+    print "        "+combined_list[page_index][0] + ": " \
+          + seconds2timestr(combined_list[page_index][1]) \
+          + " (" + str(combined_list[page_index][3]) + " subjects, " \
+          + str(combined_list[page_index][2]) + " fragments)"
+
+
+#TODO
+# time per page in function of number of fragments (plot)
+# time per participant in function of number of pages
+# plot total time for each participant
+# plot total time
+# show 'count' per page (in order)
+
+# clear up page_index <> page_count <> page_number confusion
+
+# LaTeX -> PDF print out