nickjillings@1316: #!/usr/bin/python nickjillings@1316: # -*- coding: utf-8 -*- nickjillings@1316: nickjillings@1316: import xml.etree.ElementTree as ET nickjillings@1316: import os # for getting files from directory nickjillings@1316: import operator # for sorting data with multiple keys nickjillings@1316: import sys # for accessing command line arguments nickjillings@1316: nickjillings@1316: # Command line arguments nickjillings@1316: assert len(sys.argv)<3, "evaluation_stats takes at most 1 command line argument\n"+\ nickjillings@1316: "Use: python evaluation_stats.py [results_folder]" nickjillings@1316: nickjillings@1316: # XML results files location nickjillings@1316: if len(sys.argv) == 1: nickjillings@1316: folder_name = "../saves" # Looks in 'saves/' folder from 'scripts/' folder nickjillings@1316: print "Use: python evaluation_stats.py [results_folder]" nickjillings@1316: print "Using default path: " + folder_name nickjillings@1316: elif len(sys.argv) == 2: nickjillings@1316: folder_name = sys.argv[1] # First command line argument is folder nickjillings@1316: nickjillings@1316: # Turn number of seconds (int) to '[minutes] min [seconds] s' (string) nickjillings@1316: def seconds2timestr(time_in_seconds): nickjillings@1316: time_in_minutes = int(time_in_seconds/60) nickjillings@1316: remaining_seconds = int(time_in_seconds%60) nickjillings@1316: return str(time_in_minutes) + " min " + str(remaining_seconds) + " s" nickjillings@1316: nickjillings@1316: # stats initialisation nickjillings@1316: number_of_XML_files = 0 nickjillings@1316: number_of_pages = 0 nickjillings@1316: number_of_fragments = 0 nickjillings@1316: total_empty_comments = 0 nickjillings@1316: total_not_played = 0 nickjillings@1316: total_not_moved = 0 nickjillings@1316: time_per_page_accum = 0 nickjillings@1316: nickjillings@1316: # arrays initialisation nickjillings@1316: page_names = [] nickjillings@1316: page_count = [] nickjillings@1316: duration_page = [] # duration of experiment in function of page content nickjillings@1316: duration_order = [] # duration of experiment in function of page number nickjillings@1316: fragments_per_page = [] # number of fragments for corresponding page nickjillings@1316: nickjillings@1316: # get every XML file in folder nickjillings@1316: files_list = os.listdir(folder_name) nickjillings@1316: for file in files_list: # iterate over all files in files_list nickjillings@1316: if file.endswith(".xml"): # check if XML file nickjillings@1316: number_of_XML_files += 1 nickjillings@1316: tree = ET.parse(folder_name + '/' + file) nickjillings@1316: root = tree.getroot() nickjillings@1316: nickjillings@1316: print file # print file name (subject name) nickjillings@1316: nickjillings@1316: # reset for new subject nickjillings@1316: total_duration = 0 nickjillings@1316: page_number = 0 nickjillings@1316: nickjillings@1316: # get list of all page names nickjillings@1316: for audioholder in root.findall("./audioholder"): # iterate over pages nickjillings@1316: page_name = audioholder.get('id') # get page name nickjillings@1316: nickjillings@1316: if page_name is None: # ignore 'empty' audio_holders nickjillings@1316: print "WARNING: " + file + " contains empty audio holder. (evaluation_stats.py)" nickjillings@1316: break # move on to next nickjillings@1316: nickjillings@1316: number_of_comments = 0 # for this page nickjillings@1316: number_of_missing_comments = 0 # for this page nickjillings@1316: not_played = 0 # for this page nickjillings@1316: not_moved = 0 # for this page nickjillings@1316: nickjillings@1316: # 'testTime' keeps total duration: subtract time so far for duration of this audioholder nickjillings@1316: duration = float(audioholder.find("./metric/metricresult[@id='testTime']").text) - total_duration nickjillings@1316: nickjillings@1316: # total duration of test nickjillings@1316: total_duration += duration nickjillings@1316: nickjillings@1316: # number of audio elements nickjillings@1316: audioelements = audioholder.findall("./audioelement") # get audioelements nickjillings@1316: number_of_fragments += len(audioelements) # add length of this list to total nickjillings@1316: nickjillings@1316: # number of comments (interesting if comments not mandatory) nickjillings@1316: for audioelement in audioelements: nickjillings@1316: response = audioelement.find("./comment/response") nickjillings@1316: was_played = audioelement.find("./metric/metricresult/[@name='elementFlagListenedTo']") nickjillings@1316: was_moved = audioelement.find("./metric/metricresult/[@name='elementFlagMoved']") nickjillings@1316: if response.text is not None and len(response.text) > 1: nickjillings@1316: number_of_comments += 1 nickjillings@1316: else: nickjillings@1316: number_of_missing_comments += 1 nickjillings@1316: if was_played is not None and was_played.text == 'false': nickjillings@1316: not_played += 1 nickjillings@1316: if was_moved is not None and was_moved.text == 'false': nickjillings@1316: not_moved += 1 nickjillings@1316: nickjillings@1316: # update global counters nickjillings@1316: total_empty_comments += number_of_missing_comments nickjillings@1316: total_not_played += not_played nickjillings@1316: total_not_moved += not_moved nickjillings@1316: nickjillings@1316: # print audioholder id and duration nickjillings@1316: print " " + page_name + ": " + seconds2timestr(duration) + ", "\ nickjillings@1316: + str(number_of_comments)+"/"\ nickjillings@1316: +str(number_of_comments+number_of_missing_comments)+" comments" nickjillings@1316: nickjillings@1316: # number of audio elements not played nickjillings@1316: if not_played > 1: nickjillings@1316: print 'ATTENTION: '+str(not_played)+' fragments were not listened to!' nickjillings@1316: if not_played == 1: nickjillings@1316: print 'ATTENTION: one fragment was not listened to!' nickjillings@1316: nickjillings@1316: # number of audio element markers not moved nickjillings@1316: if not_moved > 1: nickjillings@1316: print 'ATTENTION: '+str(not_moved)+' markers were not moved!' nickjillings@1316: if not_moved == 1: nickjillings@1316: print 'ATTENTION: one marker was not moved!' nickjillings@1316: nickjillings@1316: # keep track of duration in function of page index nickjillings@1316: if len(duration_order)>page_number: nickjillings@1316: duration_order[page_number].append(duration) nickjillings@1316: else: nickjillings@1316: duration_order.append([duration]) nickjillings@1316: nickjillings@1316: # keep list of audioholder ids and count how many times each audioholder id nickjillings@1316: # was tested, how long it took, and how many fragments there were (if number of nickjillings@1316: # fragments is different, store as different audioholder id) nickjillings@1316: if page_name in page_names: nickjillings@1316: page_index = page_names.index(page_name) # get index nickjillings@1316: # check if number of audioelements the same nickjillings@1316: if len(audioelements) == fragments_per_page[page_index]: nickjillings@1316: page_count[page_index] += 1 nickjillings@1316: duration_page[page_index].append(duration) nickjillings@1316: else: # make new entry nickjillings@1316: alt_page_name = page_name+"("+str(len(audioelements))+")" nickjillings@1316: if alt_page_name in page_names: # if already there nickjillings@1316: alt_page_index = page_names.index(alt_page_name) # get index nickjillings@1316: page_count[alt_page_index] += 1 nickjillings@1316: duration_page[alt_page_index].append(duration) nickjillings@1316: else: nickjillings@1316: page_names.append(alt_page_name) nickjillings@1316: page_count.append(1) nickjillings@1316: duration_page.append([duration]) nickjillings@1316: fragments_per_page.append(len(audioelements)) nickjillings@1316: else: nickjillings@1316: page_names.append(page_name) nickjillings@1316: page_count.append(1) nickjillings@1316: duration_page.append([duration]) nickjillings@1316: fragments_per_page.append(len(audioelements)) nickjillings@1316: nickjillings@1316: # bookkeeping nickjillings@1316: page_number += 1 # increase page count for this specific test nickjillings@1316: number_of_pages += 1 # increase total number of pages nickjillings@1316: time_per_page_accum += duration # total duration (for average time spent per page) nickjillings@1316: nickjillings@1316: # print total duration of this test nickjillings@1316: print " TOTAL: " + seconds2timestr(total_duration) nickjillings@1316: nickjillings@1316: nickjillings@1316: # PRINT EVERYTHING nickjillings@1316: nickjillings@1316: print "Number of XML files: " + str(number_of_XML_files) nickjillings@1316: print "Number of pages: " + str(number_of_pages) nickjillings@1316: print "Number of fragments: " + str(number_of_fragments) nickjillings@1316: print "Number of empty comments: " + str(total_empty_comments) +\ nickjillings@1316: " (" + str(round(100.0*total_empty_comments/number_of_fragments,2)) + "%)" nickjillings@1316: print "Number of unplayed fragments: " + str(total_not_played) +\ nickjillings@1316: " (" + str(round(100.0*total_not_played/number_of_fragments,2)) + "%)" nickjillings@1316: print "Number of unmoved markers: " + str(total_not_moved) +\ nickjillings@1316: " (" + str(round(100.0*total_not_moved/number_of_fragments,2)) + "%)" nickjillings@1316: print "Average time per page: " + seconds2timestr(time_per_page_accum/number_of_pages) nickjillings@1316: nickjillings@1316: # Pages and number of times tested nickjillings@1316: page_count_strings = list(str(x) for x in page_count) nickjillings@1316: count_list = page_names + page_count_strings nickjillings@1316: count_list[::2] = page_names nickjillings@1316: count_list[1::2] = page_count_strings nickjillings@1316: print "Pages tested: " + str(count_list) nickjillings@1316: nickjillings@1316: # Average duration for first, second, ... page nickjillings@1316: print "Average duration per page:" nickjillings@1316: for page_number in range(len(duration_order)): nickjillings@1316: print " page " + str(page_number+1) + ": " +\ nickjillings@1316: seconds2timestr(sum(duration_order[page_number])/len(duration_order[page_number])) +\ nickjillings@1316: " ("+str(len(duration_order[page_number]))+" subjects)" nickjillings@1316: nickjillings@1316: nickjillings@1316: # Sort pages by number of audioelements, then by duration nickjillings@1316: nickjillings@1316: # average duration and number of subjects per page nickjillings@1316: average_duration_page = [] nickjillings@1316: number_of_subjects_page = [] nickjillings@1316: for line in duration_page: nickjillings@1316: number_of_subjects_page.append(len(line)) nickjillings@1316: average_duration_page.append(sum(line)/len(line)) nickjillings@1316: nickjillings@1316: # combine and sort in function of number of audioelements and duration nickjillings@1316: combined_list = [page_names, average_duration_page, fragments_per_page, number_of_subjects_page] nickjillings@1316: combined_list = sorted(zip(*combined_list), key=operator.itemgetter(1, 2)) # sort nickjillings@1316: nickjillings@1316: # Show average duration for all songs nickjillings@1316: print "Average duration per audioholder:" nickjillings@1316: for page_index in range(len(page_names)): nickjillings@1316: print " "+combined_list[page_index][0] + ": " \ nickjillings@1316: + seconds2timestr(combined_list[page_index][1]) \ nickjillings@1316: + " (" + str(combined_list[page_index][3]) + " subjects, " \ nickjillings@1316: + str(combined_list[page_index][2]) + " fragments)" nickjillings@1316: nickjillings@1316: nickjillings@1316: #TODO nickjillings@1316: # time per page in function of number of fragments (plot) nickjillings@1316: # time per participant in function of number of pages nickjillings@1316: # plot total time for each participant nickjillings@1316: # plot total time nickjillings@1316: # show 'count' per page (in order) nickjillings@1316: nickjillings@1316: # clear up page_index <> page_count <> page_number confusion nickjillings@1316: nickjillings@1316: # LaTeX -> PDF print out