b@1810: #!/usr/bin/python
b@1810: # -*- coding: utf-8 -*-
b@1810: 
b@1810: import xml.etree.ElementTree as ET
b@1810: import os       # for getting files from directory
b@1810: import operator # for sorting data with multiple keys
b@1810: import sys      # for accessing command line arguments
b@1810: import subprocess # for calling pdflatex
b@1810: import shlex # for calling pdflatex
b@1810: import matplotlib.pyplot as plt # plots
b@1810: import numpy as np # numbers
b@1810: 
b@1810: # Command line arguments
b@1810: assert len(sys.argv)<3, "evaluation_stats takes at most 1 command line argument\n"+\
b@1810:                         "Use: python evaluation_stats.py [results_folder]"
b@1810: 
b@1810: # XML results files location
b@1810: if len(sys.argv) == 1:
b@1810:     folder_name = "../saves"    # Looks in 'saves/' folder from 'scripts/' folder
b@1810:     print "Use: python evaluation_stats.py [results_folder]"
b@1810:     print "Using default path: " + folder_name
b@1810: elif len(sys.argv) == 2:
b@1810:     folder_name = sys.argv[1]   # First command line argument is folder
b@1810: 
b@1810: # Turn number of seconds (int) to '[minutes] min [seconds] s' (string)
b@1810: def seconds2timestr(time_in_seconds):
b@1810:     time_in_minutes = int(time_in_seconds/60)
b@1810:     remaining_seconds = int(time_in_seconds%60)
b@1810:     return str(time_in_minutes) + " min " + str(remaining_seconds) + " s"
b@1810: 
b@1810: # stats initialisation
b@1810: number_of_XML_files  = 0
b@1810: number_of_pages      = 0
b@1810: number_of_fragments  = 0
b@1810: total_empty_comments = 0
b@1810: total_not_played     = 0
b@1810: total_not_moved      = 0
b@1810: time_per_page_accum  = 0
b@1810: 
b@1810: # arrays initialisation
b@1810: page_names       = []
b@1810: page_count       = []
b@1810: duration_page    = []      # duration of experiment in function of page content
b@1810: duration_order   = []      # duration of experiment in function of page number
b@1810: fragments_per_page = []    # number of fragments for corresponding page
b@1810: 
b@1810: # get username if available
b@1810: for name in ('LOGNAME', 'USER', 'LNAME', 'USERNAME'):
b@1810:     user = os.environ.get(name)
b@1810:     if user:
b@1810:         break
b@1810:     else:
b@1810:         user = ''
b@1810: 
b@1810: 
b@1810: # begin LaTeX document
b@1810: header = r'''\documentclass[11pt, oneside]{article} 
b@1810:           \usepackage{geometry}
b@1810:           \geometry{letterpaper}
b@1810:           \usepackage[parfill]{parskip}
b@1810:           \usepackage{graphicx}
b@1810:           \title{Report}
b@1810:           \author{'''+\
b@1810:           user+\
b@1810:           r'''}
b@1810:           \graphicspath{{'''+\
b@1810:           folder_name+\
b@1810:           r'''/}}
b@1810:           \begin{document}
b@1810:           \maketitle
b@1810:           \tableofcontents
b@1810:           '''
b@1810:           
b@1810: footer = '\end{document}'
b@1810: 
b@1810: body = ''
b@1810: 
b@1810: # generate images for later use
b@1810: subprocess.call("timeline_view_movement.py", shell=True)
b@1810: 
b@1810: # get every XML file in folder
b@1810: files_list = os.listdir(folder_name)
b@1810: for file in files_list: # iterate over all files in files_list
b@1810:     if file.endswith(".xml"): # check if XML file
b@1810:         number_of_XML_files += 1
b@1810:         tree = ET.parse(folder_name + '/' + file)
b@1810:         root = tree.getroot()
b@1810:         
b@1810:         # PRINT name as section
b@1810:         body+= '\section{'+file[:-4].capitalize()+'}\n' # make section header from name without extension
b@1810:         
b@1810:         # reset for new subject
b@1810:         total_duration = 0
b@1810:         page_number = 0
b@1810:         
b@1810:         individual_table = '' # table with stats for this individual test file
b@1810:         
b@1810:         # get list of all page names
b@1810:         for audioholder in root.findall("./audioholder"):   # iterate over pages
b@1810:             page_name = audioholder.get('id')               # get page name
b@1810:             
b@1810:             if page_name is None: # ignore 'empty' audio_holders
b@1810:                 print "WARNING: " + file + " contains empty audio holder. (evaluation_stats.py)"
b@1810:                 break # move on to next
b@1810:             
b@1810:             number_of_comments = 0 # for this page
b@1810:             number_of_missing_comments = 0 # for this page
b@1810:             not_played = 0 # for this page
b@1810:             not_moved = 0 # for this page
b@1810:             
b@1810:             # 'testTime' keeps total duration: subtract time so far for duration of this audioholder
b@1810:             duration = float(audioholder.find("./metric/metricresult[@id='testTime']").text) - total_duration
b@1810:             
b@1810:             # total duration of test
b@1810:             total_duration += duration
b@1810:             
b@1810:             # number of audio elements
b@1810:             audioelements = audioholder.findall("./audioelement") # get audioelements
b@1810:             number_of_fragments += len(audioelements) # add length of this list to total
b@1810:             
b@1810:             # number of comments (interesting if comments not mandatory)
b@1810:             for audioelement in audioelements:
b@1810:                 response = audioelement.find("./comment/response")
b@1810:                 was_played = audioelement.find("./metric/metricresult/[@name='elementFlagListenedTo']")
b@1810:                 was_moved = audioelement.find("./metric/metricresult/[@name='elementFlagMoved']")
b@1810:                 if response.text is not None and len(response.text) > 1: 
b@1810:                     number_of_comments += 1
b@1810:                 else: 
b@1810:                     number_of_missing_comments += 1
b@1810:                 if was_played is not None and was_played.text == 'false': 
b@1810:                     not_played += 1
b@1810:                 if was_moved is not None and was_moved.text == 'false': 
b@1810:                     not_moved += 1
b@1810:             
b@1810:             # update global counters
b@1810:             total_empty_comments += number_of_missing_comments
b@1810:             total_not_played += not_played
b@1810:             total_not_moved += not_moved
b@1810:             
b@1810:             # PRINT alerts when elements not played or markers not moved
b@1810:             # number of audio elements not played
b@1810:             if not_played > 1:
b@1810:                 body += '\\emph{\\textbf{ATTENTION: '+str(not_played)+' fragments were not listened to in '+page_name+'!}} \\\\ \n'
b@1810:             if not_played == 1: 
b@1810:                 body += '\\emph{\\textbf{ATTENTION: one fragment was not listened to in '+page_name+'!}} \\\\ \n '
b@1810:             
b@1810:             # number of audio element markers not moved
b@1810:             if not_moved > 1:
b@1810:                 body += '\\emph{\\textbf{ATTENTION: '+str(not_moved)+' markers were not moved in '+page_name+'!}} \\\\ \n'
b@1810:             if not_moved == 1: 
b@1810:                 body += '\\emph{\\textbf{ATTENTION: one marker was not moved in '+page_name+'!}} \\\\ \n'
b@1810:                 
b@1810:             #TODO which one not moved/listened to? 
b@1810:             
b@1810:             # PRINT song-specific statistic
b@1810:             individual_table += page_name+'&'+\
b@1810:                                 str(number_of_comments) + '/' +\
b@1810:                                 str(number_of_comments+number_of_missing_comments)+'&'+\
b@1810:                                 seconds2timestr(duration)+'\\\\'
b@1810:             
b@1810:             # get timeline for this audioholder
b@1810:             img_path = 'timelines_movement/'+file[:-4]+'-'+page_name+'.pdf'
b@1810:             
b@1810:             # check if available
b@1810:             if os.path.isfile(folder_name+'/'+img_path):
b@1810:                 # SHOW timeline image
b@1810:                 body += r'''\begin{figure}[htbp]
b@1810:                          \begin{center}
b@1810:                          \includegraphics[width=\textwidth]{'''+\
b@1810:                          folder_name+'/'+img_path+\
b@1810:                         r'''}
b@1810:                         \caption{Timeline of '''+\
b@1810:                          page_name+' by '+ file[:-4].capitalize() +\
b@1810:                         r'''.}
b@1810:                          \end{center}
b@1810:                          \end{figure}
b@1810:                          '''
b@1810:             
b@1810:             # keep track of duration in function of page index
b@1810:             if len(duration_order)>page_number:
b@1810:                 duration_order[page_number].append(duration)
b@1810:             else:
b@1810:                 duration_order.append([duration])
b@1810:                 
b@1810:             # keep list of audioholder ids and count how many times each audioholder id
b@1810:             # was tested, how long it took, and how many fragments there were (if number of 
b@1810:             # fragments is different, store as different audioholder id)
b@1810:             if page_name in page_names: 
b@1810:                 page_index = page_names.index(page_name) # get index
b@1810:                 # check if number of audioelements the same
b@1810:                 if len(audioelements) == fragments_per_page[page_index]: 
b@1810:                     page_count[page_index] += 1
b@1810:                     duration_page[page_index].append(duration)
b@1810:                 else: # make new entry
b@1810:                     alt_page_name = page_name+"("+str(len(audioelements))+")"
b@1810:                     if alt_page_name in page_names: # if already there
b@1810:                         alt_page_index = page_names.index(alt_page_name) # get index
b@1810:                         page_count[alt_page_index] += 1
b@1810:                         duration_page[alt_page_index].append(duration)
b@1810:                     else: 
b@1810:                         page_names.append(alt_page_name)
b@1810:                         page_count.append(1)
b@1810:                         duration_page.append([duration])
b@1810:                         fragments_per_page.append(len(audioelements))
b@1810:             else: 
b@1810:                 page_names.append(page_name)
b@1810:                 page_count.append(1)
b@1810:                 duration_page.append([duration])
b@1810:                 fragments_per_page.append(len(audioelements))
b@1810:                 
b@1810:             # bookkeeping
b@1810:             page_number += 1 # increase page count for this specific test
b@1810:             number_of_pages += 1 # increase total number of pages
b@1810:             time_per_page_accum += duration # total duration (for average time spent per page)
b@1810: 
b@1810:         # PRINT table with statistics about this test
b@1810:         body += r'''\begin{tabular}{|p{3.5cm}|c|p{2.5cm}|}
b@1810:                  \hline
b@1810:                  \textbf{Song name} & \textbf{Comments} & \textbf{Duration} \\ \hline '''+\
b@1810:                  individual_table+\
b@1810:                  r'''\hline
b@1810:                   \textbf{TOTAL} & & \textbf{'''+\
b@1810:                   seconds2timestr(total_duration)+\
b@1810:                  r'''}\\
b@1810:                   \hline 
b@1810:                   \end{tabular}'''
b@1810: 
b@1810: # join to footer
b@1810: footer = body + footer
b@1810: 
b@1810: # empty body again
b@1810: body = ''
b@1810: 
b@1810: # PRINT summary of everything (at start)
b@1810: body += '\section{Summary}\n'
b@1810: 
b@1810: # PRINT table with statistics
b@1810: body += '\\begin{tabular}{ll}'
b@1810: body += r'Number of XML files: &' + str(number_of_XML_files) + r'\\'
b@1810: body += r'Number of pages: &' + str(number_of_pages) + r'\\'
b@1810: body += r'Number of fragments: &' + str(number_of_fragments) + r'\\'
b@1810: body += r'Number of empty comments: &' + str(total_empty_comments) +\
b@1810:       " (" + str(round(100.0*total_empty_comments/number_of_fragments,2)) + r"\%)\\"
b@1810: body += r'Number of unplayed fragments: &' + str(total_not_played) +\
b@1810:       " (" + str(round(100.0*total_not_played/number_of_fragments,2)) + r"\%)\\"
b@1810: body += r'Number of unmoved markers: &' + str(total_not_moved) +\
b@1810:       " (" + str(round(100.0*total_not_moved/number_of_fragments,2)) + r"\%)\\"
b@1810: body += r'Average time per page: &' + seconds2timestr(time_per_page_accum/number_of_pages) + r"\\"
b@1810: 
b@1810: 
b@1810: # Pages and number of times tested
b@1810: page_count_strings = list(str(x) for x in page_count)
b@1810: count_list = page_names + page_count_strings
b@1810: count_list[::2] = page_names
b@1810: count_list[1::2] = page_count_strings
b@1810: #body +=  r'Pages tested: &' + str(count_list) + r"\\"
b@1810: 
b@1810: body += '\\end{tabular} \\vspace{1.5cm} \\\\ \n'
b@1810: 
b@1810: # Average duration for first, second, ... page
b@1810: body += " \\vspace{.5cm} Average duration per page (see also Figure \\ref{fig:avgtimeperpage}): \\\\ \n"
b@1810: body += r'''\begin{tabular}{lll}
b@1810:         \textbf{Page} & \textbf{Duration} & \textbf{\# subjects}\\
b@1810:         '''
b@1810: tpp_averages = [] # store average time per page
b@1810: for page_number in range(len(duration_order)): 
b@1810:     body += str(page_number+1) + "&" +\
b@1810:         seconds2timestr(sum(duration_order[page_number])/len(duration_order[page_number])) +\
b@1810:             "&"+str(len(duration_order[page_number]))+r"\\"
b@1810:     tpp_averages.append(sum(duration_order[page_number])/len(duration_order[page_number]))
b@1810:             
b@1810: body += '\\end{tabular} \\vspace{1.5cm} \\\\ \n'
b@1810: 
b@1810: # SHOW bar plot of average time per page
b@1810: plt.bar(range(1,len(duration_order)+1), tpp_averages)
b@1810: plt.xlabel('Page')
b@1810: plt.xlim(.8, len(duration_order)+1)
b@1810: plt.xticks(np.arange(1,len(duration_order)+1)+.4, range(1,len(duration_order)+1))
b@1810: plt.ylabel('Time [seconds]')
b@1810: plt.savefig(folder_name+"/time_per_page.pdf", bbox_inches='tight')
b@1810: plt.close()
b@1810: body += r'''\begin{figure}[htbp]
b@1810:          \begin{center}
b@1810:          \includegraphics[width=\textwidth]{'''+\
b@1810:          folder_name+"/time_per_page.pdf"+\
b@1810:         r'''}
b@1810:         \caption{Average time spent per audioholder page.}
b@1810:         \label{fig:avgtimeperpage}
b@1810:          \end{center}
b@1810:          \end{figure}
b@1810:          '''
b@1810: #TODO add error bars
b@1810: 
b@1810: 
b@1810: # Sort pages by number of audioelements, then by duration
b@1810: 
b@1810: # average duration and number of subjects per page
b@1810: average_duration_page = []
b@1810: number_of_subjects_page = []
b@1810: for line in duration_page:
b@1810:     number_of_subjects_page.append(len(line))
b@1810:     average_duration_page.append(sum(line)/len(line))
b@1810: 
b@1810: # combine and sort in function of number of audioelements and duration
b@1810: combined_list = [page_names, average_duration_page, fragments_per_page, number_of_subjects_page]
b@1810: combined_list = sorted(zip(*combined_list), key=operator.itemgetter(1, 2)) # sort
b@1810: 
b@1810: # Show average duration for all songs
b@1810: body += r'''\vspace{.5cm} Average duration per audioholder: \\
b@1810:         \begin{tabular}{llll}
b@1810:         \textbf{Audioholder} & \textbf{Duration} & \textbf{\# subjects} & \textbf{\# fragments} \\
b@1810:         '''
b@1810: for page_index in range(len(page_names)):
b@1810:     body +=  combined_list[page_index][0] + "&" +\
b@1810:              seconds2timestr(combined_list[page_index][1]) + "&" +\
b@1810:              str(combined_list[page_index][3]) + "&" +\
b@1810:              str(combined_list[page_index][2]) + r"\\"
b@1810: body += '\\end{tabular}\n'
b@1810: 
b@1810: #TODO
b@1810: # time per page in function of number of fragments (plot)
b@1810: # time per participant in function of number of pages
b@1810: # plot total time for each participant
b@1810: # plot total time
b@1810: # show 'count' per page (in order)
b@1810: 
b@1810: # clear up page_index <> page_count <> page_number confusion
b@1810: 
b@1810: 
b@1810: texfile = header+body+footer
b@1810: 
b@1810: # write TeX file
b@1810: with open(folder_name + '/' + 'test.tex','w') as f:
b@1810:     f.write(texfile)
b@1810: proc=subprocess.Popen(shlex.split('pdflatex -output-directory='+folder_name+' '+ folder_name + '/test.tex'))
b@1810: proc.communicate()
b@1810: # run again
b@1810: proc=subprocess.Popen(shlex.split('pdflatex -output-directory='+folder_name+' '+ folder_name + '/test.tex'))
b@1810: proc.communicate()
b@1810: 
b@1810: #TODO remove auxiliary LaTeX files