comparison scripts/generate_report.py @ 1316:279930a008ca

All interfaces support comment boxes. Comment box identification matches presented tag (for instance, AB will be Comment on fragment A, rather than 1). Tighter buffer loading protocol, audioObjects register with the buffer rather than checking for buffer existence (which can be buggy depending on the buffer state). Buffers now have a state to ensure exact location in loading chain (downloading, decoding, LUFS, ready).

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+import xml.etree.ElementTree as ET
+import os       # for getting files from directory
+import operator # for sorting data with multiple keys
+import sys      # for accessing command line arguments
+import subprocess # for calling pdflatex
+import shlex # for calling pdflatex
+import matplotlib.pyplot as plt # plots
+import numpy as np # numbers
+
+# Command line arguments
+assert len(sys.argv)<4, "generate_report takes at most 2 command line arguments\n"+\
+                        "Use: python generate_report.py [results_folder] [no_render | -nr]"
+
+render_figures = True
+
+# XML results files location
+if len(sys.argv) == 1:
+    folder_name = "../saves/"    # Looks in 'saves/' folder from 'scripts/' folder
+    print "Use: python generate_report.py [results_folder] [no_render | -nr]"
+    print "Using default path: " + folder_name
+elif len(sys.argv) == 2:
+    folder_name = sys.argv[1]   # First command line argument is folder
+elif len(sys.argv) == 3:
+    folder_name = sys.argv[1]   # First command line argument is folder
+    assert sys.argv[2] in ('no_render','-nr'), "Second argument not recognised. \n" +\
+           "Use: python generate_report.py [results_folder] [no_render | -nr]"
+    # Second command line argument is [no_render | -nr]
+    render_figures = False
+
+def isNaN(num):
+    return num != num
+
+# Turn number of seconds (int) to '[minutes] min [seconds] s' (string)
+def seconds2timestr(time_in_seconds):
+    if time_in_seconds is not None and not isNaN(time_in_seconds): 
+        time_in_minutes = int(time_in_seconds/60)
+        remaining_seconds = int(time_in_seconds%60)
+        return str(time_in_minutes) + " min " + str(remaining_seconds) + " s"
+    else:
+        return 'N/A'
+
+# stats initialisation
+number_of_XML_files  = 0
+number_of_pages      = 0
+number_of_fragments  = 0
+total_empty_comments = 0
+total_not_played     = 0
+total_not_moved      = 0
+time_per_page_accum  = 0
+
+# arrays initialisation
+page_names       = []
+real_page_names  = [] # regardless of differing numbers of fragments
+subject_count    = [] # subjects per audioholder name
+page_count       = []
+duration_page    = []      # duration of experiment in function of page content
+duration_order   = []      # duration of experiment in function of page number
+fragments_per_page = []    # number of fragments for corresponding page
+
+# survey stats
+gender = []
+age    = []
+
+# get username if available
+for name in ('LOGNAME', 'USER', 'LNAME', 'USERNAME'):
+    user = os.environ.get(name)
+    if user:
+        break
+    else:
+        user = ''
+
+
+# begin LaTeX document
+header = r'''\documentclass[11pt, oneside]{article} 
+          \usepackage{geometry}
+          \geometry{a4paper}
+          \usepackage[parfill]{parskip} % empty line instead of indent
+          \usepackage{graphicx}    % figures
+          \usepackage[space]{grffile} % include figures with spaces in paths
+          \usepackage{hyperref}
+          \usepackage{tikz}           % pie charts
+          \title{Report}
+          \author{'''+\
+          user+\
+          r'''}
+          \graphicspath{{'''+\
+          folder_name+\
+          r'''}}
+          %\setcounter{section}{-1} % Summary section 0 so number of sections equals number of files
+          \begin{document}
+          \maketitle
+          This is an automatically generated report using the `generate\_report.py' Python script 
+          included with the Web Audio Evaluation Tool \cite{WAET} distribution which can be found 
+          at \texttt{code.soundsoftware.ac.uk/projects/webaudioevaluationtool}.
+          \tableofcontents
+          
+          '''
+          
+footer = '\n\t\t'+r'''\begin{thebibliography}{9}
+         \bibitem{WAET} % reference to accompanying publication
+        Nicholas Jillings, Brecht De Man, David Moffat and Joshua D. Reiss, 
+        ``Web Audio Evaluation Tool: A browser-based listening test environment,'' 
+        presented at the 12th Sound and Music Computing Conference, July 2015.
+        \end{thebibliography}
+        \end{document}'''
+
+body = ''
+
+# make sure folder_name ends in '/'
+folder_name = os.path.join(folder_name, '')
+
+# generate images for later use
+if render_figures:
+    subprocess.call("python timeline_view_movement.py '"+folder_name+"'", shell=True)
+    subprocess.call("python score_parser.py '"+folder_name+"'", shell=True)
+    subprocess.call("python score_plot.py '"+folder_name+"ratings/'", shell=True)
+
+# get every XML file in folder
+files_list = os.listdir(folder_name)
+for file in files_list: # iterate over all files in files_list
+    if file.endswith(".xml"): # check if XML file
+        number_of_XML_files += 1
+        tree = ET.parse(folder_name + file)
+        root = tree.getroot()
+        
+        # PRINT name as section
+        body+= '\n\section{'+file[:-4].capitalize()+'}\n' # make section header from name without extension
+        
+        # reset for new subject
+        total_duration = 0
+        page_number = 0
+        
+        individual_table = '\n' # table with stats for this individual test file
+        timeline_plots = '' # plots of timeline (movements and plays)
+        
+        # DEMO survey stats
+        # get gender
+        this_subjects_gender = root.find("./posttest/radio/[@id='gender']")
+        if this_subjects_gender is not None:
+            gender.append(this_subjects_gender.get("name"))
+        else:
+            gender.append('UNAVAILABLE')
+        # get age
+        this_subjects_age = root.find("./posttest/number/[@id='age']")
+        if this_subjects_age is not None:
+            age.append(this_subjects_age.text)
+        #TODO add plot of age
+                
+        # get list of all page names
+        for audioholder in root.findall("./audioholder"):   # iterate over pages
+            page_name = audioholder.get('id')               # get page name
+            
+            if page_name is None: # ignore 'empty' audio_holders
+                print "WARNING: " + file + " contains empty audio holder. (evaluation_stats.py)"
+                break # move on to next
+            
+            number_of_comments = 0 # for this page
+            number_of_missing_comments = 0 # for this page
+            not_played = [] # for this page
+            not_moved = [] # for this page
+            
+            if audioholder.find("./metric/metricresult[@id='testTime']") is not None: # check if time is included
+                # 'testTime' keeps total duration: subtract time so far for duration of this audioholder
+                duration = float(audioholder.find("./metric/metricresult[@id='testTime']").text) - total_duration
+            
+                # total duration of test
+                total_duration += duration
+            else: 
+                duration = float('nan')
+                total_duration = float('nan')
+            
+            # number of audio elements
+            audioelements = audioholder.findall("./audioelement") # get audioelements
+            number_of_fragments += len(audioelements) # add length of this list to total
+            
+            # number of comments (interesting if comments not mandatory)
+            for audioelement in audioelements:
+                response = audioelement.find("./comment/response")
+                was_played = audioelement.find("./metric/metricresult/[@name='elementFlagListenedTo']")
+                was_moved = audioelement.find("./metric/metricresult/[@name='elementFlagMoved']")
+                if response.text is not None and len(response.text) > 1: 
+                    number_of_comments += 1
+                else: 
+                    number_of_missing_comments += 1
+                if was_played is not None and was_played.text == 'false': 
+                    not_played.append(audioelement.get('id'))
+                if was_moved is not None and was_moved.text == 'false': 
+                    not_moved.append(audioelement.get('id'))
+            
+            # update global counters
+            total_empty_comments += number_of_missing_comments
+            total_not_played += len(not_played)
+            total_not_moved += len(not_moved)
+            
+            # PRINT alerts when elements not played or markers not moved
+            # number of audio elements not played
+            if len(not_played) > 1:
+                body += '\t\t\\emph{\\textbf{ATTENTION: '+str(len(not_played))+\
+                        ' fragments were not listened to in '+page_name+'! }}'+\
+                        ', '.join(not_played)+'\\\\ \n'
+            if len(not_played) == 1: 
+                body += '\t\t\\emph{\\textbf{ATTENTION: one fragment was not listened to in '+page_name+'! }}'+\
+                        not_played[0]+'\\\\ \n'
+            
+            # number of audio element markers not moved
+            if len(not_moved) > 1:
+                body += '\t\t\\emph{\\textbf{ATTENTION: '+str(len(not_moved))+\
+                        ' markers were not moved in '+page_name+'! }}'+\
+                        ', '.join(not_moved)+'\\\\ \n'
+            if len(not_moved) == 1: 
+                body += '\t\t\\emph{\\textbf{ATTENTION: one marker was not moved in '+page_name+'! }}'+\
+                        not_moved[0]+'\\\\ \n'
+            
+            # PRINT song-specific statistic
+            individual_table += '\t\t'+page_name+'&'+\
+                                str(number_of_comments) + '/' +\
+                                str(number_of_comments+number_of_missing_comments)+'&'+\
+                                seconds2timestr(duration)+'\\\\\n'
+            
+            # get timeline for this audioholder
+            img_path = 'timelines_movement/'+file[:-4]+'-'+page_name+'.pdf'
+            
+            # check if available
+            if os.path.isfile(folder_name+img_path):
+                # SHOW timeline image
+                timeline_plots += '\\includegraphics[width=\\textwidth]{'+\
+                         folder_name+img_path+'}\n\t\t'
+            
+            # keep track of duration in function of page index
+            if len(duration_order)>page_number:
+                duration_order[page_number].append(duration)
+            else:
+                duration_order.append([duration])
+            
+            # keep list of audioholder ids and count how many times each audioholder id
+            # was tested, how long it took, and how many fragments there were 
+            # (if number of fragments is different, store as different audioholder id)
+            if page_name in page_names: 
+                page_index = page_names.index(page_name) # get index
+                # check if number of audioelements the same
+                if len(audioelements) == fragments_per_page[page_index]: 
+                    page_count[page_index] += 1
+                    duration_page[page_index].append(duration)
+                else: # make new entry
+                    alt_page_name = page_name+"("+str(len(audioelements))+")"
+                    if alt_page_name in page_names: # if already there
+                        alt_page_index = page_names.index(alt_page_name) # get index
+                        page_count[alt_page_index] += 1
+                        duration_page[alt_page_index].append(duration)
+                    else: 
+                        page_names.append(alt_page_name)
+                        page_count.append(1)
+                        duration_page.append([duration])
+                        fragments_per_page.append(len(audioelements))
+            else: 
+                page_names.append(page_name)
+                page_count.append(1)
+                duration_page.append([duration])
+                fragments_per_page.append(len(audioelements))
+            
+            # number of subjects per audioholder regardless of differing numbers of 
+            # fragments (for inclusion in box plots)
+            if page_name in real_page_names:
+                page_index = real_page_names.index(page_name) # get index
+                subject_count[page_index] += 1
+            else: 
+                real_page_names.append(page_name)
+                subject_count.append(1)
+            
+            # bookkeeping
+            page_number += 1 # increase page count for this specific test
+            number_of_pages += 1 # increase total number of pages
+            time_per_page_accum += duration # total duration (for average time spent per page)
+
+        # PRINT table with statistics about this test
+        body += '\t\t'+r'''\begin{tabular}{|p{3.5cm}|c|p{2.5cm}|}
+                 \hline
+                 \textbf{Song name} & \textbf{Comments} & \textbf{Duration} \\ \hline '''+\
+                 individual_table+'\t\t'+\
+                 r'''\hline
+                  \textbf{TOTAL} & & \textbf{'''+\
+                  seconds2timestr(total_duration)+\
+                 r'''}\\
+                  \hline 
+                  \end{tabular}
+                  
+                  '''
+        # PRINT timeline plots
+        body += timeline_plots
+
+# join to footer
+footer = body + footer
+
+# empty body again
+body = ''
+
+# PRINT summary of everything (at start) 
+#       unnumbered so that number of sections equals number of files
+body += '\section*{Summary}\n\t\t\\addcontentsline{toc}{section}{Summary}\n'
+
+# PRINT table with statistics
+body += '\t\t\\begin{tabular}{ll}\n\t\t\t'
+body += r'Number of XML files: &' + str(number_of_XML_files) + r'\\'+'\n\t\t\t'
+body += r'Number of pages: &' + str(number_of_pages) + r'\\'+'\n\t\t\t'
+body += r'Number of fragments: &' + str(number_of_fragments) + r'\\'+'\n\t\t\t'
+body += r'Number of empty comments: &' + str(total_empty_comments) +\
+      " (" + str(round(100.0*total_empty_comments/number_of_fragments,2)) + r"\%)\\"+'\n\t\t\t'
+body += r'Number of unplayed fragments: &' + str(total_not_played) +\
+      " (" + str(round(100.0*total_not_played/number_of_fragments,2)) + r"\%)\\"+'\n\t\t\t'
+body += r'Number of unmoved markers: &' + str(total_not_moved) +\
+      " (" + str(round(100.0*total_not_moved/number_of_fragments,2)) + r"\%)\\"+'\n\t\t\t'
+body += r'Average time per page: &' + seconds2timestr(time_per_page_accum/number_of_pages) + r"\\"+'\n\t\t'
+body += '\\end{tabular} \\vspace{1.5cm} \\\\ \n'
+
+# Average duration for first, second, ... page
+body += "\t\t\\vspace{.5cm} \n\n\t\tAverage duration per page (see also Figure \\ref{fig:avgtimeperpage}): \\\\ \n\t\t"
+body += r'''\begin{tabular}{lll}
+                    \textbf{Page} & \textbf{Duration} & \textbf{\# subjects}\\'''
+tpp_averages = [] # store average time per page
+for page_number in range(len(duration_order)): 
+    body += '\n\t\t\t'+str(page_number+1) + "&" +\
+        seconds2timestr(sum(duration_order[page_number])/len(duration_order[page_number])) +\
+            "&"+str(len(duration_order[page_number]))+r"\\"
+    tpp_averages.append(sum(duration_order[page_number])/len(duration_order[page_number]))
+            
+body += '\n\t\t\\end{tabular} \\vspace{1.5cm} \\\\ \n\n\t\t'
+
+# SHOW bar plot of average time per page
+plt.bar(range(1,len(duration_order)+1), np.array(tpp_averages)/60)
+plt.xlabel('Page order')
+plt.xlim(.8, len(duration_order)+1)
+plt.xticks(np.arange(1,len(duration_order)+1)+.4, range(1,len(duration_order)+1))
+plt.ylabel('Average time [minutes]')
+plt.savefig(folder_name+"time_per_page.pdf", bbox_inches='tight')
+plt.close()
+#TODO add error bars
+
+
+# Sort pages by number of audioelements, then by duration
+
+# average duration and number of subjects per page
+average_duration_page = []
+number_of_subjects_page = []
+for line in duration_page:
+    number_of_subjects_page.append(len(line))
+    average_duration_page.append(sum(line)/len(line))
+
+# combine and sort in function of number of audioelements and duration
+combined_list = [page_names, average_duration_page, fragments_per_page, number_of_subjects_page]
+combined_list = sorted(zip(*combined_list), key=operator.itemgetter(1, 2)) # sort
+
+# Show average duration for all songs
+body += r'''\vspace{.5cm}
+                Average duration per audioholder (see also Figure \ref{fig:avgtimeperaudioholder}): \\
+                \begin{tabular}{llll}
+                        \textbf{Audioholder} & \textbf{Duration} & \textbf{\# subjects} & \textbf{\# fragments} \\'''
+audioholder_names_ordered = []
+average_duration_audioholder_ordered = []
+number_of_subjects = []
+for page_index in range(len(page_names)):
+    audioholder_names_ordered.append(combined_list[page_index][0])
+    average_duration_audioholder_ordered.append(combined_list[page_index][1])
+    number_of_subjects.append(combined_list[page_index][3])
+    body +=  '\n\t\t\t'+combined_list[page_index][0] + "&" +\
+             seconds2timestr(combined_list[page_index][1]) + "&" +\
+             str(combined_list[page_index][3]) + "&" +\
+             str(combined_list[page_index][2]) + r"\\"
+body += '\n\t\t\\end{tabular}\n'
+
+# SHOW bar plot of average time per page
+plt.bar(range(1,len(audioholder_names_ordered)+1), np.array(average_duration_audioholder_ordered)/60)
+plt.xlabel('Audioholder')
+plt.xlim(.8, len(audioholder_names_ordered)+1)
+plt.xticks(np.arange(1,len(audioholder_names_ordered)+1)+.4, audioholder_names_ordered, rotation=90)
+plt.ylabel('Average time [minutes]')
+plt.savefig(folder_name+"time_per_audioholder.pdf", bbox_inches='tight')
+plt.close()
+
+# SHOW bar plot of average time per page
+plt.bar(range(1,len(audioholder_names_ordered)+1), number_of_subjects)
+plt.xlabel('Audioholder')
+plt.xlim(.8, len(audioholder_names_ordered)+1)
+plt.xticks(np.arange(1,len(audioholder_names_ordered)+1)+.4, audioholder_names_ordered, rotation=90)
+plt.ylabel('Number of subjects')
+ax = plt.gca()
+ylims = ax.get_ylim()
+yint = np.arange(int(np.floor(ylims[0])), int(np.ceil(ylims[1]))+1)
+plt.yticks(yint)
+plt.savefig(folder_name+"subjects_per_audioholder.pdf", bbox_inches='tight')
+plt.close()
+
+# SHOW both figures
+body += r'''
+         \begin{figure}[htbp]
+         \begin{center}
+         \includegraphics[width=.65\textwidth]{'''+\
+         folder_name+'time_per_page.pdf'+\
+        r'''}
+        \caption{Average time spent per page.}
+        \label{fig:avgtimeperpage}
+         \end{center}
+         \end{figure}
+         
+         '''
+body += r'''\begin{figure}[htbp]
+         \begin{center}
+         \includegraphics[width=.65\textwidth]{'''+\
+         folder_name+'time_per_audioholder.pdf'+\
+        r'''}
+        \caption{Average time spent per audioholder.}
+        \label{fig:avgtimeperaudioholder}
+         \end{center}
+         \end{figure}
+         
+         '''
+body += r'''\begin{figure}[htbp]
+         \begin{center}
+         \includegraphics[width=.65\textwidth]{'''+\
+         folder_name+'subjects_per_audioholder.pdf'+\
+        r'''}
+        \caption{Number of subjects per audioholder.}
+        \label{fig:subjectsperaudioholder}
+         \end{center}
+         \end{figure}
+         
+         '''
+#TODO add error bars
+#TODO layout of figures
+
+# SHOW boxplot per audioholder
+#TODO order in decreasing order of participants
+for audioholder_name in page_names: # get each name
+    # plot boxplot if exists (not so for the 'alt' names)
+    if os.path.isfile(folder_name+'ratings/'+audioholder_name+'-ratings-box.pdf'):
+        body += r'''\begin{figure}[htbp]
+             \begin{center}
+             \includegraphics[width=.65\textwidth]{'''+\
+             folder_name+"ratings/"+audioholder_name+'-ratings-box.pdf'+\
+            r'''}
+            \caption{Box plot of ratings for audioholder '''+\
+            audioholder_name+' ('+str(subject_count[real_page_names.index(audioholder_name)])+\
+            ''' participants).}
+            \label{fig:boxplot'''+audioholder_name.replace(" ", "")+'''}
+             \end{center}
+             \end{figure}
+             
+             '''
+
+# DEMO pie chart of gender distribution among subjects
+genders = ['male', 'female', 'other', 'preferNotToSay', 'UNAVAILABLE']
+# TODO: get the above automatically
+gender_distribution = ''
+for item in genders:
+    number = gender.count(item)
+    if number>0:
+        gender_distribution += str("{:.2f}".format((100.0*number)/len(gender)))+\
+                               '/'+item.capitalize()+' ('+str(number)+'),\n'
+
+body += r'''
+        % Pie chart of gender distribution
+        \def\angle{0}
+        \def\radius{3}
+        \def\cyclelist{{"orange","blue","red","green"}}
+        \newcount\cyclecount \cyclecount=-1
+        \newcount\ind \ind=-1
+        \begin{figure}[htbp]
+        \begin{center}\begin{tikzpicture}[nodes = {font=\sffamily}]
+        \foreach \percent/\name in {'''+\
+        gender_distribution+\
+        r'''} {\ifx\percent\empty\else               % If \percent is empty, do nothing
+        \global\advance\cyclecount by 1     % Advance cyclecount
+        \global\advance\ind by 1            % Advance list index
+        \ifnum6<\cyclecount                 % If cyclecount is larger than list
+          \global\cyclecount=0              %   reset cyclecount and
+          \global\ind=0                     %   reset list index
+        \fi
+        \pgfmathparse{\cyclelist[\the\ind]} % Get color from cycle list
+        \edef\color{\pgfmathresult}         %   and store as \color
+        % Draw angle and set labels
+        \draw[fill={\color!50},draw={\color}] (0,0) -- (\angle:\radius)
+          arc (\angle:\angle+\percent*3.6:\radius) -- cycle;
+        \node at (\angle+0.5*\percent*3.6:0.7*\radius) {\percent\,\%};
+        \node[pin=\angle+0.5*\percent*3.6:\name]
+          at (\angle+0.5*\percent*3.6:\radius) {};
+        \pgfmathparse{\angle+\percent*3.6}  % Advance angle
+        \xdef\angle{\pgfmathresult}         %   and store in \angle
+        \fi
+        };
+        \end{tikzpicture}
+        \caption{Representation of gender across subjects}
+        \label{default}
+        \end{center}
+        \end{figure}
+        
+        '''
+# problem: some people entered twice? 
+
+#TODO
+# time per page in function of number of fragments (plot)
+# time per participant in function of number of pages
+# plot total time for each participant
+# show 'count' per page (in order)
+
+# clear up page_index <> page_count <> page_number confusion
+
+
+texfile = header+body+footer # add bits together
+
+print 'pdflatex -output-directory="'+folder_name+'"" "'+ folder_name + 'Report.tex"' # DEBUG
+
+# write TeX file
+with open(folder_name + 'Report.tex','w') as f:
+    f.write(texfile)
+proc=subprocess.Popen(shlex.split('pdflatex -output-directory="'+folder_name+'" "'+ folder_name + 'Report.tex"'))
+proc.communicate()
+# run again
+proc=subprocess.Popen(shlex.split('pdflatex -output-directory="'+folder_name+'" "'+ folder_name + 'Report.tex"'))
+proc.communicate()
+
+#TODO remove auxiliary LaTeX files
+try:
+    os.remove(folder_name + 'Report.aux')
+    os.remove(folder_name + 'Report.log')
+    os.remove(folder_name + 'Report.out')
+    os.remove(folder_name + 'Report.toc')
+except OSError:
+    pass
+