comparison scripts/generate_report.py @ 1072:2ea78697aadf

Scripts: comment_parser and score_parser start new CSV files each time; various plots added to generated PDF report

 import shlex # for calling pdflatex
 import matplotlib.pyplot as plt # plots
 import numpy as np # numbers
 
 # Command line arguments
-assert len(sys.argv)<3, "evaluation_stats takes at most 1 command line argument\n"+\
-                        "Use: python evaluation_stats.py [results_folder]"
+assert len(sys.argv)<4, "evaluation_stats takes at most 2 command line argument\n"+\
+                        "Use: python generate_report.py [results_folder] [no_render | -nr]"
+
+render_figures = True
 
 # XML results files location
 if len(sys.argv) == 1:
     folder_name = "../saves"    # Looks in 'saves/' folder from 'scripts/' folder
-    print "Use: python evaluation_stats.py [results_folder]"
+    print "Use: python generate_report.py [results_folder] [no_render | -nr]"
     print "Using default path: " + folder_name
 elif len(sys.argv) == 2:
     folder_name = sys.argv[1]   # First command line argument is folder
+elif len(sys.argv) == 3:
+    folder_name = sys.argv[1]   # First command line argument is folder
+    assert sys.argv[2] in ('no_render','-nr'), "Second argument not recognised. \n" +\
+           "Use: python generate_report.py [results_folder] [no_render | -nr]"
+    # Second command line argument is [no_render | -nr]
+    render_figures = False
+    
+    
+#TODO add 'skip regenerating figures'
 
 # Turn number of seconds (int) to '[minutes] min [seconds] s' (string)
 def seconds2timestr(time_in_seconds):
     time_in_minutes = int(time_in_seconds/60)
     remaining_seconds = int(time_in_seconds%60)

 total_not_moved      = 0
 time_per_page_accum  = 0
 
 # arrays initialisation
 page_names       = []
+real_page_names  = [] # regardless of differing numbers of fragments
+subject_count    = [] # subjects per audioholder name
 page_count       = []
 duration_page    = []      # duration of experiment in function of page content
 duration_order   = []      # duration of experiment in function of page number
 fragments_per_page = []    # number of fragments for corresponding page
+
+# survey stats
+gender = []
+age    = []
 
 # get username if available
 for name in ('LOGNAME', 'USER', 'LNAME', 'USERNAME'):
     user = os.environ.get(name)
     if user:

 
 
 # begin LaTeX document
 header = r'''\documentclass[11pt, oneside]{article} 
           \usepackage{geometry}
-          \geometry{letterpaper}
-          \usepackage[parfill]{parskip}
-          \usepackage{graphicx}
+          \geometry{a4paper}
+          \usepackage[parfill]{parskip} % empty line instead of indent
+          \usepackage{graphicx}    % figures
+          \usepackage{hyperref}
+          \usepackage{tikz}		   % pie charts
           \title{Report}
           \author{'''+\
           user+\
           r'''}
           \graphicspath{{'''+\
           folder_name+\
           r'''/}}
+          %\setcounter{section}{-1} % Summary section 0 so number of sections equals number of files
           \begin{document}
           \maketitle
+          This is an automatically generated report using the `generate\_report.py' Python script 
+          included with the Web Audio Evaluation Tool \cite{WAET} distribution which can be found 
+          at \texttt{code.soundsoftware.ac.uk/projects/webaudioevaluationtool}.
           \tableofcontents
+          
           '''
           
-footer = '\end{document}'
+footer = r'''\begin{thebibliography}{9}
+         \bibitem{WAET} % reference to accompanying publication
+        Nicholas Jillings, Brecht De Man, David Moffat and Joshua D. Reiss, 
+        ``Web Audio Evaluation Tool: A browser-based listening test environment,'' 
+        presented at the 12th Sound and Music Computing Conference, July 2015.
+        \end{thebibliography}
+        \end{document}'''
 
 body = ''
 
 # generate images for later use
-subprocess.call("timeline_view_movement.py", shell=True)
+if render_figures:
+    subprocess.call("python timeline_view_movement.py", shell=True)
+    subprocess.call("python score_parser.py", shell=True)
+    subprocess.call("python score_plot.py", shell=True)
 
 # get every XML file in folder
 files_list = os.listdir(folder_name)
 for file in files_list: # iterate over all files in files_list
     if file.endswith(".xml"): # check if XML file

         # reset for new subject
         total_duration = 0
         page_number = 0
         
         individual_table = '' # table with stats for this individual test file
+        timeline_plots = '' # plots of timeline (movements and plays)
+        
+        # DEMO survey stats
+        # get gender
+        this_subjects_gender = root.find("./posttest/radio/[@id='gender']")
+        if this_subjects_gender is not None:
+            gender.append(this_subjects_gender.get("name"))
+        else:
+            gender.append('UNAVAILABLE')
+        # get age
+        this_subjects_age = root.find("./posttest/number/[@id='age']")
+        if this_subjects_age is not None:
+            age.append(this_subjects_age.text)
         
         # get list of all page names
         for audioholder in root.findall("./audioholder"):   # iterate over pages
             page_name = audioholder.get('id')               # get page name
             

             # number of audio element markers not moved
             if not_moved > 1:
                 body += '\\emph{\\textbf{ATTENTION: '+str(not_moved)+' markers were not moved in '+page_name+'!}} \\\\ \n'
             if not_moved == 1: 
                 body += '\\emph{\\textbf{ATTENTION: one marker was not moved in '+page_name+'!}} \\\\ \n'
-                
             #TODO which one not moved/listened to? 
             
             # PRINT song-specific statistic
             individual_table += page_name+'&'+\
                                 str(number_of_comments) + '/' +\

             img_path = 'timelines_movement/'+file[:-4]+'-'+page_name+'.pdf'
             
             # check if available
             if os.path.isfile(folder_name+'/'+img_path):
                 # SHOW timeline image
-                body += r'''\begin{figure}[htbp]
-                         \begin{center}
-                         \includegraphics[width=\textwidth]{'''+\
-                         folder_name+'/'+img_path+\
-                        r'''}
-                        \caption{Timeline of '''+\
-                         page_name+' by '+ file[:-4].capitalize() +\
-                        r'''.}
-                         \end{center}
-                         \end{figure}
-                         '''
+                timeline_plots += '\\includegraphics[width=\\textwidth]{'+\
+                         folder_name+'/'+img_path+'}\n\n'
             
             # keep track of duration in function of page index
             if len(duration_order)>page_number:
                 duration_order[page_number].append(duration)
             else:
                 duration_order.append([duration])
-                
+            
             # keep list of audioholder ids and count how many times each audioholder id
-            # was tested, how long it took, and how many fragments there were (if number of 
-            # fragments is different, store as different audioholder id)
+            # was tested, how long it took, and how many fragments there were 
+            # (if number of fragments is different, store as different audioholder id)
             if page_name in page_names: 
                 page_index = page_names.index(page_name) # get index
                 # check if number of audioelements the same
                 if len(audioelements) == fragments_per_page[page_index]: 
                     page_count[page_index] += 1

             else: 
                 page_names.append(page_name)
                 page_count.append(1)
                 duration_page.append([duration])
                 fragments_per_page.append(len(audioelements))
-                
+            
+            # number of subjects per audioholder regardless of differing numbers of 
+            # fragments (for inclusion in box plots)
+            if page_name in real_page_names:
+                page_index = real_page_names.index(page_name) # get index
+                subject_count[page_index] += 1
+            else: 
+                real_page_names.append(page_name)
+                subject_count.append(1)
+            
             # bookkeeping
             page_number += 1 # increase page count for this specific test
             number_of_pages += 1 # increase total number of pages
             time_per_page_accum += duration # total duration (for average time spent per page)
 

                  r'''\hline
                   \textbf{TOTAL} & & \textbf{'''+\
                   seconds2timestr(total_duration)+\
                  r'''}\\
                   \hline 
-                  \end{tabular}'''
+                  \end{tabular}
+                  
+                  '''
+        # PRINT timeline plots
+        body += timeline_plots
 
 # join to footer
 footer = body + footer
 
 # empty body again
 body = ''
 
-# PRINT summary of everything (at start)
-body += '\section{Summary}\n'
+# PRINT summary of everything (at start) 
+#       unnumbered so that number of sections equals number of files
+body += '\section*{Summary}\n\\addcontentsline{toc}{section}{Summary}'
 
 # PRINT table with statistics
 body += '\\begin{tabular}{ll}'
 body += r'Number of XML files: &' + str(number_of_XML_files) + r'\\'
 body += r'Number of pages: &' + str(number_of_pages) + r'\\'

 body += r'Number of unplayed fragments: &' + str(total_not_played) +\
       " (" + str(round(100.0*total_not_played/number_of_fragments,2)) + r"\%)\\"
 body += r'Number of unmoved markers: &' + str(total_not_moved) +\
       " (" + str(round(100.0*total_not_moved/number_of_fragments,2)) + r"\%)\\"
 body += r'Average time per page: &' + seconds2timestr(time_per_page_accum/number_of_pages) + r"\\"
-
-
-# Pages and number of times tested
-page_count_strings = list(str(x) for x in page_count)
-count_list = page_names + page_count_strings
-count_list[::2] = page_names
-count_list[1::2] = page_count_strings
-#body +=  r'Pages tested: &' + str(count_list) + r"\\"
-
 body += '\\end{tabular} \\vspace{1.5cm} \\\\ \n'
 
 # Average duration for first, second, ... page
 body += " \\vspace{.5cm} Average duration per page (see also Figure \\ref{fig:avgtimeperpage}): \\\\ \n"
 body += r'''\begin{tabular}{lll}

     tpp_averages.append(sum(duration_order[page_number])/len(duration_order[page_number]))
             
 body += '\\end{tabular} \\vspace{1.5cm} \\\\ \n'
 
 # SHOW bar plot of average time per page
-plt.bar(range(1,len(duration_order)+1), tpp_averages)
-plt.xlabel('Page')
+plt.bar(range(1,len(duration_order)+1), np.array(tpp_averages)/60)
+plt.xlabel('Page order')
 plt.xlim(.8, len(duration_order)+1)
 plt.xticks(np.arange(1,len(duration_order)+1)+.4, range(1,len(duration_order)+1))
-plt.ylabel('Time [seconds]')
+plt.ylabel('Average time [minutes]')
 plt.savefig(folder_name+"/time_per_page.pdf", bbox_inches='tight')
 plt.close()
-body += r'''\begin{figure}[htbp]
-         \begin{center}
-         \includegraphics[width=\textwidth]{'''+\
-         folder_name+"/time_per_page.pdf"+\
-        r'''}
-        \caption{Average time spent per audioholder page.}
-        \label{fig:avgtimeperpage}
-         \end{center}
-         \end{figure}
-         '''
 #TODO add error bars
 
 
 # Sort pages by number of audioelements, then by duration
 

 # combine and sort in function of number of audioelements and duration
 combined_list = [page_names, average_duration_page, fragments_per_page, number_of_subjects_page]
 combined_list = sorted(zip(*combined_list), key=operator.itemgetter(1, 2)) # sort
 
 # Show average duration for all songs
-body += r'''\vspace{.5cm} Average duration per audioholder: \\
+body += r'''\vspace{.5cm} Average duration per audioholder (see also Figure \ref{fig:avgtimeperaudioholder}): \\
         \begin{tabular}{llll}
         \textbf{Audioholder} & \textbf{Duration} & \textbf{\# subjects} & \textbf{\# fragments} \\
         '''
+audioholder_names_ordered = []
+average_duration_audioholder_ordered = []
+number_of_subjects = []
 for page_index in range(len(page_names)):
+    audioholder_names_ordered.append(combined_list[page_index][0])
+    average_duration_audioholder_ordered.append(combined_list[page_index][1])
+    number_of_subjects.append(combined_list[page_index][3])
     body +=  combined_list[page_index][0] + "&" +\
              seconds2timestr(combined_list[page_index][1]) + "&" +\
              str(combined_list[page_index][3]) + "&" +\
              str(combined_list[page_index][2]) + r"\\"
 body += '\\end{tabular}\n'
 
+# SHOW bar plot of average time per page
+plt.bar(range(1,len(audioholder_names_ordered)+1), np.array(average_duration_audioholder_ordered)/60)
+plt.xlabel('Audioholder')
+plt.xlim(.8, len(audioholder_names_ordered)+1)
+plt.xticks(np.arange(1,len(audioholder_names_ordered)+1)+.4, audioholder_names_ordered, rotation=90)
+plt.ylabel('Average time [minutes]')
+plt.savefig(folder_name+"/time_per_audioholder.pdf", bbox_inches='tight')
+plt.close()
+
+# SHOW bar plot of average time per page
+plt.bar(range(1,len(audioholder_names_ordered)+1), number_of_subjects)
+plt.xlabel('Audioholder')
+plt.xlim(.8, len(audioholder_names_ordered)+1)
+plt.xticks(np.arange(1,len(audioholder_names_ordered)+1)+.4, audioholder_names_ordered, rotation=90)
+plt.ylabel('Number of subjects')
+ax = plt.gca()
+ylims = ax.get_ylim()
+yint = np.arange(int(np.floor(ylims[0])), int(np.ceil(ylims[1]))+1)
+plt.yticks(yint)
+plt.savefig(folder_name+"/subjects_per_audioholder.pdf", bbox_inches='tight')
+plt.close()
+
+# SHOW both figures
+body += r'''\begin{figure}[htbp]
+         \begin{center}
+         \includegraphics[width=.65\textwidth]{'''+\
+         folder_name+"/time_per_page.pdf"+\
+        r'''}
+        \caption{Average time spent per page.}
+        \label{fig:avgtimeperpage}
+         \end{center}
+         \end{figure}
+         '''
+body += r'''\begin{figure}[htbp]
+         \begin{center}
+         \includegraphics[width=.65\textwidth]{'''+\
+         folder_name+"/time_per_audioholder.pdf"+\
+        r'''}
+        \caption{Average time spent per audioholder.}
+        \label{fig:avgtimeperaudioholder}
+         \end{center}
+         \end{figure}
+         '''
+body += r'''\begin{figure}[htbp]
+         \begin{center}
+         \includegraphics[width=.65\textwidth]{'''+\
+         folder_name+"/subjects_per_audioholder.pdf"+\
+        r'''}
+        \caption{Number of subjects per audioholder.}
+        \label{fig:avgtimeperaudioholder}
+         \end{center}
+         \end{figure}
+         '''
+#TODO add error bars
+#TODO layout of figures
+
+# SHOW boxplot per audioholder
+#TODO order in decreasing order of participants
+for audioholder_name in page_names: # get each name
+    # plot boxplot if exists (not so for the 'alt' names)
+    if os.path.isfile(folder_name+'/ratings/'+audioholder_name+'-ratings-box.pdf'):
+        body += r'''\begin{figure}[htbp]
+             \begin{center}
+             \includegraphics[width=.65\textwidth]{'''+\
+             folder_name+"/ratings/"+audioholder_name+'-ratings-box.pdf'+\
+            r'''}
+            \caption{Box plot of ratings for audioholder '''+\
+            audioholder_name+' ('+str(subject_count[real_page_names.index(audioholder_name)])+\
+            ''' participants).}
+            \label{fig:avgtimeperpage}
+             \end{center}
+             \end{figure}
+             '''
+
+# DEMO pie chart of gender distribution among subjects
+genders = ['male', 'female', 'other', 'preferNotToSay', 'UNAVAILABLE']
+# TODO: get the above automatically
+gender_distribution = ''
+for item in genders:
+    number = gender.count(item)
+    if number>0:
+        gender_distribution += str("{:.2f}".format((100.0*number)/len(gender)))+\
+                               '/'+item.capitalize()+' ('+str(number)+'),\n'
+
+body += r'''
+        \def\angle{0}
+        \def\radius{3}
+        \def\cyclelist{{"orange","blue","red","green"}}
+        \newcount\cyclecount \cyclecount=-1
+        \newcount\ind \ind=-1
+        \begin{figure}[htbp]
+        \begin{center}\begin{tikzpicture}[nodes = {font=\sffamily}]
+        \foreach \percent/\name in {'''+\
+        gender_distribution+\
+        r'''} {\ifx\percent\empty\else               % If \percent is empty, do nothing
+        \global\advance\cyclecount by 1     % Advance cyclecount
+        \global\advance\ind by 1            % Advance list index
+        \ifnum6<\cyclecount                 % If cyclecount is larger than list
+          \global\cyclecount=0              %   reset cyclecount and
+          \global\ind=0                     %   reset list index
+        \fi
+        \pgfmathparse{\cyclelist[\the\ind]} % Get color from cycle list
+        \edef\color{\pgfmathresult}         %   and store as \color
+        % Draw angle and set labels
+        \draw[fill={\color!50},draw={\color}] (0,0) -- (\angle:\radius)
+          arc (\angle:\angle+\percent*3.6:\radius) -- cycle;
+        \node at (\angle+0.5*\percent*3.6:0.7*\radius) {\percent\,\%};
+        \node[pin=\angle+0.5*\percent*3.6:\name]
+          at (\angle+0.5*\percent*3.6:\radius) {};
+        \pgfmathparse{\angle+\percent*3.6}  % Advance angle
+        \xdef\angle{\pgfmathresult}         %   and store in \angle
+        \fi
+        };
+        \end{tikzpicture}
+        \caption{Representation of gender across subjects}
+        \label{default}
+        \end{center}
+        \end{figure}
+        '''
+# problem: some people entered twice? 
+
 #TODO
 # time per page in function of number of fragments (plot)
 # time per participant in function of number of pages
 # plot total time for each participant
-# plot total time
 # show 'count' per page (in order)
 
 # clear up page_index <> page_count <> page_number confusion
 
 
-texfile = header+body+footer
+texfile = header+body+footer # add bits together
 
 # write TeX file
-with open(folder_name + '/' + 'test.tex','w') as f:
+with open(folder_name + '/' + 'Report.tex','w') as f:
     f.write(texfile)
-proc=subprocess.Popen(shlex.split('pdflatex -output-directory='+folder_name+' '+ folder_name + '/test.tex'))
+proc=subprocess.Popen(shlex.split('pdflatex -output-directory='+folder_name+' '+ folder_name + '/Report.tex'))
 proc.communicate()
 # run again
-proc=subprocess.Popen(shlex.split('pdflatex -output-directory='+folder_name+' '+ folder_name + '/test.tex'))
+proc=subprocess.Popen(shlex.split('pdflatex -output-directory='+folder_name+' '+ folder_name + '/Report.tex'))
 proc.communicate()
 
 #TODO remove auxiliary LaTeX files
+try:
+    os.remove(folder_name + '/' + 'Report.aux')
+    os.remove(folder_name + '/' + 'Report.log')
+    os.remove(folder_name + '/' + 'Report.out')
+    os.remove(folder_name + '/' + 'Report.toc')
+except OSError:
+    pass
+