changeset 2524:dde81c372fdc

Merge score_parser2 into score_parser

--- a/python/score_parser.py	Sat Nov 05 12:04:34 2016 +0000
+++ b/python/score_parser.py	Tue Nov 08 09:58:31 2016 +0000
@@ -26,26 +26,28 @@
 elif not os.access(os.path.dirname(folder_name), os.W_OK):
     #the file does exist but write privileges are not given
     print("No write privileges in folder '"+folder_name+"'.")
+    
 
-    
 # CODE
 
-# remember which files have been opened this time
-file_history = []
+storage = {}
 
-# get every XML file in folder
+# create folder 'ratings' if not yet created
+if not os.path.exists(folder_name + '/ratings'):
+    os.makedirs(folder_name + '/ratings')
+
+# Get every XML file in the folder
 for file_name in os.listdir(folder_name):
-    if file_name.endswith(".xml"):
+    if (file_name.endswith(".xml")):
         tree = ET.parse(folder_name + '/' + file_name)
         root = tree.getroot()
-
-        # get subject ID from XML file
-        subject_id = file_name[:-4] # file name (without extension) as subject ID
-
-        # get list of all pages this subject evaluated
-        for page in root.findall("./page"):    # iterate over pages
-            page_name = page.get('ref') # get page reference ID
-                       
+        
+        subject_id = root.get('key');
+        
+        # get the list of the pages this subject evaluated
+        for page in root.findall("./page"):     # iterate over pages
+            page_name = page.get('ref') # get page ID
+            
             if page_name is None: # ignore 'empty' audio_holders
                 print("WARNING: " + file_name + " contains empty audio holder. (score_parser.py)")
                 break
@@ -53,84 +55,64 @@
             if page.get('state') != "complete":
                 print("WARNING: " + file_name + " contains incomplete page " +page_name+ ". (score_parser.py)")
                 break;
-
-            file_name = folder_name+'/ratings/'+page_name+'-ratings.csv' # score file name
-
-            # create folder 'ratings' if not yet created
-            if not os.path.exists(folder_name + '/ratings'):
-                os.makedirs(folder_name + '/ratings')
-
+            
+            # Check if page in the store
+            if storage.get(page_name) == None:
+                storage[page_name] = {'header':[], 'axis':{}} # add to the store
+            
+            # Get the axis names
+            pageConfig = root.find('./waet/page/[@id="'+page_name+'"]')
+            for interface in pageConfig.findall('./interface'):    # Get the <interface> noeds
+                interfaceName = interface.get("name"); # Get the axis name
+                if interfaceName == None:
+                    interfaceName = "default"   # If name not set, make name 'default'
+                if storage[page_name]['axis'].get(interfaceName) == None:
+                    storage[page_name]['axis'][interfaceName] = {}  # If not in store for page, add empty dict
+                storage[page_name]['axis'][interfaceName][subject_id] = [] # Add the store for the session
+                    
             # header: fragment IDs in 'alphabetical' order
             # go to fragment column, or create new column if it doesn't exist yet
-
-            # get array of audio elements and number of audio elements
-            audiolist = page.findall("./audioelement")
-            n_fragments = len(audiolist)
-
+            
             # get alphabetical array of fragment IDs from this subject's XML
             fragmentnamelist = []    # make empty list
-            for audioelement in audiolist: # iterate over all audioelements
+            for audioelement in page.findall("./audioelement"): # iterate over all audioelements
                 fragmentnamelist.append(audioelement.get('ref')) # add to list
+            
+            fragmentnamelist = sorted(fragmentnamelist);    # Sort the list
+            storage[page_name]['header'] = fragmentnamelist;
+            
+            for fragmentname in fragmentnamelist:
+                audioElement = page.find("./audioelement/[@ref='"+ fragmentname+ "']") # Get the element
+                for value in audioElement.findall('./value'):
+                    axisName = value.get('interface-name')
+                    if axisName == None:
+                        axisName = 'default'
+                    axisStore = storage[page_name]['axis'][axisName]
+                    if hasattr(value, 'text'):
+                        axisStore[subject_id].append(value.text)
+                    else:
+                        axisStore[subject_id].append('')
 
-
-            # if file exists, get header and add any 'new' fragments not yet in the header
-            if os.path.isfile(file_name):
-                with open(file_name, 'r') as readfile:
-                    filereader = csv.reader(readfile, delimiter=',')
-                    headerrow = next(filereader)
-
-                # If file hasn't been opened yet this time, remove all rows except header
-                if file_name not in file_history:
-                    with open(file_name, 'w') as writefile:
-                        filewriter = csv.writer(writefile, delimiter=',')
-                        headerrow = sorted(headerrow)
-                        filewriter.writerow(headerrow)
-                    file_history.append(file_name)
-
-                # Which of the fragments are in fragmentnamelist but not in headerrow?
-                newfragments = list(set(fragmentnamelist)-set(headerrow))
-                newfragments = sorted(newfragments) # new fragments in alphabetical order
-                # If not empty, read file and rewrite adding extra columns
-                if newfragments: # if not empty
-                    with open('temp.csv', 'w') as writefile:
-                        filewriter = csv.writer(writefile, delimiter=',')
-                        filewriter.writerow(headerrow + newfragments) # write new header
-                        with open(file_name, 'r') as readfile:
-                            filereader = csv.reader(readfile, delimiter=',')
-                            next(filereader) # skip header
-                            for row in filereader: # rewrite row plus empty cells for every new fragment name
-                                filewriter.writerow(row + ['']*len(newfragments))
-                    os.rename('temp.csv', file_name) # replace old file with temp file
-                    headerrow = headerrow + newfragments
-                    
-
-            # if file does not exist yet, create file and make header
-            else:
-                headerrow = sorted(fragmentnamelist) # sort alphabetically
-                headerrow.insert(0,'')
-                fragmentnamelist = fragmentnamelist[1:] #HACKY FIX inserting in firstrow also affects fragmentnamelist
-                with open(file_name, 'w') as writefile:
-                    filewriter = csv.writer(writefile, delimiter=',')
-                    filewriter.writerow(headerrow)
-                file_history.append(file_name)
-
-            # open file to write for this page
-            writefile = open(file_name, 'a')
+# Now create the individual files
+for page_name in storage:
+    for axis_name in storage[page_name]['axis']:
+        
+        file_name = folder_name+'/ratings/'+page_name+'-'+axis_name+'-ratings.csv' # score file name
+        
+        # I'm not as elegant, I say burn the files and start again
+        headerrow = list(storage[page_name]['header'])  # Extract the element IDs
+        headerrow.insert(0,'file_keys')
+        with open(file_name, 'w') as writefile:
             filewriter = csv.writer(writefile, delimiter=',')
-
-            # prepare row to be written for this subject for this page
-            ratingrow = [subject_id]
-
-            # get scores related to fragment [id]
-            for fragmentname in headerrow[1:]: # iterate over fragments in header (skip first empty column)
-                elementvalue = page.find("./audioelement/[@ref='"
-                                       + fragmentname
-                                       + "']/value")
-                if hasattr(elementvalue, 'text'): # if rating for this fragment exists
-                    ratingrow.append(elementvalue.text) # add to rating row
-                else: # if this subject has not rated this fragment
-                    ratingrow.append('') # append empty cell
-
-            # write row: [subject ID, rating fragment ID 1, ..., rating fragment ID M]
-            if any(ratingrow[1:]): # append to file if row non-empty (except subject name)
-                filewriter.writerow(ratingrow)
+            filewriter.writerow(headerrow)
+        
+        # open file to write the page
+        writefile = open(file_name, 'a')
+        filewriter = csv.writer(writefile, delimiter=',')
+        
+        for subject_id in storage[page_name]['axis'][axis_name]:
+            entry = [subject_id]
+            for value in storage[page_name]['axis'][axis_name][subject_id]:
+                entry.append(value)
+            filewriter.writerow(entry)
+        writefile.close()
\ No newline at end of file