comparison dml-cla/python/chord_seq_spmf_helper.py @ 0:718306e29690 tip

commiting public release

+#!/usr/bin/python
+# Part of DML (Digital Music Laboratory)
+# Copyright 2014-2015 Daniel Wolff, City University
+ 
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+# -*- coding: utf-8 -*-
+#
+# This is a data conversion wrapper for the spmf toolkit
+__author__="Daniel Wolff"
+
+import chord_seq_key_relative as c2f
+import csv
+import re
+import tempfile
+import subprocess
+import os
+import platform
+from aggregate import *
+from csvutils import *
+
+# command for threading
+import subprocess, threading
+import signal
+
+# limit for sequences read
+max_lines = 10000000
+
+
+class Command(object):
+    def __init__(self, cmd):
+        self.cmd = cmd
+        self.process = None
+        self.text = 'SPMF terminated unexpectedly'
+
+    def run(self, timeout):
+        def target():
+            print_status('Thread started')
+            if 'Win' in platform.system():
+                self.process = subprocess.Popen(self.cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,shell=False)
+            else:
+                self.process = subprocess.Popen(self.cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,shell=False, preexec_fn=os.setsid)
+                
+            self.text = self.process.stdout.read()
+            self.process.communicate()
+
+            print_status('Thread finished')
+
+        thread = threading.Thread(target=target)
+        thread.start()
+        
+        # wait until timeout if specified
+        if timeout > 0:
+            thread.join(timeout)
+            if thread.is_alive():
+                print_status('Terminating process')
+                if 'Win' in platform.system():
+                    self.text = 'Terminating SPMF after ' + str(timeout) + ' seconds'
+                    self.process.kill()
+                else:
+                    self.text = 'Terminating SPMF after ' + str(timeout) + ' seconds'
+                    os.killpg(self.process.pid, signal.SIGTERM)
+                    thread.join()
+                    
+        else:
+            thread.join()
+            
+        # return retcode
+        return (self.process.returncode, self.text)
+
+
+# runs the spmf java with method and parameters as specified
+# 1st parameter: usually minimal support of sequence
+# 2nd parameter: minimal length of sequence
+# run spmf with java -jar spmf.jar run CM-SPADE Beethoven.spmf output.txt 50% 3
+def spmf(file,method = "CM-SPADE", params=["70%", "3"], timeout=10):
+    
+    # create outpput file name
+    outfile = tempfile.mktemp()
+    command = ["java"]
+    command.extend(["-Xmx1g","-jar","spmf.jar","run"])
+    command.extend([method, file, outfile])
+    command.extend(params)
+    
+    #print_status('CWD:' + os.getcwd())
+    #print_status('Calling SPMF: ' + ' '.join(command))
+    
+    proc = Command(command)
+    retcode, text = proc.run(timeout=timeout)
+    
+    if (retcode==0):
+        #print_status("Finished")
+        return outfile
+    else:
+        print_status( "Terminated with errors" + text)
+        return outfile
+    
+
+# takes a dictionary of chords for one or multiple files 
+# in the form of dict[clipid] = [ (time,key,mode,fun,typ,bfun) ]
+# and converts it into spmf
+#
+# output: tempfile of spmf output
+def relchords2spmf(input):
+
+    # choose random filename for spmf location
+    # open spmf file
+    fspmf = tempfile.NamedTemporaryFile(delete=False)
+    
+    # ---
+    # this is writing the spmf format
+    for track,trackdata in input.iteritems():
+        # write chord sequence as one line in spmf file
+        for (time,key,mode,fun,typ,bfun) in trackdata:
+            chord = c2f.fun2num(fun,typ,bfun,mode)
+            
+            # -1 is the spearator of items or itemsets
+            fspmf.write(str(chord) + ' -1 ') 
+            
+        # the sequence is closed with -2
+        fspmf.write('-2\n') 
+            
+    fspmf.close()
+    
+    return fspmf
+
+
+## takes a dictionary of chords for one or multiple files 
+## in the form of dict[clipid] = [ (time,key,mode,fun,typ,bfun) ]
+## and converts it into spmf
+#def folder2spmf(folderin = 'D:/mirg/Chord_Analysis20141216/', fileout = 'D:/mirg/Chord_Analysis20141216/Beethoven.spmf'):
+#
+#    # get chords for all files                    
+#    output  = c2f.folder2functions(folderin)
+#
+#    # open log
+#    logfile = fileout + '.dic'
+#    csvfile = open(logfile, "w+b") #opens the file for updating
+#    w = csv.writer(csvfile)
+#    w.writerow(["track","key","mode","sequence length"])
+#    
+#    # open spmf file
+#    fspmf = open(fileout,'w')
+#    # ---
+#    # this is writing the spmf format
+#    for track,trackdata in output.iteritems():
+#        # write chord sequence as one line in spmf file
+#        for (time,key,mode,fun,typ,bfun) in trackdata:
+#            chord = c2f.fun2num(fun,typ,bfun,mode)
+#            
+#            # -1 is the spearator of items or itemsets
+#            fspmf.write(str(chord) + ' -1 ') 
+#            
+#        # the sequence is closed with -2
+#        fspmf.write('-2\n') 
+#        w.writerow([track, str(key), str(mode),str(len(trackdata))])
+#            
+#    fspmf.close()
+#    csvfile.close()
+
+# read an spmf file
+# def parsespmf(filein = 'D:/mirg/Chord_Analysis20141216/Beethoven.txt'):
+
+# string sourcefile path to the source spmf file with chords from records
+# string patternfile path to the pattern spmf file
+# matches each of the patterns in patternfile
+#  to the chord sequences in sourcefile
+def match(sourcefile = 'D:/mirg/Chord_Analysis20141216/Beethoven.spmf',sourcedict = 'D:/mirg/Chord_Analysis20141216/Beethoven.spmf.dic', patternfile = 'D:/mirg/Chord_Analysis20141216/Beethoven_70.txt'):
+    
+    # define regular expressions for matching
+    # closed sequence
+    
+    # ---
+    # we here assume that there are more files than patterns,
+    # as display of patterns is somehow limited
+    # therefore parallelisation will be 1 pattern/multiple files
+    # per instance
+    # ---
+    
+    patterns = spmf2table(patternfile)
+        
+    # ---
+    # now for the input sequences
+    # ---
+    # first: read track dictionary and get the input sequence names
+    tracks = getClipDict(sourcedict)
+       
+    # read the input sequences
+    source = open(sourcefile, 'r')
+    patterns_tracks = dict()
+    tracks_patterns = dict()
+    
+    # iterate over all tracks - to be parallelised
+    for track,count in tracks.iteritems():
+        sequence = readSequence(next(source))
+        print track
+        for p in range(0,len(patterns)):
+            # match open or closed pattern
+            if openPatternInSequence(sequence,patterns[p]):
+                if patterns_tracks.has_key(p):
+                    patterns_tracks[p].append(track)
+                else:
+                    patterns_tracks[p] = [track]
+
+                if tracks_patterns.has_key(track):
+                    tracks_patterns[track].append(p)
+                else:
+                    tracks_patterns[track] = [p]  
+    
+    # write clip index to files
+    writeAllPatternsForClips('D:/mirg/Chord_Analysis20141216/',tracks_patterns)
+    #print patterns_tracks[p]
+
+# writes results to disk per key
+def writeAllPatternsForClips(path = 'D:/mirg/Chord_Analysis20141216/',tracks_patterns = dict()):
+
+    for name, contents in tracks_patterns.iteritems():
+        # create new file
+        csvfile = open(path + '/' + name + '_patterns.csv', "w+b") #opens the file for updating
+        w = csv.writer(csvfile)
+   
+        # compress pattern data ?
+        # e.g. 2 columns from-to for the long series of atomic increments
+        
+        w.writerow(contents)
+        csvfile.close()
+
+# reads output of spmf to table
+def spmf2table(patternfile):
+    
+    patterns = []
+    supports = []
+    patterns_raw = []
+    linecnt = 0
+    # read all patterns
+    with open(patternfile, 'r') as f:
+        for line in f:
+            # a line looks like this:
+            # 1120401 -1 1120101 -1 #SUP: 916
+
+            # save pattern
+            #patterns.append(pattern)
+            #numeric? or just regex?
+            # we'll use string, so any representation works
+
+            pattern,support = readPattern(line)
+            patterns.append(pattern)
+            supports.append(support)
+
+            # here's the regex
+            # first the spacer
+            #spacer = '((\s-1\s)|((\s-1\s)*[0-9]+\s-1\s)+)'
+            #repattern = r'(' + spacer + '*' + spacer.join(pattern) + spacer + '*' + '.*)'
+            #print repattern
+            #patterns.append(re.compile(repattern))
+            linecnt +=1
+            
+            if linecnt > max_lines:
+                print_status('Not reading more than ' + str(max_lines) + ' lines :(')
+                break
+            
+    return patterns,supports
+
+# @param line: reads a line in the spmf output file with frequent patterns
+# returns list of strings "pattern" and int "support"
+def readPattern(line):
+    # locate support 
+    suploc = line.find('#SUP:')
+    
+    # test whether we have a broken file
+    if suploc > 6:
+        support = int(line[suploc+5:-1])
+    else:
+        support = -1
+
+    # extract pattern
+    pattern = line[:suploc].split(' -1 ')[:-1]
+    return (pattern,support)
+   
+# @param line: reads a line in the spmf input file with chord sequence
+# returns list of strings "pattern" and int "support"
+def readSequence(line):
+    # locate support 
+    suploc = line.find('-2')
+
+    # extract pattern
+    sequence = line[:suploc].split(' -1 ')[:-1] 
+    return sequence
+
+# finds open pattern in sequences
+# @param [string] sequence input sequence
+# @param [string] pattern pattern to be found
+def openPatternInSequence(sequence,pattern):
+    patidx = 0
+    for item in sequence:
+        if item == pattern[patidx]:
+            patidx +=1
+            
+            # did we complet the pattern?
+            if patidx >= (len(pattern)-1):
+                # could also return the start index
+                return 1
+    # finished the sequence before finishing pattern
+    return 0
+            
+# finds closed pattern in sequences
+# @param [string] sequence input sequence
+# @param [string] pattern pattern to be found
+def closedPatternInSequence(sequence,pattern):
+    # alternatively use KnuthMorrisPratt with unsplit string
+    return ''.join(map(str, pattern)) in ''.join(map(str, sequence))  
+            
+# reads all track names from the dictionary created by folder2spmf
+# @param sourcedict path to dictionary
+def getClipDict(sourcedict):
+    
+    f = open(sourcedict, 'rt')
+    reader = csv.reader(f)
+    
+    # skip first roow that contains legend
+    next(reader)
+    
+    # get following rows
+    tracks = dict()
+    for (track,key,mode,seqlen) in reader:
+        tracks[track]= (key,mode,seqlen)
+        #tracks.append((track,count))
+    
+    f.close()
+    return tracks 
+    
+            
+# run spmf afterwards with java -jar spmf.jar run CM-SPADE Beethoven.spmf output.txt 50% 3
+if __name__ == "__main__":
+    #folder2spmf()
+    #match()
+    print "huhu"