comparison pyspark/sonic-annotator-notimeside/test_sonic_annotator_notimeside.py @ 0:e34cf1b6fe09 tip

commit

+# Part of DML (Digital Music Laboratory)
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+#!/usr/local/spark-1.0.0-bin-hadoop2/bin/spark-submit
+# -*- coding: utf-8 -*-
+__author__="wolffd"
+__date__ ="$11-Jul-2014 15:31:01$"
+
+from pyspark import SparkConf, SparkContext
+import sys
+import os
+from sonic_annotator_vamp import *
+
+# this is the main routine to be submmitted as a spark job
+#
+#
+# Running python applications through ./bin/pyspark is deprecated as of Spark 1.0.
+# Use ./bin/spark-submit <python file> --py-files sonic_annotator_vamp.py
+# you can also provide a zip of all necessary python files
+#
+# @param string audiopath root of the folder structure to be traversed
+# @param string transform_file path to the .n3 turtle file describing the transform
+#def main(audiopath = '/home/wolffd/Documents/python/dml/TimeSide/tests/samples/',
+#         transform_file = '/home/wolffd/Documents/python/dml/pyspark/sonic-annotator-notimeside/silvet_settings.n3',
+#         masterip = '10.2.165.101'):
+def main(audiopath = '/CHARM-Collection',
+         transform_file = 'bbc_speechmusic.n3',
+         masterip = '0.0.0.0'):
+    print "PySpark Telemeta and Vamp Test"
+
+    # configure spark, cave: local profile uses just 1 core
+    conf = (SparkConf()
+            #.setMaster("local")
+            .setMaster("spark://" + masterip + ":7077")
+            .setAppName("Sonic Annotating")
+            .set("spark.executor.memory", "40g")
+            .set("spark.cores.max", "35"));
+    sc = SparkContext(conf = conf)
+    
+    # here traverse the file structure
+    data = []
+    for (dirpath, dirnames, filenames) in os.walk(audiopath):
+        for file in filenames:
+            if file.endswith(".wav") or file.endswith(".mp3") or file.endswith(".flac"):
+                data.append(os.path.join(dirpath, file))
+    njobs = len(data)
+    donejobs = sc.accumulator(0)
+    print "Total: " + str(njobs) + " files"
+
+    # define distributed dataset
+    distData = sc.parallelize(data)
+    
+    # define map 
+    m1 = distData.map(lambda x: transform(wav_file=x,transform_file=transform_file))
+
+    # reduce (just do the maps ;) )
+    result = m1.collect()
+
+if __name__ == "__main__":
+    if len(sys.argv) >= 3:
+        main(sys.argv[1],sys.argv[2])
+    else:
+        main()