comparison pyspark/test_timeside_vamp_spark.py @ 0:e34cf1b6fe09 tip

commit

+# Part of DML (Digital Music Laboratory)
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+#!/usr/local/spark-1.0.0-bin-hadoop2/bin/spark-submit
+# -*- coding: utf-8 -*-
+__author__="wolffd"
+__date__ ="$11-Jul-2014 15:31:01$"
+
+from pyspark import SparkConf, SparkContext
+# @todo: timeside has to be packed for multi-pc usage
+from timeside_vamp import *
+from os import walk
+
+# Running python applications through ./bin/pyspark is deprecated as of Spark 1.0.
+# Use ./bin/spark-submit <python file>
+
+
+def main():
+    print "PySpark Telemeta and Vamp Test"
+    conf = (SparkConf()
+            .setMaster("local")
+            .setAppName("My app")
+            .set("spark.executor.memory", "1g"))
+    sc = SparkContext(conf = conf)
+    
+    # here come the wav file names
+    
+    mypath = '../../TimeSide/tests/samples/'
+    data = []
+    for (dirpath, dirnames, filenames) in walk(mypath):
+        for file in filenames:
+            if file.endswith(".wav"):
+                data.append(os.path.join(dirpath, file))
+    
+    # define distributed dataset
+    distData = sc.parallelize(data)
+    
+    # define map 
+    m1 = distData.map(lambda x: transform(wav_file=x))
+   
+    #process 2
+    m1.take(2)
+
+if __name__ == "__main__":
+    main()
+