comparison pyspark/ilm/spark_feat_extract.py @ 0:e34cf1b6fe09 tip

commit

+# Part of DML (Digital Music Laboratory)
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+#!/usr/local/spark-1.0.0-bin-hadoop2/bin/spark-submit
+# -*- coding: utf-8 -*-
+__author__="hargreavess"
+
+from assetDB import assetDB
+from pyspark import SparkConf, SparkContext
+import ConfigParser
+import logging
+from transform import *
+import os
+import time
+import shutil
+
+def main():
+    start_complete = time.time();
+
+    # get config
+    config = ConfigParser.ConfigParser()
+    config.read('server.cfg')
+
+    #vamp_transform = [config.get('Sonic Annotator', 'vamp-transform')]
+    vamp_transform_list = config.get('Sonic Annotator', 'vamp-transform-list')
+    genre_id = config.getint('Queries', 'genre-id')
+
+    output_dir = config.get('Sonic Annotator', 'output-dir')
+    ltime = time.localtime()
+    output_dir = output_dir + '_' + str(ltime.tm_mday) + '_' + str(ltime.tm_mon) + '_' + str(ltime.tm_year)
+    output_dir = output_dir + '_' + str(ltime.tm_hour) + str(ltime.tm_min) + '_' + str(ltime.tm_sec)
+    output_dir = output_dir + '_genre_id_' + str(genre_id)
+    # create output directory, if it doesn't exist
+    if not os.access(output_dir, os.F_OK):
+        os.makedirs(output_dir)
+
+    # copy vamp_transform_list file to output directory
+    shutil.copy(vamp_transform_list, output_dir)
+        
+    # create logger
+    #logger = logging.getLogger('spark_feat_extract')
+    logger = logging.getLogger('spark_feat_extract')
+    logger.setLevel(logging.DEBUG)
+
+    # create file handler and set level to debug
+    fh = logging.FileHandler(output_dir + "/ilm.assets.spark.features.log")
+    fh.setLevel(logging.DEBUG)
+
+    # create formatter
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+    # add formatter to fh
+    fh.setFormatter(formatter)
+
+    # add fh to logger
+    logger.addHandler(fh)
+
+    logger.info('starting new spark_feat_extract job')
+    logger.info("using vamp transform list: " + vamp_transform_list)
+    logger.info('audio-file-size-limit: ' + config.get('Audio Files', 'audio-file-size-limit'))
+    logger.info("audio-prefix: " + config.get('Audio Files', 'audio-prefix'))
+    logger.info('num-cores: ' + config.get('Spark', 'num-cores'))
+    logger.info("spark memory: " + config.get('Spark', 'memory'))
+    logger.info("genre_id: " + str(genre_id))
+
+    # create a spark context
+    conf = (SparkConf()
+            .setMaster("local[" + config.get('Spark', 'num-cores') + "]")
+            .setAppName("spark feature extractor")
+            .set("spark.executor.memory", "" + config.get('Spark', 'memory') + ""))
+    sc = SparkContext(conf = conf)
+
+    SQL_start = config.getint('Queries', 'sql-start')
+    SQL_limit = config.getint('Queries', 'sql-limit')
+    local_SQL_start = SQL_start
+    logger.info('SQL_start = %i', SQL_start)
+    logger.info('SQL_limit = %i', SQL_limit)
+
+    array_step_size = config.getint('Application', 'array-step-size')
+    logger.info('array-step-size = %i', array_step_size)
+    local_SQL_limit = min(SQL_limit, array_step_size)
+
+    while local_SQL_limit <= SQL_limit:
+
+        # query db for assets (song tracks)
+        db = assetDB(prefix=config.get('Audio Files', 'audio-prefix'),config=config)
+        db.connect()
+
+        data = []
+        logger.info('local_start = %i', local_SQL_start)
+        logger.info('local_SQL_limit = %i', local_SQL_limit)
+
+        for path, asset in db.get_assets_by_genre(genre_id, local_SQL_start, local_SQL_limit):
+            if path == None:
+                logger.warning("Asset not found for: %s. (Album ID: %i Track No: %i)",asset.song_title,asset.album_id,asset.track_no)
+            else:
+                data.append(path)
+
+        db.close
+
+        # If the db query returned no results, stop here
+        if len(data) == 0:
+            break
+
+        batch_output_dir = output_dir + '/batch' + str(local_SQL_start) + '-' + str(local_SQL_limit)
+        os.makedirs(batch_output_dir)
+        logger.info('created results directory ' + batch_output_dir)
+
+        logger.info("calling sc.parallelize(data)...")
+        start = time.time();
+
+        # define distributed dataset
+        distData = sc.parallelize(data)
+        end = time.time();
+        logger.info("finished in " + (str)(end - start))
+
+        logger.info("calling distData.map...")
+        start = time.time();
+
+        # define map 
+        m1 = distData.map(lambda x: transform(audio_file=x,
+            vamp_transform_list=vamp_transform_list,
+            output_dir=batch_output_dir))
+        end = time.time();
+        logger.info("finished in " + (str)(end - start))
+
+        logger.info("calling m1.collect()...")
+        start = time.time();
+
+        # collect results
+        theResult = m1.collect()
+
+        end = time.time();
+
+        logger.info("finished in " + (str)(end - start))
+
+        local_SQL_start += array_step_size
+        local_SQL_limit += min(SQL_limit, array_step_size)
+
+    print "finished all in " + (str)(end - start_complete)
+    logger.info("finished all in " + (str)(end - start_complete))
+
+if __name__ == "__main__":
+    main()
+