annotate ipcluster/tools/zip_features_folderwise.py @ 0:e34cf1b6fe09 tip

commit

#!/usr/local/python
# -*- coding: utf-8 -*-
__author__="wolffd"
import os
import sys
import time
import shutil
import hashlib
import re
import errno
import subprocess


# this copies  features which used to be stored with the audio    
# usage: copy_features source_path dest_path transform_substr ext move?
# params:
#        transfrom_substr can be either a substring or hash of the transform file
#                         or the filename.n3 of a transform file


def main(audiopath, out_path, transform_substr = "", move = 0):
    
    
    
    # replace transform_substr by pluginhash 
    if transform_substr.endswith (".n3"):
        BLOCKSIZE = 65536
        hasher = hashlib.sha1()
        with open(transform_substr, 'rb') as afile:
            buf = afile.read(BLOCKSIZE)
            while len(buf) > 0:
                hasher.update(buf)
                buf = afile.read(BLOCKSIZE)
        transform_substr = str(hasher.hexdigest())

    # ---
    # we traverse the file structure
    # and list files to copy
    # ---
    data = []
    count = 0
    count2 = 0
    for (dirpath, dirnames, filenames) in os.walk(audiopath):
        for dir in dirnames:
            print '\rChecked %d, gathered %d files' % (count, count2), 
            count += 1
            
            # we copy all requested files and the transform files as well!
            if (transform_substr and (transform_substr in dir)):
                source = os.path.join(dirpath, dir).replace('\\','/')
                data.append(source)
                count2 +=1
#        if count2 > 1:
#            break
    
    # count jobs
    njobs = len(data)
    print '\r\nAbout to copy %d directories' % (njobs) 
    
               
    count = 0
    # copy individual items
    for x in data:
        spath = os.path.split(x)
        folders = spath[0].split('/')
        
        collectionname = folders[-1]

         # create the target folder
        #try:
        #    os.makedirs(out_path)
        #except OSError as exception:
        #    if exception.errno!= errno.EEXIST:
        #        raise   
                
        command = '7z -r -mmt=30 -mx5 -v50g -mm=bzip2 a '  + '"' + os.path.join(out_path,collectionname + '_'+ transform_substr + '.zip') +  '"' + ' ' + x
        print command
        print " Currently zipping " + collectionname
        
        p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
        text = p.stdout.read()
        retcode = p.wait()
        if not (retcode==0):
            print "Error: "  + text
        
        #os.system(command)
        #print 'Removing ' + dest
        #shutil.rmtree(dest)
            
        count = count + 1
        
        # progress indicator
        print '\r%3.2f percent. %d done, %d pending' % (count/(njobs*1.0) * 100.0,count, njobs-count),
        
    print '\rCopied %d of %d files.' % (count,njobs)
    print '\n'
    
def copytree(src, dst, symlinks=False, ignore=None):
    for item in os.listdir(src):
        s = os.path.join(src, item)
        d = os.path.join(dst, item)
        if os.path.isdir(s):
            shutil.copytree(s, d, symlinks, ignore)
        else:
            os.makedirs(d)
            shutil.copy2(s, d)

if __name__ == "__main__":
    if len(sys.argv) >= 5:
        main(sys.argv[1],sys.argv[2], sys.argv[3], int(sys.argv[4]))
    else:
        main(audiopath = 'D:/_Audio/',out_path = 'D:/_Audio_Analysis/', transform_substr = "",  ext = "")