comparison ipcluster/tools/zip_features_folderwise.py @ 0:e34cf1b6fe09 tip

commit

+#!/usr/local/python
+# -*- coding: utf-8 -*-
+__author__="wolffd"
+import os
+import sys
+import time
+import shutil
+import hashlib
+import re
+import errno
+import subprocess
+
+
+# this copies  features which used to be stored with the audio    
+# usage: copy_features source_path dest_path transform_substr ext move?
+# params:
+#        transfrom_substr can be either a substring or hash of the transform file
+#                         or the filename.n3 of a transform file
+
+
+def main(audiopath, out_path, transform_substr = "", move = 0):
+    
+    
+    
+    # replace transform_substr by pluginhash 
+    if transform_substr.endswith (".n3"):
+        BLOCKSIZE = 65536
+        hasher = hashlib.sha1()
+        with open(transform_substr, 'rb') as afile:
+            buf = afile.read(BLOCKSIZE)
+            while len(buf) > 0:
+                hasher.update(buf)
+                buf = afile.read(BLOCKSIZE)
+        transform_substr = str(hasher.hexdigest())
+
+    # ---
+    # we traverse the file structure
+    # and list files to copy
+    # ---
+    data = []
+    count = 0
+    count2 = 0
+    for (dirpath, dirnames, filenames) in os.walk(audiopath):
+        for dir in dirnames:
+            print '\rChecked %d, gathered %d files' % (count, count2), 
+            count += 1
+            
+            # we copy all requested files and the transform files as well!
+            if (transform_substr and (transform_substr in dir)):
+                source = os.path.join(dirpath, dir).replace('\\','/')
+                data.append(source)
+                count2 +=1
+#        if count2 > 1:
+#            break
+    
+    # count jobs
+    njobs = len(data)
+    print '\r\nAbout to copy %d directories' % (njobs) 
+    
+               
+    count = 0
+    # copy individual items
+    for x in data:
+        spath = os.path.split(x)
+        folders = spath[0].split('/')
+        
+        collectionname = folders[-1]
+
+         # create the target folder
+        #try:
+        #    os.makedirs(out_path)
+        #except OSError as exception:
+        #    if exception.errno!= errno.EEXIST:
+        #        raise   
+                
+        command = '7z -r -mmt=30 -mx5 -v50g -mm=bzip2 a '  + '"' + os.path.join(out_path,collectionname + '_'+ transform_substr + '.zip') +  '"' + ' ' + x
+        print command
+        print " Currently zipping " + collectionname
+        
+        p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
+        text = p.stdout.read()
+        retcode = p.wait()
+        if not (retcode==0):
+            print "Error: "  + text
+        
+        #os.system(command)
+        #print 'Removing ' + dest
+        #shutil.rmtree(dest)
+            
+        count = count + 1
+        
+        # progress indicator
+        print '\r%3.2f percent. %d done, %d pending' % (count/(njobs*1.0) * 100.0,count, njobs-count),
+        
+    print '\rCopied %d of %d files.' % (count,njobs)
+    print '\n'
+    
+def copytree(src, dst, symlinks=False, ignore=None):
+    for item in os.listdir(src):
+        s = os.path.join(src, item)
+        d = os.path.join(dst, item)
+        if os.path.isdir(s):
+            shutil.copytree(s, d, symlinks, ignore)
+        else:
+            os.makedirs(d)
+            shutil.copy2(s, d)
+
+if __name__ == "__main__":
+    if len(sys.argv) >= 5:
+        main(sys.argv[1],sys.argv[2], sys.argv[3], int(sys.argv[4]))
+    else:
+        main(audiopath = 'D:/_Audio/',out_path = 'D:/_Audio_Analysis/', transform_substr = "",  ext = "")
+