view collection_analysis/tools/vampstats.py @ 0:e34cf1b6fe09 tip

commit

# Part of DML (Digital Music Laboratory)
# Copyright 2014-2015 Daniel Wolff, City University
 
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

#!/usr/bin/python
# -*- coding: utf-8 -*-

# creates a histogram from given input files or folder

__author__="Daniel Wolff"
__date__ ="$11-Feb-2015 18:18:47$"

import sys
import os
import csv
import numpy
import csv2json as c2j
import re


# global feature extensions
#ext = tuple([".n3",".csv",".mid"])
ext = tuple([".csv"])

floater = re.compile("((\d+)(.\d+)*)")
# reads in any csv and returns a list of structure
# time(float), data1, data2 ....data2
def read_vamp_csv(filein = '', datapos = 0):
    output = []
    badcount = 0
    with open(filein, 'rb') as csvfile:
        contents = csv.reader(csvfile, delimiter=',', quotechar='"')
        for row in contents:
	    if len(row) >= datapos + 2:
	        output.append([float(row[0])] + row[1:])
            else: 
                badcount += 1
    print "Ignored " + str(badcount) + " short rows" 
    return output

#calculates the histogram
def histogram(data, datapos = 1, nbins = -1):
    
    # symbols or numerical input?
    if not nbins == -1:
        
        #convert to numpy data
	ddata = string2numpy(data,datapos)
        
        count,index = numpy.histogram(ddata,nbins-1)
        count = count.tolist()
        index = index.tolist()
        
    # here for strings
    else: 
        # build histogram on strings
        histo = dict()
        for row in data:
            histo[row[datapos+1]] = histo.get(row[datapos+1], 0) + 1 
            index = histo.keys()
            count = histo.values()
            
    # return histogram
    return {"count":count, "index":index}
    
#calculates statistics for  numerical input
def numstats(data,datapos):
    
    #convert to numpy data
    ddata = string2numpy(data,datapos)

    avg = numpy.average(ddata).tolist()
    med = numpy.median(ddata).tolist()
    std = numpy.std(ddata).tolist()
    
    # return data
    return {"average": avg, "median": med, "std": std}

def featurefilesinpath(path):
    # ---
    # we traverse the file structure
    # and list files to copy
    # ---
    files = []
    for (dirpath, dirnames, filenames) in os.walk(path):
        for file in filenames:        
            # we copy all requested files and the transform files as well!
            if (file.endswith(ext)):
                source = os.path.join(dirpath, file).replace('\\','/')
                files.append(source)
    return files

# convert to numpy
def string2numpy(data,datapos):
    try:
        ddata = numpy.array(data, dtype=float)[:, datapos+1]
    except:
        edata = []
        for row in data:
            # account for verbatim units
            m = re.search("[a-zA-Z]", row[datapos+1])
            if m is not None:
                # take only the specified column datapos+1
                edata.append(row[datapos+1][:(m.start()-1)])
            else:
                # take only the specified column datapos+1
                edata.append(row[datapos+1])
        ddata = numpy.array(edata,dtype=float)
    return ddata

# main entry point
if __name__ == "__main__":
    print "Usage: vampstats datapos nbins file1/dir1 file2/dir2 ...."
    print "datapos: column of data after timecode to process"
    print "nbins: -1 for categorical data, otherwise number of bins for histogram"
    
    datapos = int(sys.argv[1])
    nbins = int(sys.argv[2])
    
    # check and collate files
    files = []
    for path in sys.argv[3:]:
        if os.path.isdir(path):
            files.extend(featurefilesinpath(path))
        else: 
            if os.path.isfile(path):
                files.extend(path)
    print "Number of files now loading: " + str(len(files))
    
    # we collate all data first and then count. 
    # @todo: read all files and create dictionary first for large tasks
    data = []
    for file in files:
        print file
        data.extend(read_vamp_csv(file, datapos))
    
    print "Total data size in memory: " + str(sys.getsizeof(data))
    
    # now get the histogram for all data
    histo = histogram(data,datapos,nbins)
    print histo
    print "Please input a description for the histogram analysis features"
    c2j.data2json(histo)
    
    # further numerical analysis if this is not categorical data
    if not nbins == -1: 
        ns = numstats(data,datapos)
        print ns
        print "Please input a description for the general statistics features"
        c2j.data2json(ns)