annotate pyspark/transforms/tuningFrequencyStatistics.py @ 0:e34cf1b6fe09 tip

commit

# Part of DML (Digital Music Laboratory)
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

# -*- coding: utf-8 -*-
__author__="Daniel Wolff, hargreaves"

# this script derives standard statistics for tuning frequency, 
# in particular:
# average
# standard deviation

from rdflib import Graph, Namespace, BNode, RDF, Literal
import codecs
import warnings
import numpy
import csv
from n3Parser import get_rdf_graph_from_n3, uri2path
# from csvParser import get_dict_from_csv, get_array_from_csv

# statistics per clip ?
perfilestats = 1

# dml namespace
dml_ns = Namespace("http://dml.org/dml/cla#")

# Add triples representing a 'key histogram' result to
# an RDF graph
def add_tf_statistics_to_graph(statistics, output_rdf_graph, transform, sample_count, input_f_files):
    
    # add base
    output_bnode = BNode()
    output_rdf_graph.add((transform, dml_ns.output, output_bnode))
    for input_f_file in input_f_files:
        output_rdf_graph.add((transform, dml_ns.input, input_f_file))
    output_rdf_graph.add((output_bnode, RDF.type, dml_ns.TuningFrequencyStatistics))
    output_rdf_graph.add((output_bnode, dml_ns.sample_count, Literal(sample_count)))

    # add mean and std
    output_rdf_graph.add((output_bnode, dml_ns.mean, Literal(statistics["mean"])))
    output_rdf_graph.add((output_bnode, dml_ns.std_dev, Literal(statistics["std-dev"])))
    
    # add histogram
    for i in range(0,len(statistics["histogram"]["count"])):

        bin_bnode = BNode()
        output_rdf_graph.add((output_bnode, dml_ns.bin, bin_bnode))
        output_rdf_graph.add((bin_bnode, dml_ns.bin_number, Literal(i+1)))
        output_rdf_graph.add((bin_bnode, dml_ns.bin_value, Literal(statistics["histogram"]["count"][i])))
        output_rdf_graph.add((bin_bnode, dml_ns.bin_name, Literal(statistics["histogram"]["index"][i])))

    return output_rdf_graph

# Parse the input_f_files n3 files, and generate
# a key histogram
def find_cla_tf_statistics(input_f_files):


    sample_count = len(input_f_files)
    
    all_data = []
    perfile_freq = []
    perfile_hist = []
    hist_index =[]
    for input_f_file in input_f_files:

        # get all data from feature file
        data = file_to_table(input_f_file)

        # filter those rows which have an A
        # returns duration, frequency
        data = filter_norm_A(data)
        
        if perfilestats:
            # get frequency and duration columns
            freq = string2numpy(data,2)
            dur = string2numpy(data,1)
            # get mean values per clip now,
            # then statistics over clips later
            avg, std = numstats(freq, weights = dur)
            hist = histogram(freq, nbins = 100, lb=390, ub=490, weights = dur)
            
            # remember statistics
            perfile_freq.append(avg) 
            perfile_hist.append(hist["count"])
            
            # remember histogram index
            if len(hist_index) == 0:
                hist_index = hist["index"]
            
        else:
            # this version just adds everything per collection,
            # recordings are not treated as seperate entities
            all_data.extend(data)
            

    if perfilestats:
        avg, std = histostats(numpy.array(perfile_freq,dtype=float))
        hist_avg, hist_std = histostats(numpy.array(perfile_hist,dtype=float))
    
    else:
        # get frequency and duration columns
        freq = string2numpy(all_data,2)
        dur = string2numpy(all_data,1)

        # get basic statistics
        avg, std = numstats(freq, weights = dur)

        # get histogram weighted by duration
        hist = histogram(freq, nbins = 100, lb=390, ub=490, weights = dur)
        
    return {"mean": avg, "std-dev": std, "histogram": hist}, sample_count#(key_hist, num_f_files)

# convert one column, specified by datapos, to numpy
def string2numpy(data,datapos):
    
    edata = []
    for row in data:
        edata.append(row[datapos])
        
    colu = numpy.array(edata,dtype=float)
    return colu

#calculates the histogram
# nbins: number of bins
# lb: lower bound
# ub: upper bound
def histogram(colu, nbins = 100, lb=-1, ub=-1, weights = []):
    
    # lower bounds defined?
    if lb == -1 or ub == -1:
        lb = colu.min()
        ub = colu.max()
        
    # get histogram
    count,index = numpy.histogram(colu,bins=nbins,range = [lb, ub],weights = weights)
    count = count.tolist()
    index = index.tolist()
    
    # normalise for clip
    count = count / numpy.max(count)
    
    # return histogram
    return {"count":count, "index":index}


# calculates unweighted statistics for the histograms
def histostats(counts):
    avg = numpy.average(counts, axis = 0).tolist()

    #weighted standard deviation
    std = numpy.std(counts, axis =0)

    #med = numpy.median(colu, weights = weights).tolist()
    # could use https://pypi.python.org/pypi/wquantiles for weighted median

    return (avg,std)
    
#calculates weighted statistics for  numerical input
def numstats(colu, weights = []):
    
    # we want to always use the last dimension
    # get average
    avg = numpy.average(colu, axis = 0 ,weights = weights)

    #weighted standard deviation
    std = numpy.sqrt(numpy.average((colu-avg)**2, axis = 0, weights=weights))
    #std = numpy.std(colu, weights = weights).tolist()
    
    #med = numpy.median(colu, weights = weights).tolist()
    # could use https://pypi.python.org/pypi/wquantiles for weighted median

    return (avg,std)


# only returns data columns which refer to the note A
# the frequencies are folded up / down to A4
# returns time, duration, frequency
def filter_norm_A(data):
    Adata = []
    for row in data:
        # we assume format time , duration , pitch, ingeger_pitch, label 
        if 'A3' in row[4]:
            Adata.append(row[:2] + [2*row[2]])
        elif 'A4' in row[4]:
            Adata.append(row[:3])
        elif 'A5' in row[4]:
            Adata.append(row[:2] + [0.5*row[2]])
            
    return Adata
    
    
# Read named features into table of format
# time, feature[0], feature[1} ...
def file_to_table(input_f_file):
    if input_f_file.endswith('.n3'):
        data = n3_to_table(input_f_file)
    elif input_f_file.endswith('.csv'):    
        data = csv_to_table(input_f_file)
        #data = get_array_from_csv(input_f_file)
        #data = get_dict_from_csv(input_f_file,columtype = ['time','duration','pitch','velocity','label'])
    return data


# Read named features into table of format
# time, feature[0], feature[1} ...
def n3_to_table(input_f_file):

    # read feature file
    feature_graph = get_rdf_graph_from_n3(input_f_file)

    # we construct a generic search string that gets all 
    # necessary features 

    q = """prefix dml:     <http://dml.org/dml/cla#>
            SELECT ?event ?tl_time ?tl_duration ?feature ?label
            WHERE {
                ?event event:time ?event_time .
                ?event_time tl:beginsAt ?tl_time .
                ?event_time tl:duration ?tl_duration .
                ?event rdfs:label ?label .
                ?event af:feature ?feature .
            }"""       
    
    # query parsed file
    qres = feature_graph.query(q)
    data = []
    for row in qres:
        # parse time
        tl_time_str_len = len(row.tl_time)
        tl_time = float(row.tl_time[2:tl_time_str_len-1])
        
        # parse duration
        tl_dur_str_len = len(row.tl_duration)
        tl_duration = row.tl_duration[2:tl_dur_str_len-1]
        # parse feature
        data.append([tl_time, tl_duration] + [float(i) for i in row.feature.split(' ') ] + [row.label])
        
    #data = numpy.array(data, dtype=float)
    # print data
    # we assume format time , duration , pitch, velocity, label 
    return data #int(last_key)

# todo: do the same conversion for csv, should allow to use the same script with csv
def csv_to_table(input_f_file):
    
    
    output = []
    badcount = 0
    
    # keep track of column names
    ncols = 0
    with open(uri2path(input_f_file), 'rb') as csvfile:
        contents = csv.reader(csvfile, delimiter=',', quotechar='"')
        for row in contents:
            if ncols == 0:
                ncols = len(row)
                
	    if len(row) >= ncols:
                # we assume format time , duration , pitch, velocity, label 
	        output.append([float(row[0]), float(row[1]),  float(row[2])] + row[3:])
            else: 
                badcount += 1
                
    if badcount > 0:
        warnings.warn("Incomplete csv file, ignoring " + str(badcount) + " entries")
            
    return output