annotate runner/BinaryFeatureWriter.cpp @ 71:9600f5b8076f tip

latest version before copying fextractor

/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */

/*
    Sonic Annotator
    A utility for batch feature extraction from audio files.
    Mark Levy, Chris Sutton and Chris Cannam, Queen Mary, University of London.
    Copyright 2007-2008 QMUL.

    This program is free software; you can redistribute it and/or
    modify it under the terms of the GNU General Public License as
    published by the Free Software Foundation; either version 2 of the
    License, or (at your option) any later version.  See the file
    COPYING included with this distribution for more information.
*/

#include <fstream>

#include <QFileInfo>

#include <ctime>

#include <stdint.h>

#include "BinaryFeatureWriter.h"

#include "base/RealTime.h"

#include "../version.h"

#ifdef _WIN32
#define platform "Windows"
#elif __APPLE__
#define platform "MacOS"
#elif __linux__
#define platform "Linux"
#else
#define platform "Unix"
#endif
#ifdef __LP64__ //__x86_64__
#define arch "64" 
#elif _WIN64
#define arch "64" 
#else
#define arch "32"
#endif

# define MAJOR_VERSION_PY 0

using namespace std;
using namespace Vamp;

// Parameter names
string
BinaryFeatureWriter::outputFileParam = "output";

struct BinaryFeatureWriter::OutputStream
{
    ofstream* stream;
	bool newtransform;
	const Transform *transform;
	OutputStream() : newtransform(true),stream(NULL),transform(NULL) { }
	~OutputStream() { if (stream != NULL) {stream->close(); delete stream;} }
	
	struct header_t { 
		int16_t BOM16;          // 16-bit BOM (FEFF as in UTF-16) 2 bytes
		int32_t BOM32;          // 32-bit BOM (human readable: e.g. ABCD) 4 bytes
		char major_version;     // check for binary compatibility
		char minor_version;     // changes in txt parts only
		char compression;       // use of stream compression (e.g. gzip)
		char reserved1;         // reserved byte
		char reserved2;         // reserved byte
		char float_size;        // size of float
		char int_size;          // size of int
		char info[160];         // 160 byte text field
		char null; };           // NULL

	bool open(string filename, bool append = true) {

		if (stream) return true;
		
		header_t header = {(int16_t) 0xFEFF, (int32_t) 0x41424344,MAJOR_VERSION_PY,2,0,NULL,NULL,(char)sizeof(float),(char)sizeof(int),{NULL},NULL}; 	
		char* p_header = reinterpret_cast<char*>(&header);

		if (append) 
			stream = new ofstream(filename.c_str(), fstream::binary | ios_base::out | ios_base::in | ofstream::ate);
		else 
			stream = new ofstream(filename.c_str(), fstream::binary);
		
		if (!stream || !stream->is_open()) {
			if (append) {
				cerr << endl << "NOTE: Writing new binary output file: " << filename << endl;
				delete stream; 
				stream = NULL;
				return open(filename,false);
			} else {
				cerr << "ERROR: BinaryFeatureWriter::OutputStream::open(): can't open file " << filename << endl;
		        return false;
			}
		}
		
		// verify input file format
		if (append) {
			ifstream istream;
			istream.open(filename.c_str(), fstream::binary | ios::in);
			if (!istream.is_open()) { 
				cerr << "ERROR: BinaryFeatureWriter::OutputStream::open(): Can not verify supplied output stream." << endl;
				return false;
			}
			header_t iheader;
			istream.read(reinterpret_cast<char*>(&iheader),sizeof(header_t));
			istream.close();
			int16_t FEFF = 0xFEFF;
			if (iheader.BOM16 != FEFF) {
				if (iheader.BOM16 == (int16_t) 0xFFFE) {
					cerr << "ERROR: BinaryFeatureWriter::OutputStream::open(): This file apperas to have created on a different platform. Can not be appended. " 
					<< "Byte order mark: " << iheader.BOM32 << endl;
					return false;
				} else {
					cerr << "ERROR: BinaryFeatureWriter::OutputStream::open(): Invalid target file for this writer." << endl;
					return false;
				}
			}
			if (iheader.major_version != (char) MAJOR_VERSION_PY || iheader.BOM32 != (int32_t) 0x41424344) {
				cerr << "ERROR: BinaryFeatureWriter::OutputStream::open(): This file is not binary compatible with this version of the writer." 
				<< "file version: " << iheader.major_version << " required: " << MAJOR_VERSION_PY << endl;
				return false;
			}
		}

		if (append) stream->seekp(0,ios_base::end);	

		if (!append) { 
			time_t now = time(0);
			tm* gmtm = gmtime(&now);
			QString timestamp;
			if (gmtm != NULL) 
				timestamp = QString("%1 %2").arg(", Created: ").arg(asctime(gmtm)).trimmed();
			string info = QString("_SONIC ANNOTATOR v%1 PYTHON BINARY V0.2, Platform: %2-%3bit%4")
				.arg(RUNNER_VERSION)
				.arg(platform)
				.arg(arch)
				.arg(timestamp).toStdString();
			strncpy(reinterpret_cast<char*>(&header.info),info.c_str(), info.length() <= 160 ? info.length() : 160);
			stream->write(p_header,sizeof(header_t));
		}
		return true;
	}    
};


BinaryFeatureWriter::BinaryFeatureWriter() : 
    outputFile("features")
{
	binary = new OutputStream();
}

BinaryFeatureWriter::~BinaryFeatureWriter()
{
	if (binary) delete binary;
}

BinaryFeatureWriter::ParameterList
BinaryFeatureWriter::getSupportedParameters() const
{
    ParameterList pl;
    Parameter p;

    p.name = outputFileParam;
    p.description = "Binary output file path";
    p.hasArg = true;
    pl.push_back(p);

    return pl;
}

void
BinaryFeatureWriter::setParameters(map<string, string> &params)
{
    if (params.find(outputFileParam) != params.end()) {
        setOutputFile(params[outputFileParam]);
        params.erase(outputFileParam);
    }
}

void
BinaryFeatureWriter::setOutputFile(const string &file)
{
    outputFile = file;
}

void BinaryFeatureWriter::write(QString trackid,
                                 const Transform &transform,
                                 const Vamp::Plugin::OutputDescriptor& output, 
                                 const Vamp::Plugin::FeatureList& featureList,
                                 std::string summaryType)
{
    //!!! use summaryType
    if (summaryType != "") {
        //!!! IMPLEMENT
        cerr << "ERROR: BinaryFeatureWriter::write: Writing summaries is not yet implemented!" << endl;
        exit(1);
    }

	// TODO: Consider writing out NumPy arrays directly following this documentation:
	// https://github.com/numpy/numpy/blob/master/doc/neps/npy-format.txt 
	// and using the .npy format
		
	// return if file could not be opened
	if(!openBinaryFile()) {
		cerr << "ERROR: BinaryFeatureWriter::write: Error opening binary output file!" << endl;
	    exit(1);	        
	}	
		
	ofstream &ofs = *(binary->stream); 
	
	// The manager does not call finish() after writing different outputs from the same plugin, but we need this behaviour here:
	if (!binary->newtransform && binary->transform != NULL && binary->transform != &transform) finish();
	
	// write a python dictionary string containing (some) metadata needed to interpret the results
	// this can be evaluated in python using the expression : d = eval(f.readline())
	// given f is an open file, which should yield a valid dictionary. 
/*	
	enum SampleType {
	
	    /// Results from each process() align with that call's block start
	    0: OneSamplePerStep, 
	
	    /// Results are evenly spaced in time 
	    1: FixedSampleRate,
	
	    /// Results are unevenly spaced and have individual timestamps
	    2: VariableSampleRate 
	};
*/
	
	if (binary->newtransform) {
		binary->newtransform = false;
		output_binCount = output.binCount;
		feature_count = 0;
		element_count = 0;
		binary->transform = &transform;
	
	ofs << endl << "{"
	<< "\"track_id\":\"" << trackid << "\"," 
	<< "\"transform_id\":\"" << transform.getIdentifier() << "\","
	<< "\"sample_rate\":" << transform.getSampleRate() << ","
	<< "\"step_size\":" << transform.getStepSize() << ","
	<< "\"block_size\":" << transform.getBlockSize() << ","
	<< "\"window_type\":" << transform.getWindowType() << ","
	
	<< "\"features_list\":" << featureList.size() << ","
	<< "\"bin_count\":" << output.binCount << ","
	// << "\"output_description\":\"" << output.description << "\","
	<< "\"output_sample_type\":" << output.sampleType << ","
	<< "\"output_sample_rate\":" << output.sampleRate << ",";
	
	// Write start time and duration if the transform is not for the whole file
	if (transform.getDuration().toString() != "0.000000000") {
		ofs << "\"start_time\":\"" << transform.getStartTime().toString() << "\",";	
		ofs << "\"duration\":\"" << transform.getDuration().toString() << "\",";
	}
	// Write plugin version if known. (NOTE: using RDF transforms, it remains empty for some reason)
	if (!transform.getPluginVersion().isEmpty())
		ofs << "\"plugin_version\":\"" << transform.getPluginVersion() << "\",";
	
	// write transform parameters into a dict: parameters:{"parameter_name":value,...} where value is float	
	ofs << "\"parameters\":{" ;
	ParameterMap m = transform.getParameters();	
	
	for (ParameterMap::const_iterator i = m.begin(); i != m.end(); ++i) 
		// note last comma is ignored by python
		if (i == m.begin())
			ofs << QString("\"%1\":%2").arg(i->first).arg(i->second);
		else
			ofs << QString(",\"%1\":%2").arg(i->first).arg(i->second);
	ofs << "}"; 
	
	// write the data size last, and close the line.
	data_size_pos = ofs.tellp();	
	ofs << "                                                   }" << endl;	
	}
	
	// BUG: File corruption after about 34000 runs due to beat spectrum plugin returning less data than advertised !!!
	
	// write the feature data
	feature_count += featureList.size();	
	for (size_t i = 0; i < featureList.size(); ++i) {
		for (size_t j = 0; j < featureList[i].values.size(); ++j) {
			ofs.write( (const char*) &featureList[i].values[j], sizeof(featureList[i].values[j]) );
			element_count++;
		}
	}
	// ofs << endl;
	
	// // write time stamp data
	// for (int i = 0; i < featureList.size(); ++i) {
	// 	for (int j = 0; j < featureList[i].values.size(); ++j) {
	// 		// float sec = (int) featureList[i].timestamp.sec;
	// 		// float nsec = (int) featureList[i].timestamp.nsec;
	// 		// (*dbfiles[id].ofs).write( (const char*) &sec, sizeof(int));
	// 		// (*dbfiles[id].ofs).write( (const char*) &nsec, sizeof(int));
	// 		ofs.write( (const char*) &featureList[i].timestamp.sec, sizeof(int));
	// 		ofs.write( (const char*) &featureList[i].timestamp.nsec, sizeof(int));
	// 		
	// 	}
	// 		
	// }
	
	// -- UNCOMMENT - TO - HERE --
		
	

}

bool BinaryFeatureWriter::openBinaryFile()
{
	if (outputFile.rfind(".") == string::npos)
		return binary->open(outputFile + ".bin");
	else 
		return binary->open(outputFile);
}

void BinaryFeatureWriter::finish()
{
	ofstream &ofs = *(binary->stream);
	binary->newtransform = true;
	// ofs << endl;
	long t = ofs.tellp();
	ofs.seekp(data_size_pos);
	// fill in the missing information in the transform python dict that is required to read the output into an array
	ofs << ",\"feature_count\":" 
		<< feature_count
		<< ",\"data_size\":" 	
		// << feature_count * output_binCount * sizeof(float);	
		<< element_count * sizeof(float);
	ofs.seekp(t);
	if (element_count != feature_count * output_binCount) 
		cerr << "ERROR: The number of feature values written is different from the expected value given by the output bin count and the number of features returned by the plugin. \n"
		<< "Otput bin count: " << output_binCount << ", Feature count: " << feature_count << ", Expected elements: " << feature_count * output_binCount 
		<< ", Written elements: " << element_count << ", Resulting float32 data size: " << element_count * sizeof(float) << " bytes."
		<< endl;
    // FileFeatureWriter::finish();
}