changeset 51:c1e4f706ca9a

Fix numpy version incompatibility issues and updated some example plugins.

--- a/PyOutputDescriptor.cpp	Tue Oct 06 12:37:01 2009 +0000
+++ b/PyOutputDescriptor.cpp	Thu Oct 08 08:47:28 2009 +0000
@@ -88,6 +88,7 @@
 			Py_INCREF(v);
 			return v;
 		}
+		PyErr_SetString(PyExc_AttributeError,"non-existing OutputDescriptor attribute");
 	}
 	return NULL;
 }

--- a/PyParameterDescriptor.cpp	Tue Oct 06 12:37:01 2009 +0000
+++ b/PyParameterDescriptor.cpp	Thu Oct 08 08:47:28 2009 +0000
@@ -89,6 +89,7 @@
 			Py_INCREF(v);
 			return v;
 		}
+		PyErr_SetString(PyExc_AttributeError,"non-existing ParameterDescriptor attribute");
 	}
 	return NULL;
 }

--- a/PyPlugin.cpp	Tue Oct 06 12:37:01 2009 +0000
+++ b/PyPlugin.cpp	Thu Oct 08 08:47:28 2009 +0000
@@ -30,7 +30,7 @@
 
 Mutex PyPlugin::m_pythonInterpreterMutex;
 
-PyPlugin::PyPlugin(std::string pluginKey, float inputSampleRate, PyObject *pyClass, int &instcount) :
+PyPlugin::PyPlugin(std::string pluginKey, float inputSampleRate, PyObject *pyClass, int &instcount, bool &numpyInstalled) :
 	Plugin(inputSampleRate),
 	m_pyClass(pyClass),
 	m_instcount(instcount),
@@ -44,7 +44,9 @@
 	m_pyProcess(NULL),
 	m_inputDomain(TimeDomain),
 	m_quitOnErrorFlag(false),
-	m_debugFlag(false)
+	m_debugFlag(false),
+	m_numpyInstalled(numpyInstalled),
+	m_processFailure(false)
 {	
 	m_ti.setInputSampleRate(inputSampleRate);
 	MutexLocker locker(&m_pythonInterpreterMutex);
@@ -86,6 +88,7 @@
    
 	if (m_debugFlag && st_flag) cerr << "Strict type conversion ON for: " << m_class << endl;
 	m_ti.setStrictTypingFlag(st_flag);
+	m_ti.setNumpyInstalled(m_numpyInstalled);
 
 }
 
@@ -188,6 +191,7 @@
 PyPlugin::reset()
 {
 	MutexLocker locker(&m_pythonInterpreterMutex);
+	m_processFailure = false;
 	genericMethodCall("reset");
 }
 
@@ -290,6 +294,8 @@
 	return FeatureSet();
 	}
 	
+	if (m_processFailure) return FeatureSet();
+	
 	return processMethodCall(inputBuffers,timestamp);
 
 }
@@ -298,6 +304,7 @@
 PyPlugin::getRemainingFeatures()
 {
 	MutexLocker locker(&m_pythonInterpreterMutex);
+	if (m_processFailure) return FeatureSet();
 	FeatureSet rValue;
 	return genericMethodCall("getRemainingFeatures",rValue); 
 }
@@ -355,6 +362,7 @@
 	//quering process implementation type
 	char legacyMethod[]="process";
 	char numpyMethod[]="processN";
+	m_processFailure = false;
 
 	if (PyObject_HasAttrString(m_pyInstance,legacyMethod) &&
 	    m_processType == 0) 
@@ -384,33 +392,68 @@
 
     if (m_vampyFlags & vf_ARRAY) {
 #ifdef HAVE_NUMPY
-		m_processType = numpy_arrayProcess;
-		if (m_debugFlag) cerr << "Process using numpy array interface." << endl;
+		if (m_numpyInstalled) { m_processType = numpy_arrayProcess;
+			if (m_debugFlag) 
+				cerr << "Process using numpy array interface." << endl;
+		}
+		else {
+			m_processFailure = true;
+			char method[]="initialise::setProcessType";
+			cerr << PLUGIN_ERROR
+			<< "This plugin requests the Numpy array interface by setting "
+			<< " the vf_ARRAY flag in its __init__() function." << endl 
+			<< "However, we could not found a version of Numpy compatible with this build of Vampy." << endl
+			<< "If you have a numerical library installed that supports the buffer interface, " << endl
+			<< "you can request this interface instead by setting the vf_BUFFER flag." << endl;
+		}
 #else
-		cerr << "Error: This version of vampy was compiled without numpy support, "
-			 << "however the vf_ARRAY flag is set for plugin: " << m_class << endl
-			 << "The default behaviour is: passing a python list of samples for each channel in process() "
-			 << "or a list of memory buffers in processN(). " << endl 
-			 << "This can be used create numpy arrays using the numpy.frombuffer() command." << endl;
+		m_processFailure = true;
+		char method[]="initialise::setProcessType";
+		cerr << PLUGIN_ERROR
+		<< "Error: This version of vampy was compiled without numpy support, "
+		<< "however the vf_ARRAY flag is set for plugin: " << m_class << endl
+		<< "The default behaviour is: passing a python list of samples for each channel in process() "
+		<< "or a list of memory buffers in processN(). " << endl 
+		<< "This can be used create numpy arrays using the numpy.frombuffer() command." << endl;
 #endif		
 	}
 	
-	if (!m_processType)
+	if (!m_pyProcessCallable)
 	{
 		m_processType = not_implemented;
 		m_pyProcess = NULL;
-		m_pyProcessCallable = NULL;
 		char method[]="initialise::setProcessType";
 		cerr << PLUGIN_ERROR << " No process implementation found. Plugin will do nothing." << endl;
+		m_processFailure = true;
 	}
 }
 
 void
-PyPlugin::typeErrorHandler(char *method) const
+PyPlugin::typeErrorHandler(char *method, bool process) const
 {
 	bool strict = false;
 	while (m_ti.error) { 
 		PyTypeInterface::ValueError e = m_ti.getError();
+#ifdef HAVE_NUMPY
+		// disable the process completely if numpy types are returned 
+		// but a compatible version was not loaded.
+		// This is required because if an object is returned from
+		// the wrong build, malloc complains about its size
+		// (i.e. the interpreter doesn't free it properly)
+		// and the process may be leaking.
+		// Note: this only happens in the obscure situation when
+		// someone forces to return wrong numpy types from an 
+		// incompatible version using the buffer interface.
+		// In this case the incampatible library is still usable,
+		// but manual conversion to python builtins is required.
+		// If the ARRAY interface is set but Numpy is not installed
+		// the process will be disabled already at initialisation.
+		if (process && !m_numpyInstalled && e.str().find("numpy")!=std::string::npos) 
+		{
+			m_processFailure = true;
+			cerr << "Warning: incompatible numpy type encountered. Disabling process." << endl;
+		}
+#endif		
 		cerr << PLUGIN_ERROR << e.str() << endl;
 		if (e.strict) strict = true;
 		// e.print();
@@ -422,5 +465,9 @@
 	/// It would be best if hosts could catch an exception instead
 	/// and display something meaningful to the user.
 	if (strict && m_quitOnErrorFlag) exit(EXIT_FAILURE);
+
+	// this would disable all outputs even if some are valid
+	// if (process) m_processFailure = true;
+	
 }
 

--- a/PyPlugin.h	Tue Oct 06 12:37:01 2009 +0000
+++ b/PyPlugin.h	Thu Oct 08 08:47:28 2009 +0000
@@ -73,7 +73,7 @@
 class PyPlugin : public Vamp::Plugin
 {
 public:
-	PyPlugin(std::string plugin,float inputSampleRate, PyObject *pyClass, int &instcount);
+	PyPlugin(std::string plugin,float inputSampleRate, PyObject *pyClass, int &instcount, bool &numpyInstalled);
 	virtual ~PyPlugin();
 
 	bool initialise(size_t channels, size_t stepSize, size_t blockSize);
@@ -122,6 +122,8 @@
 	bool m_quitOnErrorFlag;
 	bool m_debugFlag;
 	bool m_useRealTimeFlag;
+	bool m_numpyInstalled;
+	mutable bool m_processFailure;
 
 	void setProcessType();
 	
@@ -129,7 +131,7 @@
 
 	bool getBooleanFlag(char flagName[],bool) const;
 	int getBinaryFlags(char flagName[], eVampyFlags) const;
-	void typeErrorHandler(char *method) const;
+	void typeErrorHandler(char *method, bool process = false) const;
 
 	/// simple 'void return' call with no args
 	void genericMethodCall(char *method) const
@@ -397,7 +399,7 @@
 inline PyPlugin::FeatureSet
 PyPlugin::processMethodCall(const float *const *inputBuffers,Vamp::RealTime timestamp)
 {
-	
+
 	/// Optimizations: 1) we avoid ...ObjArg functions since we know
 	/// the number of arguments, and we don't like va_list parsing 
 	/// in the process. 2) Also: we're supposed to incref args, 
@@ -410,16 +412,19 @@
 	PyObject *pyChannelList = NULL;
 
 	if (m_processType == numpy_bufferProcess) {
-		pyChannelList = m_ti.InputBuffers_As_SharedMemoryList(inputBuffers,m_channels,m_blockSize,m_inputDomain);
+		pyChannelList = m_ti.InputBuffers_As_SharedMemoryList(
+				inputBuffers,m_channels,m_blockSize,m_inputDomain);
 	} 
 
 	if (m_processType == legacyProcess) {
-		pyChannelList = m_ti.InputBuffers_As_PythonLists(inputBuffers,m_channels,m_blockSize,m_inputDomain);
+		pyChannelList = m_ti.InputBuffers_As_PythonLists(
+			inputBuffers,m_channels,m_blockSize,m_inputDomain);
 	}
 
 #ifdef HAVE_NUMPY
 	if (m_processType == numpy_arrayProcess) {
-		pyChannelList = m_ti.InputBuffers_As_NumpyArray(inputBuffers,m_channels,m_blockSize,m_inputDomain);
+		pyChannelList = m_ti.InputBuffers_As_NumpyArray(
+			inputBuffers,m_channels,m_blockSize,m_inputDomain);
 	}
 #endif
 
@@ -481,7 +486,7 @@
 		Py_DECREF(pyValue);
 		Py_DECREF(pyArgs);
 	} else {
-		typeErrorHandler(PyString_AsString(m_pyProcess));
+		typeErrorHandler(PyString_AsString(m_pyProcess),true);
 		Py_CLEAR(pyValue);
 		Py_CLEAR(pyArgs);
 	}

--- a/PyTypeInterface.cpp	Tue Oct 06 12:37:01 2009 +0000
+++ b/PyTypeInterface.cpp	Thu Oct 08 08:47:28 2009 +0000
@@ -45,6 +45,7 @@
 PyTypeInterface::PyTypeInterface() : 
 	m_strict(false),
 	m_error(false),
+	m_numpyInstalled(false),
 	error(m_error) // const public reference for easy access
 {
 }
@@ -64,13 +65,13 @@
 	
 	if (pyValue == NULL)
 	{
-		setValueError("Error while converting float object.",m_strict);
+		setValueError("Error while converting object " + PyValue_Get_TypeName(pyValue) + " to float. ",m_strict);
 		return 0.0;		
 	}
 		
 	// in strict mode we will not try harder
 	if (m_strict) {
-		setValueError("Strict conversion error: object is not float.",m_strict);
+		setValueError("Strict conversion error: object" + PyValue_Get_TypeName(pyValue) +" is not float.",m_strict);
 		return 0.0;
 	}
 
@@ -560,9 +561,10 @@
 		}
 		return Output;
 	}
-#ifdef _DEBUG
-	cerr << "PyTypeInterface::PyValue_To_StringVector: Warning: Value is not list of strings." << endl;
-#endif
+
+// #ifdef _DEBUG
+// 	cerr << "PyTypeInterface::PyValue_To_StringVector: Warning: Value is not list of strings." << endl;
+// #endif
 
 	/// Assume a single value that can be casted as string 
 	/// this allows to write e.g. Feature.label = 5.2 instead of ['5.2']
@@ -583,6 +585,8 @@
 {
 
 #ifdef HAVE_NUMPY
+if (m_numpyInstalled) 
+{
 	// there are four types of values we may receive from a numpy process:
 	// * a python scalar, 
 	// * an array scalar, (e.g. numpy.float32)
@@ -604,7 +608,7 @@
 	/// numpy array
 	if (PyArray_CheckExact(pyValue)) 
 		return PyArray_To_FloatVector(pyValue);
-
+}
 #endif
 
 	/// python list of floats (backward compatible)
@@ -621,7 +625,7 @@
 		std::string msg = "Value is not list or array of floats nor can be casted as float. ";
 		setValueError(msg,m_strict);
 #ifdef _DEBUG
-	cerr << "PyTypeInterface::PyValue_To_FloatVector failed." << msg << endl;
+	cerr << "PyTypeInterface::PyValue_To_FloatVector failed. " << msg << endl;
 #endif
 	}
 	return Output;
@@ -674,7 +678,9 @@
 	return Output;
 }
 
-#ifdef HAVE_NUMPY
+// if numpy is not installed this will not be called, 
+// therefor we do not check again
+#ifdef HAVE_NUMPY 
 std::vector<float> 
 PyTypeInterface::PyArray_To_FloatVector (PyObject *pyValue) const 
 {
@@ -744,7 +750,7 @@
 #endif
 
 
-/// FeatureSet (an integer map of OutputLists)
+/// FeatureSet (an integer map of FeatureLists)
 Vamp::Plugin::FeatureSet
 PyTypeInterface::PyValue_To_FeatureSet(PyObject* pyValue) const
 {
@@ -798,7 +804,7 @@
 		return rFeatureSet;
 	}
 
-	/// accept no return values
+	/// accept None return values
 	if (pyValue == Py_None) return rFeatureSet;
 
 	/// give up

--- a/PyTypeInterface.h	Tue Oct 06 12:37:01 2009 +0000
+++ b/PyTypeInterface.h	Thu Oct 08 08:47:28 2009 +0000
@@ -124,6 +124,7 @@
 	
 	// Utilities
 	void setStrictTypingFlag(bool b) {m_strict = b;}
+	void setNumpyInstalled(bool b) {m_numpyInstalled = b;}
 	ValueError getError() const;
 	std::string PyValue_Get_TypeName(PyObject*) const;
 	bool initMaps() const;
@@ -392,6 +393,7 @@
 	mutable bool m_error;
 	mutable std::queue<ValueError> m_errorQueue;
 	unsigned int m_inputSampleRate; 
+	bool m_numpyInstalled;
 	
 	void setValueError(std::string,bool) const;
 	ValueError& lastError() const;
@@ -432,7 +434,7 @@
 
 /* 		   		  Convert Sample Buffers to Python 	         		*/
 
-/// passing the sample buffers as buitin python lists
+/// passing the sample buffers as builtin python lists
 /// Optimization: using fast sequence protocol
 inline PyObject*
 PyTypeInterface::InputBuffers_As_PythonLists(const float *const *inputBuffers,const size_t& channels, const size_t& blockSize, const Vamp::Plugin::InputDomain& dtype)
@@ -446,22 +448,23 @@
 	
 	PyObject **pyChannelListArray =  PySequence_Fast_ITEMS(pyChannelList);
 	for (size_t i=0; i < channels; ++i) {
-
-		PyObject *pySampleList = PyList_New((Py_ssize_t) blockSize);
-		PyObject **pySampleListArray =  PySequence_Fast_ITEMS(pySampleList);
-		size_t arraySize;
-
+		
+        size_t arraySize;
 		if (dtype==Vamp::Plugin::FrequencyDomain) 
-			arraySize = blockSize + 2;
+			arraySize = (blockSize / 2) + 1; //blockSize + 2; if cplx list isn't used
 		else 
 			arraySize = blockSize;
+
+		PyObject *pySampleList = PyList_New((Py_ssize_t) arraySize);
+		PyObject **pySampleListArray =  PySequence_Fast_ITEMS(pySampleList);
 		
 		// Note: passing a complex list crashes the C-style plugin
 		// when it tries to convert it to a numpy array directly.
 		// This plugin will be obsolete, but we have to find a way
-		// to prevent such crash.
+		// to prevent such crash: possibly a numpy bug, 
+		// works fine above 1.0.4
 		
-		switch (Vamp::Plugin::TimeDomain) //(dtype)
+		switch (dtype) //(Vamp::Plugin::TimeDomain)
 		{
 			case Vamp::Plugin::TimeDomain :
 
@@ -475,7 +478,7 @@
 			case Vamp::Plugin::FrequencyDomain :
 
 			size_t k = 0;
-			for (size_t j = 0; j < arraySize/2; ++j) {
+			for (size_t j = 0; j < arraySize; ++j) {
 				PyObject *pyComplex=PyComplex_FromDoubles(
 					(double) inputBuffers[i][k], 
 					(double) inputBuffers[i][k+1]);

--- a/README	Tue Oct 06 12:37:01 2009 +0000
+++ b/README	Thu Oct 08 08:47:28 2009 +0000
@@ -127,15 +127,15 @@
 HISTORY:
 
 	v1:
-	* added support for NumPy arrays in processN()
+	* added support for Numpy arrays in processN()
 	* framecount is now passed also to legacy process() and fixed resulting bugs in the PyZeroCrossing plugin
 	* added two examples which use Frequency Domain input in processN()
 
 	v2.0:
 	* complete rewrite (using generic functions implementing full error checking)
 	* added extension module : support RealTime and other Vamp type wrappers
-	* added numpy Array interface
-	* added falgs
+	* added Numpy Array interface
+	* added flags
 	* added environment variables
 	* recognise byte compiled python scripts
 	

--- a/vampy-main.cpp	Tue Oct 06 12:37:01 2009 +0000
+++ b/vampy-main.cpp	Thu Oct 08 08:47:28 2009 +0000
@@ -12,8 +12,25 @@
 #include <Python.h>
 
 #ifdef HAVE_NUMPY
-#define PY_ARRAY_UNIQUE_SYMBOL VAMPY_ARRAY_API
+
+// define a unique API pointer 
+#define PY_ARRAY_UNIQUE_SYMBOL VAMPY_ARRAY_API 
 #include "numpy/arrayobject.h"
+
+// prevent building with very old versions of numpy
+#ifndef NPY_VERSION 
+#undef HAVE_NUMPY
+#endif
+
+#endif
+
+// this is not part of the API, but we will require it for a bug workaround
+// define this symbol if you use another version of numpy in the makefile
+// Vampy will not attempt to load a lower version than specified
+#ifdef HAVE_NUMPY
+#ifndef NUMPY_SHORTVERSION
+#define NUMPY_SHORTVERSION 1.1 
+#endif
 #endif
 
 #include "vamp/vamp.h"
@@ -41,6 +58,8 @@
 
 static int adinstcount;
 static int totinstcount;
+static bool numpyInstalled = false;
+static bool arrayApiInitialised = false;
 
 class PyPluginAdapter : public Vamp::PluginAdapterBase
 {
@@ -66,7 +85,7 @@
     Vamp::Plugin *createPlugin(float inputSampleRate)
     {
         try {
-            PyPlugin *plugin = new PyPlugin(m_plug, inputSampleRate, m_pyClass, totinstcount);
+            PyPlugin *plugin = new PyPlugin(m_plug, inputSampleRate, m_pyClass, totinstcount, numpyInstalled);
             return plugin;
         } catch (...) {
             cerr << "PyPluginAdapter::createPlugin: Failed to construct PyPlugin" << endl;
@@ -81,15 +100,106 @@
 	bool m_failed;  
 };
 
+
 static void array_API_initialiser()
 {
-/// numpy C-API requirement
+	if (arrayApiInitialised) return; 
+
+/* Numpy 1.3 build note: there seems to be a bug 
+in this version (at least on OS/X) which will cause memory 
+access error in the array API import function if an earlier runtime 
+version of Numpy is used when loading the library.
+(below is a horrible workaround)
+*/
+
 #ifdef HAVE_NUMPY
+
+	string ver;
+	float numpyVersion;
+
+	/// attmept to test numpy version before importing the array API
+	cerr << "Numpy build information: ABI level: " << NPY_VERSION 
+	<< " Numpy version: " << NUMPY_SHORTVERSION << endl;
+	
+	PyObject *pyModule, *pyDict, *pyVer;
+	
+	pyModule = PyImport_ImportModule("numpy"); //numpy.core.multiarray
+	if (!pyModule) {
+		cerr << "Vampy was compiled with Numpy support but Numpy does not seem to be installed." << endl;
+#ifdef __APPLE__
+		cerr << "Hint: Check if Numpy is installed for the particular setup of Python used by Vampy (given by Python exec prefix)." << endl;
+#endif		
+		goto numpyFailure;
+	}
+
+	pyDict = PyModule_GetDict(pyModule); // borrowed ref
+	if (!pyDict) {
+		cerr << "Can not access Numpy module dictionary." << endl;
+		goto numpyFailure;
+	}
+
+	pyVer = PyDict_GetItemString(pyDict,"__version__"); //borrowed ref
+	if (!pyVer) {
+		cerr << "Can not access Numpy version information." << endl;
+		goto numpyFailure;
+	}
+
+	ver = PyString_AsString(pyVer);
+	ver = ver.substr(0,ver.rfind("."));
+	if(EOF == sscanf(ver.c_str(), "%f", &numpyVersion))
+	{
+		cerr << "Could not parse Numpy version information." << endl;
+		goto numpyFailure;
+	}
+
+	cerr << "Numpy runtime version: " << numpyVersion << endl;
+	if (numpyVersion < (float) NUMPY_SHORTVERSION) {
+		cerr << "Incompatible Numpy version found: " << numpyVersion << endl;
+		goto numpyFailure;
+	}
+
+	Py_DECREF(pyModule);
+
+	// At least we catch import errors, but if binary compatibility
+	// has changed without notice, this would still fail.
+	// However, we should never get to this point now anyway.
 	import_array();
-	if(NPY_VERSION != PyArray_GetNDArrayCVersion())
-		cerr << "Warning: Numpy ABI version mismatch. (Build version: " 
-		<< NPY_VERSION << " Runtime version: " << PyArray_GetNDArrayCVersion() << ")" << endl;
+	if (PyErr_Occurred()) { 
+		cerr << "Import error while loading the Numpy Array API." << endl;
+		PyErr_Print(); PyErr_Clear(); 
+		goto numpyFailure;
+	}
+	else {
+
+#ifdef _DEBUG		
+		if (NPY_VERSION != PyArray_GetNDArrayCVersion()) {  
+			// the Import function does this check already.
+			cerr << "Warning: Numpy version mismatch. (Build version: " 
+				<< NPY_VERSION << " Runtime version: " << PyArray_GetNDArrayCVersion() << ")" << endl;
+			goto numpyFailure; 
+		}
 #endif
+
+		numpyInstalled = true;
+		arrayApiInitialised = true;
+		return;
+  	}
+
+
+numpyFailure: 
+	cerr << "Please make sure you have Numpy " << NUMPY_SHORTVERSION << " or greater installed." << endl;
+	cerr << "Vampy: Numpy support disabled." << endl;
+	numpyInstalled = false;
+	arrayApiInitialised = true;
+	if (pyModule) Py_XDECREF(pyModule);
+	return;
+
+/*HAVE_NUMPY*/
+#endif 
+
+    numpyInstalled = false;
+	arrayApiInitialised = true;
+	return;
 }
 
 
@@ -187,6 +297,7 @@
 			if (PyImport_AppendInittab("vampy",initvampy) != 0)
 				cerr << "Warning: Extension module could not be added to module inittab." << endl;
 			Py_Initialize();
+			array_API_initialiser();
 			initvampy();
 #ifdef _DEBUG			
 		    cerr << "# isPythonInitialized after initialize: " << Py_IsInitialized() << endl;