annotate vampy-main.cpp @ 92:a6718f9fe942

If a module appears to redefine one of our own types, refuse to load it. Also clear out the class dict for all refused modules now, so that we don't get stale names on the next scan due to not having cleared the module on unload
author Chris Cannam
date Mon, 14 Jan 2019 16:19:44 +0000
parents 39c5f644152d
children c85d26cb9dab
rev   line source
Chris@66 1 /* -*- c-basic-offset: 8 indent-tabs-mode: t -*- */
fazekasgy@37 2 /*
fazekasgy@37 3
fazekasgy@37 4 * Vampy : This plugin is a wrapper around the Vamp plugin API.
fazekasgy@37 5 * It allows for writing Vamp plugins in Python.
fazekasgy@37 6
fazekasgy@37 7 * Centre for Digital Music, Queen Mary University of London.
fazekasgy@37 8 * Copyright (C) 2008-2009 Gyorgy Fazekas, QMUL. (See Vamp sources
fazekasgy@37 9 * for licence information.)
fazekasgy@37 10
fazekasgy@37 11 */
fazekasgy@37 12
fazekasgy@37 13 #include <Python.h>
fazekasgy@37 14
fazekasgy@37 15 #ifdef HAVE_NUMPY
fazekasgy@51 16
fazekasgy@51 17 // define a unique API pointer
Chris@66 18 #define PY_ARRAY_UNIQUE_SYMBOL VAMPY_ARRAY_API
Chris@66 19 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
fazekasgy@37 20 #include "numpy/arrayobject.h"
fazekasgy@51 21
fazekasgy@51 22 // prevent building with very old versions of numpy
fazekasgy@51 23 #ifndef NPY_VERSION
fazekasgy@51 24 #undef HAVE_NUMPY
fazekasgy@51 25 #endif
fazekasgy@51 26
fazekasgy@51 27 #endif
fazekasgy@51 28
fazekasgy@51 29 // this is not part of the API, but we will require it for a bug workaround
fazekasgy@51 30 // define this symbol if you use another version of numpy in the makefile
fazekasgy@51 31 // Vampy will not attempt to load a lower version than specified
fazekasgy@51 32 #ifdef HAVE_NUMPY
Chris@81 33 #ifndef NUMPY_MAJORVERSION
Chris@81 34 #define NUMPY_MAJORVERSION 1
fazekasgy@51 35 #endif
Chris@81 36 #ifndef NUMPY_MINORVERSION
Chris@81 37 #define NUMPY_MINORVERSION 9
Chris@81 38 #endif
Chris@81 39
fazekasgy@37 40 #endif
fazekasgy@37 41
fazekasgy@37 42 #include "vamp/vamp.h"
fazekasgy@37 43 #include "vamp-sdk/PluginAdapter.h"
fazekasgy@37 44 #include "PyPlugScanner.h"
fazekasgy@37 45 #include "PyPlugin.h"
fazekasgy@37 46 #include "PyExtensionModule.h"
fazekasgy@37 47 #include "PyExtensionManager.h"
Chris@67 48 #include "Debug.h"
gyorgyf@63 49 #include <sstream>
fazekasgy@37 50
fazekasgy@37 51 #ifdef _WIN32
fazekasgy@37 52 #define pathsep ('\\')
fazekasgy@37 53 #include <windows.h>
fazekasgy@37 54 #include <tchar.h>
fazekasgy@37 55 #else
fazekasgy@37 56 #define pathsep ('/')
fazekasgy@37 57 #include <dirent.h>
fazekasgy@37 58 #include <dlfcn.h>
fazekasgy@37 59 #endif
fazekasgy@37 60
fazekasgy@37 61 using std::cerr;
fazekasgy@37 62 using std::endl;
fazekasgy@37 63 using std::string;
fazekasgy@37 64 using std::vector;
fazekasgy@37 65
fazekasgy@37 66 static int adinstcount;
fazekasgy@37 67 static int totinstcount;
fazekasgy@51 68 static bool numpyInstalled = false;
fazekasgy@51 69 static bool arrayApiInitialised = false;
fazekasgy@37 70
fazekasgy@37 71 class PyPluginAdapter : public Vamp::PluginAdapterBase
fazekasgy@37 72 {
fazekasgy@37 73 public:
fazekasgy@37 74 PyPluginAdapter(std::string pyPlugId, PyObject* pyClass) :
fazekasgy@37 75 PluginAdapterBase(),
fazekasgy@37 76 m_plug(pyPlugId),
fazekasgy@37 77 m_pyClass(pyClass),
fazekasgy@37 78 m_failed(false)
fazekasgy@37 79 {
Chris@67 80 DSTREAM << "PyPluginAdapter:ctor:"<< adinstcount << ": " << m_plug << endl;
fazekasgy@37 81 adinstcount++;
fazekasgy@37 82 }
fazekasgy@37 83
fazekasgy@37 84 ~PyPluginAdapter()
fazekasgy@37 85 {
fazekasgy@37 86 }
fazekasgy@37 87
fazekasgy@37 88 bool failed() { return m_failed; }
fazekasgy@37 89 std::string getPlugKey() { return m_plug; }
fazekasgy@37 90
fazekasgy@37 91 protected:
fazekasgy@37 92 Vamp::Plugin *createPlugin(float inputSampleRate)
fazekasgy@37 93 {
fazekasgy@37 94 try {
fazekasgy@51 95 PyPlugin *plugin = new PyPlugin(m_plug, inputSampleRate, m_pyClass, totinstcount, numpyInstalled);
fazekasgy@37 96 return plugin;
fazekasgy@37 97 } catch (...) {
Chris@67 98 cerr << "ERROR: PyPluginAdapter::createPlugin: Failed to construct PyPlugin" << endl;
Chris@67 99 // any plugin with syntax errors will fail to construct
Chris@67 100 m_failed = true;
fazekasgy@37 101 return 0;
fazekasgy@37 102 }
fazekasgy@37 103 }
fazekasgy@37 104
fazekasgy@37 105 std::string m_plug;
fazekasgy@37 106 PyObject *m_pyClass;
fazekasgy@37 107 bool m_failed;
fazekasgy@37 108 };
fazekasgy@37 109
fazekasgy@51 110
fazekasgy@37 111 static void array_API_initialiser()
fazekasgy@37 112 {
fazekasgy@51 113 if (arrayApiInitialised) return;
fazekasgy@51 114
fazekasgy@51 115 /* Numpy 1.3 build note: there seems to be a bug
fazekasgy@51 116 in this version (at least on OS/X) which will cause memory
fazekasgy@51 117 access error in the array API import function if an earlier runtime
fazekasgy@51 118 version of Numpy is used when loading the library.
fazekasgy@51 119 (below is a horrible workaround)
fazekasgy@51 120 */
fazekasgy@51 121
fazekasgy@37 122 #ifdef HAVE_NUMPY
fazekasgy@51 123
Chris@81 124 string ver, majorver, minorver;
gyorgyf@63 125 std::istringstream verStream;
Chris@81 126 int numpyVersionMajor, numpyVersionMinor;
fazekasgy@51 127
fazekasgy@51 128 /// attmept to test numpy version before importing the array API
Chris@67 129 DSTREAM << "Numpy build information: ABI level: " << NPY_VERSION
Chris@81 130 << " Numpy version: " << NUMPY_MAJORVERSION << "." << NUMPY_MINORVERSION << endl;
fazekasgy@51 131
fazekasgy@51 132 PyObject *pyModule, *pyDict, *pyVer;
fazekasgy@51 133
fazekasgy@51 134 pyModule = PyImport_ImportModule("numpy"); //numpy.core.multiarray
fazekasgy@51 135 if (!pyModule) {
Chris@67 136 cerr << "ERROR: Vampy was compiled with Numpy support but Numpy does not seem to be installed." << endl;
fazekasgy@51 137 #ifdef __APPLE__
fazekasgy@51 138 cerr << "Hint: Check if Numpy is installed for the particular setup of Python used by Vampy (given by Python exec prefix)." << endl;
fazekasgy@51 139 #endif
fazekasgy@51 140 goto numpyFailure;
fazekasgy@51 141 }
fazekasgy@51 142
fazekasgy@51 143 pyDict = PyModule_GetDict(pyModule); // borrowed ref
fazekasgy@51 144 if (!pyDict) {
Chris@67 145 cerr << "ERROR: Can not access Numpy module dictionary." << endl;
fazekasgy@51 146 goto numpyFailure;
fazekasgy@51 147 }
fazekasgy@51 148
fazekasgy@51 149 pyVer = PyDict_GetItemString(pyDict,"__version__"); //borrowed ref
fazekasgy@51 150 if (!pyVer) {
Chris@67 151 cerr << "ERROR: Can not access Numpy version information." << endl;
fazekasgy@51 152 goto numpyFailure;
fazekasgy@51 153 }
fazekasgy@51 154
fazekasgy@51 155 ver = PyString_AsString(pyVer);
fazekasgy@51 156 ver = ver.substr(0,ver.rfind("."));
Chris@81 157 majorver = ver.substr(0,ver.rfind("."));
Chris@81 158 minorver = ver.substr(ver.rfind(".")+1);
Chris@81 159
Chris@81 160 // parse version string to float
Chris@81 161 verStream.str(majorver);
Chris@81 162 verStream >> numpyVersionMajor;
Chris@81 163 verStream.str(minorver);
Chris@81 164 verStream >> numpyVersionMinor;
Chris@67 165
Chris@81 166 DSTREAM << "Numpy runtime version: " << numpyVersionMajor << "." << numpyVersionMinor << endl;
Chris@81 167
Chris@81 168 if(numpyVersionMajor < NUMPY_MAJORVERSION ||
Chris@81 169 (numpyVersionMajor < NUMPY_MAJORVERSION &&
Chris@81 170 numpyVersionMinor < NUMPY_MINORVERSION)) {
Chris@81 171 cerr << "ERROR: Incompatible Numpy version found: " << ver << endl;
fazekasgy@51 172 goto numpyFailure;
fazekasgy@51 173 }
fazekasgy@51 174
fazekasgy@51 175 Py_DECREF(pyModule);
fazekasgy@51 176
fazekasgy@51 177 // At least we catch import errors, but if binary compatibility
fazekasgy@51 178 // has changed without notice, this would still fail.
fazekasgy@51 179 // However, we should never get to this point now anyway.
fazekasgy@37 180 import_array();
fazekasgy@51 181 if (PyErr_Occurred()) {
Chris@67 182 cerr << "ERROR: Import error while loading the Numpy Array API." << endl;
fazekasgy@51 183 PyErr_Print(); PyErr_Clear();
fazekasgy@51 184 goto numpyFailure;
fazekasgy@51 185 }
fazekasgy@51 186 else {
fazekasgy@51 187 numpyInstalled = true;
fazekasgy@51 188 arrayApiInitialised = true;
fazekasgy@51 189 return;
fazekasgy@51 190 }
fazekasgy@51 191
fazekasgy@51 192
fazekasgy@51 193 numpyFailure:
Chris@81 194 cerr << "Please make sure you have Numpy " << NUMPY_MAJORVERSION << "." << NUMPY_MINORVERSION << " or greater installed." << endl;
fazekasgy@51 195 cerr << "Vampy: Numpy support disabled." << endl;
fazekasgy@51 196 numpyInstalled = false;
fazekasgy@51 197 arrayApiInitialised = true;
fazekasgy@51 198 if (pyModule) Py_XDECREF(pyModule);
fazekasgy@51 199 return;
fazekasgy@51 200
fazekasgy@51 201 /*HAVE_NUMPY*/
fazekasgy@51 202 #endif
fazekasgy@51 203
fazekasgy@51 204 numpyInstalled = false;
fazekasgy@51 205 arrayApiInitialised = true;
fazekasgy@51 206 return;
fazekasgy@37 207 }
fazekasgy@37 208
fazekasgy@37 209
fazekasgy@37 210 static std::vector<PyPluginAdapter *> adapters;
fazekasgy@37 211 static bool haveScannedPlugins = false;
fazekasgy@37 212
fazekasgy@37 213 static bool tryPreload(string name)
fazekasgy@37 214 {
cannam@53 215 // cerr << "tryPreload: " << name << endl;
fazekasgy@37 216 #ifdef _WIN32
fazekasgy@37 217 void *lib = LoadLibrary(name.c_str());
fazekasgy@37 218 if (!lib) {
fazekasgy@37 219 return false;
fazekasgy@37 220 }
fazekasgy@37 221 #else
fazekasgy@37 222 void *lib = dlopen(name.c_str(), RTLD_NOW | RTLD_GLOBAL);
fazekasgy@37 223 if (!lib) {
cannam@56 224 //perror("dlopen");
fazekasgy@37 225 return false;
fazekasgy@37 226 }
fazekasgy@37 227 #endif
Chris@67 228 DSTREAM << "Preloaded Python from " << name << endl;
fazekasgy@37 229 return true;
fazekasgy@37 230 }
fazekasgy@37 231
fazekasgy@37 232 static bool preloadPython()
fazekasgy@37 233 {
fazekasgy@37 234 #ifdef _WIN32
fazekasgy@37 235 // this doesn't seem to be necessary at all on Windows
fazekasgy@37 236 return true;
fazekasgy@37 237 #endif
fazekasgy@37 238
fazekasgy@37 239 string pyver = Py_GetVersion();
fazekasgy@37 240 int dots = 2;
fazekasgy@37 241 string shortver;
fazekasgy@37 242 for (size_t i = 0; i < pyver.length(); ++i) {
fazekasgy@37 243 if (pyver[i] == '.') {
fazekasgy@37 244 if (--dots == 0) {
fazekasgy@37 245 shortver = pyver.substr(0, i);
fazekasgy@37 246 break;
fazekasgy@37 247 }
fazekasgy@37 248 }
fazekasgy@37 249 }
Chris@67 250 DSTREAM << "Short version: " << shortver << endl;
Chris@67 251 // this is useful to find out where the loaded library might be loaded from
Chris@67 252 DSTREAM << "Python exec prefix: " << Py_GetExecPrefix() << endl;
fazekasgy@37 253
cannam@55 254 char *pylib = getenv("VAMPY_PYLIB");
cannam@54 255 if (pylib && *pylib) {
Chris@67 256 DSTREAM << "Trying to preload Python from specified location " << pylib
Chris@67 257 << "..." << endl;
cannam@54 258 return tryPreload(string(pylib));
cannam@54 259 }
cannam@54 260
fazekasgy@37 261 vector<string> pfxs;
cannam@53 262 pfxs.push_back(string(Py_GetExecPrefix()) + "/");
cannam@53 263 pfxs.push_back(string(Py_GetExecPrefix()) + "/lib/");
fazekasgy@37 264 pfxs.push_back("");
fazekasgy@37 265 pfxs.push_back("/usr/lib/");
fazekasgy@37 266 pfxs.push_back("/usr/local/lib/");
fazekasgy@37 267 char buffer[5];
fazekasgy@37 268
fazekasgy@37 269 // hahaha! grossness is like a brother to us
fazekasgy@37 270 #ifdef __APPLE__
fazekasgy@37 271 for (size_t pfxidx = 0; pfxidx < pfxs.size(); ++pfxidx) {
cannam@53 272 // cerr << "prefix: " << pfxs[pfxidx] << endl;
cannam@53 273 if (tryPreload(pfxs[pfxidx] + string("Python"))) return true;
fazekasgy@37 274 for (int minor = 8; minor >= 0; --minor) {
fazekasgy@37 275 sprintf(buffer, "%d", minor);
fazekasgy@37 276 if (tryPreload(pfxs[pfxidx] + string("libpython") + shortver + ".dylib." + buffer)) return true;
fazekasgy@37 277 }
fazekasgy@37 278 if (tryPreload(pfxs[pfxidx] + string("libpython") + shortver + ".dylib")) return true;
fazekasgy@37 279 if (tryPreload(pfxs[pfxidx] + string("libpython.dylib"))) return true;
fazekasgy@37 280 }
fazekasgy@37 281 #else
fazekasgy@37 282 for (size_t pfxidx = 0; pfxidx < pfxs.size(); ++pfxidx) {
fazekasgy@37 283 for (int minor = 8; minor >= 0; --minor) {
fazekasgy@37 284 sprintf(buffer, "%d", minor);
fazekasgy@37 285 if (tryPreload(pfxs[pfxidx] + string("libpython") + shortver + ".so." + buffer)) return true;
fazekasgy@37 286 }
fazekasgy@37 287 if (tryPreload(pfxs[pfxidx] + string("libpython") + shortver + ".so")) return true;
fazekasgy@37 288 if (tryPreload(pfxs[pfxidx] + string("libpython.so"))) return true;
fazekasgy@37 289 }
fazekasgy@37 290 #endif
fazekasgy@37 291
fazekasgy@37 292 return false;
fazekasgy@37 293 }
fazekasgy@37 294
fazekasgy@37 295
fazekasgy@37 296 static PyExtensionManager pyExtensionManager;
fazekasgy@37 297
fazekasgy@37 298 const VampPluginDescriptor
fazekasgy@37 299 *vampGetPluginDescriptor(unsigned int version,unsigned int index)
Chris@79 300 {
Chris@79 301 DSTREAM << "# vampGetPluginDescriptor(" << version << "," << index << ")" << endl;
Chris@79 302
fazekasgy@37 303 if (version < 1) return 0;
fazekasgy@37 304
fazekasgy@37 305 int isPythonInitialized = Py_IsInitialized();
Chris@67 306 DSTREAM << "# isPythonInitialized: " << isPythonInitialized << endl;
Chris@67 307 DSTREAM << "# haveScannedPlugins: " << haveScannedPlugins << endl;
fazekasgy@37 308
fazekasgy@37 309 if (!haveScannedPlugins) {
fazekasgy@37 310
fazekasgy@37 311 if (!isPythonInitialized){
fazekasgy@37 312
fazekasgy@37 313 if (!preloadPython())
fazekasgy@37 314 cerr << "Warning: Could not preload Python. Dynamic loading in scripts will fail." << endl;
fazekasgy@37 315 if (PyImport_AppendInittab("vampy",initvampy) != 0)
fazekasgy@37 316 cerr << "Warning: Extension module could not be added to module inittab." << endl;
fazekasgy@37 317 Py_Initialize();
fazekasgy@51 318 array_API_initialiser();
fazekasgy@37 319 initvampy();
Chris@67 320 DSTREAM << "# isPythonInitialized after initialize: " << Py_IsInitialized() << endl;
fazekasgy@37 321 }
fazekasgy@37 322
fazekasgy@37 323 vector<string> pyPlugs;
fazekasgy@37 324 vector<string> pyPath;
fazekasgy@37 325 vector<PyObject *> pyClasses;
fazekasgy@37 326 static PyPlugScanner *scanner;
fazekasgy@37 327
fazekasgy@37 328 //Scanning Plugins
Chris@67 329 DSTREAM << "Scanning Vampy Plugins" << endl;
fazekasgy@37 330 scanner = PyPlugScanner::getInstance();
fazekasgy@37 331
fazekasgy@37 332 // added env. varable support VAMPY_EXTPATH
fazekasgy@37 333 pyPath=scanner->getAllValidPath();
fazekasgy@37 334 scanner->setPath(pyPath);
fazekasgy@37 335
fazekasgy@37 336 // added env. variable support:
fazekasgy@37 337 // VAMPY_COMPILED=1 to recognise .pyc files (default is 1)
fazekasgy@37 338 pyPlugs = scanner->getPyPlugs();
fazekasgy@37 339
Chris@67 340 DSTREAM << "Found " << pyPlugs.size() << " Scripts." << endl;
fazekasgy@37 341 //TODO: should this support multiple classes per script (?)
fazekasgy@37 342 pyClasses = scanner->getPyClasses();
Chris@67 343 DSTREAM << "Found " << pyClasses.size() << " Classes." << endl;
fazekasgy@37 344
fazekasgy@37 345 for (size_t i = 0; i < pyPlugs.size(); ++i) {
fazekasgy@37 346 adapters.push_back( new PyPluginAdapter(pyPlugs[i],pyClasses[i]));
fazekasgy@37 347 }
fazekasgy@37 348 pyExtensionManager.setPlugModuleNames(pyPlugs);
fazekasgy@37 349 pyExtensionManager.initExtension();
fazekasgy@37 350 array_API_initialiser();
fazekasgy@37 351 haveScannedPlugins=true;
fazekasgy@37 352 }
fazekasgy@37 353
Chris@67 354 DSTREAM << "Accessing adapter index: " << index << " (adapters: " << adapters.size() << ")" << endl;
fazekasgy@37 355
Chris@84 356 while (index < adapters.size()) {
fazekasgy@37 357
fazekasgy@37 358 const VampPluginDescriptor *tmp = adapters[index]->getDescriptor();
fazekasgy@37 359
fazekasgy@37 360 if (adapters[index]->failed()) {
fazekasgy@37 361 cerr << "\nERROR: [in vampGetPluginDescriptor] Removing adapter of: \n'"
Chris@87 362 << adapters[index]->getPlugKey() << "'\n"
Chris@87 363 << "The plugin has failed to construct. Hint: Check __init__() function." << endl;
fazekasgy@37 364 pyExtensionManager.deleteModuleName(adapters[index]->getPlugKey());
fazekasgy@37 365 delete adapters[index];
Chris@87 366 adapters.erase(adapters.begin() + index);
Chris@84 367 continue;
fazekasgy@37 368 }
fazekasgy@37 369
fazekasgy@37 370 return tmp;
Chris@84 371 }
fazekasgy@37 372
Chris@84 373 return 0;
fazekasgy@37 374 }
fazekasgy@37 375
fazekasgy@37 376
fazekasgy@37 377
fazekasgy@37 378
fazekasgy@37 379
fazekasgy@37 380
fazekasgy@37 381
fazekasgy@37 382