annotate rdf/PluginRDFIndexer.cpp @ 482:f54381e01141

* Fix a deadlock * Make debug output more suited to datastore rdf implementation
author Chris Cannam
date Fri, 14 Nov 2008 12:29:48 +0000
parents a82645e788fc
children b13213785a6f
rev   line source
Chris@439 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@439 2
Chris@439 3 /*
Chris@439 4 Sonic Visualiser
Chris@439 5 An audio file viewer and annotation editor.
Chris@439 6 Centre for Digital Music, Queen Mary, University of London.
Chris@439 7 This file copyright 2008 QMUL.
Chris@439 8
Chris@439 9 This program is free software; you can redistribute it and/or
Chris@439 10 modify it under the terms of the GNU General Public License as
Chris@439 11 published by the Free Software Foundation; either version 2 of the
Chris@439 12 License, or (at your option) any later version. See the file
Chris@439 13 COPYING included with this distribution for more information.
Chris@439 14 */
Chris@439 15
Chris@439 16 #include "PluginRDFIndexer.h"
Chris@439 17
Chris@439 18 #include "SimpleSPARQLQuery.h"
Chris@439 19
Chris@467 20 #include "data/fileio/CachedFile.h"
Chris@471 21 #include "data/fileio/FileSource.h"
Chris@461 22 #include "data/fileio/PlaylistFileReader.h"
Chris@439 23 #include "plugin/PluginIdentifier.h"
Chris@439 24
Chris@457 25 #include "base/Profiler.h"
Chris@457 26
Chris@475 27 #include <vamp-hostsdk/PluginHostAdapter.h>
Chris@439 28
Chris@439 29 #include <QFileInfo>
Chris@439 30 #include <QDir>
Chris@439 31 #include <QUrl>
Chris@461 32 #include <QDateTime>
Chris@461 33 #include <QSettings>
Chris@461 34 #include <QFile>
Chris@439 35
Chris@439 36 #include <iostream>
Chris@439 37 using std::cerr;
Chris@439 38 using std::endl;
Chris@439 39 using std::vector;
Chris@439 40 using std::string;
Chris@439 41 using Vamp::PluginHostAdapter;
Chris@439 42
Chris@439 43 PluginRDFIndexer *
Chris@439 44 PluginRDFIndexer::m_instance = 0;
Chris@439 45
Chris@439 46 PluginRDFIndexer *
Chris@439 47 PluginRDFIndexer::getInstance()
Chris@439 48 {
Chris@439 49 if (!m_instance) m_instance = new PluginRDFIndexer();
Chris@439 50 return m_instance;
Chris@439 51 }
Chris@439 52
Chris@439 53 PluginRDFIndexer::PluginRDFIndexer()
Chris@439 54 {
Chris@477 55 indexInstalledURLs();
Chris@477 56 }
Chris@477 57
Chris@477 58 PluginRDFIndexer::~PluginRDFIndexer()
Chris@477 59 {
Chris@477 60 QMutexLocker locker(&m_mutex);
Chris@477 61 }
Chris@477 62
Chris@477 63 void
Chris@477 64 PluginRDFIndexer::indexInstalledURLs()
Chris@477 65 {
Chris@439 66 vector<string> paths = PluginHostAdapter::getPluginPath();
Chris@439 67
Chris@439 68 QStringList filters;
Chris@439 69 filters << "*.n3";
Chris@439 70 filters << "*.N3";
Chris@439 71 filters << "*.rdf";
Chris@439 72 filters << "*.RDF";
Chris@439 73
Chris@439 74 // Search each Vamp plugin path for a .rdf file that either has
Chris@439 75 // name "soname", "soname:label" or "soname/label" plus RDF
Chris@439 76 // extension. Use that order of preference, and prefer n3 over
Chris@439 77 // rdf extension.
Chris@439 78
Chris@439 79 for (vector<string>::const_iterator i = paths.begin(); i != paths.end(); ++i) {
Chris@439 80
Chris@439 81 QDir dir(i->c_str());
Chris@439 82 if (!dir.exists()) continue;
Chris@439 83
Chris@439 84 QStringList entries = dir.entryList
Chris@439 85 (filters, QDir::Files | QDir::Readable);
Chris@439 86
Chris@439 87 for (QStringList::const_iterator j = entries.begin();
Chris@439 88 j != entries.end(); ++j) {
Chris@439 89 QFileInfo fi(dir.filePath(*j));
Chris@439 90 indexFile(fi.absoluteFilePath());
Chris@439 91 }
Chris@439 92
Chris@439 93 QStringList subdirs = dir.entryList
Chris@439 94 (QDir::AllDirs | QDir::NoDotAndDotDot | QDir::Readable);
Chris@439 95
Chris@439 96 for (QStringList::const_iterator j = subdirs.begin();
Chris@439 97 j != subdirs.end(); ++j) {
Chris@439 98 QDir subdir(dir.filePath(*j));
Chris@439 99 if (subdir.exists()) {
Chris@439 100 entries = subdir.entryList
Chris@439 101 (filters, QDir::Files | QDir::Readable);
Chris@439 102 for (QStringList::const_iterator k = entries.begin();
Chris@439 103 k != entries.end(); ++k) {
Chris@439 104 QFileInfo fi(subdir.filePath(*k));
Chris@439 105 indexFile(fi.absoluteFilePath());
Chris@439 106 }
Chris@439 107 }
Chris@439 108 }
Chris@439 109 }
Chris@439 110 }
Chris@439 111
Chris@461 112 bool
Chris@461 113 PluginRDFIndexer::indexConfiguredURLs()
Chris@461 114 {
Chris@461 115 std::cerr << "PluginRDFIndexer::indexConfiguredURLs" << std::endl;
Chris@461 116
Chris@461 117 QSettings settings;
Chris@461 118 settings.beginGroup("RDF");
Chris@461 119
Chris@461 120 QString indexKey("rdf-indices");
Chris@461 121 QStringList indices = settings.value(indexKey).toStringList();
Chris@461 122
Chris@461 123 for (int i = 0; i < indices.size(); ++i) {
Chris@461 124
Chris@461 125 QString index = indices[i];
Chris@461 126
Chris@461 127 std::cerr << "PluginRDFIndexer::indexConfiguredURLs: index url is "
Chris@461 128 << index.toStdString() << std::endl;
Chris@461 129
Chris@467 130 CachedFile cf(index);
Chris@467 131 if (!cf.isOK()) continue;
Chris@467 132
Chris@467 133 FileSource indexSource(cf.getLocalFilename());
Chris@461 134
Chris@461 135 PlaylistFileReader reader(indexSource);
Chris@461 136 if (!reader.isOK()) continue;
Chris@461 137
Chris@461 138 PlaylistFileReader::Playlist list = reader.load();
Chris@461 139 for (PlaylistFileReader::Playlist::const_iterator j = list.begin();
Chris@461 140 j != list.end(); ++j) {
Chris@461 141 std::cerr << "PluginRDFIndexer::indexConfiguredURLs: url is "
Chris@461 142 << j->toStdString() << std::endl;
Chris@461 143 indexURL(*j);
Chris@461 144 }
Chris@461 145 }
Chris@461 146
Chris@461 147 QString urlListKey("rdf-urls");
Chris@461 148 QStringList urls = settings.value(urlListKey).toStringList();
Chris@461 149
Chris@461 150 for (int i = 0; i < urls.size(); ++i) {
Chris@461 151 indexURL(urls[i]);
Chris@461 152 }
Chris@461 153
Chris@461 154 settings.endGroup();
Chris@461 155 return true;
Chris@461 156 }
Chris@461 157
Chris@439 158 QString
Chris@439 159 PluginRDFIndexer::getURIForPluginId(QString pluginId)
Chris@439 160 {
Chris@461 161 QMutexLocker locker(&m_mutex);
Chris@461 162
Chris@439 163 if (m_idToUriMap.find(pluginId) == m_idToUriMap.end()) return "";
Chris@439 164 return m_idToUriMap[pluginId];
Chris@439 165 }
Chris@439 166
Chris@439 167 QString
Chris@439 168 PluginRDFIndexer::getIdForPluginURI(QString uri)
Chris@439 169 {
Chris@476 170 m_mutex.lock();
Chris@461 171
Chris@439 172 if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) {
Chris@439 173
Chris@476 174 m_mutex.unlock();
Chris@476 175
Chris@439 176 // Haven't found this uri referenced in any document on the
Chris@439 177 // local filesystem; try resolving the pre-fragment part of
Chris@439 178 // the uri as a document URL and reading that if possible.
Chris@439 179
Chris@439 180 // Because we may want to refer to this document again, we
Chris@439 181 // cache it locally if it turns out to exist.
Chris@439 182
Chris@439 183 cerr << "PluginRDFIndexer::getIdForPluginURI: NOTE: Failed to find a local RDF document describing plugin <" << uri.toStdString() << ">: attempting to retrieve one remotely by guesswork" << endl;
Chris@439 184
Chris@439 185 QString baseUrl = QUrl(uri).toString(QUrl::RemoveFragment);
Chris@439 186
Chris@457 187 indexURL(baseUrl);
Chris@439 188
Chris@476 189 m_mutex.lock();
Chris@476 190
Chris@439 191 if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) {
Chris@439 192 m_uriToIdMap[uri] = "";
Chris@439 193 }
Chris@439 194 }
Chris@439 195
Chris@476 196 QString id = m_uriToIdMap[uri];
Chris@476 197 m_mutex.unlock();
Chris@476 198 return id;
Chris@439 199 }
Chris@439 200
Chris@439 201 QString
Chris@439 202 PluginRDFIndexer::getDescriptionURLForPluginId(QString pluginId)
Chris@439 203 {
Chris@461 204 QMutexLocker locker(&m_mutex);
Chris@461 205
Chris@439 206 if (m_idToDescriptionMap.find(pluginId) == m_idToDescriptionMap.end()) return "";
Chris@439 207 return m_idToDescriptionMap[pluginId];
Chris@439 208 }
Chris@439 209
Chris@439 210 QString
Chris@439 211 PluginRDFIndexer::getDescriptionURLForPluginURI(QString uri)
Chris@439 212 {
Chris@461 213 QMutexLocker locker(&m_mutex);
Chris@461 214
Chris@439 215 QString id = getIdForPluginURI(uri);
Chris@439 216 if (id == "") return "";
Chris@439 217 return getDescriptionURLForPluginId(id);
Chris@439 218 }
Chris@439 219
Chris@456 220 QStringList
Chris@456 221 PluginRDFIndexer::getIndexedPluginIds()
Chris@456 222 {
Chris@461 223 QMutexLocker locker(&m_mutex);
Chris@461 224
Chris@456 225 QStringList ids;
Chris@456 226 for (StringMap::const_iterator i = m_idToDescriptionMap.begin();
Chris@456 227 i != m_idToDescriptionMap.end(); ++i) {
Chris@456 228 ids.push_back(i->first);
Chris@456 229 }
Chris@456 230 return ids;
Chris@456 231 }
Chris@456 232
Chris@439 233 bool
Chris@439 234 PluginRDFIndexer::indexFile(QString filepath)
Chris@439 235 {
Chris@439 236 QUrl url = QUrl::fromLocalFile(filepath);
Chris@439 237 QString urlString = url.toString();
Chris@439 238 return indexURL(urlString);
Chris@439 239 }
Chris@461 240
Chris@439 241 bool
Chris@439 242 PluginRDFIndexer::indexURL(QString urlString)
Chris@439 243 {
Chris@457 244 Profiler profiler("PluginRDFIndexer::indexURL");
Chris@457 245
Chris@461 246 std::cerr << "PluginRDFIndexer::indexURL(" << urlString.toStdString() << ")" << std::endl;
Chris@461 247
Chris@461 248 QMutexLocker locker(&m_mutex);
Chris@461 249
Chris@457 250 QString localString = urlString;
Chris@457 251
Chris@457 252 if (FileSource::isRemote(urlString) &&
Chris@457 253 FileSource::canHandleScheme(urlString)) {
Chris@457 254
Chris@467 255 CachedFile cf(urlString);
Chris@467 256 if (!cf.isOK()) {
Chris@467 257 return false;
Chris@467 258 }
Chris@467 259
Chris@480 260 localString = "file://" + cf.getLocalFilename(); //!!! crud - fix!
Chris@457 261 }
Chris@457 262
Chris@439 263 // cerr << "PluginRDFIndexer::indexURL: url = <" << urlString.toStdString() << ">" << endl;
Chris@481 264 /*!!!
Chris@439 265 SimpleSPARQLQuery query
Chris@480 266 (localString,
Chris@480 267 QString
Chris@439 268 (
Chris@439 269 " PREFIX vamp: <http://purl.org/ontology/vamp/> "
Chris@439 270
Chris@439 271 " SELECT ?plugin ?library_id ?plugin_id "
Chris@439 272 " FROM <%1> "
Chris@439 273
Chris@439 274 " WHERE { "
Chris@439 275 " ?plugin a vamp:Plugin . "
Chris@439 276
Chris@439 277 // Make the identifier and library parts optional, so
Chris@439 278 // that we can check and report helpfully if one or both
Chris@439 279 // is absent instead of just getting no results
Chris@439 280
Chris@440 281 //!!! No -- because of rasqal's inability to correctly
Chris@440 282 // handle more than one OPTIONAL graph in a query, let's
Chris@440 283 // make identifier compulsory after all
Chris@440 284 //" OPTIONAL { ?plugin vamp:identifier ?plugin_id } . "
Chris@440 285
Chris@440 286 " ?plugin vamp:identifier ?plugin_id . "
Chris@439 287
Chris@439 288 " OPTIONAL { "
Chris@439 289 " ?library a vamp:PluginLibrary ; "
Chris@439 290 " vamp:available_plugin ?plugin ; "
Chris@439 291 " vamp:identifier ?library_id "
Chris@439 292 " } "
Chris@439 293 " } "
Chris@439 294 )
Chris@457 295 .arg(localString));
Chris@481 296 */
Chris@481 297 SimpleSPARQLQuery query
Chris@481 298 (localString,
Chris@481 299 QString
Chris@481 300 (
Chris@481 301 " PREFIX vamp: <http://purl.org/ontology/vamp/> "
Chris@481 302
Chris@481 303 " SELECT ?plugin ?library ?plugin_id "
Chris@481 304 " FROM <%1> "
Chris@481 305
Chris@481 306 " WHERE { "
Chris@481 307 " ?plugin a vamp:Plugin . "
Chris@481 308 " ?plugin vamp:identifier ?plugin_id . "
Chris@481 309
Chris@481 310 " OPTIONAL { "
Chris@481 311 " ?library vamp:available_plugin ?plugin "
Chris@481 312 " } "
Chris@481 313 " } "
Chris@481 314 )
Chris@481 315 .arg(localString));
Chris@439 316
Chris@439 317 SimpleSPARQLQuery::ResultList results = query.execute();
Chris@439 318
Chris@439 319 if (!query.isOK()) {
Chris@439 320 cerr << "ERROR: PluginRDFIndexer::indexURL: ERROR: Failed to index document at <"
Chris@439 321 << urlString.toStdString() << ">: "
Chris@439 322 << query.getErrorString().toStdString() << endl;
Chris@439 323 return false;
Chris@439 324 }
Chris@439 325
Chris@439 326 if (results.empty()) {
Chris@439 327 cerr << "PluginRDFIndexer::indexURL: NOTE: Document at <"
Chris@439 328 << urlString.toStdString()
Chris@439 329 << "> does not describe any vamp:Plugin resources" << endl;
Chris@439 330 return false;
Chris@439 331 }
Chris@439 332
Chris@439 333 bool foundSomething = false;
Chris@439 334 bool addedSomething = false;
Chris@439 335
Chris@439 336 for (SimpleSPARQLQuery::ResultList::iterator i = results.begin();
Chris@439 337 i != results.end(); ++i) {
Chris@439 338
Chris@439 339 QString pluginUri = (*i)["plugin"].value;
Chris@481 340 //!!! QString soname = (*i)["library_id"].value;
Chris@481 341 QString soUri = (*i)["library"].value;
Chris@439 342 QString identifier = (*i)["plugin_id"].value;
Chris@439 343
Chris@439 344 if (identifier == "") {
Chris@482 345 cerr << "PluginRDFIndexer::indexURL: NOTE: No vamp:identifier for plugin <"
Chris@439 346 << pluginUri.toStdString() << ">"
Chris@439 347 << endl;
Chris@439 348 continue;
Chris@439 349 }
Chris@481 350 if (soUri == "") {
Chris@482 351 cerr << "PluginRDFIndexer::indexURL: NOTE: No implementation library for plugin <"
Chris@482 352 << pluginUri.toStdString() << ">"
Chris@439 353 << endl;
Chris@439 354 continue;
Chris@439 355 }
Chris@481 356
Chris@481 357 QString sonameQuery =
Chris@481 358 QString(
Chris@481 359 " PREFIX vamp: <http://purl.org/ontology/vamp/> "
Chris@481 360 " SELECT ?library_id "
Chris@481 361 " FROM <%1> "
Chris@481 362 " WHERE { "
Chris@481 363 " <%2> vamp:identifier ?library_id "
Chris@481 364 " } "
Chris@481 365 )
Chris@481 366 .arg(localString)
Chris@481 367 .arg(soUri);
Chris@481 368
Chris@481 369 SimpleSPARQLQuery::Value sonameValue =
Chris@481 370 SimpleSPARQLQuery::singleResultQuery(localString, sonameQuery, "library_id");
Chris@481 371 QString soname = sonameValue.value;
Chris@481 372 if (soname == "") {
Chris@482 373 cerr << "PluginRDFIndexer::indexURL: NOTE: No identifier for library <"
Chris@481 374 << soUri.toStdString() << ">"
Chris@481 375 << endl;
Chris@481 376 continue;
Chris@481 377 }
Chris@481 378
Chris@481 379
Chris@439 380 /*
Chris@439 381 cerr << "PluginRDFIndexer::indexURL: Document for plugin \""
Chris@439 382 << soname.toStdString() << ":" << identifier.toStdString()
Chris@439 383 << "\" (uri <" << pluginUri.toStdString() << ">) is at url <"
Chris@439 384 << urlString.toStdString() << ">" << endl;
Chris@439 385 */
Chris@439 386 QString pluginId = PluginIdentifier::createIdentifier
Chris@439 387 ("vamp", soname, identifier);
Chris@439 388
Chris@439 389 foundSomething = true;
Chris@439 390
Chris@439 391 if (m_idToDescriptionMap.find(pluginId) != m_idToDescriptionMap.end()) {
Chris@482 392 /*!!!
Chris@482 393
Chris@482 394 This can happen quite legitimately when using an RDF datastore rather
Chris@482 395 than querying individual files, as of course the datastore contains
Chris@482 396 all plugin data found so far, and each time a file is added to it,
Chris@482 397 subsequent queries will return all older plugins as well.
Chris@482 398
Chris@482 399 It would be more efficient to add everything at once and then do all
Chris@482 400 queries, of course.
Chris@482 401
Chris@439 402 cerr << "PluginRDFIndexer::indexURL: NOTE: Plugin id \""
Chris@439 403 << pluginId.toStdString() << "\", described in document at <"
Chris@439 404 << urlString.toStdString()
Chris@439 405 << ">, has already been described in document <"
Chris@439 406 << m_idToDescriptionMap[pluginId].toStdString()
Chris@439 407 << ">: ignoring this new description" << endl;
Chris@482 408 */
Chris@439 409 continue;
Chris@439 410 }
Chris@439 411
Chris@439 412 m_idToDescriptionMap[pluginId] = urlString;
Chris@439 413 m_idToUriMap[pluginId] = pluginUri;
Chris@439 414
Chris@439 415 addedSomething = true;
Chris@439 416
Chris@439 417 if (pluginUri != "") {
Chris@439 418 if (m_uriToIdMap.find(pluginUri) != m_uriToIdMap.end()) {
Chris@439 419 cerr << "PluginRDFIndexer::indexURL: WARNING: Found multiple plugins with the same URI:" << endl;
Chris@439 420 cerr << " 1. Plugin id \"" << m_uriToIdMap[pluginUri].toStdString() << "\"" << endl;
Chris@439 421 cerr << " described in <" << m_idToDescriptionMap[m_uriToIdMap[pluginUri]].toStdString() << ">" << endl;
Chris@439 422 cerr << " 2. Plugin id \"" << pluginId.toStdString() << "\"" << endl;
Chris@439 423 cerr << " described in <" << urlString.toStdString() << ">" << endl;
Chris@439 424 cerr << "both claim URI <" << pluginUri.toStdString() << ">" << endl;
Chris@439 425 } else {
Chris@439 426 m_uriToIdMap[pluginUri] = pluginId;
Chris@439 427 }
Chris@439 428 }
Chris@439 429 }
Chris@439 430
Chris@439 431 if (!foundSomething) {
Chris@439 432 cerr << "PluginRDFIndexer::indexURL: NOTE: Document at <"
Chris@439 433 << urlString.toStdString()
Chris@439 434 << "> does not sufficiently describe any plugins" << endl;
Chris@439 435 }
Chris@439 436
Chris@439 437 return addedSomething;
Chris@439 438 }
Chris@439 439
Chris@439 440
Chris@439 441