annotate rdf/PluginRDFIndexer.cpp @ 1272:6a7ea3bd0e10

Send RDF-indexer warnings to debug rather than cerr
author Chris Cannam
date Mon, 21 Nov 2016 16:32:58 +0000
parents f5cd33909744
children 70e172e6cc59
rev   line source
Chris@439 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@439 2
Chris@439 3 /*
Chris@439 4 Sonic Visualiser
Chris@439 5 An audio file viewer and annotation editor.
Chris@439 6 Centre for Digital Music, Queen Mary, University of London.
Chris@727 7 This file copyright 2008-2012 QMUL.
Chris@439 8
Chris@439 9 This program is free software; you can redistribute it and/or
Chris@439 10 modify it under the terms of the GNU General Public License as
Chris@439 11 published by the Free Software Foundation; either version 2 of the
Chris@439 12 License, or (at your option) any later version. See the file
Chris@439 13 COPYING included with this distribution for more information.
Chris@439 14 */
Chris@439 15
Chris@439 16 #include "PluginRDFIndexer.h"
Chris@439 17
Chris@467 18 #include "data/fileio/CachedFile.h"
Chris@471 19 #include "data/fileio/FileSource.h"
Chris@461 20 #include "data/fileio/PlaylistFileReader.h"
Chris@439 21 #include "plugin/PluginIdentifier.h"
Chris@439 22
Chris@457 23 #include "base/Profiler.h"
Chris@1272 24 #include "base/Debug.h"
Chris@457 25
Chris@475 26 #include <vamp-hostsdk/PluginHostAdapter.h>
Chris@439 27
Chris@725 28 #include <dataquay/BasicStore.h>
Chris@725 29 #include <dataquay/RDFException.h>
Chris@725 30
Chris@439 31 #include <QFileInfo>
Chris@439 32 #include <QDir>
Chris@439 33 #include <QUrl>
Chris@461 34 #include <QDateTime>
Chris@461 35 #include <QSettings>
Chris@461 36 #include <QFile>
Chris@439 37
Chris@439 38 #include <iostream>
Chris@843 39
Chris@439 40 using std::vector;
Chris@439 41 using std::string;
Chris@439 42 using Vamp::PluginHostAdapter;
Chris@439 43
Chris@725 44 using Dataquay::Uri;
Chris@725 45 using Dataquay::Node;
Chris@725 46 using Dataquay::Nodes;
Chris@725 47 using Dataquay::Triple;
Chris@725 48 using Dataquay::Triples;
Chris@725 49 using Dataquay::BasicStore;
Chris@725 50 using Dataquay::RDFException;
Chris@725 51 using Dataquay::RDFDuplicateImportException;
Chris@725 52
Chris@439 53 PluginRDFIndexer *
Chris@439 54 PluginRDFIndexer::m_instance = 0;
Chris@439 55
Chris@439 56 PluginRDFIndexer *
Chris@439 57 PluginRDFIndexer::getInstance()
Chris@439 58 {
Chris@439 59 if (!m_instance) m_instance = new PluginRDFIndexer();
Chris@439 60 return m_instance;
Chris@439 61 }
Chris@439 62
Chris@725 63 PluginRDFIndexer::PluginRDFIndexer() :
Chris@725 64 m_index(new Dataquay::BasicStore)
Chris@439 65 {
Chris@725 66 m_index->addPrefix("vamp", Uri("http://purl.org/ontology/vamp/"));
Chris@725 67 m_index->addPrefix("foaf", Uri("http://xmlns.com/foaf/0.1/"));
Chris@725 68 m_index->addPrefix("dc", Uri("http://purl.org/dc/elements/1.1/"));
Chris@477 69 indexInstalledURLs();
Chris@477 70 }
Chris@477 71
Chris@725 72 const BasicStore *
Chris@725 73 PluginRDFIndexer::getIndex()
Chris@725 74 {
Chris@725 75 return m_index;
Chris@725 76 }
Chris@725 77
Chris@477 78 PluginRDFIndexer::~PluginRDFIndexer()
Chris@477 79 {
Chris@477 80 QMutexLocker locker(&m_mutex);
Chris@477 81 }
Chris@477 82
Chris@477 83 void
Chris@477 84 PluginRDFIndexer::indexInstalledURLs()
Chris@477 85 {
Chris@439 86 vector<string> paths = PluginHostAdapter::getPluginPath();
Chris@439 87
Chris@1272 88 // SVDEBUG << "\nPluginRDFIndexer::indexInstalledURLs: pid is " << getpid() << endl;
Chris@730 89
Chris@439 90 QStringList filters;
Chris@731 91 filters << "*.ttl";
Chris@731 92 filters << "*.TTL";
Chris@439 93 filters << "*.n3";
Chris@439 94 filters << "*.N3";
Chris@439 95 filters << "*.rdf";
Chris@439 96 filters << "*.RDF";
Chris@439 97
Chris@731 98 // Search each Vamp plugin path for an RDF file that either has
Chris@439 99 // name "soname", "soname:label" or "soname/label" plus RDF
Chris@731 100 // extension. Use that order of preference, and prefer ttl over
Chris@731 101 // n3 over rdf extension.
Chris@439 102
Chris@439 103 for (vector<string>::const_iterator i = paths.begin(); i != paths.end(); ++i) {
Chris@718 104
Chris@439 105 QDir dir(i->c_str());
Chris@439 106 if (!dir.exists()) continue;
Chris@439 107
Chris@439 108 QStringList entries = dir.entryList
Chris@439 109 (filters, QDir::Files | QDir::Readable);
Chris@439 110
Chris@439 111 for (QStringList::const_iterator j = entries.begin();
Chris@439 112 j != entries.end(); ++j) {
Chris@718 113
Chris@439 114 QFileInfo fi(dir.filePath(*j));
Chris@489 115 pullFile(fi.absoluteFilePath());
Chris@439 116 }
Chris@439 117
Chris@439 118 QStringList subdirs = dir.entryList
Chris@439 119 (QDir::AllDirs | QDir::NoDotAndDotDot | QDir::Readable);
Chris@439 120
Chris@439 121 for (QStringList::const_iterator j = subdirs.begin();
Chris@439 122 j != subdirs.end(); ++j) {
Chris@718 123
Chris@439 124 QDir subdir(dir.filePath(*j));
Chris@439 125 if (subdir.exists()) {
Chris@439 126 entries = subdir.entryList
Chris@439 127 (filters, QDir::Files | QDir::Readable);
Chris@439 128 for (QStringList::const_iterator k = entries.begin();
Chris@439 129 k != entries.end(); ++k) {
Chris@439 130 QFileInfo fi(subdir.filePath(*k));
Chris@489 131 pullFile(fi.absoluteFilePath());
Chris@439 132 }
Chris@439 133 }
Chris@439 134 }
Chris@439 135 }
Chris@489 136
Chris@489 137 reindex();
Chris@439 138 }
Chris@439 139
Chris@461 140 bool
Chris@461 141 PluginRDFIndexer::indexConfiguredURLs()
Chris@461 142 {
Chris@690 143 SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs" << endl;
Chris@461 144
Chris@461 145 QSettings settings;
Chris@461 146 settings.beginGroup("RDF");
Chris@461 147
Chris@461 148 QString indexKey("rdf-indices");
Chris@461 149 QStringList indices = settings.value(indexKey).toStringList();
Chris@461 150
Chris@461 151 for (int i = 0; i < indices.size(); ++i) {
Chris@461 152
Chris@461 153 QString index = indices[i];
Chris@461 154
Chris@690 155 SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs: index url is "
Chris@687 156 << index << endl;
Chris@461 157
Chris@467 158 CachedFile cf(index);
Chris@467 159 if (!cf.isOK()) continue;
Chris@467 160
Chris@467 161 FileSource indexSource(cf.getLocalFilename());
Chris@461 162
Chris@461 163 PlaylistFileReader reader(indexSource);
Chris@461 164 if (!reader.isOK()) continue;
Chris@461 165
Chris@461 166 PlaylistFileReader::Playlist list = reader.load();
Chris@461 167 for (PlaylistFileReader::Playlist::const_iterator j = list.begin();
Chris@461 168 j != list.end(); ++j) {
Chris@690 169 SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs: url is "
Chris@844 170 << *j << endl;
Chris@489 171 pullURL(*j);
Chris@461 172 }
Chris@461 173 }
Chris@461 174
Chris@461 175 QString urlListKey("rdf-urls");
Chris@461 176 QStringList urls = settings.value(urlListKey).toStringList();
Chris@461 177
Chris@461 178 for (int i = 0; i < urls.size(); ++i) {
Chris@489 179 pullURL(urls[i]);
Chris@461 180 }
Chris@461 181
Chris@461 182 settings.endGroup();
Chris@489 183 reindex();
Chris@461 184 return true;
Chris@461 185 }
Chris@461 186
Chris@439 187 QString
Chris@439 188 PluginRDFIndexer::getURIForPluginId(QString pluginId)
Chris@439 189 {
Chris@461 190 QMutexLocker locker(&m_mutex);
Chris@461 191
Chris@439 192 if (m_idToUriMap.find(pluginId) == m_idToUriMap.end()) return "";
Chris@439 193 return m_idToUriMap[pluginId];
Chris@439 194 }
Chris@439 195
Chris@439 196 QString
Chris@439 197 PluginRDFIndexer::getIdForPluginURI(QString uri)
Chris@439 198 {
Chris@476 199 m_mutex.lock();
Chris@461 200
Chris@439 201 if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) {
Chris@439 202
Chris@476 203 m_mutex.unlock();
Chris@476 204
Chris@439 205 // Haven't found this uri referenced in any document on the
Chris@439 206 // local filesystem; try resolving the pre-fragment part of
Chris@439 207 // the uri as a document URL and reading that if possible.
Chris@439 208
Chris@439 209 // Because we may want to refer to this document again, we
Chris@439 210 // cache it locally if it turns out to exist.
Chris@439 211
Chris@1272 212 SVDEBUG << "PluginRDFIndexer::getIdForPluginURI: NOTE: Failed to find a local RDF document describing plugin <" << uri << ">: attempting to retrieve one remotely by guesswork" << endl;
Chris@439 213
Chris@439 214 QString baseUrl = QUrl(uri).toString(QUrl::RemoveFragment);
Chris@439 215
Chris@457 216 indexURL(baseUrl);
Chris@439 217
Chris@476 218 m_mutex.lock();
Chris@476 219
Chris@439 220 if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) {
Chris@439 221 m_uriToIdMap[uri] = "";
Chris@439 222 }
Chris@439 223 }
Chris@439 224
Chris@476 225 QString id = m_uriToIdMap[uri];
Chris@476 226 m_mutex.unlock();
Chris@476 227 return id;
Chris@439 228 }
Chris@439 229
Chris@456 230 QStringList
Chris@456 231 PluginRDFIndexer::getIndexedPluginIds()
Chris@456 232 {
Chris@461 233 QMutexLocker locker(&m_mutex);
Chris@461 234
Chris@456 235 QStringList ids;
Chris@489 236 for (StringMap::const_iterator i = m_idToUriMap.begin();
Chris@489 237 i != m_idToUriMap.end(); ++i) {
Chris@456 238 ids.push_back(i->first);
Chris@456 239 }
Chris@456 240 return ids;
Chris@456 241 }
Chris@456 242
Chris@439 243 bool
Chris@489 244 PluginRDFIndexer::pullFile(QString filepath)
Chris@439 245 {
Chris@439 246 QUrl url = QUrl::fromLocalFile(filepath);
Chris@439 247 QString urlString = url.toString();
Chris@489 248 return pullURL(urlString);
Chris@439 249 }
Chris@461 250
Chris@439 251 bool
Chris@439 252 PluginRDFIndexer::indexURL(QString urlString)
Chris@439 253 {
Chris@489 254 bool pulled = pullURL(urlString);
Chris@489 255 if (!pulled) return false;
Chris@489 256 reindex();
Chris@489 257 return true;
Chris@489 258 }
Chris@489 259
Chris@489 260 bool
Chris@489 261 PluginRDFIndexer::pullURL(QString urlString)
Chris@489 262 {
Chris@457 263 Profiler profiler("PluginRDFIndexer::indexURL");
Chris@457 264
Chris@1272 265 // SVDEBUG << "PluginRDFIndexer::indexURL(" << urlString << ")" << endl;
Chris@461 266
Chris@461 267 QMutexLocker locker(&m_mutex);
Chris@461 268
Chris@725 269 QUrl local = urlString;
Chris@457 270
Chris@457 271 if (FileSource::isRemote(urlString) &&
Chris@457 272 FileSource::canHandleScheme(urlString)) {
Chris@457 273
Chris@520 274 CachedFile cf(urlString, 0, "application/rdf+xml");
Chris@467 275 if (!cf.isOK()) {
Chris@467 276 return false;
Chris@467 277 }
Chris@467 278
Chris@725 279 local = QUrl::fromLocalFile(cf.getLocalFilename());
Chris@725 280
Chris@730 281 } else if (urlString.startsWith("file:")) {
Chris@730 282
Chris@730 283 local = QUrl(urlString);
Chris@730 284
Chris@725 285 } else {
Chris@725 286
Chris@725 287 local = QUrl::fromLocalFile(urlString);
Chris@457 288 }
Chris@457 289
Chris@725 290 try {
Chris@725 291 m_index->import(local, BasicStore::ImportFailOnDuplicates);
Chris@725 292 } catch (RDFDuplicateImportException &e) {
Chris@1272 293 SVDEBUG << e.what() << endl;
Chris@1272 294 SVDEBUG << "PluginRDFIndexer::pullURL: Document at " << urlString
Chris@1272 295 << " duplicates triples found in earlier loaded document -- skipping it" << endl;
Chris@725 296 return false;
Chris@725 297 } catch (RDFException &e) {
Chris@1272 298 SVDEBUG << e.what() << endl;
Chris@1272 299 SVDEBUG << "PluginRDFIndexer::pullURL: Failed to import document from "
Chris@1272 300 << urlString << ": " << e.what() << endl;
Chris@725 301 return false;
Chris@725 302 }
Chris@725 303 return true;
Chris@489 304 }
Chris@489 305
Chris@489 306 bool
Chris@489 307 PluginRDFIndexer::reindex()
Chris@489 308 {
Chris@725 309 Triples tt = m_index->match
Chris@730 310 (Triple(Node(), Uri("a"), m_index->expand("vamp:Plugin")));
Chris@730 311 Nodes plugins = tt.subjects();
Chris@439 312
Chris@439 313 bool foundSomething = false;
Chris@439 314 bool addedSomething = false;
Chris@439 315
Chris@725 316 foreach (Node plugin, plugins) {
Chris@725 317
Chris@725 318 if (plugin.type != Node::URI) {
Chris@1272 319 SVDEBUG << "PluginRDFIndexer::reindex: Plugin has no URI: node is "
Chris@725 320 << plugin << endl;
Chris@439 321 continue;
Chris@439 322 }
Chris@725 323
Chris@730 324 Node idn = m_index->complete
Chris@730 325 (Triple(plugin, m_index->expand("vamp:identifier"), Node()));
Chris@730 326
Chris@730 327 if (idn.type != Node::Literal) {
Chris@1272 328 SVDEBUG << "PluginRDFIndexer::reindex: Plugin " << plugin
Chris@725 329 << " lacks vamp:identifier literal" << endl;
Chris@439 330 continue;
Chris@439 331 }
Chris@481 332
Chris@730 333 Node libn = m_index->complete
Chris@730 334 (Triple(Node(), m_index->expand("vamp:available_plugin"), plugin));
Chris@481 335
Chris@730 336 if (libn.type != Node::URI) {
Chris@1272 337 SVDEBUG << "PluginRDFIndexer::reindex: Plugin " << plugin
Chris@725 338 << " is not vamp:available_plugin in any library" << endl;
Chris@481 339 continue;
Chris@481 340 }
Chris@481 341
Chris@730 342 Node son = m_index->complete
Chris@730 343 (Triple(libn, m_index->expand("vamp:identifier"), Node()));
Chris@725 344
Chris@730 345 if (son.type != Node::Literal) {
Chris@1272 346 SVDEBUG << "PluginRDFIndexer::reindex: Library " << libn
Chris@725 347 << " lacks vamp:identifier for soname" << endl;
Chris@725 348 continue;
Chris@725 349 }
Chris@725 350
Chris@725 351 QString pluginUri = plugin.value;
Chris@730 352 QString identifier = idn.value;
Chris@730 353 QString soname = son.value;
Chris@725 354
Chris@439 355 QString pluginId = PluginIdentifier::createIdentifier
Chris@439 356 ("vamp", soname, identifier);
Chris@439 357
Chris@439 358 foundSomething = true;
Chris@439 359
Chris@489 360 if (m_idToUriMap.find(pluginId) != m_idToUriMap.end()) {
Chris@439 361 continue;
Chris@439 362 }
Chris@439 363
Chris@439 364 m_idToUriMap[pluginId] = pluginUri;
Chris@439 365
Chris@439 366 addedSomething = true;
Chris@439 367
Chris@439 368 if (pluginUri != "") {
Chris@439 369 if (m_uriToIdMap.find(pluginUri) != m_uriToIdMap.end()) {
Chris@1272 370 SVDEBUG << "PluginRDFIndexer::reindex: WARNING: Found multiple plugins with the same URI:" << endl;
Chris@1272 371 SVDEBUG << " 1. Plugin id \"" << m_uriToIdMap[pluginUri] << "\"" << endl;
Chris@1272 372 SVDEBUG << " 2. Plugin id \"" << pluginId << "\"" << endl;
Chris@1272 373 SVDEBUG << "both claim URI <" << pluginUri << ">" << endl;
Chris@439 374 } else {
Chris@439 375 m_uriToIdMap[pluginUri] = pluginId;
Chris@439 376 }
Chris@439 377 }
Chris@439 378 }
Chris@439 379
Chris@439 380 if (!foundSomething) {
Chris@1272 381 SVDEBUG << "PluginRDFIndexer::reindex: NOTE: Plugins found, but none sufficiently described" << endl;
Chris@439 382 }
Chris@439 383
Chris@439 384 return addedSomething;
Chris@439 385 }