annotate rdf/PluginRDFIndexer.cpp @ 1869:cb9209ef373a startup-timing

Merge from default branch
author Chris Cannam
date Tue, 16 Jun 2020 17:44:06 +0100
parents 5b1b03c1d8d4
children
rev   line source
Chris@439 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@439 2
Chris@439 3 /*
Chris@439 4 Sonic Visualiser
Chris@439 5 An audio file viewer and annotation editor.
Chris@439 6 Centre for Digital Music, Queen Mary, University of London.
Chris@727 7 This file copyright 2008-2012 QMUL.
Chris@439 8
Chris@439 9 This program is free software; you can redistribute it and/or
Chris@439 10 modify it under the terms of the GNU General Public License as
Chris@439 11 published by the Free Software Foundation; either version 2 of the
Chris@439 12 License, or (at your option) any later version. See the file
Chris@439 13 COPYING included with this distribution for more information.
Chris@439 14 */
Chris@439 15
Chris@439 16 #include "PluginRDFIndexer.h"
Chris@439 17
Chris@467 18 #include "data/fileio/CachedFile.h"
Chris@471 19 #include "data/fileio/FileSource.h"
Chris@461 20 #include "data/fileio/PlaylistFileReader.h"
Chris@439 21 #include "plugin/PluginIdentifier.h"
Chris@439 22
Chris@457 23 #include "base/Profiler.h"
Chris@1272 24 #include "base/Debug.h"
Chris@457 25
Chris@475 26 #include <vamp-hostsdk/PluginHostAdapter.h>
Chris@439 27
Chris@725 28 #include <dataquay/BasicStore.h>
Chris@725 29 #include <dataquay/RDFException.h>
Chris@725 30
Chris@439 31 #include <QFileInfo>
Chris@439 32 #include <QDir>
Chris@439 33 #include <QUrl>
Chris@461 34 #include <QDateTime>
Chris@461 35 #include <QSettings>
Chris@461 36 #include <QFile>
Chris@439 37
Chris@439 38 #include <iostream>
Chris@843 39
Chris@439 40 using std::vector;
Chris@439 41 using std::string;
Chris@439 42 using Vamp::PluginHostAdapter;
Chris@439 43
Chris@725 44 using Dataquay::Uri;
Chris@725 45 using Dataquay::Node;
Chris@725 46 using Dataquay::Nodes;
Chris@725 47 using Dataquay::Triple;
Chris@725 48 using Dataquay::Triples;
Chris@725 49 using Dataquay::BasicStore;
Chris@725 50 using Dataquay::RDFException;
Chris@725 51 using Dataquay::RDFDuplicateImportException;
Chris@725 52
Chris@439 53 PluginRDFIndexer *
Chris@1582 54 PluginRDFIndexer::m_instance = nullptr;
Chris@439 55
Chris@439 56 PluginRDFIndexer *
Chris@439 57 PluginRDFIndexer::getInstance()
Chris@439 58 {
Chris@439 59 if (!m_instance) m_instance = new PluginRDFIndexer();
Chris@439 60 return m_instance;
Chris@439 61 }
Chris@439 62
Chris@725 63 PluginRDFIndexer::PluginRDFIndexer() :
Chris@725 64 m_index(new Dataquay::BasicStore)
Chris@439 65 {
Chris@725 66 m_index->addPrefix("vamp", Uri("http://purl.org/ontology/vamp/"));
Chris@725 67 m_index->addPrefix("foaf", Uri("http://xmlns.com/foaf/0.1/"));
Chris@725 68 m_index->addPrefix("dc", Uri("http://purl.org/dc/elements/1.1/"));
Chris@477 69 indexInstalledURLs();
Chris@477 70 }
Chris@477 71
Chris@725 72 const BasicStore *
Chris@725 73 PluginRDFIndexer::getIndex()
Chris@725 74 {
Chris@725 75 return m_index;
Chris@725 76 }
Chris@725 77
Chris@477 78 PluginRDFIndexer::~PluginRDFIndexer()
Chris@477 79 {
Chris@477 80 QMutexLocker locker(&m_mutex);
Chris@477 81 }
Chris@477 82
Chris@477 83 void
Chris@477 84 PluginRDFIndexer::indexInstalledURLs()
Chris@477 85 {
Chris@439 86 vector<string> paths = PluginHostAdapter::getPluginPath();
Chris@439 87
Chris@1272 88 // SVDEBUG << "\nPluginRDFIndexer::indexInstalledURLs: pid is " << getpid() << endl;
Chris@730 89
Chris@439 90 QStringList filters;
Chris@731 91 filters << "*.ttl";
Chris@731 92 filters << "*.TTL";
Chris@439 93 filters << "*.n3";
Chris@439 94 filters << "*.N3";
Chris@439 95 filters << "*.rdf";
Chris@439 96 filters << "*.RDF";
Chris@439 97
Chris@731 98 // Search each Vamp plugin path for an RDF file that either has
Chris@439 99 // name "soname", "soname:label" or "soname/label" plus RDF
Chris@731 100 // extension. Use that order of preference, and prefer ttl over
Chris@731 101 // n3 over rdf extension.
Chris@439 102
Chris@439 103 for (vector<string>::const_iterator i = paths.begin(); i != paths.end(); ++i) {
Chris@718 104
Chris@439 105 QDir dir(i->c_str());
Chris@439 106 if (!dir.exists()) continue;
Chris@439 107
Chris@439 108 QStringList entries = dir.entryList
Chris@439 109 (filters, QDir::Files | QDir::Readable);
Chris@439 110
Chris@439 111 for (QStringList::const_iterator j = entries.begin();
Chris@439 112 j != entries.end(); ++j) {
Chris@718 113
Chris@439 114 QFileInfo fi(dir.filePath(*j));
Chris@489 115 pullFile(fi.absoluteFilePath());
Chris@439 116 }
Chris@439 117
Chris@439 118 QStringList subdirs = dir.entryList
Chris@439 119 (QDir::AllDirs | QDir::NoDotAndDotDot | QDir::Readable);
Chris@439 120
Chris@439 121 for (QStringList::const_iterator j = subdirs.begin();
Chris@439 122 j != subdirs.end(); ++j) {
Chris@718 123
Chris@439 124 QDir subdir(dir.filePath(*j));
Chris@439 125 if (subdir.exists()) {
Chris@439 126 entries = subdir.entryList
Chris@439 127 (filters, QDir::Files | QDir::Readable);
Chris@439 128 for (QStringList::const_iterator k = entries.begin();
Chris@439 129 k != entries.end(); ++k) {
Chris@439 130 QFileInfo fi(subdir.filePath(*k));
Chris@489 131 pullFile(fi.absoluteFilePath());
Chris@439 132 }
Chris@439 133 }
Chris@439 134 }
Chris@439 135 }
Chris@489 136
Chris@489 137 reindex();
Chris@439 138 }
Chris@439 139
Chris@461 140 bool
Chris@461 141 PluginRDFIndexer::indexConfiguredURLs()
Chris@461 142 {
Chris@690 143 SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs" << endl;
Chris@461 144
Chris@461 145 QSettings settings;
Chris@461 146 settings.beginGroup("RDF");
Chris@461 147
Chris@461 148 QString indexKey("rdf-indices");
Chris@461 149 QStringList indices = settings.value(indexKey).toStringList();
Chris@461 150
Chris@461 151 for (int i = 0; i < indices.size(); ++i) {
Chris@461 152
Chris@461 153 QString index = indices[i];
Chris@461 154
Chris@690 155 SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs: index url is "
Chris@687 156 << index << endl;
Chris@461 157
Chris@467 158 CachedFile cf(index);
Chris@467 159 if (!cf.isOK()) continue;
Chris@467 160
Chris@467 161 FileSource indexSource(cf.getLocalFilename());
Chris@461 162
Chris@461 163 PlaylistFileReader reader(indexSource);
Chris@461 164 if (!reader.isOK()) continue;
Chris@461 165
Chris@461 166 PlaylistFileReader::Playlist list = reader.load();
Chris@461 167 for (PlaylistFileReader::Playlist::const_iterator j = list.begin();
Chris@461 168 j != list.end(); ++j) {
Chris@690 169 SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs: url is "
Chris@844 170 << *j << endl;
Chris@489 171 pullURL(*j);
Chris@461 172 }
Chris@461 173 }
Chris@461 174
Chris@461 175 QString urlListKey("rdf-urls");
Chris@461 176 QStringList urls = settings.value(urlListKey).toStringList();
Chris@461 177
Chris@461 178 for (int i = 0; i < urls.size(); ++i) {
Chris@489 179 pullURL(urls[i]);
Chris@461 180 }
Chris@461 181
Chris@461 182 settings.endGroup();
Chris@489 183 reindex();
Chris@1844 184
Chris@461 185 return true;
Chris@461 186 }
Chris@461 187
Chris@1844 188 void
Chris@1844 189 PluginRDFIndexer::performConsistencyChecks()
Chris@1844 190 {
Chris@1844 191 // Add more here!
Chris@1844 192
Chris@1844 193 Triples packs = m_index->match
Chris@1844 194 (Triple(Node(), m_index->expand("vamp:available_library"), Node()));
Chris@1844 195
Chris@1844 196 for (Triple packt: packs) {
Chris@1844 197 Triples libraries = m_index->match
Chris@1844 198 (Triple(packt.object(), m_index->expand("a"),
Chris@1844 199 m_index->expand("vamp:PluginLibrary")));
Chris@1844 200 if (libraries.empty()) {
Chris@1844 201 SVCERR << "WARNING: Plugin pack " << packt.subject()
Chris@1844 202 << " claims to contain library " << packt.object()
Chris@1844 203 << " which is not known to us as a vamp:PluginLibrary"
Chris@1844 204 << endl;
Chris@1844 205 }
Chris@1844 206 }
Chris@1844 207 }
Chris@1844 208
Chris@439 209 QString
Chris@439 210 PluginRDFIndexer::getURIForPluginId(QString pluginId)
Chris@439 211 {
Chris@461 212 QMutexLocker locker(&m_mutex);
Chris@461 213
Chris@439 214 if (m_idToUriMap.find(pluginId) == m_idToUriMap.end()) return "";
Chris@439 215 return m_idToUriMap[pluginId];
Chris@439 216 }
Chris@439 217
Chris@439 218 QString
Chris@439 219 PluginRDFIndexer::getIdForPluginURI(QString uri)
Chris@439 220 {
Chris@476 221 m_mutex.lock();
Chris@461 222
Chris@439 223 if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) {
Chris@439 224
Chris@476 225 m_mutex.unlock();
Chris@476 226
Chris@439 227 // Haven't found this uri referenced in any document on the
Chris@439 228 // local filesystem; try resolving the pre-fragment part of
Chris@439 229 // the uri as a document URL and reading that if possible.
Chris@439 230
Chris@439 231 // Because we may want to refer to this document again, we
Chris@439 232 // cache it locally if it turns out to exist.
Chris@439 233
Chris@1272 234 SVDEBUG << "PluginRDFIndexer::getIdForPluginURI: NOTE: Failed to find a local RDF document describing plugin <" << uri << ">: attempting to retrieve one remotely by guesswork" << endl;
Chris@439 235
Chris@439 236 QString baseUrl = QUrl(uri).toString(QUrl::RemoveFragment);
Chris@439 237
Chris@457 238 indexURL(baseUrl);
Chris@439 239
Chris@476 240 m_mutex.lock();
Chris@476 241
Chris@439 242 if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) {
Chris@439 243 m_uriToIdMap[uri] = "";
Chris@439 244 }
Chris@439 245 }
Chris@439 246
Chris@476 247 QString id = m_uriToIdMap[uri];
Chris@476 248 m_mutex.unlock();
Chris@476 249 return id;
Chris@439 250 }
Chris@439 251
Chris@456 252 QStringList
Chris@456 253 PluginRDFIndexer::getIndexedPluginIds()
Chris@456 254 {
Chris@461 255 QMutexLocker locker(&m_mutex);
Chris@461 256
Chris@456 257 QStringList ids;
Chris@489 258 for (StringMap::const_iterator i = m_idToUriMap.begin();
Chris@489 259 i != m_idToUriMap.end(); ++i) {
Chris@456 260 ids.push_back(i->first);
Chris@456 261 }
Chris@456 262 return ids;
Chris@456 263 }
Chris@456 264
Chris@439 265 bool
Chris@489 266 PluginRDFIndexer::pullFile(QString filepath)
Chris@439 267 {
Chris@439 268 QUrl url = QUrl::fromLocalFile(filepath);
Chris@439 269 QString urlString = url.toString();
Chris@489 270 return pullURL(urlString);
Chris@439 271 }
Chris@461 272
Chris@439 273 bool
Chris@439 274 PluginRDFIndexer::indexURL(QString urlString)
Chris@439 275 {
Chris@489 276 bool pulled = pullURL(urlString);
Chris@489 277 if (!pulled) return false;
Chris@489 278 reindex();
Chris@489 279 return true;
Chris@489 280 }
Chris@489 281
Chris@489 282 bool
Chris@489 283 PluginRDFIndexer::pullURL(QString urlString)
Chris@489 284 {
Chris@457 285 Profiler profiler("PluginRDFIndexer::indexURL");
Chris@457 286
Chris@1272 287 // SVDEBUG << "PluginRDFIndexer::indexURL(" << urlString << ")" << endl;
Chris@461 288
Chris@461 289 QMutexLocker locker(&m_mutex);
Chris@461 290
Chris@725 291 QUrl local = urlString;
Chris@457 292
Chris@457 293 if (FileSource::isRemote(urlString) &&
Chris@457 294 FileSource::canHandleScheme(urlString)) {
Chris@457 295
Chris@1582 296 CachedFile cf(urlString, nullptr, "application/rdf+xml");
Chris@467 297 if (!cf.isOK()) {
Chris@467 298 return false;
Chris@467 299 }
Chris@467 300
Chris@725 301 local = QUrl::fromLocalFile(cf.getLocalFilename());
Chris@725 302
Chris@730 303 } else if (urlString.startsWith("file:")) {
Chris@730 304
Chris@730 305 local = QUrl(urlString);
Chris@730 306
Chris@725 307 } else {
Chris@725 308
Chris@725 309 local = QUrl::fromLocalFile(urlString);
Chris@457 310 }
Chris@457 311
Chris@725 312 try {
Chris@1844 313 m_index->import(local, BasicStore::ImportIgnoreDuplicates);
Chris@725 314 } catch (RDFException &e) {
Chris@1272 315 SVDEBUG << e.what() << endl;
Chris@1272 316 SVDEBUG << "PluginRDFIndexer::pullURL: Failed to import document from "
Chris@1272 317 << urlString << ": " << e.what() << endl;
Chris@725 318 return false;
Chris@725 319 }
Chris@725 320 return true;
Chris@489 321 }
Chris@489 322
Chris@489 323 bool
Chris@489 324 PluginRDFIndexer::reindex()
Chris@489 325 {
Chris@725 326 Triples tt = m_index->match
Chris@730 327 (Triple(Node(), Uri("a"), m_index->expand("vamp:Plugin")));
Chris@730 328 Nodes plugins = tt.subjects();
Chris@439 329
Chris@439 330 bool foundSomething = false;
Chris@439 331 bool addedSomething = false;
Chris@439 332
Chris@725 333 foreach (Node plugin, plugins) {
Chris@725 334
Chris@725 335 if (plugin.type != Node::URI) {
Chris@1272 336 SVDEBUG << "PluginRDFIndexer::reindex: Plugin has no URI: node is "
Chris@725 337 << plugin << endl;
Chris@439 338 continue;
Chris@439 339 }
Chris@725 340
Chris@730 341 Node idn = m_index->complete
Chris@730 342 (Triple(plugin, m_index->expand("vamp:identifier"), Node()));
Chris@730 343
Chris@730 344 if (idn.type != Node::Literal) {
Chris@1272 345 SVDEBUG << "PluginRDFIndexer::reindex: Plugin " << plugin
Chris@725 346 << " lacks vamp:identifier literal" << endl;
Chris@439 347 continue;
Chris@439 348 }
Chris@481 349
Chris@730 350 Node libn = m_index->complete
Chris@730 351 (Triple(Node(), m_index->expand("vamp:available_plugin"), plugin));
Chris@481 352
Chris@730 353 if (libn.type != Node::URI) {
Chris@1272 354 SVDEBUG << "PluginRDFIndexer::reindex: Plugin " << plugin
Chris@725 355 << " is not vamp:available_plugin in any library" << endl;
Chris@481 356 continue;
Chris@481 357 }
Chris@481 358
Chris@730 359 Node son = m_index->complete
Chris@730 360 (Triple(libn, m_index->expand("vamp:identifier"), Node()));
Chris@725 361
Chris@730 362 if (son.type != Node::Literal) {
Chris@1272 363 SVDEBUG << "PluginRDFIndexer::reindex: Library " << libn
Chris@725 364 << " lacks vamp:identifier for soname" << endl;
Chris@725 365 continue;
Chris@725 366 }
Chris@725 367
Chris@725 368 QString pluginUri = plugin.value;
Chris@730 369 QString identifier = idn.value;
Chris@730 370 QString soname = son.value;
Chris@725 371
Chris@439 372 QString pluginId = PluginIdentifier::createIdentifier
Chris@439 373 ("vamp", soname, identifier);
Chris@439 374
Chris@439 375 foundSomething = true;
Chris@439 376
Chris@489 377 if (m_idToUriMap.find(pluginId) != m_idToUriMap.end()) {
Chris@439 378 continue;
Chris@439 379 }
Chris@439 380
Chris@439 381 m_idToUriMap[pluginId] = pluginUri;
Chris@439 382
Chris@439 383 addedSomething = true;
Chris@439 384
Chris@439 385 if (pluginUri != "") {
Chris@439 386 if (m_uriToIdMap.find(pluginUri) != m_uriToIdMap.end()) {
Chris@1272 387 SVDEBUG << "PluginRDFIndexer::reindex: WARNING: Found multiple plugins with the same URI:" << endl;
Chris@1272 388 SVDEBUG << " 1. Plugin id \"" << m_uriToIdMap[pluginUri] << "\"" << endl;
Chris@1272 389 SVDEBUG << " 2. Plugin id \"" << pluginId << "\"" << endl;
Chris@1272 390 SVDEBUG << "both claim URI <" << pluginUri << ">" << endl;
Chris@439 391 } else {
Chris@439 392 m_uriToIdMap[pluginUri] = pluginId;
Chris@439 393 }
Chris@439 394 }
Chris@439 395 }
Chris@439 396
Chris@439 397 if (!foundSomething) {
Chris@1272 398 SVDEBUG << "PluginRDFIndexer::reindex: NOTE: Plugins found, but none sufficiently described" << endl;
Chris@439 399 }
Chris@439 400
Chris@439 401 return addedSomething;
Chris@439 402 }