Chris@439: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ Chris@439: Chris@439: /* Chris@439: Sonic Visualiser Chris@439: An audio file viewer and annotation editor. Chris@439: Centre for Digital Music, Queen Mary, University of London. Chris@727: This file copyright 2008-2012 QMUL. Chris@439: Chris@439: This program is free software; you can redistribute it and/or Chris@439: modify it under the terms of the GNU General Public License as Chris@439: published by the Free Software Foundation; either version 2 of the Chris@439: License, or (at your option) any later version. See the file Chris@439: COPYING included with this distribution for more information. Chris@439: */ Chris@439: Chris@439: #include "PluginRDFIndexer.h" Chris@439: Chris@467: #include "data/fileio/CachedFile.h" Chris@471: #include "data/fileio/FileSource.h" Chris@461: #include "data/fileio/PlaylistFileReader.h" Chris@439: #include "plugin/PluginIdentifier.h" Chris@439: Chris@457: #include "base/Profiler.h" Chris@457: Chris@475: #include Chris@439: Chris@725: #include Chris@725: #include Chris@725: Chris@439: #include Chris@439: #include Chris@439: #include Chris@461: #include Chris@461: #include Chris@461: #include Chris@439: Chris@439: #include Chris@439: using std::cerr; Chris@439: using std::endl; Chris@439: using std::vector; Chris@439: using std::string; Chris@439: using Vamp::PluginHostAdapter; Chris@439: Chris@725: using Dataquay::Uri; Chris@725: using Dataquay::Node; Chris@725: using Dataquay::Nodes; Chris@725: using Dataquay::Triple; Chris@725: using Dataquay::Triples; Chris@725: using Dataquay::BasicStore; Chris@725: using Dataquay::RDFException; Chris@725: using Dataquay::RDFDuplicateImportException; Chris@725: Chris@439: PluginRDFIndexer * Chris@439: PluginRDFIndexer::m_instance = 0; Chris@439: Chris@439: PluginRDFIndexer * Chris@439: PluginRDFIndexer::getInstance() Chris@439: { Chris@439: if (!m_instance) m_instance = new PluginRDFIndexer(); Chris@439: return m_instance; Chris@439: } Chris@439: Chris@725: PluginRDFIndexer::PluginRDFIndexer() : Chris@725: m_index(new Dataquay::BasicStore) Chris@439: { Chris@725: m_index->addPrefix("vamp", Uri("http://purl.org/ontology/vamp/")); Chris@725: m_index->addPrefix("foaf", Uri("http://xmlns.com/foaf/0.1/")); Chris@725: m_index->addPrefix("dc", Uri("http://purl.org/dc/elements/1.1/")); Chris@477: indexInstalledURLs(); Chris@477: } Chris@477: Chris@725: const BasicStore * Chris@725: PluginRDFIndexer::getIndex() Chris@725: { Chris@725: return m_index; Chris@725: } Chris@725: Chris@477: PluginRDFIndexer::~PluginRDFIndexer() Chris@477: { Chris@477: QMutexLocker locker(&m_mutex); Chris@477: } Chris@477: Chris@477: void Chris@477: PluginRDFIndexer::indexInstalledURLs() Chris@477: { Chris@439: vector paths = PluginHostAdapter::getPluginPath(); Chris@439: Chris@730: // std::cerr << "\nPluginRDFIndexer::indexInstalledURLs: pid is " << getpid() << std::endl; Chris@730: Chris@439: QStringList filters; Chris@731: filters << "*.ttl"; Chris@731: filters << "*.TTL"; Chris@439: filters << "*.n3"; Chris@439: filters << "*.N3"; Chris@439: filters << "*.rdf"; Chris@439: filters << "*.RDF"; Chris@439: Chris@731: // Search each Vamp plugin path for an RDF file that either has Chris@439: // name "soname", "soname:label" or "soname/label" plus RDF Chris@731: // extension. Use that order of preference, and prefer ttl over Chris@731: // n3 over rdf extension. Chris@439: Chris@439: for (vector::const_iterator i = paths.begin(); i != paths.end(); ++i) { Chris@718: Chris@439: QDir dir(i->c_str()); Chris@439: if (!dir.exists()) continue; Chris@439: Chris@439: QStringList entries = dir.entryList Chris@439: (filters, QDir::Files | QDir::Readable); Chris@439: Chris@439: for (QStringList::const_iterator j = entries.begin(); Chris@439: j != entries.end(); ++j) { Chris@718: Chris@439: QFileInfo fi(dir.filePath(*j)); Chris@489: pullFile(fi.absoluteFilePath()); Chris@439: } Chris@439: Chris@439: QStringList subdirs = dir.entryList Chris@439: (QDir::AllDirs | QDir::NoDotAndDotDot | QDir::Readable); Chris@439: Chris@439: for (QStringList::const_iterator j = subdirs.begin(); Chris@439: j != subdirs.end(); ++j) { Chris@718: Chris@439: QDir subdir(dir.filePath(*j)); Chris@439: if (subdir.exists()) { Chris@439: entries = subdir.entryList Chris@439: (filters, QDir::Files | QDir::Readable); Chris@439: for (QStringList::const_iterator k = entries.begin(); Chris@439: k != entries.end(); ++k) { Chris@439: QFileInfo fi(subdir.filePath(*k)); Chris@489: pullFile(fi.absoluteFilePath()); Chris@439: } Chris@439: } Chris@439: } Chris@439: } Chris@489: Chris@489: reindex(); Chris@439: } Chris@439: Chris@461: bool Chris@461: PluginRDFIndexer::indexConfiguredURLs() Chris@461: { Chris@690: SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs" << endl; Chris@461: Chris@461: QSettings settings; Chris@461: settings.beginGroup("RDF"); Chris@461: Chris@461: QString indexKey("rdf-indices"); Chris@461: QStringList indices = settings.value(indexKey).toStringList(); Chris@461: Chris@461: for (int i = 0; i < indices.size(); ++i) { Chris@461: Chris@461: QString index = indices[i]; Chris@461: Chris@690: SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs: index url is " Chris@687: << index << endl; Chris@461: Chris@467: CachedFile cf(index); Chris@467: if (!cf.isOK()) continue; Chris@467: Chris@467: FileSource indexSource(cf.getLocalFilename()); Chris@461: Chris@461: PlaylistFileReader reader(indexSource); Chris@461: if (!reader.isOK()) continue; Chris@461: Chris@461: PlaylistFileReader::Playlist list = reader.load(); Chris@461: for (PlaylistFileReader::Playlist::const_iterator j = list.begin(); Chris@461: j != list.end(); ++j) { Chris@690: SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs: url is " Chris@687: << j->toStdString() << endl; Chris@489: pullURL(*j); Chris@461: } Chris@461: } Chris@461: Chris@461: QString urlListKey("rdf-urls"); Chris@461: QStringList urls = settings.value(urlListKey).toStringList(); Chris@461: Chris@461: for (int i = 0; i < urls.size(); ++i) { Chris@489: pullURL(urls[i]); Chris@461: } Chris@461: Chris@461: settings.endGroup(); Chris@489: reindex(); Chris@461: return true; Chris@461: } Chris@461: Chris@439: QString Chris@439: PluginRDFIndexer::getURIForPluginId(QString pluginId) Chris@439: { Chris@461: QMutexLocker locker(&m_mutex); Chris@461: Chris@439: if (m_idToUriMap.find(pluginId) == m_idToUriMap.end()) return ""; Chris@439: return m_idToUriMap[pluginId]; Chris@439: } Chris@439: Chris@439: QString Chris@439: PluginRDFIndexer::getIdForPluginURI(QString uri) Chris@439: { Chris@476: m_mutex.lock(); Chris@461: Chris@439: if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) { Chris@439: Chris@476: m_mutex.unlock(); Chris@476: Chris@439: // Haven't found this uri referenced in any document on the Chris@439: // local filesystem; try resolving the pre-fragment part of Chris@439: // the uri as a document URL and reading that if possible. Chris@439: Chris@439: // Because we may want to refer to this document again, we Chris@439: // cache it locally if it turns out to exist. Chris@439: Chris@686: cerr << "PluginRDFIndexer::getIdForPluginURI: NOTE: Failed to find a local RDF document describing plugin <" << uri << ">: attempting to retrieve one remotely by guesswork" << endl; Chris@439: Chris@439: QString baseUrl = QUrl(uri).toString(QUrl::RemoveFragment); Chris@439: Chris@457: indexURL(baseUrl); Chris@439: Chris@476: m_mutex.lock(); Chris@476: Chris@439: if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) { Chris@439: m_uriToIdMap[uri] = ""; Chris@439: } Chris@439: } Chris@439: Chris@476: QString id = m_uriToIdMap[uri]; Chris@476: m_mutex.unlock(); Chris@476: return id; Chris@439: } Chris@439: Chris@456: QStringList Chris@456: PluginRDFIndexer::getIndexedPluginIds() Chris@456: { Chris@461: QMutexLocker locker(&m_mutex); Chris@461: Chris@456: QStringList ids; Chris@489: for (StringMap::const_iterator i = m_idToUriMap.begin(); Chris@489: i != m_idToUriMap.end(); ++i) { Chris@456: ids.push_back(i->first); Chris@456: } Chris@456: return ids; Chris@456: } Chris@456: Chris@439: bool Chris@489: PluginRDFIndexer::pullFile(QString filepath) Chris@439: { Chris@439: QUrl url = QUrl::fromLocalFile(filepath); Chris@439: QString urlString = url.toString(); Chris@489: return pullURL(urlString); Chris@439: } Chris@461: Chris@439: bool Chris@439: PluginRDFIndexer::indexURL(QString urlString) Chris@439: { Chris@489: bool pulled = pullURL(urlString); Chris@489: if (!pulled) return false; Chris@489: reindex(); Chris@489: return true; Chris@489: } Chris@489: Chris@489: bool Chris@489: PluginRDFIndexer::pullURL(QString urlString) Chris@489: { Chris@457: Profiler profiler("PluginRDFIndexer::indexURL"); Chris@457: Chris@730: // std::cerr << "PluginRDFIndexer::indexURL(" << urlString.toStdString() << ")" << std::endl; Chris@461: Chris@461: QMutexLocker locker(&m_mutex); Chris@461: Chris@725: QUrl local = urlString; Chris@457: Chris@457: if (FileSource::isRemote(urlString) && Chris@457: FileSource::canHandleScheme(urlString)) { Chris@457: Chris@520: CachedFile cf(urlString, 0, "application/rdf+xml"); Chris@467: if (!cf.isOK()) { Chris@467: return false; Chris@467: } Chris@467: Chris@725: local = QUrl::fromLocalFile(cf.getLocalFilename()); Chris@725: Chris@730: } else if (urlString.startsWith("file:")) { Chris@730: Chris@730: local = QUrl(urlString); Chris@730: Chris@725: } else { Chris@725: Chris@725: local = QUrl::fromLocalFile(urlString); Chris@457: } Chris@457: Chris@725: try { Chris@725: m_index->import(local, BasicStore::ImportFailOnDuplicates); Chris@725: } catch (RDFDuplicateImportException &e) { Chris@730: cerr << e.what() << endl; Chris@725: cerr << "PluginRDFIndexer::pullURL: Document at " << urlString Chris@730: << " duplicates triples found in earlier loaded document -- skipping it" << endl; Chris@725: return false; Chris@725: } catch (RDFException &e) { Chris@730: cerr << e.what() << endl; Chris@725: cerr << "PluginRDFIndexer::pullURL: Failed to import document from " Chris@725: << urlString << ": " << e.what() << endl; Chris@725: return false; Chris@725: } Chris@725: return true; Chris@489: } Chris@489: Chris@489: bool Chris@489: PluginRDFIndexer::reindex() Chris@489: { Chris@725: Triples tt = m_index->match Chris@730: (Triple(Node(), Uri("a"), m_index->expand("vamp:Plugin"))); Chris@730: Nodes plugins = tt.subjects(); Chris@439: Chris@439: bool foundSomething = false; Chris@439: bool addedSomething = false; Chris@439: Chris@725: foreach (Node plugin, plugins) { Chris@725: Chris@725: if (plugin.type != Node::URI) { Chris@725: cerr << "PluginRDFIndexer::reindex: Plugin has no URI: node is " Chris@725: << plugin << endl; Chris@439: continue; Chris@439: } Chris@725: Chris@730: Node idn = m_index->complete Chris@730: (Triple(plugin, m_index->expand("vamp:identifier"), Node())); Chris@730: Chris@730: if (idn.type != Node::Literal) { Chris@725: cerr << "PluginRDFIndexer::reindex: Plugin " << plugin Chris@725: << " lacks vamp:identifier literal" << endl; Chris@439: continue; Chris@439: } Chris@481: Chris@730: Node libn = m_index->complete Chris@730: (Triple(Node(), m_index->expand("vamp:available_plugin"), plugin)); Chris@481: Chris@730: if (libn.type != Node::URI) { Chris@725: cerr << "PluginRDFIndexer::reindex: Plugin " << plugin Chris@725: << " is not vamp:available_plugin in any library" << endl; Chris@481: continue; Chris@481: } Chris@481: Chris@730: Node son = m_index->complete Chris@730: (Triple(libn, m_index->expand("vamp:identifier"), Node())); Chris@725: Chris@730: if (son.type != Node::Literal) { Chris@730: cerr << "PluginRDFIndexer::reindex: Library " << libn Chris@725: << " lacks vamp:identifier for soname" << endl; Chris@725: continue; Chris@725: } Chris@725: Chris@725: QString pluginUri = plugin.value; Chris@730: QString identifier = idn.value; Chris@730: QString soname = son.value; Chris@725: Chris@439: QString pluginId = PluginIdentifier::createIdentifier Chris@439: ("vamp", soname, identifier); Chris@439: Chris@439: foundSomething = true; Chris@439: Chris@489: if (m_idToUriMap.find(pluginId) != m_idToUriMap.end()) { Chris@439: continue; Chris@439: } Chris@439: Chris@439: m_idToUriMap[pluginId] = pluginUri; Chris@439: Chris@439: addedSomething = true; Chris@439: Chris@439: if (pluginUri != "") { Chris@439: if (m_uriToIdMap.find(pluginUri) != m_uriToIdMap.end()) { Chris@718: cerr << "PluginRDFIndexer::reindex: WARNING: Found multiple plugins with the same URI:" << endl; Chris@686: cerr << " 1. Plugin id \"" << m_uriToIdMap[pluginUri] << "\"" << endl; Chris@686: cerr << " 2. Plugin id \"" << pluginId << "\"" << endl; Chris@686: cerr << "both claim URI <" << pluginUri << ">" << endl; Chris@439: } else { Chris@439: m_uriToIdMap[pluginUri] = pluginId; Chris@439: } Chris@439: } Chris@439: } Chris@439: Chris@439: if (!foundSomething) { Chris@718: cerr << "PluginRDFIndexer::reindex: NOTE: Plugins found, but none sufficiently described" << endl; Chris@439: } Chris@439: Chris@439: return addedSomething; Chris@439: }