Chris@439: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
Chris@439: 
Chris@439: /*
Chris@439:     Sonic Visualiser
Chris@439:     An audio file viewer and annotation editor.
Chris@439:     Centre for Digital Music, Queen Mary, University of London.
Chris@727:     This file copyright 2008-2012 QMUL.
Chris@439:    
Chris@439:     This program is free software; you can redistribute it and/or
Chris@439:     modify it under the terms of the GNU General Public License as
Chris@439:     published by the Free Software Foundation; either version 2 of the
Chris@439:     License, or (at your option) any later version.  See the file
Chris@439:     COPYING included with this distribution for more information.
Chris@439: */
Chris@439: 
Chris@439: #include "PluginRDFIndexer.h"
Chris@439: 
Chris@467: #include "data/fileio/CachedFile.h"
Chris@471: #include "data/fileio/FileSource.h"
Chris@461: #include "data/fileio/PlaylistFileReader.h"
Chris@439: #include "plugin/PluginIdentifier.h"
Chris@439: 
Chris@457: #include "base/Profiler.h"
Chris@1272: #include "base/Debug.h"
Chris@457: 
Chris@475: #include <vamp-hostsdk/PluginHostAdapter.h>
Chris@439: 
Chris@725: #include <dataquay/BasicStore.h>
Chris@725: #include <dataquay/RDFException.h>
Chris@725: 
Chris@439: #include <QFileInfo>
Chris@439: #include <QDir>
Chris@439: #include <QUrl>
Chris@461: #include <QDateTime>
Chris@461: #include <QSettings>
Chris@461: #include <QFile>
Chris@439: 
Chris@439: #include <iostream>
Chris@843: 
Chris@439: using std::vector;
Chris@439: using std::string;
Chris@439: using Vamp::PluginHostAdapter;
Chris@439: 
Chris@725: using Dataquay::Uri;
Chris@725: using Dataquay::Node;
Chris@725: using Dataquay::Nodes;
Chris@725: using Dataquay::Triple;
Chris@725: using Dataquay::Triples;
Chris@725: using Dataquay::BasicStore;
Chris@725: using Dataquay::RDFException;
Chris@725: using Dataquay::RDFDuplicateImportException;
Chris@725: 
Chris@439: PluginRDFIndexer *
Chris@1582: PluginRDFIndexer::m_instance = nullptr;
Chris@439: 
Chris@439: PluginRDFIndexer *
Chris@439: PluginRDFIndexer::getInstance() 
Chris@439: {
Chris@439:     if (!m_instance) m_instance = new PluginRDFIndexer();
Chris@439:     return m_instance;
Chris@439: }
Chris@439: 
Chris@725: PluginRDFIndexer::PluginRDFIndexer() :
Chris@725:     m_index(new Dataquay::BasicStore)
Chris@439: {
Chris@725:     m_index->addPrefix("vamp", Uri("http://purl.org/ontology/vamp/"));
Chris@725:     m_index->addPrefix("foaf", Uri("http://xmlns.com/foaf/0.1/"));
Chris@725:     m_index->addPrefix("dc", Uri("http://purl.org/dc/elements/1.1/"));
Chris@477:     indexInstalledURLs();
Chris@477: }
Chris@477: 
Chris@725: const BasicStore *
Chris@725: PluginRDFIndexer::getIndex()
Chris@725: {
Chris@725:     return m_index;
Chris@725: }
Chris@725: 
Chris@477: PluginRDFIndexer::~PluginRDFIndexer()
Chris@477: {
Chris@477:     QMutexLocker locker(&m_mutex);
Chris@477: }
Chris@477: 
Chris@477: void
Chris@477: PluginRDFIndexer::indexInstalledURLs()
Chris@477: {
Chris@439:     vector<string> paths = PluginHostAdapter::getPluginPath();
Chris@439: 
Chris@1272: //    SVDEBUG << "\nPluginRDFIndexer::indexInstalledURLs: pid is " << getpid() << endl;
Chris@730: 
Chris@439:     QStringList filters;
Chris@731:     filters << "*.ttl";
Chris@731:     filters << "*.TTL";
Chris@439:     filters << "*.n3";
Chris@439:     filters << "*.N3";
Chris@439:     filters << "*.rdf";
Chris@439:     filters << "*.RDF";
Chris@439: 
Chris@731:     // Search each Vamp plugin path for an RDF file that either has
Chris@439:     // name "soname", "soname:label" or "soname/label" plus RDF
Chris@731:     // extension.  Use that order of preference, and prefer ttl over
Chris@731:     // n3 over rdf extension.
Chris@439: 
Chris@439:     for (vector<string>::const_iterator i = paths.begin(); i != paths.end(); ++i) {
Chris@718: 
Chris@439:         QDir dir(i->c_str());
Chris@439:         if (!dir.exists()) continue;
Chris@439: 
Chris@439:         QStringList entries = dir.entryList
Chris@439:             (filters, QDir::Files | QDir::Readable);
Chris@439: 
Chris@439:         for (QStringList::const_iterator j = entries.begin();
Chris@439:              j != entries.end(); ++j) {
Chris@718: 
Chris@439:             QFileInfo fi(dir.filePath(*j));
Chris@489:             pullFile(fi.absoluteFilePath());
Chris@439:         }
Chris@439: 
Chris@439:         QStringList subdirs = dir.entryList
Chris@439:             (QDir::AllDirs | QDir::NoDotAndDotDot | QDir::Readable);
Chris@439: 
Chris@439:         for (QStringList::const_iterator j = subdirs.begin();
Chris@439:              j != subdirs.end(); ++j) {
Chris@718: 
Chris@439:             QDir subdir(dir.filePath(*j));
Chris@439:             if (subdir.exists()) {
Chris@439:                 entries = subdir.entryList
Chris@439:                     (filters, QDir::Files | QDir::Readable);
Chris@439:                 for (QStringList::const_iterator k = entries.begin();
Chris@439:                      k != entries.end(); ++k) {
Chris@439:                     QFileInfo fi(subdir.filePath(*k));
Chris@489:                     pullFile(fi.absoluteFilePath());
Chris@439:                 }
Chris@439:             }
Chris@439:         }
Chris@439:     }
Chris@489: 
Chris@489:     reindex();
Chris@439: }
Chris@439: 
Chris@461: bool
Chris@461: PluginRDFIndexer::indexConfiguredURLs()
Chris@461: {
Chris@690:     SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs" << endl;
Chris@461: 
Chris@461:     QSettings settings;
Chris@461:     settings.beginGroup("RDF");
Chris@461:     
Chris@461:     QString indexKey("rdf-indices");
Chris@461:     QStringList indices = settings.value(indexKey).toStringList();
Chris@461:     
Chris@461:     for (int i = 0; i < indices.size(); ++i) {
Chris@461: 
Chris@461:         QString index = indices[i];
Chris@461: 
Chris@690:         SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs: index url is "
Chris@687:                   << index << endl;
Chris@461: 
Chris@467:         CachedFile cf(index);
Chris@467:         if (!cf.isOK()) continue;
Chris@467: 
Chris@467:         FileSource indexSource(cf.getLocalFilename());
Chris@461: 
Chris@461:         PlaylistFileReader reader(indexSource);
Chris@461:         if (!reader.isOK()) continue;
Chris@461: 
Chris@461:         PlaylistFileReader::Playlist list = reader.load();
Chris@461:         for (PlaylistFileReader::Playlist::const_iterator j = list.begin();
Chris@461:              j != list.end(); ++j) {
Chris@690:             SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs: url is "
Chris@844:                   << *j << endl;
Chris@489:             pullURL(*j);
Chris@461:         }
Chris@461:     }
Chris@461: 
Chris@461:     QString urlListKey("rdf-urls");
Chris@461:     QStringList urls = settings.value(urlListKey).toStringList();
Chris@461: 
Chris@461:     for (int i = 0; i < urls.size(); ++i) {
Chris@489:         pullURL(urls[i]);
Chris@461:     }
Chris@461:     
Chris@461:     settings.endGroup();
Chris@489:     reindex();
Chris@1844: 
Chris@461:     return true;
Chris@461: }
Chris@461: 
Chris@1844: void
Chris@1844: PluginRDFIndexer::performConsistencyChecks()
Chris@1844: {
Chris@1844:     // Add more here!
Chris@1844:     
Chris@1844:     Triples packs = m_index->match
Chris@1844:         (Triple(Node(), m_index->expand("vamp:available_library"), Node()));
Chris@1844: 
Chris@1844:     for (Triple packt: packs) {
Chris@1844:         Triples libraries = m_index->match
Chris@1844:             (Triple(packt.object(), m_index->expand("a"),
Chris@1844:                     m_index->expand("vamp:PluginLibrary")));
Chris@1844:         if (libraries.empty()) {
Chris@1844:             SVCERR << "WARNING: Plugin pack " << packt.subject()
Chris@1844:                    << " claims to contain library " << packt.object()
Chris@1844:                    << " which is not known to us as a vamp:PluginLibrary"
Chris@1844:                    << endl;
Chris@1844:         }
Chris@1844:     }
Chris@1844: }
Chris@1844: 
Chris@439: QString
Chris@439: PluginRDFIndexer::getURIForPluginId(QString pluginId)
Chris@439: {
Chris@461:     QMutexLocker locker(&m_mutex);
Chris@461: 
Chris@439:     if (m_idToUriMap.find(pluginId) == m_idToUriMap.end()) return "";
Chris@439:     return m_idToUriMap[pluginId];
Chris@439: }
Chris@439: 
Chris@439: QString
Chris@439: PluginRDFIndexer::getIdForPluginURI(QString uri)
Chris@439: {
Chris@476:     m_mutex.lock();
Chris@461: 
Chris@439:     if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) {
Chris@439: 
Chris@476:         m_mutex.unlock();
Chris@476: 
Chris@439:         // Haven't found this uri referenced in any document on the
Chris@439:         // local filesystem; try resolving the pre-fragment part of
Chris@439:         // the uri as a document URL and reading that if possible.
Chris@439: 
Chris@439:         // Because we may want to refer to this document again, we
Chris@439:         // cache it locally if it turns out to exist.
Chris@439: 
Chris@1272:         SVDEBUG << "PluginRDFIndexer::getIdForPluginURI: NOTE: Failed to find a local RDF document describing plugin <" << uri << ">: attempting to retrieve one remotely by guesswork" << endl;
Chris@439: 
Chris@439:         QString baseUrl = QUrl(uri).toString(QUrl::RemoveFragment);
Chris@439: 
Chris@457:         indexURL(baseUrl);
Chris@439: 
Chris@476:         m_mutex.lock();
Chris@476: 
Chris@439:         if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) {
Chris@439:             m_uriToIdMap[uri] = "";
Chris@439:         }
Chris@439:     }
Chris@439: 
Chris@476:     QString id = m_uriToIdMap[uri];
Chris@476:     m_mutex.unlock();
Chris@476:     return id;
Chris@439: }
Chris@439: 
Chris@456: QStringList
Chris@456: PluginRDFIndexer::getIndexedPluginIds() 
Chris@456: {
Chris@461:     QMutexLocker locker(&m_mutex);
Chris@461: 
Chris@456:     QStringList ids;
Chris@489:     for (StringMap::const_iterator i = m_idToUriMap.begin();
Chris@489:          i != m_idToUriMap.end(); ++i) {
Chris@456:         ids.push_back(i->first);
Chris@456:     }
Chris@456:     return ids;
Chris@456: }
Chris@456: 
Chris@439: bool
Chris@489: PluginRDFIndexer::pullFile(QString filepath)
Chris@439: {
Chris@439:     QUrl url = QUrl::fromLocalFile(filepath);
Chris@439:     QString urlString = url.toString();
Chris@489:     return pullURL(urlString);
Chris@439: }
Chris@461: 
Chris@439: bool
Chris@439: PluginRDFIndexer::indexURL(QString urlString)
Chris@439: {
Chris@489:     bool pulled = pullURL(urlString);
Chris@489:     if (!pulled) return false;
Chris@489:     reindex();
Chris@489:     return true;
Chris@489: }
Chris@489: 
Chris@489: bool
Chris@489: PluginRDFIndexer::pullURL(QString urlString)
Chris@489: {
Chris@457:     Profiler profiler("PluginRDFIndexer::indexURL");
Chris@457: 
Chris@1272: //    SVDEBUG << "PluginRDFIndexer::indexURL(" << urlString << ")" << endl;
Chris@461: 
Chris@461:     QMutexLocker locker(&m_mutex);
Chris@461: 
Chris@725:     QUrl local = urlString;
Chris@457: 
Chris@457:     if (FileSource::isRemote(urlString) &&
Chris@457:         FileSource::canHandleScheme(urlString)) {
Chris@457: 
Chris@1582:         CachedFile cf(urlString, nullptr, "application/rdf+xml");
Chris@467:         if (!cf.isOK()) {
Chris@467:             return false;
Chris@467:         }
Chris@467: 
Chris@725:         local = QUrl::fromLocalFile(cf.getLocalFilename());
Chris@725: 
Chris@730:     } else if (urlString.startsWith("file:")) {
Chris@730: 
Chris@730:         local = QUrl(urlString);
Chris@730: 
Chris@725:     } else {
Chris@725: 
Chris@725:         local = QUrl::fromLocalFile(urlString);
Chris@457:     }
Chris@457: 
Chris@725:     try {
Chris@1844:         m_index->import(local, BasicStore::ImportIgnoreDuplicates);
Chris@725:     } catch (RDFException &e) {
Chris@1272:         SVDEBUG << e.what() << endl;
Chris@1272:         SVDEBUG << "PluginRDFIndexer::pullURL: Failed to import document from "
Chris@1272:                  << urlString << ": " << e.what() << endl;
Chris@725:         return false;
Chris@725:     }
Chris@725:     return true;
Chris@489: }
Chris@489: 
Chris@489: bool
Chris@489: PluginRDFIndexer::reindex()
Chris@489: {
Chris@725:     Triples tt = m_index->match
Chris@730:         (Triple(Node(), Uri("a"), m_index->expand("vamp:Plugin")));
Chris@730:     Nodes plugins = tt.subjects();
Chris@439: 
Chris@439:     bool foundSomething = false;
Chris@439:     bool addedSomething = false;
Chris@439: 
Chris@725:     foreach (Node plugin, plugins) {
Chris@725:         
Chris@725:         if (plugin.type != Node::URI) {
Chris@1272:             SVDEBUG << "PluginRDFIndexer::reindex: Plugin has no URI: node is "
Chris@725:                  << plugin << endl;
Chris@439:             continue;
Chris@439:         }
Chris@725:         
Chris@730:         Node idn = m_index->complete
Chris@730:             (Triple(plugin, m_index->expand("vamp:identifier"), Node()));
Chris@730: 
Chris@730:         if (idn.type != Node::Literal) {
Chris@1272:             SVDEBUG << "PluginRDFIndexer::reindex: Plugin " << plugin
Chris@725:                  << " lacks vamp:identifier literal" << endl;
Chris@439:             continue;
Chris@439:         }
Chris@481: 
Chris@730:         Node libn = m_index->complete
Chris@730:             (Triple(Node(), m_index->expand("vamp:available_plugin"), plugin));
Chris@481: 
Chris@730:         if (libn.type != Node::URI) {
Chris@1272:             SVDEBUG << "PluginRDFIndexer::reindex: Plugin " << plugin 
Chris@725:                  << " is not vamp:available_plugin in any library" << endl;
Chris@481:             continue;
Chris@481:         }
Chris@481: 
Chris@730:         Node son = m_index->complete
Chris@730:             (Triple(libn, m_index->expand("vamp:identifier"), Node()));
Chris@725: 
Chris@730:         if (son.type != Node::Literal) {
Chris@1272:             SVDEBUG << "PluginRDFIndexer::reindex: Library " << libn
Chris@725:                  << " lacks vamp:identifier for soname" << endl;
Chris@725:             continue;
Chris@725:         }
Chris@725: 
Chris@725:         QString pluginUri = plugin.value;
Chris@730:         QString identifier = idn.value;
Chris@730:         QString soname = son.value;
Chris@725: 
Chris@439:         QString pluginId = PluginIdentifier::createIdentifier
Chris@439:             ("vamp", soname, identifier);
Chris@439: 
Chris@439:         foundSomething = true;
Chris@439: 
Chris@489:         if (m_idToUriMap.find(pluginId) != m_idToUriMap.end()) {
Chris@439:             continue;
Chris@439:         }
Chris@439: 
Chris@439:         m_idToUriMap[pluginId] = pluginUri;
Chris@439: 
Chris@439:         addedSomething = true;
Chris@439: 
Chris@439:         if (pluginUri != "") {
Chris@439:             if (m_uriToIdMap.find(pluginUri) != m_uriToIdMap.end()) {
Chris@1272:                 SVDEBUG << "PluginRDFIndexer::reindex: WARNING: Found multiple plugins with the same URI:" << endl;
Chris@1272:                 SVDEBUG << "  1. Plugin id \"" << m_uriToIdMap[pluginUri] << "\"" << endl;
Chris@1272:                 SVDEBUG << "  2. Plugin id \"" << pluginId << "\"" << endl;
Chris@1272:                 SVDEBUG << "both claim URI <" << pluginUri << ">" << endl;
Chris@439:             } else {
Chris@439:                 m_uriToIdMap[pluginUri] = pluginId;
Chris@439:             }
Chris@439:         }
Chris@439:     }
Chris@439: 
Chris@439:     if (!foundSomething) {
Chris@1272:         SVDEBUG << "PluginRDFIndexer::reindex: NOTE: Plugins found, but none sufficiently described" << endl;
Chris@439:     }
Chris@439:     
Chris@439:     return addedSomething;
Chris@439: }