Chris@439: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
Chris@439: 
Chris@439: /*
Chris@439:     Sonic Visualiser
Chris@439:     An audio file viewer and annotation editor.
Chris@439:     Centre for Digital Music, Queen Mary, University of London.
Chris@439:     This file copyright 2008 QMUL.
Chris@439:    
Chris@439:     This program is free software; you can redistribute it and/or
Chris@439:     modify it under the terms of the GNU General Public License as
Chris@439:     published by the Free Software Foundation; either version 2 of the
Chris@439:     License, or (at your option) any later version.  See the file
Chris@439:     COPYING included with this distribution for more information.
Chris@439: */
Chris@439: 
Chris@439: #include "PluginRDFIndexer.h"
Chris@439: 
Chris@439: #include "SimpleSPARQLQuery.h"
Chris@439: 
Chris@467: #include "data/fileio/CachedFile.h"
Chris@471: #include "data/fileio/FileSource.h"
Chris@461: #include "data/fileio/PlaylistFileReader.h"
Chris@439: #include "plugin/PluginIdentifier.h"
Chris@439: 
Chris@457: #include "base/Profiler.h"
Chris@457: 
Chris@475: #include <vamp-hostsdk/PluginHostAdapter.h>
Chris@439: 
Chris@439: #include <QFileInfo>
Chris@439: #include <QDir>
Chris@439: #include <QUrl>
Chris@461: #include <QDateTime>
Chris@461: #include <QSettings>
Chris@461: #include <QFile>
Chris@439: 
Chris@439: #include <iostream>
Chris@439: using std::cerr;
Chris@439: using std::endl;
Chris@439: using std::vector;
Chris@439: using std::string;
Chris@439: using Vamp::PluginHostAdapter;
Chris@439: 
Chris@439: PluginRDFIndexer *
Chris@439: PluginRDFIndexer::m_instance = 0;
Chris@439: 
Chris@520: bool
Chris@520: PluginRDFIndexer::m_prefixesLoaded = false;
Chris@520: 
Chris@439: PluginRDFIndexer *
Chris@439: PluginRDFIndexer::getInstance() 
Chris@439: {
Chris@439:     if (!m_instance) m_instance = new PluginRDFIndexer();
Chris@439:     return m_instance;
Chris@439: }
Chris@439: 
Chris@439: PluginRDFIndexer::PluginRDFIndexer()
Chris@439: {
Chris@477:     indexInstalledURLs();
Chris@477: }
Chris@477: 
Chris@477: PluginRDFIndexer::~PluginRDFIndexer()
Chris@477: {
Chris@477:     QMutexLocker locker(&m_mutex);
Chris@477: }
Chris@477: 
Chris@477: void
Chris@477: PluginRDFIndexer::indexInstalledURLs()
Chris@477: {
Chris@439:     vector<string> paths = PluginHostAdapter::getPluginPath();
Chris@439: 
Chris@439:     QStringList filters;
Chris@439:     filters << "*.n3";
Chris@439:     filters << "*.N3";
Chris@439:     filters << "*.rdf";
Chris@439:     filters << "*.RDF";
Chris@439: 
Chris@439:     // Search each Vamp plugin path for a .rdf file that either has
Chris@439:     // name "soname", "soname:label" or "soname/label" plus RDF
Chris@439:     // extension.  Use that order of preference, and prefer n3 over
Chris@439:     // rdf extension.
Chris@439: 
Chris@439:     for (vector<string>::const_iterator i = paths.begin(); i != paths.end(); ++i) {
Chris@439:         
Chris@439:         QDir dir(i->c_str());
Chris@439:         if (!dir.exists()) continue;
Chris@439: 
Chris@439:         QStringList entries = dir.entryList
Chris@439:             (filters, QDir::Files | QDir::Readable);
Chris@439: 
Chris@439:         for (QStringList::const_iterator j = entries.begin();
Chris@439:              j != entries.end(); ++j) {
Chris@439:             QFileInfo fi(dir.filePath(*j));
Chris@489:             pullFile(fi.absoluteFilePath());
Chris@439:         }
Chris@439: 
Chris@439:         QStringList subdirs = dir.entryList
Chris@439:             (QDir::AllDirs | QDir::NoDotAndDotDot | QDir::Readable);
Chris@439: 
Chris@439:         for (QStringList::const_iterator j = subdirs.begin();
Chris@439:              j != subdirs.end(); ++j) {
Chris@439:             QDir subdir(dir.filePath(*j));
Chris@439:             if (subdir.exists()) {
Chris@439:                 entries = subdir.entryList
Chris@439:                     (filters, QDir::Files | QDir::Readable);
Chris@439:                 for (QStringList::const_iterator k = entries.begin();
Chris@439:                      k != entries.end(); ++k) {
Chris@439:                     QFileInfo fi(subdir.filePath(*k));
Chris@489:                     pullFile(fi.absoluteFilePath());
Chris@439:                 }
Chris@439:             }
Chris@439:         }
Chris@439:     }
Chris@489: 
Chris@489:     reindex();
Chris@439: }
Chris@439: 
Chris@461: bool
Chris@461: PluginRDFIndexer::indexConfiguredURLs()
Chris@461: {
Chris@461:     std::cerr << "PluginRDFIndexer::indexConfiguredURLs" << std::endl;
Chris@461: 
Chris@461:     QSettings settings;
Chris@461:     settings.beginGroup("RDF");
Chris@461:     
Chris@461:     QString indexKey("rdf-indices");
Chris@461:     QStringList indices = settings.value(indexKey).toStringList();
Chris@461:     
Chris@461:     for (int i = 0; i < indices.size(); ++i) {
Chris@461: 
Chris@461:         QString index = indices[i];
Chris@461: 
Chris@461:         std::cerr << "PluginRDFIndexer::indexConfiguredURLs: index url is "
Chris@461:                   << index.toStdString() << std::endl;
Chris@461: 
Chris@467:         CachedFile cf(index);
Chris@467:         if (!cf.isOK()) continue;
Chris@467: 
Chris@467:         FileSource indexSource(cf.getLocalFilename());
Chris@461: 
Chris@461:         PlaylistFileReader reader(indexSource);
Chris@461:         if (!reader.isOK()) continue;
Chris@461: 
Chris@461:         PlaylistFileReader::Playlist list = reader.load();
Chris@461:         for (PlaylistFileReader::Playlist::const_iterator j = list.begin();
Chris@461:              j != list.end(); ++j) {
Chris@461:             std::cerr << "PluginRDFIndexer::indexConfiguredURLs: url is "
Chris@461:                       << j->toStdString() << std::endl;
Chris@489:             pullURL(*j);
Chris@461:         }
Chris@461:     }
Chris@461: 
Chris@461:     QString urlListKey("rdf-urls");
Chris@461:     QStringList urls = settings.value(urlListKey).toStringList();
Chris@461: 
Chris@461:     for (int i = 0; i < urls.size(); ++i) {
Chris@489:         pullURL(urls[i]);
Chris@461:     }
Chris@461:     
Chris@461:     settings.endGroup();
Chris@489:     reindex();
Chris@461:     return true;
Chris@461: }
Chris@461: 
Chris@439: QString
Chris@439: PluginRDFIndexer::getURIForPluginId(QString pluginId)
Chris@439: {
Chris@461:     QMutexLocker locker(&m_mutex);
Chris@461: 
Chris@439:     if (m_idToUriMap.find(pluginId) == m_idToUriMap.end()) return "";
Chris@439:     return m_idToUriMap[pluginId];
Chris@439: }
Chris@439: 
Chris@439: QString
Chris@439: PluginRDFIndexer::getIdForPluginURI(QString uri)
Chris@439: {
Chris@476:     m_mutex.lock();
Chris@461: 
Chris@439:     if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) {
Chris@439: 
Chris@476:         m_mutex.unlock();
Chris@476: 
Chris@439:         // Haven't found this uri referenced in any document on the
Chris@439:         // local filesystem; try resolving the pre-fragment part of
Chris@439:         // the uri as a document URL and reading that if possible.
Chris@439: 
Chris@439:         // Because we may want to refer to this document again, we
Chris@439:         // cache it locally if it turns out to exist.
Chris@439: 
Chris@439:         cerr << "PluginRDFIndexer::getIdForPluginURI: NOTE: Failed to find a local RDF document describing plugin <" << uri.toStdString() << ">: attempting to retrieve one remotely by guesswork" << endl;
Chris@439: 
Chris@439:         QString baseUrl = QUrl(uri).toString(QUrl::RemoveFragment);
Chris@439: 
Chris@457:         indexURL(baseUrl);
Chris@439: 
Chris@476:         m_mutex.lock();
Chris@476: 
Chris@439:         if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) {
Chris@439:             m_uriToIdMap[uri] = "";
Chris@439:         }
Chris@439:     }
Chris@439: 
Chris@476:     QString id = m_uriToIdMap[uri];
Chris@476:     m_mutex.unlock();
Chris@476:     return id;
Chris@439: }
Chris@439: 
Chris@456: QStringList
Chris@456: PluginRDFIndexer::getIndexedPluginIds() 
Chris@456: {
Chris@461:     QMutexLocker locker(&m_mutex);
Chris@461: 
Chris@456:     QStringList ids;
Chris@489:     for (StringMap::const_iterator i = m_idToUriMap.begin();
Chris@489:          i != m_idToUriMap.end(); ++i) {
Chris@456:         ids.push_back(i->first);
Chris@456:     }
Chris@456:     return ids;
Chris@456: }
Chris@456: 
Chris@439: bool
Chris@489: PluginRDFIndexer::pullFile(QString filepath)
Chris@439: {
Chris@439:     QUrl url = QUrl::fromLocalFile(filepath);
Chris@439:     QString urlString = url.toString();
Chris@489:     return pullURL(urlString);
Chris@439: }
Chris@461: 
Chris@439: bool
Chris@439: PluginRDFIndexer::indexURL(QString urlString)
Chris@439: {
Chris@489:     bool pulled = pullURL(urlString);
Chris@489:     if (!pulled) return false;
Chris@489:     reindex();
Chris@489:     return true;
Chris@489: }
Chris@489: 
Chris@489: bool
Chris@489: PluginRDFIndexer::pullURL(QString urlString)
Chris@489: {
Chris@457:     Profiler profiler("PluginRDFIndexer::indexURL");
Chris@457: 
Chris@520:     loadPrefixes();
Chris@520: 
Chris@508: //    std::cerr << "PluginRDFIndexer::indexURL(" << urlString.toStdString() << ")" << std::endl;
Chris@461: 
Chris@461:     QMutexLocker locker(&m_mutex);
Chris@461: 
Chris@457:     QString localString = urlString;
Chris@457: 
Chris@457:     if (FileSource::isRemote(urlString) &&
Chris@457:         FileSource::canHandleScheme(urlString)) {
Chris@457: 
Chris@520:         CachedFile cf(urlString, 0, "application/rdf+xml");
Chris@467:         if (!cf.isOK()) {
Chris@467:             return false;
Chris@467:         }
Chris@467: 
Chris@483:         localString = QUrl::fromLocalFile(cf.getLocalFilename()).toString();
Chris@457:     }
Chris@457: 
Chris@489:     return SimpleSPARQLQuery::addSourceToModel(localString);
Chris@489: }
Chris@489: 
Chris@489: bool
Chris@489: PluginRDFIndexer::reindex()
Chris@489: {
Chris@489:     SimpleSPARQLQuery::QueryType m = SimpleSPARQLQuery::QueryFromModel;
Chris@489: 
Chris@439:     SimpleSPARQLQuery query
Chris@489:         (m,
Chris@481:          QString
Chris@481:          (
Chris@481:              " PREFIX vamp: <http://purl.org/ontology/vamp/> "
Chris@481: 
Chris@481:              " SELECT ?plugin ?library ?plugin_id "
Chris@481: 
Chris@481:              " WHERE { "
Chris@481:              "   ?plugin a vamp:Plugin . "
Chris@481:              "   ?plugin vamp:identifier ?plugin_id . "
Chris@481: 
Chris@481:              "   OPTIONAL { "
Chris@481:              "     ?library vamp:available_plugin ?plugin "
Chris@481:              "   } "
Chris@481:              " } "
Chris@489:              ));
Chris@439: 
Chris@439:     SimpleSPARQLQuery::ResultList results = query.execute();
Chris@439: 
Chris@439:     if (!query.isOK()) {
Chris@489:         cerr << "ERROR: PluginRDFIndexer::reindex: ERROR: Failed to query plugins from model: "
Chris@439:              << query.getErrorString().toStdString() << endl;
Chris@439:         return false;
Chris@439:     }
Chris@439: 
Chris@439:     if (results.empty()) {
Chris@489:         cerr << "PluginRDFIndexer::reindex: NOTE: no vamp:Plugin resources found in indexed documents" << endl;
Chris@439:         return false;
Chris@439:     }
Chris@439: 
Chris@439:     bool foundSomething = false;
Chris@439:     bool addedSomething = false;
Chris@439: 
Chris@439:     for (SimpleSPARQLQuery::ResultList::iterator i = results.begin();
Chris@439:          i != results.end(); ++i) {
Chris@439: 
Chris@439:         QString pluginUri = (*i)["plugin"].value;
Chris@481:         QString soUri = (*i)["library"].value;
Chris@439:         QString identifier = (*i)["plugin_id"].value;
Chris@439: 
Chris@439:         if (identifier == "") {
Chris@489:             cerr << "PluginRDFIndexer::reindex: NOTE: No vamp:identifier for plugin <"
Chris@439:                  << pluginUri.toStdString() << ">"
Chris@439:                  << endl;
Chris@439:             continue;
Chris@439:         }
Chris@481:         if (soUri == "") {
Chris@489:             cerr << "PluginRDFIndexer::reindex: NOTE: No implementation library for plugin <"
Chris@482:                  << pluginUri.toStdString() << ">"
Chris@439:                  << endl;
Chris@439:             continue;
Chris@439:         }
Chris@481: 
Chris@481:         QString sonameQuery =
Chris@481:             QString(
Chris@481:                 " PREFIX vamp: <http://purl.org/ontology/vamp/> "
Chris@481:                 " SELECT ?library_id "
Chris@481:                 " WHERE { "
Chris@489:                 "   <%1> vamp:identifier ?library_id "
Chris@481:                 " } "
Chris@481:                 )
Chris@481:             .arg(soUri);
Chris@481: 
Chris@481:         SimpleSPARQLQuery::Value sonameValue = 
Chris@489:             SimpleSPARQLQuery::singleResultQuery(m, sonameQuery, "library_id");
Chris@481:         QString soname = sonameValue.value;
Chris@481:         if (soname == "") {
Chris@489:             cerr << "PluginRDFIndexer::reindex: NOTE: No identifier for library <"
Chris@481:                  << soUri.toStdString() << ">"
Chris@481:                  << endl;
Chris@481:             continue;
Chris@481:         }
Chris@481: 
Chris@439:         QString pluginId = PluginIdentifier::createIdentifier
Chris@439:             ("vamp", soname, identifier);
Chris@439: 
Chris@439:         foundSomething = true;
Chris@439: 
Chris@489:         if (m_idToUriMap.find(pluginId) != m_idToUriMap.end()) {
Chris@439:             continue;
Chris@439:         }
Chris@439: 
Chris@439:         m_idToUriMap[pluginId] = pluginUri;
Chris@439: 
Chris@439:         addedSomething = true;
Chris@439: 
Chris@439:         if (pluginUri != "") {
Chris@439:             if (m_uriToIdMap.find(pluginUri) != m_uriToIdMap.end()) {
Chris@489:                 cerr << "PluginRDFIndexer::reindex: WARNING: Found multiple plugins with the same URI:" << endl;
Chris@439:                 cerr << "  1. Plugin id \"" << m_uriToIdMap[pluginUri].toStdString() << "\"" << endl;
Chris@439:                 cerr << "  2. Plugin id \"" << pluginId.toStdString() << "\"" << endl;
Chris@439:                 cerr << "both claim URI <" << pluginUri.toStdString() << ">" << endl;
Chris@439:             } else {
Chris@439:                 m_uriToIdMap[pluginUri] = pluginId;
Chris@439:             }
Chris@439:         }
Chris@439:     }
Chris@439: 
Chris@439:     if (!foundSomething) {
Chris@489:         cerr << "PluginRDFIndexer::reindex: NOTE: Plugins found, but none sufficiently described" << endl;
Chris@439:     }
Chris@439:     
Chris@439:     return addedSomething;
Chris@439: }
Chris@439: 
Chris@520: void
Chris@520: PluginRDFIndexer::loadPrefixes()
Chris@520: {
Chris@520:     return;
Chris@520: //!!!
Chris@520:     if (m_prefixesLoaded) return;
Chris@520:     const char *prefixes[] = {
Chris@520:         "http://purl.org/ontology/vamp/"
Chris@520:     };
Chris@520:     for (size_t i = 0; i < sizeof(prefixes)/sizeof(prefixes[0]); ++i) {
Chris@520:         CachedFile cf(prefixes[i], 0, "application/rdf+xml");
Chris@520:         if (!cf.isOK()) continue;
Chris@520:         SimpleSPARQLQuery::addSourceToModel
Chris@520:             (QUrl::fromLocalFile(cf.getLocalFilename()).toString());
Chris@520:     }
Chris@520:     m_prefixesLoaded = true;
Chris@520: }
Chris@439: 
Chris@439: