Chris@439: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
Chris@439: 
Chris@439: /*
Chris@439:     Sonic Visualiser
Chris@439:     An audio file viewer and annotation editor.
Chris@439:     Centre for Digital Music, Queen Mary, University of London.
Chris@439:     This file copyright 2008 QMUL.
Chris@439:    
Chris@439:     This program is free software; you can redistribute it and/or
Chris@439:     modify it under the terms of the GNU General Public License as
Chris@439:     published by the Free Software Foundation; either version 2 of the
Chris@439:     License, or (at your option) any later version.  See the file
Chris@439:     COPYING included with this distribution for more information.
Chris@439: */
Chris@439: 
Chris@439: #include "PluginRDFIndexer.h"
Chris@439: 
Chris@439: #include "SimpleSPARQLQuery.h"
Chris@439: 
Chris@467: #include "data/fileio/CachedFile.h"
Chris@471: #include "data/fileio/FileSource.h"
Chris@461: #include "data/fileio/PlaylistFileReader.h"
Chris@439: #include "plugin/PluginIdentifier.h"
Chris@439: 
Chris@457: #include "base/Profiler.h"
Chris@457: 
Chris@475: #include <vamp-hostsdk/PluginHostAdapter.h>
Chris@439: 
Chris@439: #include <QFileInfo>
Chris@439: #include <QDir>
Chris@439: #include <QUrl>
Chris@461: #include <QDateTime>
Chris@461: #include <QSettings>
Chris@461: #include <QFile>
Chris@439: 
Chris@439: #include <iostream>
Chris@439: using std::cerr;
Chris@439: using std::endl;
Chris@439: using std::vector;
Chris@439: using std::string;
Chris@439: using Vamp::PluginHostAdapter;
Chris@439: 
Chris@439: PluginRDFIndexer *
Chris@439: PluginRDFIndexer::m_instance = 0;
Chris@439: 
Chris@439: PluginRDFIndexer *
Chris@439: PluginRDFIndexer::getInstance() 
Chris@439: {
Chris@439:     if (!m_instance) m_instance = new PluginRDFIndexer();
Chris@439:     return m_instance;
Chris@439: }
Chris@439: 
Chris@439: PluginRDFIndexer::PluginRDFIndexer()
Chris@439: {
Chris@477:     indexInstalledURLs();
Chris@477: }
Chris@477: 
Chris@477: PluginRDFIndexer::~PluginRDFIndexer()
Chris@477: {
Chris@477:     QMutexLocker locker(&m_mutex);
Chris@477: }
Chris@477: 
Chris@477: void
Chris@477: PluginRDFIndexer::indexInstalledURLs()
Chris@477: {
Chris@439:     vector<string> paths = PluginHostAdapter::getPluginPath();
Chris@439: 
Chris@439:     QStringList filters;
Chris@439:     filters << "*.n3";
Chris@439:     filters << "*.N3";
Chris@439:     filters << "*.rdf";
Chris@439:     filters << "*.RDF";
Chris@439: 
Chris@439:     // Search each Vamp plugin path for a .rdf file that either has
Chris@439:     // name "soname", "soname:label" or "soname/label" plus RDF
Chris@439:     // extension.  Use that order of preference, and prefer n3 over
Chris@439:     // rdf extension.
Chris@439: 
Chris@439:     for (vector<string>::const_iterator i = paths.begin(); i != paths.end(); ++i) {
Chris@439:         
Chris@439:         QDir dir(i->c_str());
Chris@439:         if (!dir.exists()) continue;
Chris@439: 
Chris@439:         QStringList entries = dir.entryList
Chris@439:             (filters, QDir::Files | QDir::Readable);
Chris@439: 
Chris@439:         for (QStringList::const_iterator j = entries.begin();
Chris@439:              j != entries.end(); ++j) {
Chris@439:             QFileInfo fi(dir.filePath(*j));
Chris@439:             indexFile(fi.absoluteFilePath());
Chris@439:         }
Chris@439: 
Chris@439:         QStringList subdirs = dir.entryList
Chris@439:             (QDir::AllDirs | QDir::NoDotAndDotDot | QDir::Readable);
Chris@439: 
Chris@439:         for (QStringList::const_iterator j = subdirs.begin();
Chris@439:              j != subdirs.end(); ++j) {
Chris@439:             QDir subdir(dir.filePath(*j));
Chris@439:             if (subdir.exists()) {
Chris@439:                 entries = subdir.entryList
Chris@439:                     (filters, QDir::Files | QDir::Readable);
Chris@439:                 for (QStringList::const_iterator k = entries.begin();
Chris@439:                      k != entries.end(); ++k) {
Chris@439:                     QFileInfo fi(subdir.filePath(*k));
Chris@439:                     indexFile(fi.absoluteFilePath());
Chris@439:                 }
Chris@439:             }
Chris@439:         }
Chris@439:     }
Chris@439: }
Chris@439: 
Chris@461: bool
Chris@461: PluginRDFIndexer::indexConfiguredURLs()
Chris@461: {
Chris@461:     std::cerr << "PluginRDFIndexer::indexConfiguredURLs" << std::endl;
Chris@461: 
Chris@461:     QSettings settings;
Chris@461:     settings.beginGroup("RDF");
Chris@461:     
Chris@461:     QString indexKey("rdf-indices");
Chris@461:     QStringList indices = settings.value(indexKey).toStringList();
Chris@461:     
Chris@461:     for (int i = 0; i < indices.size(); ++i) {
Chris@461: 
Chris@461:         QString index = indices[i];
Chris@461: 
Chris@461:         std::cerr << "PluginRDFIndexer::indexConfiguredURLs: index url is "
Chris@461:                   << index.toStdString() << std::endl;
Chris@461: 
Chris@467:         CachedFile cf(index);
Chris@467:         if (!cf.isOK()) continue;
Chris@467: 
Chris@467:         FileSource indexSource(cf.getLocalFilename());
Chris@461: 
Chris@461:         PlaylistFileReader reader(indexSource);
Chris@461:         if (!reader.isOK()) continue;
Chris@461: 
Chris@461:         PlaylistFileReader::Playlist list = reader.load();
Chris@461:         for (PlaylistFileReader::Playlist::const_iterator j = list.begin();
Chris@461:              j != list.end(); ++j) {
Chris@461:             std::cerr << "PluginRDFIndexer::indexConfiguredURLs: url is "
Chris@461:                       << j->toStdString() << std::endl;
Chris@461:             indexURL(*j);
Chris@461:         }
Chris@461:     }
Chris@461: 
Chris@461:     QString urlListKey("rdf-urls");
Chris@461:     QStringList urls = settings.value(urlListKey).toStringList();
Chris@461: 
Chris@461:     for (int i = 0; i < urls.size(); ++i) {
Chris@461:         indexURL(urls[i]);
Chris@461:     }
Chris@461:     
Chris@461:     settings.endGroup();
Chris@461:     return true;
Chris@461: }
Chris@461: 
Chris@439: QString
Chris@439: PluginRDFIndexer::getURIForPluginId(QString pluginId)
Chris@439: {
Chris@461:     QMutexLocker locker(&m_mutex);
Chris@461: 
Chris@439:     if (m_idToUriMap.find(pluginId) == m_idToUriMap.end()) return "";
Chris@439:     return m_idToUriMap[pluginId];
Chris@439: }
Chris@439: 
Chris@439: QString
Chris@439: PluginRDFIndexer::getIdForPluginURI(QString uri)
Chris@439: {
Chris@476:     m_mutex.lock();
Chris@461: 
Chris@439:     if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) {
Chris@439: 
Chris@476:         m_mutex.unlock();
Chris@476: 
Chris@439:         // Haven't found this uri referenced in any document on the
Chris@439:         // local filesystem; try resolving the pre-fragment part of
Chris@439:         // the uri as a document URL and reading that if possible.
Chris@439: 
Chris@439:         // Because we may want to refer to this document again, we
Chris@439:         // cache it locally if it turns out to exist.
Chris@439: 
Chris@439:         cerr << "PluginRDFIndexer::getIdForPluginURI: NOTE: Failed to find a local RDF document describing plugin <" << uri.toStdString() << ">: attempting to retrieve one remotely by guesswork" << endl;
Chris@439: 
Chris@439:         QString baseUrl = QUrl(uri).toString(QUrl::RemoveFragment);
Chris@439: 
Chris@457:         indexURL(baseUrl);
Chris@439: 
Chris@476:         m_mutex.lock();
Chris@476: 
Chris@439:         if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) {
Chris@439:             m_uriToIdMap[uri] = "";
Chris@439:         }
Chris@439:     }
Chris@439: 
Chris@476:     QString id = m_uriToIdMap[uri];
Chris@476:     m_mutex.unlock();
Chris@476:     return id;
Chris@439: }
Chris@439: 
Chris@439: QString
Chris@439: PluginRDFIndexer::getDescriptionURLForPluginId(QString pluginId)
Chris@439: {
Chris@461:     QMutexLocker locker(&m_mutex);
Chris@461: 
Chris@439:     if (m_idToDescriptionMap.find(pluginId) == m_idToDescriptionMap.end()) return "";
Chris@439:     return m_idToDescriptionMap[pluginId];
Chris@439: }
Chris@439: 
Chris@439: QString
Chris@439: PluginRDFIndexer::getDescriptionURLForPluginURI(QString uri)
Chris@439: {
Chris@461:     QMutexLocker locker(&m_mutex);
Chris@461: 
Chris@439:     QString id = getIdForPluginURI(uri);
Chris@439:     if (id == "") return "";
Chris@439:     return getDescriptionURLForPluginId(id);
Chris@439: }
Chris@439: 
Chris@456: QStringList
Chris@456: PluginRDFIndexer::getIndexedPluginIds() 
Chris@456: {
Chris@461:     QMutexLocker locker(&m_mutex);
Chris@461: 
Chris@456:     QStringList ids;
Chris@456:     for (StringMap::const_iterator i = m_idToDescriptionMap.begin();
Chris@456:          i != m_idToDescriptionMap.end(); ++i) {
Chris@456:         ids.push_back(i->first);
Chris@456:     }
Chris@456:     return ids;
Chris@456: }
Chris@456: 
Chris@439: bool
Chris@439: PluginRDFIndexer::indexFile(QString filepath)
Chris@439: {
Chris@439:     QUrl url = QUrl::fromLocalFile(filepath);
Chris@439:     QString urlString = url.toString();
Chris@439:     return indexURL(urlString);
Chris@439: }
Chris@461: 
Chris@439: bool
Chris@439: PluginRDFIndexer::indexURL(QString urlString)
Chris@439: {
Chris@457:     Profiler profiler("PluginRDFIndexer::indexURL");
Chris@457: 
Chris@461:     std::cerr << "PluginRDFIndexer::indexURL(" << urlString.toStdString() << ")" << std::endl;
Chris@461: 
Chris@461:     QMutexLocker locker(&m_mutex);
Chris@461: 
Chris@457:     QString localString = urlString;
Chris@457: 
Chris@457:     if (FileSource::isRemote(urlString) &&
Chris@457:         FileSource::canHandleScheme(urlString)) {
Chris@457: 
Chris@467:         CachedFile cf(urlString);
Chris@467:         if (!cf.isOK()) {
Chris@467:             return false;
Chris@467:         }
Chris@467: 
Chris@483:         localString = QUrl::fromLocalFile(cf.getLocalFilename()).toString();
Chris@483: //        localString = "file://" + cf.getLocalFilename(); //!!! crud - fix!
Chris@457:     }
Chris@457: 
Chris@439: //    cerr << "PluginRDFIndexer::indexURL: url = <" << urlString.toStdString() << ">" << endl;
Chris@481: /*!!!
Chris@439:     SimpleSPARQLQuery query
Chris@480:         (localString,
Chris@480:          QString
Chris@439:          (
Chris@439:              " PREFIX vamp: <http://purl.org/ontology/vamp/> "
Chris@439: 
Chris@439:              " SELECT ?plugin ?library_id ?plugin_id "
Chris@439:              " FROM <%1> "
Chris@439: 
Chris@439:              " WHERE { "
Chris@439:              "   ?plugin a vamp:Plugin . "
Chris@439: 
Chris@439:              // Make the identifier and library parts optional, so
Chris@439:              // that we can check and report helpfully if one or both
Chris@439:              // is absent instead of just getting no results
Chris@439: 
Chris@440:              //!!! No -- because of rasqal's inability to correctly
Chris@440:              // handle more than one OPTIONAL graph in a query, let's
Chris@440:              // make identifier compulsory after all
Chris@440:              //"   OPTIONAL { ?plugin vamp:identifier ?plugin_id } . "
Chris@440: 
Chris@440:              "   ?plugin vamp:identifier ?plugin_id . "
Chris@439: 
Chris@439:              "   OPTIONAL { "
Chris@439:              "     ?library a vamp:PluginLibrary ; "
Chris@439:              "              vamp:available_plugin ?plugin ; "
Chris@439:              "              vamp:identifier ?library_id "
Chris@439:              "   } "
Chris@439:              " } "
Chris@439:              )
Chris@457:          .arg(localString));
Chris@481: */
Chris@481:     SimpleSPARQLQuery query
Chris@481:         (localString,
Chris@481:          QString
Chris@481:          (
Chris@481:              " PREFIX vamp: <http://purl.org/ontology/vamp/> "
Chris@481: 
Chris@481:              " SELECT ?plugin ?library ?plugin_id "
Chris@481:              " FROM <%1> "
Chris@481: 
Chris@481:              " WHERE { "
Chris@481:              "   ?plugin a vamp:Plugin . "
Chris@481:              "   ?plugin vamp:identifier ?plugin_id . "
Chris@481: 
Chris@481:              "   OPTIONAL { "
Chris@481:              "     ?library vamp:available_plugin ?plugin "
Chris@481:              "   } "
Chris@481:              " } "
Chris@481:              )
Chris@481:          .arg(localString));
Chris@439: 
Chris@439:     SimpleSPARQLQuery::ResultList results = query.execute();
Chris@439: 
Chris@439:     if (!query.isOK()) {
Chris@439:         cerr << "ERROR: PluginRDFIndexer::indexURL: ERROR: Failed to index document at <"
Chris@439:              << urlString.toStdString() << ">: "
Chris@439:              << query.getErrorString().toStdString() << endl;
Chris@439:         return false;
Chris@439:     }
Chris@439: 
Chris@439:     if (results.empty()) {
Chris@439:         cerr << "PluginRDFIndexer::indexURL: NOTE: Document at <"
Chris@439:              << urlString.toStdString()
Chris@439:              << "> does not describe any vamp:Plugin resources" << endl;
Chris@439:         return false;
Chris@439:     }
Chris@439: 
Chris@439:     bool foundSomething = false;
Chris@439:     bool addedSomething = false;
Chris@439: 
Chris@439:     for (SimpleSPARQLQuery::ResultList::iterator i = results.begin();
Chris@439:          i != results.end(); ++i) {
Chris@439: 
Chris@439:         QString pluginUri = (*i)["plugin"].value;
Chris@481: //!!!        QString soname = (*i)["library_id"].value;
Chris@481:         QString soUri = (*i)["library"].value;
Chris@439:         QString identifier = (*i)["plugin_id"].value;
Chris@439: 
Chris@439:         if (identifier == "") {
Chris@482:             cerr << "PluginRDFIndexer::indexURL: NOTE: No vamp:identifier for plugin <"
Chris@439:                  << pluginUri.toStdString() << ">"
Chris@439:                  << endl;
Chris@439:             continue;
Chris@439:         }
Chris@481:         if (soUri == "") {
Chris@482:             cerr << "PluginRDFIndexer::indexURL: NOTE: No implementation library for plugin <"
Chris@482:                  << pluginUri.toStdString() << ">"
Chris@439:                  << endl;
Chris@439:             continue;
Chris@439:         }
Chris@481: 
Chris@481:         QString sonameQuery =
Chris@481:             QString(
Chris@481:                 " PREFIX vamp: <http://purl.org/ontology/vamp/> "
Chris@481:                 " SELECT ?library_id "
Chris@481:                 " FROM <%1> "
Chris@481:                 " WHERE { "
Chris@481:                 "   <%2> vamp:identifier ?library_id "
Chris@481:                 " } "
Chris@481:                 )
Chris@481:             .arg(localString)
Chris@481:             .arg(soUri);
Chris@481: 
Chris@481:         SimpleSPARQLQuery::Value sonameValue = 
Chris@481:             SimpleSPARQLQuery::singleResultQuery(localString, sonameQuery, "library_id");
Chris@481:         QString soname = sonameValue.value;
Chris@481:         if (soname == "") {
Chris@482:             cerr << "PluginRDFIndexer::indexURL: NOTE: No identifier for library <"
Chris@481:                  << soUri.toStdString() << ">"
Chris@481:                  << endl;
Chris@481:             continue;
Chris@481:         }
Chris@481: 
Chris@481: 
Chris@439: /*
Chris@439:         cerr << "PluginRDFIndexer::indexURL: Document for plugin \""
Chris@439:              << soname.toStdString() << ":" << identifier.toStdString()
Chris@439:              << "\" (uri <" << pluginUri.toStdString() << ">) is at url <"
Chris@439:              << urlString.toStdString() << ">" << endl;
Chris@439: */
Chris@439:         QString pluginId = PluginIdentifier::createIdentifier
Chris@439:             ("vamp", soname, identifier);
Chris@439: 
Chris@439:         foundSomething = true;
Chris@439: 
Chris@439:         if (m_idToDescriptionMap.find(pluginId) != m_idToDescriptionMap.end()) {
Chris@482: /*!!!
Chris@482: 
Chris@482:   This can happen quite legitimately when using an RDF datastore rather
Chris@482:   than querying individual files, as of course the datastore contains
Chris@482:   all plugin data found so far, and each time a file is added to it,
Chris@482:   subsequent queries will return all older plugins as well.
Chris@482: 
Chris@482:   It would be more efficient to add everything at once and then do all
Chris@482:   queries, of course.
Chris@482: 
Chris@439:             cerr << "PluginRDFIndexer::indexURL: NOTE: Plugin id \""
Chris@439:                  << pluginId.toStdString() << "\", described in document at <"
Chris@439:                  << urlString.toStdString()
Chris@439:                  << ">, has already been described in document <"
Chris@439:                  << m_idToDescriptionMap[pluginId].toStdString()
Chris@439:                  << ">: ignoring this new description" << endl;
Chris@482: */
Chris@439:             continue;
Chris@439:         }
Chris@439: 
Chris@439:         m_idToDescriptionMap[pluginId] = urlString;
Chris@439:         m_idToUriMap[pluginId] = pluginUri;
Chris@439: 
Chris@439:         addedSomething = true;
Chris@439: 
Chris@439:         if (pluginUri != "") {
Chris@439:             if (m_uriToIdMap.find(pluginUri) != m_uriToIdMap.end()) {
Chris@439:                 cerr << "PluginRDFIndexer::indexURL: WARNING: Found multiple plugins with the same URI:" << endl;
Chris@439:                 cerr << "  1. Plugin id \"" << m_uriToIdMap[pluginUri].toStdString() << "\"" << endl;
Chris@439:                 cerr << "     described in <" << m_idToDescriptionMap[m_uriToIdMap[pluginUri]].toStdString() << ">" << endl;
Chris@439:                 cerr << "  2. Plugin id \"" << pluginId.toStdString() << "\"" << endl;
Chris@439:                 cerr << "     described in <" << urlString.toStdString() << ">" << endl;
Chris@439:                 cerr << "both claim URI <" << pluginUri.toStdString() << ">" << endl;
Chris@439:             } else {
Chris@439:                 m_uriToIdMap[pluginUri] = pluginId;
Chris@439:             }
Chris@439:         }
Chris@439:     }
Chris@439: 
Chris@439:     if (!foundSomething) {
Chris@439:         cerr << "PluginRDFIndexer::indexURL: NOTE: Document at <"
Chris@439:              << urlString.toStdString()
Chris@439:              << "> does not sufficiently describe any plugins" << endl;
Chris@439:     }
Chris@439:     
Chris@439:     return addedSomething;
Chris@439: }
Chris@439: 
Chris@439: 
Chris@439: