Mercurial > hg > svcore
changeset 489:82ab61fa9223
* Reorganise our sparql queries on the basis that Redland must be
available, not only optional. So for anything querying the pool
of data about plugins, we use a single datastore and model which
is initialised at the outset by PluginRDFIndexer and then queried
directly; for anything that "reads from a file" (e.g. loading
annotations) we query directly using Rasqal, going to the
datastore when we need additional plugin-related information.
This may improve performance, but mostly it simplifies the code
and fixes a serious issue with RDF import in the previous versions
(namely that multiple sequential RDF imports would end up sharing
the same RDF data pool!)
| author | Chris Cannam |
|---|---|
| date | Fri, 21 Nov 2008 16:12:29 +0000 |
| parents | 1c66e199e7d9 |
| children | c3fb8258e34d |
| files | rdf/PluginRDFDescription.cpp rdf/PluginRDFDescription.h rdf/PluginRDFIndexer.cpp rdf/PluginRDFIndexer.h rdf/RDFImporter.cpp rdf/RDFTransformFactory.cpp rdf/SimpleSPARQLQuery.cpp rdf/SimpleSPARQLQuery.h transform/TransformFactory.cpp |
| diffstat | 9 files changed, 306 insertions(+), 422 deletions(-) [+] |
line wrap: on
line diff
--- a/rdf/PluginRDFDescription.cpp Fri Nov 21 14:25:33 2008 +0000 +++ b/rdf/PluginRDFDescription.cpp Fri Nov 21 16:12:29 2008 +0000 @@ -34,15 +34,14 @@ m_haveDescription(false) { PluginRDFIndexer *indexer = PluginRDFIndexer::getInstance(); - QString url = indexer->getDescriptionURLForPluginId(pluginId); - if (url == "") { + m_pluginUri = indexer->getURIForPluginId(pluginId); + if (m_pluginUri == "") { cerr << "PluginRDFDescription: WARNING: No RDF description available for plugin ID \"" << pluginId.toStdString() << "\"" << endl; } else { - if (!indexURL(url)) { - cerr << "PluginRDFDescription: ERROR: Failed to query RDF description for plugin ID \"" - << pluginId.toStdString() << "\"" << endl; - } else { + // All the data we need should be in our RDF model already: + // if it's not there, we don't know where to find it anyway + if (index()) { m_haveDescription = true; } } @@ -151,83 +150,63 @@ } bool -PluginRDFDescription::indexURL(QString url) +PluginRDFDescription::index() { - Profiler profiler("PluginRDFDescription::indexURL"); - - QString type, soname, label; - PluginIdentifier::parseIdentifier(m_pluginId, type, soname, label); + Profiler profiler("PluginRDFDescription::index"); bool success = true; - - QString local = url; - - if (FileSource::isRemote(url) && - FileSource::canHandleScheme(url)) { - - CachedFile cf(url); - if (!cf.isOK()) { - return false; - } - - local = QUrl::fromLocalFile(cf.getLocalFilename()).toString(); - } - - if (!indexMetadata(local, label)) success = false; - if (!indexOutputs(local, label)) success = false; + if (!indexMetadata()) success = false; + if (!indexOutputs()) success = false; return success; } bool -PluginRDFDescription::indexMetadata(QString url, QString label) +PluginRDFDescription::indexMetadata() { - Profiler profiler("PluginRDFDescription::indexMetadata"); + Profiler profiler("PluginRDFDescription::index"); + + SimpleSPARQLQuery::QueryType m = SimpleSPARQLQuery::QueryFromModel; QString queryTemplate = QString( " PREFIX vamp: <http://purl.org/ontology/vamp/> " " PREFIX foaf: <http://xmlns.com/foaf/0.1/> " " PREFIX dc: <http://purl.org/dc/elements/1.1/> " - " SELECT ?%4 FROM <%1> " + " SELECT ?%3 " " WHERE { " - " ?plugin vamp:identifier \"%2\" ; " - " a vamp:Plugin ; " - " %3 ?%4 . " + " <%1> %2 ?%3 . " " }") - .arg(url) - .arg(label); + .arg(m_pluginUri); SimpleSPARQLQuery::Value v; v = SimpleSPARQLQuery::singleResultQuery - (url, queryTemplate.arg("vamp:name").arg("name"), "name"); + (m, queryTemplate.arg("vamp:name").arg("name"), "name"); if (v.type == SimpleSPARQLQuery::LiteralValue && v.value != "") { m_pluginName = v.value; } v = SimpleSPARQLQuery::singleResultQuery - (url, queryTemplate.arg("dc:description").arg("description"), "description"); + (m, queryTemplate.arg("dc:description").arg("description"), "description"); if (v.type == SimpleSPARQLQuery::LiteralValue && v.value != "") { m_pluginDescription = v.value; } v = SimpleSPARQLQuery::singleResultQuery - (url, + (m, QString( " PREFIX vamp: <http://purl.org/ontology/vamp/> " " PREFIX foaf: <http://xmlns.com/foaf/0.1/> " - " SELECT ?name FROM <%1> " + " SELECT ?name " " WHERE { " - " ?plugin vamp:identifier \"%2\" ; " - " a vamp:Plugin ; " - " foaf:maker ?maker . " + " <%1> foaf:maker ?maker . " " ?maker foaf:name ?name . " " }") - .arg(url) - .arg(label), "name"); + .arg(m_pluginUri), + "name"); if (v.type == SimpleSPARQLQuery::LiteralValue && v.value != "") { m_pluginMaker = v.value; @@ -240,18 +219,16 @@ // perhaps that would be unwise v = SimpleSPARQLQuery::singleResultQuery - (url, + (m, QString( " PREFIX vamp: <http://purl.org/ontology/vamp/> " " PREFIX foaf: <http://xmlns.com/foaf/0.1/> " - " SELECT ?page from <%1> " + " SELECT ?page " " WHERE { " - " ?plugin vamp:identifier \"%2\" ; " - " a vamp:Plugin ; " - " foaf:page ?page . " + " <%1> foaf:page ?page . " " }") - .arg(url) - .arg(label), "page"); + .arg(m_pluginUri), + "page"); if (v.type == SimpleSPARQLQuery::URIValue && v.value != "") { @@ -260,20 +237,18 @@ } else { v = SimpleSPARQLQuery::singleResultQuery - (url, + (m, QString( " PREFIX vamp: <http://purl.org/ontology/vamp/> " " PREFIX foaf: <http://xmlns.com/foaf/0.1/> " - " SELECT ?page from <%1> " + " SELECT ?page " " WHERE { " - " ?library a vamp:PluginLibrary ; " - " vamp:available_plugin ?plugin ; " + " ?library vamp:available_plugin <%1> ; " + " a vamp:PluginLibrary ; " " foaf:page ?page . " - " ?plugin a vamp:Plugin ; " - " vamp:identifier \"%2\" . " " }") - .arg(url) - .arg(label), "page"); + .arg(m_pluginUri), + "page"); if (v.type == SimpleSPARQLQuery::URIValue && v.value != "") { @@ -285,24 +260,23 @@ } bool -PluginRDFDescription::indexOutputs(QString url, QString label) +PluginRDFDescription::indexOutputs() { Profiler profiler("PluginRDFDescription::indexOutputs"); + + SimpleSPARQLQuery::QueryType m = SimpleSPARQLQuery::QueryFromModel; SimpleSPARQLQuery query - (url, + (m, QString ( " PREFIX vamp: <http://purl.org/ontology/vamp/> " " SELECT ?output ?output_id ?output_type ?unit " - " FROM <%1> " " WHERE { " - " ?plugin vamp:identifier \"%2\" ; " - " a vamp:Plugin ; " - " vamp:output ?output . " + " <%1> vamp:output ?output . " " ?output vamp:identifier ?output_id ; " " a ?output_type . " @@ -313,23 +287,20 @@ " } " ) - .arg(url) - .arg(label)); + .arg(m_pluginUri)); SimpleSPARQLQuery::ResultList results = query.execute(); if (!query.isOK()) { - cerr << "ERROR: PluginRDFDescription::indexURL: ERROR: Failed to query document at <" - << url.toStdString() << ">: " + cerr << "ERROR: PluginRDFDescription::index: ERROR: Failed to query outputs for <" + << m_pluginUri.toStdString() << ">: " << query.getErrorString().toStdString() << endl; return false; } if (results.empty()) { - cerr << "ERROR: PluginRDFDescription::indexURL: NOTE: Document at <" - << url.toStdString() - << "> does not appear to describe any outputs for plugin with id \"" - << label.toStdString() << "\"" << endl; + cerr << "ERROR: PluginRDFDescription::indexURL: NOTE: No outputs defined for <" + << m_pluginUri.toStdString() << ">" << endl; return false; } @@ -365,12 +336,12 @@ SimpleSPARQLQuery::Value v; v = SimpleSPARQLQuery::singleResultQuery - (url, + (m, QString(" PREFIX vamp: <http://purl.org/ontology/vamp/> " " PREFIX dc: <http://purl.org/dc/elements/1.1/> " - " SELECT ?title FROM <%1> " + " SELECT ?title " " WHERE { <%2> dc:title ?title } ") - .arg(url).arg(outputUri), "title"); + .arg(outputUri), "title"); if (v.type == SimpleSPARQLQuery::LiteralValue && v.value != "") { m_outputNames[outputId] = v.value; @@ -378,26 +349,26 @@ QString queryTemplate = QString(" PREFIX vamp: <http://purl.org/ontology/vamp/> " - " SELECT ?%3 FROM <%1> " + " SELECT ?%3 " " WHERE { <%2> vamp:computes_%3 ?%3 } ") - .arg(url).arg(outputUri); + .arg(outputUri); v = SimpleSPARQLQuery::singleResultQuery - (url, queryTemplate.arg("event_type"), "event_type"); + (m, queryTemplate.arg("event_type"), "event_type"); if (v.type == SimpleSPARQLQuery::URIValue && v.value != "") { m_outputEventTypeURIMap[outputId] = v.value; } v = SimpleSPARQLQuery::singleResultQuery - (url, queryTemplate.arg("feature_attribute"), "feature_attribute"); + (m, queryTemplate.arg("feature_attribute"), "feature_attribute"); if (v.type == SimpleSPARQLQuery::URIValue && v.value != "") { m_outputFeatureAttributeURIMap[outputId] = v.value; } v = SimpleSPARQLQuery::singleResultQuery - (url, queryTemplate.arg("signal_type"), "signal_type"); + (m, queryTemplate.arg("signal_type"), "signal_type"); if (v.type == SimpleSPARQLQuery::URIValue && v.value != "") { m_outputSignalTypeURIMap[outputId] = v.value;
--- a/rdf/PluginRDFDescription.h Fri Nov 21 14:25:33 2008 +0000 +++ b/rdf/PluginRDFDescription.h Fri Nov 21 16:12:29 2008 +0000 @@ -55,6 +55,7 @@ typedef std::map<QString, QString> OutputStringMap; QString m_pluginId; + QString m_pluginUri; bool m_haveDescription; QString m_pluginName; QString m_pluginDescription; @@ -66,9 +67,9 @@ OutputStringMap m_outputFeatureAttributeURIMap; OutputStringMap m_outputSignalTypeURIMap; OutputStringMap m_outputUnitMap; - bool indexURL(QString url); - bool indexMetadata(QString url, QString label); - bool indexOutputs(QString url, QString label); + bool index(); + bool indexMetadata(); + bool indexOutputs(); }; #endif
--- a/rdf/PluginRDFIndexer.cpp Fri Nov 21 14:25:33 2008 +0000 +++ b/rdf/PluginRDFIndexer.cpp Fri Nov 21 16:12:29 2008 +0000 @@ -87,7 +87,7 @@ for (QStringList::const_iterator j = entries.begin(); j != entries.end(); ++j) { QFileInfo fi(dir.filePath(*j)); - indexFile(fi.absoluteFilePath()); + pullFile(fi.absoluteFilePath()); } QStringList subdirs = dir.entryList @@ -102,11 +102,13 @@ for (QStringList::const_iterator k = entries.begin(); k != entries.end(); ++k) { QFileInfo fi(subdir.filePath(*k)); - indexFile(fi.absoluteFilePath()); + pullFile(fi.absoluteFilePath()); } } } } + + reindex(); } bool @@ -140,7 +142,7 @@ j != list.end(); ++j) { std::cerr << "PluginRDFIndexer::indexConfiguredURLs: url is " << j->toStdString() << std::endl; - indexURL(*j); + pullURL(*j); } } @@ -148,10 +150,11 @@ QStringList urls = settings.value(urlListKey).toStringList(); for (int i = 0; i < urls.size(); ++i) { - indexURL(urls[i]); + pullURL(urls[i]); } settings.endGroup(); + reindex(); return true; } @@ -198,49 +201,39 @@ return id; } -QString -PluginRDFIndexer::getDescriptionURLForPluginId(QString pluginId) -{ - QMutexLocker locker(&m_mutex); - - if (m_idToDescriptionMap.find(pluginId) == m_idToDescriptionMap.end()) return ""; - return m_idToDescriptionMap[pluginId]; -} - -QString -PluginRDFIndexer::getDescriptionURLForPluginURI(QString uri) -{ - QMutexLocker locker(&m_mutex); - - QString id = getIdForPluginURI(uri); - if (id == "") return ""; - return getDescriptionURLForPluginId(id); -} - QStringList PluginRDFIndexer::getIndexedPluginIds() { QMutexLocker locker(&m_mutex); QStringList ids; - for (StringMap::const_iterator i = m_idToDescriptionMap.begin(); - i != m_idToDescriptionMap.end(); ++i) { + for (StringMap::const_iterator i = m_idToUriMap.begin(); + i != m_idToUriMap.end(); ++i) { ids.push_back(i->first); } return ids; } bool -PluginRDFIndexer::indexFile(QString filepath) +PluginRDFIndexer::pullFile(QString filepath) { QUrl url = QUrl::fromLocalFile(filepath); QString urlString = url.toString(); - return indexURL(urlString); + return pullURL(urlString); } bool PluginRDFIndexer::indexURL(QString urlString) { + bool pulled = pullURL(urlString); + if (!pulled) return false; + reindex(); + return true; +} + +bool +PluginRDFIndexer::pullURL(QString urlString) +{ Profiler profiler("PluginRDFIndexer::indexURL"); std::cerr << "PluginRDFIndexer::indexURL(" << urlString.toStdString() << ")" << std::endl; @@ -258,51 +251,23 @@ } localString = QUrl::fromLocalFile(cf.getLocalFilename()).toString(); -// localString = "file://" + cf.getLocalFilename(); //!!! crud - fix! } -// cerr << "PluginRDFIndexer::indexURL: url = <" << urlString.toStdString() << ">" << endl; -/*!!! + return SimpleSPARQLQuery::addSourceToModel(localString); +} + +bool +PluginRDFIndexer::reindex() +{ + SimpleSPARQLQuery::QueryType m = SimpleSPARQLQuery::QueryFromModel; + SimpleSPARQLQuery query - (localString, - QString - ( - " PREFIX vamp: <http://purl.org/ontology/vamp/> " - - " SELECT ?plugin ?library_id ?plugin_id " - " FROM <%1> " - - " WHERE { " - " ?plugin a vamp:Plugin . " - - // Make the identifier and library parts optional, so - // that we can check and report helpfully if one or both - // is absent instead of just getting no results - - //!!! No -- because of rasqal's inability to correctly - // handle more than one OPTIONAL graph in a query, let's - // make identifier compulsory after all - //" OPTIONAL { ?plugin vamp:identifier ?plugin_id } . " - - " ?plugin vamp:identifier ?plugin_id . " - - " OPTIONAL { " - " ?library a vamp:PluginLibrary ; " - " vamp:available_plugin ?plugin ; " - " vamp:identifier ?library_id " - " } " - " } " - ) - .arg(localString)); -*/ - SimpleSPARQLQuery query - (localString, + (m, QString ( " PREFIX vamp: <http://purl.org/ontology/vamp/> " " SELECT ?plugin ?library ?plugin_id " - " FROM <%1> " " WHERE { " " ?plugin a vamp:Plugin . " @@ -312,22 +277,18 @@ " ?library vamp:available_plugin ?plugin " " } " " } " - ) - .arg(localString)); + )); SimpleSPARQLQuery::ResultList results = query.execute(); if (!query.isOK()) { - cerr << "ERROR: PluginRDFIndexer::indexURL: ERROR: Failed to index document at <" - << urlString.toStdString() << ">: " + cerr << "ERROR: PluginRDFIndexer::reindex: ERROR: Failed to query plugins from model: " << query.getErrorString().toStdString() << endl; return false; } if (results.empty()) { - cerr << "PluginRDFIndexer::indexURL: NOTE: Document at <" - << urlString.toStdString() - << "> does not describe any vamp:Plugin resources" << endl; + cerr << "PluginRDFIndexer::reindex: NOTE: no vamp:Plugin resources found in indexed documents" << endl; return false; } @@ -338,18 +299,17 @@ i != results.end(); ++i) { QString pluginUri = (*i)["plugin"].value; -//!!! QString soname = (*i)["library_id"].value; QString soUri = (*i)["library"].value; QString identifier = (*i)["plugin_id"].value; if (identifier == "") { - cerr << "PluginRDFIndexer::indexURL: NOTE: No vamp:identifier for plugin <" + cerr << "PluginRDFIndexer::reindex: NOTE: No vamp:identifier for plugin <" << pluginUri.toStdString() << ">" << endl; continue; } if (soUri == "") { - cerr << "PluginRDFIndexer::indexURL: NOTE: No implementation library for plugin <" + cerr << "PluginRDFIndexer::reindex: NOTE: No implementation library for plugin <" << pluginUri.toStdString() << ">" << endl; continue; @@ -359,69 +319,40 @@ QString( " PREFIX vamp: <http://purl.org/ontology/vamp/> " " SELECT ?library_id " - " FROM <%1> " " WHERE { " - " <%2> vamp:identifier ?library_id " + " <%1> vamp:identifier ?library_id " " } " ) - .arg(localString) .arg(soUri); SimpleSPARQLQuery::Value sonameValue = - SimpleSPARQLQuery::singleResultQuery(localString, sonameQuery, "library_id"); + SimpleSPARQLQuery::singleResultQuery(m, sonameQuery, "library_id"); QString soname = sonameValue.value; if (soname == "") { - cerr << "PluginRDFIndexer::indexURL: NOTE: No identifier for library <" + cerr << "PluginRDFIndexer::reindex: NOTE: No identifier for library <" << soUri.toStdString() << ">" << endl; continue; } - -/* - cerr << "PluginRDFIndexer::indexURL: Document for plugin \"" - << soname.toStdString() << ":" << identifier.toStdString() - << "\" (uri <" << pluginUri.toStdString() << ">) is at url <" - << urlString.toStdString() << ">" << endl; -*/ QString pluginId = PluginIdentifier::createIdentifier ("vamp", soname, identifier); foundSomething = true; - if (m_idToDescriptionMap.find(pluginId) != m_idToDescriptionMap.end()) { -/*!!! - - This can happen quite legitimately when using an RDF datastore rather - than querying individual files, as of course the datastore contains - all plugin data found so far, and each time a file is added to it, - subsequent queries will return all older plugins as well. - - It would be more efficient to add everything at once and then do all - queries, of course. - - cerr << "PluginRDFIndexer::indexURL: NOTE: Plugin id \"" - << pluginId.toStdString() << "\", described in document at <" - << urlString.toStdString() - << ">, has already been described in document <" - << m_idToDescriptionMap[pluginId].toStdString() - << ">: ignoring this new description" << endl; -*/ + if (m_idToUriMap.find(pluginId) != m_idToUriMap.end()) { continue; } - m_idToDescriptionMap[pluginId] = urlString; m_idToUriMap[pluginId] = pluginUri; addedSomething = true; if (pluginUri != "") { if (m_uriToIdMap.find(pluginUri) != m_uriToIdMap.end()) { - cerr << "PluginRDFIndexer::indexURL: WARNING: Found multiple plugins with the same URI:" << endl; + cerr << "PluginRDFIndexer::reindex: WARNING: Found multiple plugins with the same URI:" << endl; cerr << " 1. Plugin id \"" << m_uriToIdMap[pluginUri].toStdString() << "\"" << endl; - cerr << " described in <" << m_idToDescriptionMap[m_uriToIdMap[pluginUri]].toStdString() << ">" << endl; cerr << " 2. Plugin id \"" << pluginId.toStdString() << "\"" << endl; - cerr << " described in <" << urlString.toStdString() << ">" << endl; cerr << "both claim URI <" << pluginUri.toStdString() << ">" << endl; } else { m_uriToIdMap[pluginUri] = pluginId; @@ -430,9 +361,7 @@ } if (!foundSomething) { - cerr << "PluginRDFIndexer::indexURL: NOTE: Document at <" - << urlString.toStdString() - << "> does not sufficiently describe any plugins" << endl; + cerr << "PluginRDFIndexer::reindex: NOTE: Plugins found, but none sufficiently described" << endl; } return addedSomething;
--- a/rdf/PluginRDFIndexer.h Fri Nov 21 14:25:33 2008 +0000 +++ b/rdf/PluginRDFIndexer.h Fri Nov 21 16:12:29 2008 +0000 @@ -45,8 +45,6 @@ QString getURIForPluginId(QString pluginId); QString getIdForPluginURI(QString uri); - QString getDescriptionURLForPluginId(QString pluginId); - QString getDescriptionURLForPluginURI(QString uri); QStringList getIndexedPluginIds(); @@ -59,10 +57,13 @@ typedef std::map<QString, QString> StringMap; StringMap m_uriToIdMap; StringMap m_idToUriMap; - StringMap m_idToDescriptionMap; void indexInstalledURLs(); - bool indexFile(QString path); + + bool pullFile(QString path); + bool pullURL(QString urlString); + bool reindex(); + static PluginRDFIndexer *m_instance; };
--- a/rdf/RDFImporter.cpp Fri Nov 21 14:25:33 2008 +0000 +++ b/rdf/RDFImporter.cpp Fri Nov 21 16:12:29 2008 +0000 @@ -142,7 +142,7 @@ ProgressReporter *reporter) { SimpleSPARQLQuery query = SimpleSPARQLQuery - (m_uristring, + (SimpleSPARQLQuery::QueryFromSingleSource, QString ( " PREFIX mo: <http://purl.org/ontology/mo/>" @@ -258,6 +258,8 @@ int &sampleRate, int &windowLength, int &hopSize, int &width, int &height) { + SimpleSPARQLQuery::QueryType s = SimpleSPARQLQuery::QueryFromSingleSource; + QString dimensionsQuery ( " PREFIX mo: <http://purl.org/ontology/mo/>" @@ -274,10 +276,8 @@ ); SimpleSPARQLQuery::Value dimensionsValue = - SimpleSPARQLQuery::singleResultQuery(m_uristring, - dimensionsQuery - .arg(m_uristring).arg(featureUri), - "dimensions"); + SimpleSPARQLQuery::singleResultQuery + (s, dimensionsQuery.arg(m_uristring).arg(featureUri), "dimensions"); cerr << "Dimensions = \"" << dimensionsValue.value.toStdString() << "\"" << endl; @@ -316,7 +316,7 @@ // multiple optionals properly SimpleSPARQLQuery::Value srValue = - SimpleSPARQLQuery::singleResultQuery(m_uristring, + SimpleSPARQLQuery::singleResultQuery(s, queryTemplate .arg(m_uristring).arg(featureUri) .arg("sampleRate"), @@ -326,7 +326,7 @@ } SimpleSPARQLQuery::Value hopValue = - SimpleSPARQLQuery::singleResultQuery(m_uristring, + SimpleSPARQLQuery::singleResultQuery(s, queryTemplate .arg(m_uristring).arg(featureUri) .arg("hopSize"), @@ -336,7 +336,7 @@ } SimpleSPARQLQuery::Value winValue = - SimpleSPARQLQuery::singleResultQuery(m_uristring, + SimpleSPARQLQuery::singleResultQuery(s, queryTemplate .arg(m_uristring).arg(featureUri) .arg("windowLength"), @@ -352,6 +352,8 @@ RDFImporterImpl::getDataModelsSparse(std::vector<Model *> &models, ProgressReporter *reporter) { + SimpleSPARQLQuery::QueryType s = SimpleSPARQLQuery::QueryFromSingleSource; + // Our query is intended to retrieve every thing that has a time, // and every feature type and value associated with a thing that // has a time. @@ -443,7 +445,7 @@ ).arg(m_uristring); - SimpleSPARQLQuery query(m_uristring, queryString); + SimpleSPARQLQuery query(s, queryString); query.setProgressReporter(reporter); cerr << "Query will be: " << queryString.toStdString() << endl; @@ -505,9 +507,9 @@ bool haveDuration = false; QString label = SimpleSPARQLQuery::singleResultQuery - (m_uristring, labelQueryString.arg(thinguri), "label").value; + (s, labelQueryString.arg(thinguri), "label").value; - SimpleSPARQLQuery rangeQuery(m_uristring, rangeQueryString.arg(thinguri)); + SimpleSPARQLQuery rangeQuery(s, rangeQueryString.arg(thinguri)); SimpleSPARQLQuery::ResultList rangeResults = rangeQuery.execute(); if (!rangeResults.empty()) { // std::cerr << rangeResults.size() << " range results" << std::endl; @@ -520,7 +522,7 @@ haveDuration = true; } else { QString timestring = SimpleSPARQLQuery::singleResultQuery - (m_uristring, timeQueryString.arg(thinguri), "time").value; + (s, timeQueryString.arg(thinguri), "time").value; if (timestring != "") { time = RealTime::fromXsdDuration(timestring.toStdString()); haveTime = true;
--- a/rdf/RDFTransformFactory.cpp Fri Nov 21 14:25:33 2008 +0000 +++ b/rdf/RDFTransformFactory.cpp Fri Nov 21 16:12:29 2008 +0000 @@ -47,8 +47,8 @@ protected: QString m_urlString; QString m_errorString; - bool setOutput(Transform &, QString, QString); - bool setParameters(Transform &, QString, QString); + bool setOutput(Transform &, QString); + bool setParameters(Transform &, QString); }; @@ -112,38 +112,22 @@ { std::vector<Transform> transforms; - // We have to do this a very long way round, to work around - // rasqal's current inability to handle correctly more than one - // OPTIONAL graph in a query - - const char *optionals[] = { - "output", - "program", - "step_size", - "block_size", - "window_type", - "sample_rate", - "start", - "duration" - }; - std::map<QString, Transform> uriTransformMap; - QString queryTemplate = + QString query = " PREFIX vamp: <http://purl.org/ontology/vamp/> " - " SELECT ?transform ?plugin %1 " + " SELECT ?transform ?plugin " " FROM <%2> " " WHERE { " " ?transform a vamp:Transform ; " " vamp:plugin ?plugin . " - " %3 " " } "; SimpleSPARQLQuery transformsQuery - (m_urlString, queryTemplate.arg("").arg(m_urlString).arg("")); + (SimpleSPARQLQuery::QueryFromSingleSource, query.arg(m_urlString)); SimpleSPARQLQuery::ResultList transformResults = transformsQuery.execute(); @@ -157,6 +141,17 @@ return transforms; } + // There are various queries we need to make that might include + // data from iether the transform RDF or the model accumulated + // from plugin descriptions. For example, the transform RDF may + // specify the output's true URI, or it might have a blank node or + // some other URI with the appropriate vamp:identifier included in + // the file. To cover both cases, we need to add the file itself + // into the model and always query the model using the transform + // URI rather than querying the file itself subsequently. + + SimpleSPARQLQuery::addSourceToModel(m_urlString); + PluginRDFIndexer *indexer = PluginRDFIndexer::getInstance(); for (int i = 0; i < transformResults.size(); ++i) { @@ -175,93 +170,89 @@ continue; } - QString pluginDescriptionURL = - indexer->getDescriptionURLForPluginId(pluginId); - if (pluginDescriptionURL == "") { - cerr << "RDFTransformFactory: WARNING: No RDF description available for plugin <" - << pluginUri.toStdString() << ">, skipping transform <" - << transformUri.toStdString() << ">" << endl; - continue; - } - Transform transform; transform.setPluginIdentifier(pluginId); - if (!setOutput(transform, transformUri, pluginDescriptionURL)) { + if (!setOutput(transform, transformUri)) { return transforms; } - if (!setParameters(transform, transformUri, pluginDescriptionURL)) { + if (!setParameters(transform, transformUri)) { return transforms; } uriTransformMap[transformUri] = transform; - } - for (int i = 0; i < sizeof(optionals)/sizeof(optionals[0]); ++i) { + // We have to do this a very long way round, to work around + // rasqal's current inability to handle correctly more than one + // OPTIONAL graph in a query - QString optional = optionals[i]; + static const char *optionals[] = { + "output", + "program", + "step_size", + "block_size", + "window_type", + "sample_rate", + "start", + "duration" + }; + + for (int j = 0; j < sizeof(optionals)/sizeof(optionals[0]); ++j) { - SimpleSPARQLQuery query - (m_urlString, - queryTemplate - .arg(QString("?%1").arg(optional)) - .arg(m_urlString) - .arg(QString("?transform vamp:%1 ?%2") - .arg(optionals[i]).arg(optional))); + QString optional = optionals[j]; + + QString queryTemplate = + " PREFIX vamp: <http://purl.org/ontology/vamp/> " + + " SELECT ?%1 " + + " WHERE { " + " <%2> vamp:%1 ?%1 " + " } "; + + SimpleSPARQLQuery query + (SimpleSPARQLQuery::QueryFromModel, + queryTemplate.arg(optional).arg(transformUri)); - SimpleSPARQLQuery::ResultList results = query.execute(); + SimpleSPARQLQuery::ResultList results = query.execute(); - if (!query.isOK()) { - m_errorString = query.getErrorString(); - return transforms; - } - - if (results.empty()) continue; - - for (int j = 0; j < results.size(); ++j) { - - QString transformUri = results[j]["transform"].value; - - if (uriTransformMap.find(transformUri) == uriTransformMap.end()) { - cerr << "RDFTransformFactory: ERROR: Transform URI <" - << transformUri.toStdString() << "> not found in internal map!" << endl; - continue; + if (!query.isOK()) { + m_errorString = query.getErrorString(); + return transforms; } - Transform &transform = uriTransformMap[transformUri]; - const SimpleSPARQLQuery::Value &v = results[j][optional]; + if (results.empty()) continue; - if (v.type == SimpleSPARQLQuery::LiteralValue) { + for (int k = 0; k < results.size(); ++k) { + + const SimpleSPARQLQuery::Value &v = results[k][optional]; + + if (v.type == SimpleSPARQLQuery::LiteralValue) { - if (optional == "program") { - transform.setProgram(v.value); - } else if (optional == "step_size") { - transform.setStepSize(v.value.toUInt()); - } else if (optional == "block_size") { - transform.setBlockSize(v.value.toUInt()); - } else if (optional == "window_type") { - cerr << "NOTE: can't handle window type yet (value is \"" - << v.value.toStdString() << "\")" << endl; - } else if (optional == "sample_rate") { - transform.setSampleRate(v.value.toFloat()); - } else if (optional == "start") { - transform.setStartTime - (RealTime::fromXsdDuration(v.value.toStdString())); - } else if (optional == "duration") { - transform.setDuration - (RealTime::fromXsdDuration(v.value.toStdString())); - } else { - cerr << "RDFTransformFactory: ERROR: Inconsistent optionals lists (unexpected optional \"" << optional.toStdString() << "\"" << endl; + if (optional == "program") { + transform.setProgram(v.value); + } else if (optional == "step_size") { + transform.setStepSize(v.value.toUInt()); + } else if (optional == "block_size") { + transform.setBlockSize(v.value.toUInt()); + } else if (optional == "window_type") { + cerr << "NOTE: can't handle window type yet (value is \"" + << v.value.toStdString() << "\")" << endl; + } else if (optional == "sample_rate") { + transform.setSampleRate(v.value.toFloat()); + } else if (optional == "start") { + transform.setStartTime + (RealTime::fromXsdDuration(v.value.toStdString())); + } else if (optional == "duration") { + transform.setDuration + (RealTime::fromXsdDuration(v.value.toStdString())); + } else { + cerr << "RDFTransformFactory: ERROR: Inconsistent optionals lists (unexpected optional \"" << optional.toStdString() << "\"" << endl; + } } } } - } - - for (std::map<QString, Transform>::iterator i = uriTransformMap.begin(); - i != uriTransformMap.end(); ++i) { - - Transform &transform = i->second; cerr << "RDFTransformFactory: NOTE: Transform is: " << endl; cerr << transform.toXmlString().toStdString() << endl; @@ -274,45 +265,55 @@ bool RDFTransformFactoryImpl::setOutput(Transform &transform, - QString transformUri, - QString pluginDescriptionURL) + QString transformUri) { - SimpleSPARQLQuery outputQuery - (m_urlString, + SimpleSPARQLQuery::Value outputValue = + SimpleSPARQLQuery::singleResultQuery + (SimpleSPARQLQuery::QueryFromModel, + QString + ( + " PREFIX vamp: <http://purl.org/ontology/vamp/> " + + " SELECT ?output " + + " WHERE { " + " <%1> vamp:output ?output . " + " } " + ) + .arg(transformUri), + "output"); + + if (outputValue.type == SimpleSPARQLQuery::NoValue) { + return true; + } + + if (outputValue.type != SimpleSPARQLQuery::URIValue) { + m_errorString = "No vamp:output given, or not a URI"; + return false; + } + + SimpleSPARQLQuery::Value outputIdValue = + SimpleSPARQLQuery::singleResultQuery + (SimpleSPARQLQuery::QueryFromModel, QString ( " PREFIX vamp: <http://purl.org/ontology/vamp/> " " SELECT ?output_id " - " FROM <%1> " - " FROM <%2> " - " WHERE { " - " <%3> vamp:output ?output . " - " ?output vamp:identifier ?output_id " + " <%1> vamp:identifier ?output_id " " } " ) - .arg(m_urlString) - .arg(pluginDescriptionURL) - .arg(transformUri)); + .arg(outputValue.value), + "output_id"); - SimpleSPARQLQuery::ResultList outputResults = outputQuery.execute(); - - if (!outputQuery.isOK()) { - m_errorString = outputQuery.getErrorString(); + if (outputIdValue.type != SimpleSPARQLQuery::LiteralValue) { + m_errorString = "No output vamp:identifier available, or not a literal"; return false; } - - if (outputQuery.wasCancelled()) { - m_errorString = "Query cancelled"; - return false; - } - - for (int j = 0; j < outputResults.size(); ++j) { - QString outputId = outputResults[j]["output_id"].value; - transform.setOutput(outputId); - } + + transform.setOutput(outputIdValue.value); return true; } @@ -320,29 +321,23 @@ bool RDFTransformFactoryImpl::setParameters(Transform &transform, - QString transformUri, - QString pluginDescriptionURL) + QString transformUri) { SimpleSPARQLQuery paramQuery - (m_urlString, + (SimpleSPARQLQuery::QueryFromModel, QString ( " PREFIX vamp: <http://purl.org/ontology/vamp/> " " SELECT ?param_id ?param_value " - " FROM <%1> " - " FROM <%2> " - " WHERE { " - " <%3> vamp:parameter_binding ?binding . " + " <%1> vamp:parameter_binding ?binding . " " ?binding vamp:parameter ?param ; " " vamp:value ?param_value . " " ?param vamp:identifier ?param_id " " } " ) - .arg(m_urlString) - .arg(pluginDescriptionURL) .arg(transformUri)); SimpleSPARQLQuery::ResultList paramResults = paramQuery.execute();
--- a/rdf/SimpleSPARQLQuery.cpp Fri Nov 21 14:25:33 2008 +0000 +++ b/rdf/SimpleSPARQLQuery.cpp Fri Nov 21 16:12:29 2008 +0000 @@ -29,9 +29,7 @@ #include <rasqal.h> #endif -#ifdef HAVE_REDLAND #include <redland.h> -#endif //#define DEBUG_SIMPLE_SPARQL_QUERY 1 @@ -64,6 +62,8 @@ rasqal_free_world(m_world); } + bool isOK() const { return (m_world != 0); } + rasqal_world *getWorld() { return m_world; } const rasqal_world *getWorld() const { return m_world; } @@ -72,7 +72,6 @@ }; #endif -#ifdef HAVE_REDLAND class WredlandWorldWrapper { public: @@ -169,14 +168,15 @@ QMutex m_mutex; std::map<QString, librdf_uri *> m_parsedUris; }; -#endif class SimpleSPARQLQuery::Impl { public: - Impl(QString fromUri, QString query); + Impl(SimpleSPARQLQuery::QueryType, QString query); ~Impl(); + static bool addSourceToModel(QString sourceUri); + void setProgressReporter(ProgressReporter *reporter) { m_reporter = reporter; } bool wasCancelled() const { return m_cancelled; } @@ -185,10 +185,6 @@ bool isOK() const; QString getErrorString() const; - static void setBackEnd(SimpleSPARQLQuery::BackEndPreference p) { - m_preference = p; - } - protected: static void errorHandler(void *, raptor_locator *, const char *); @@ -200,16 +196,12 @@ static bool m_rasqalInitialised; #endif -#ifdef HAVE_REDLAND static WredlandWorldWrapper *m_redland; -#endif - - static SimpleSPARQLQuery::BackEndPreference m_preference; ResultList executeDirectParser(); ResultList executeDatastore(); - QString m_fromUri; + QueryType m_type; QString m_query; QString m_errorString; ProgressReporter *m_reporter; @@ -222,17 +214,12 @@ bool SimpleSPARQLQuery::Impl::m_rasqalInitialised = false; #endif -#ifdef HAVE_REDLAND WredlandWorldWrapper *SimpleSPARQLQuery::Impl::m_redland = 0; -#endif QMutex SimpleSPARQLQuery::Impl::m_mutex; -SimpleSPARQLQuery::BackEndPreference -SimpleSPARQLQuery::Impl::m_preference = SimpleSPARQLQuery::AutoSelectBackEnd; - -SimpleSPARQLQuery::SimpleSPARQLQuery(QString fromUri, QString query) : - m_impl(new Impl(fromUri, query)) +SimpleSPARQLQuery::SimpleSPARQLQuery(QueryType type, QString query) : + m_impl(new Impl(type, query)) { } @@ -271,14 +258,14 @@ return m_impl->getErrorString(); } -void -SimpleSPARQLQuery::setBackEnd(BackEndPreference p) +bool +SimpleSPARQLQuery::addSourceToModel(QString sourceUri) { - SimpleSPARQLQuery::Impl::setBackEnd(p); + return SimpleSPARQLQuery::Impl::addSourceToModel(sourceUri); } -SimpleSPARQLQuery::Impl::Impl(QString fromUri, QString query) : - m_fromUri(fromUri), +SimpleSPARQLQuery::Impl::Impl(QueryType type, QString query) : + m_type(type), m_query(query), m_reporter(0), m_cancelled(false) @@ -328,48 +315,32 @@ { ResultList list; - BackEndPreference preference; - m_mutex.lock(); - if (m_preference == AutoSelectBackEnd) { -#ifdef HAVE_REDLAND -// cerr << "librdf version: " << librdf_version_major << "." << librdf_version_minor << "." << librdf_version_release << endl; - if (librdf_version_major > 1 || - (librdf_version_major == 1 && - (librdf_version_minor > 0 || - (librdf_version_minor == 0 && - librdf_version_release > 7)))) { - cerr << "SimpleSPARQLQuery: Auto-selecting LIBRDF back-end for tree-based storage" << endl; - m_preference = DatastoreBackEnd; - } -#endif - if (m_preference == AutoSelectBackEnd) { - cerr << "SimpleSPARQLQuery: Auto-selecting RASQAL back-end" << endl; - m_preference = DirectParserBackEnd; + if (m_type == QueryFromModel) { + if (!m_redland) { + // There can be no results, because no sources have been + // added to the model yet (m_redland is only created when + // addSourceToModel is called) + cerr << "SimpleSPARQLQuery::execute: NOTE: No sources have been added to data model yet, so no results are possible" << endl; + m_mutex.unlock(); + return list; } } - if (m_preference == DatastoreBackEnd) { -#ifdef HAVE_REDLAND - if (!m_redland) { - m_redland = new WredlandWorldWrapper(); - if (!m_redland->isOK()) { - cerr << "WARNING: SimpleSPARQLQuery::execute: Failed to initialise Redland datastore, falling back to direct parser implementation" << endl; - delete m_redland; - m_preference = DirectParserBackEnd; + if (m_type == QueryFromSingleSource) { +#ifdef USE_NEW_RASQAL_API + if (!m_rasqal) { + m_rasqal = new WrasqalWorldWrapper(); + if (!m_rasqal->isOK()) { + cerr << "ERROR: SimpleSPARQLQuery::execute: Failed to initialise Rasqal query engine" << endl; + delete m_rasqal; + m_rasqal = 0; + m_mutex.unlock(); + return list; } } #else - cerr << "WARNING: SimpleSPARQLQuery::execute: Datastore implementation preference indicated, but no datastore compiled in; using direct parser" << endl; - m_preference = DirectParserBackEnd; -#endif - } - - if (m_preference == DirectParserBackEnd) { -#ifdef USE_NEW_RASQAL_API - if (!m_rasqal) m_rasqal = new WrasqalWorldWrapper(); -#else if (!m_rasqalInitialised) { rasqal_init(); m_rasqalInitialised = true; @@ -377,10 +348,9 @@ #endif } - preference = m_preference; m_mutex.unlock(); - if (preference == SimpleSPARQLQuery::DirectParserBackEnd) { + if (m_type == QueryFromSingleSource) { return executeDirectParser(); } else { return executeDatastore(); @@ -510,19 +480,9 @@ SimpleSPARQLQuery::Impl::executeDatastore() { ResultList list; -#ifndef HAVE_REDLAND - // This should have been caught by execute() - cerr << "SimpleSPARQLQuery: INTERNAL ERROR: Datastore not compiled in" << endl; - return list; -#else + Profiler profiler("SimpleSPARQLQuery::executeDatastore"); - librdf_uri *uri = m_redland->getUri(m_fromUri, m_errorString); - if (!uri) return list; - -#ifdef DEBUG_SIMPLE_SPARQL_QUERY - std::cerr << "SimpleSPARQLQuery: Query is: \"" << m_query.toStdString() << "\"" << std::endl; -#endif /*!!! static std::map<QString, int> counter; if (counter.find(m_query) == counter.end()) counter[m_query] = 1; @@ -537,7 +497,7 @@ Profiler p("SimpleSPARQLQuery: Prepare LIBRDF query"); query = librdf_new_query (m_redland->getWorld(), "sparql", NULL, - (const unsigned char *)m_query.toUtf8().data(), uri); + (const unsigned char *)m_query.toUtf8().data(), NULL); } if (!query) { @@ -650,14 +610,42 @@ #endif return list; -#endif +} + +bool +SimpleSPARQLQuery::Impl::addSourceToModel(QString sourceUri) +{ + QString err; + + m_mutex.lock(); + + if (!m_redland) { + m_redland = new WredlandWorldWrapper(); + if (!m_redland->isOK()) { + cerr << "ERROR: SimpleSPARQLQuery::addSourceToModel: Failed to initialise Redland datastore" << endl; + delete m_redland; + m_redland = 0; + m_mutex.unlock(); + return false; + } + } + + m_mutex.unlock(); + + librdf_uri *uri = m_redland->getUri(sourceUri, err); + + if (!uri) { + std::cerr << "SimpleSPARQLQuery::addSourceToModel: Failed to add source URI \"" << sourceUri.toStdString() << ": " << err.toStdString() << std::endl; + return false; + } + return true; } SimpleSPARQLQuery::Value -SimpleSPARQLQuery::singleResultQuery(QString fromUri, +SimpleSPARQLQuery::singleResultQuery(QueryType type, QString query, QString binding) { - SimpleSPARQLQuery q(fromUri, query); + SimpleSPARQLQuery q(type, query); ResultList results = q.execute(); if (!q.isOK()) { cerr << "SimpleSPARQLQuery::singleResultQuery: ERROR: "
--- a/rdf/SimpleSPARQLQuery.h Fri Nov 21 14:25:33 2008 +0000 +++ b/rdf/SimpleSPARQLQuery.h Fri Nov 21 16:12:29 2008 +0000 @@ -37,9 +37,16 @@ typedef std::map<QString, Value> KeyValueMap; typedef std::vector<KeyValueMap> ResultList; - SimpleSPARQLQuery(QString fromUri, QString query); + enum QueryType { + QueryFromModel, + QueryFromSingleSource + }; + + SimpleSPARQLQuery(QueryType type, QString query); ~SimpleSPARQLQuery(); + static bool addSourceToModel(QString sourceUri); + void setProgressReporter(ProgressReporter *reporter); bool wasCancelled() const; @@ -50,21 +57,10 @@ // Do a query and return the value for the given binding, from the // first result that has a value for it - static Value singleResultQuery(QString fromUri, + static Value singleResultQuery(QueryType type, QString query, QString binding); - enum BackEndPreference { - AutoSelectBackEnd, // pick based on likely speed of available storage - DirectParserBackEnd, // use rasqal (simpler if seldom used) - DatastoreBackEnd, // use redland (faster if version not too old) - }; - /** - * Select the preferred query back end. This should be called - * before any queries are made. The default is AutoSelectBackEnd. - */ - static void setBackEnd(BackEndPreference); - protected: class Impl; Impl *m_impl;
--- a/transform/TransformFactory.cpp Fri Nov 21 14:25:33 2008 +0000 +++ b/transform/TransformFactory.cpp Fri Nov 21 16:12:29 2008 +0000 @@ -295,6 +295,7 @@ case TransformDescription::Generator: return tr("Generator"); case TransformDescription::UnknownType: return tr("Other"); } + return tr("Other"); } void