Chris@44: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
Chris@44: 
Chris@44: #include "FeatureFileIndex.h"
Chris@44: #include "TypeRegistrar.h"
Chris@44: 
Chris@44: #include <QMutexLocker>
Chris@44: #include <QDir>
Chris@44: 
Chris@44: #include "base/TempDirectory.h"
Chris@44: #include "base/Exceptions.h"
Chris@44: 
Chris@44: using namespace Dataquay;
Chris@44: 
Chris@44: 
Chris@44: namespace ClassicalData {
Chris@44: 
Chris@44: FeatureFileIndex *
Chris@44: FeatureFileIndex::getInstance()
Chris@44: {
Chris@44:     static FeatureFileIndex instance;
Chris@44:     return &instance;
Chris@44: }
Chris@44: 
Chris@44: FeatureFileIndex::FeatureFileIndex() :
Chris@45:     m_bs(0),
Chris@44:     m_index(0)
Chris@44: {
Chris@44:     try {
Chris@44: 	m_indexFileName = getIndexFileName();
Chris@44:     } catch (DirectoryCreationFailed f) {
Chris@44:         std::cerr << "FeatureFileIndex: ERROR: Failed to find or create index directory: " << f.what() << std::endl;
Chris@44:         return;
Chris@44:     }
Chris@44: 
Chris@45:     m_bs = new BasicStore;
Chris@52:     m_bs->setBaseUri(Uri(QUrl::fromLocalFile(m_indexFileName)));
Chris@52: 
Chris@45:     m_index = new TransactionalStore(m_bs);
Chris@44: 
Chris@45:     TypeRegistrar::addMappings(m_bs, 0);
Chris@44: 
Chris@44:     if (QFile(m_indexFileName).exists()) {
Chris@45: 	m_bs->import(QUrl::fromLocalFile(m_indexFileName),
Chris@45:                      BasicStore::ImportIgnoreDuplicates);
Chris@44: 	//!!! catch
Chris@44:     }
Chris@44: }
Chris@44: 
Chris@44: FeatureFileIndex::~FeatureFileIndex()
Chris@44: {
Chris@45:     delete m_index;
Chris@45:     delete m_bs;
Chris@44: }
Chris@44: 
Chris@44: QString
Chris@44: FeatureFileIndex::getIndexFileName()
Chris@44: {
Chris@44:     QDir d = TempDirectory::getInstance()->getContainingPath();
Chris@44:     QString n("index");
Chris@44:     QFileInfo fi(d.filePath(n));
Chris@44: 
Chris@44:     if ((fi.exists() && !fi.isDir()) ||
Chris@44:         (!fi.exists() && !d.mkdir(n))) {
Chris@44:         throw DirectoryCreationFailed(fi.filePath());
Chris@44:     }
Chris@44: 
Chris@44:     return QDir(fi.filePath()).filePath("features.ttl");
Chris@44: }
Chris@44: 
Chris@44: QString
Chris@44: FeatureFileIndex::getFeatureDirectoryName()
Chris@44: {
Chris@44:     QDir d = TempDirectory::getInstance()->getContainingPath();
Chris@44:     QString n("features");
Chris@44:     QFileInfo fi(d.filePath(n));
Chris@44: 
Chris@44:     if ((fi.exists() && !fi.isDir()) ||
Chris@44:         (!fi.exists() && !d.mkdir(n))) {
Chris@44:         throw DirectoryCreationFailed(fi.filePath());
Chris@44:     }
Chris@44: 
Chris@44:     return fi.filePath();
Chris@44: }
Chris@44: 
Chris@44: void
Chris@45: FeatureFileIndex::loadFor(AudioFile *tf, Store *store)
Chris@44: {
Chris@44:     if (!m_index) {
Chris@44: 	std::cerr << "FeatureFileIndex::loadFor: No index!" << std::endl;
Chris@44: 	return;
Chris@44:     }
Chris@44:     updateIndex();
Chris@44: 
Chris@46:     QSet<Uri> fileUris;
Chris@46: 
Chris@46:     // The same file may be referred to with more than one URI; we
Chris@46:     // want to load any or all of: the URI in our file object; encoded
Chris@46:     // version of same; and any other file that is recorded as having
Chris@46:     // the same hash (i.e. it is the same file).
Chris@46: 
Chris@46:     fileUris.insert(tf->uri());
Chris@46: 
Chris@46:     // and again with encoded version of file URI
Chris@46:     QByteArray enc = QUrl(tf->uri().toString()).toEncoded();
Chris@46:     fileUris.insert(Uri(QString::fromUtf8(enc)));
Chris@46: 
Chris@46:     // and again with anything else having the same hash
Chris@46:     if (tf->hash() != "") {
Chris@52:         Triple pattern(Node(), store->expand("foaf:sha1"), Node(tf->hash()));
Chris@46:         Triples results = m_index->match(pattern);
Chris@46:         std::cerr << "FeatureFileIndex::loadFor: " << results.size() << " audio file(s) found with hash " << tf->hash().toStdString() << std::endl;
Chris@46:         foreach (Triple t, results) {
Chris@46:             fileUris.insert(Uri(t.a.value));
Chris@46:         }
Chris@46:     }
Chris@46: 
Chris@46:     foreach (Uri u, fileUris) {
Chris@46:         loadFor(tf->uri(), u, tf->hash(), store);
Chris@46:     }
Chris@46: }
Chris@46: 
Chris@46: bool
Chris@46: FeatureFileIndex::loadFor(Uri canonicalUri, Uri afuri,
Chris@46:                           QString hash, Store *store)
Chris@46: {
Chris@45:     // The AudioFile object has a URI and a hash.  Feature files
Chris@45:     // generated for this AudioFile should ideally have a matching
Chris@45:     // hash; if they have no hash, then the URI should match.  If the
Chris@45:     // hash is present in the feature file but does not match, then it
Chris@45:     // cannot be the right track even if the URI matches.
Chris@45: 
Chris@52:     Triple pattern(Node(), store->expand("foaf:primaryTopic"), afuri);
Chris@46:     Triples results = m_index->match(pattern);
Chris@46:     std::cerr << "FeatureFileIndex::loadFor: " << results.size() << " feature file(s) for audio file " << afuri << std::endl;
Chris@45: 
Chris@46:     bool loadedSomething = false;
Chris@45: 
Chris@46:     foreach (Triple t, results) {
Chris@46:         try {
Chris@46:             BasicStore *b = BasicStore::load(QUrl(t.a.value));
Chris@46:             Triples ts = b->match
Chris@52:                 (Triple(afuri, store->expand("a"), m_index->expand("mo:AudioFile")));
Chris@46:             std::cerr << "FeatureFileIndex::loadFor: feature file "
Chris@46:                       << t.a << " has " << ts.size() << " type node(s) for this audio file" << std::endl;
Chris@46:             bool someGood = false;
Chris@46:             foreach (Triple t, ts) {
Chris@46:                 bool good = true;
Chris@46:                 if (hash != "") {
Chris@46:                     Triples hashts = b->match
Chris@46:                         (Triple(afuri, m_index->expand("foaf:sha1"), Node()));
Chris@46:                     std::cerr << "FeatureFileIndex::loadFor: feature file "
Chris@46:                               << t.a << " has " << hashts.size() << " hashes for this file" << std::endl;
Chris@46:                     if (!hashts.empty()) {
Chris@46:                         good = false;
Chris@46:                         foreach (Triple hasht, hashts) {
Chris@46:                             if (hasht.c.value == hash) {
Chris@46:                                 std::cerr << "Hash " << hasht.c << " matches our hash " << hash.toStdString() << std::endl;
Chris@46:                                 good = true;
Chris@46:                                 break;
Chris@46:                             }
Chris@46:                         }
Chris@46:                         if (!good) {
Chris@46:                             std::cerr << "(no hash matches, eliminating file)" << std::endl;
Chris@46:                         }
Chris@46:                     } else {
Chris@46:                         std::cerr << "(so cannot eliminate file via hash)" << std::endl;
Chris@46:                     }
Chris@46:                 }
Chris@46:                 if (good) {
Chris@46:                     std::cerr << "...going to import this one" << std::endl;
Chris@46:                     someGood = true;
Chris@46:                 }
Chris@46:             }
Chris@46:             if (someGood) {
Chris@46:                 Triples all = b->match(Triple());
Chris@46:                 std::cerr << "Importing " << all.size() << " triple(s) into store" << std::endl;
Chris@46:                 // Replace instances of the audio file URI with our
Chris@46:                 // canonical URI (we want to make sure we're
Chris@46:                 // associating these facts with our own URI for this
Chris@46:                 // file, even if they originated from a different URI
Chris@46:                 // with the same hash)
Chris@52:                 Node from = Node(Uri(afuri.toString()));
Chris@52:                 Node to = Node(Uri(canonicalUri.toString()));
Chris@46:                 foreach (Triple t, all) {
Chris@46:                     if (t.a == from) t.a = to;
Chris@46:                     if (t.c == from) t.c = to;
Chris@46:                     store->add(t);
Chris@46:                 }
Chris@46:                 loadedSomething = true;
Chris@46:             }
Chris@46:         } catch (...) { }
Chris@45:     }
Chris@45: 
Chris@46:     return loadedSomething;
Chris@44: }
Chris@44: 
Chris@44: void
Chris@45: FeatureFileIndex::featureFileAdded(QString filepath)
Chris@45: {
Chris@45:     index(QUrl::fromLocalFile(filepath));
Chris@45: }    
Chris@45: 
Chris@45: void
Chris@44: FeatureFileIndex::updateIndex()
Chris@44: {
Chris@44:     QMutexLocker locker(&m_mutex);
Chris@44:     if (!m_index) return;
Chris@44: 
Chris@46:     std::cerr << "Generating index..." << std::endl;
Chris@46: 
Chris@44:     QDir featureDir;
Chris@44:     try {
Chris@44: 	QString s = getFeatureDirectoryName();
Chris@44:         featureDir = QDir(s);
Chris@44:     } catch (DirectoryCreationFailed f) {
Chris@44:         std::cerr << "FeatureFileIndex::updateIndex: ERROR: Failed to find or create feature directory: " << f.what() << std::endl;
Chris@44:         return;
Chris@44:     }
Chris@44: 
Chris@44:     featureDir.setFilter(QDir::Files);
Chris@44: 
Chris@44:     for (unsigned int i = 0; i < featureDir.count(); ++i) {
Chris@44:         QFileInfo fi(featureDir.filePath(featureDir[i]));
Chris@44:         if (fi.isFile() && fi.isReadable()) {
Chris@45:             index(QUrl::fromLocalFile(fi.filePath()));
Chris@44:         }
Chris@44:     }
Chris@44: 
Chris@44:     //!!! remove triples from index that refer to nonexistent files?
Chris@44: 
Chris@44:     std::cerr << "Saving index to " << m_indexFileName.toStdString() << std::endl;
Chris@45:     m_bs->save(m_indexFileName);
Chris@46: 
Chris@46:     std::cerr << "Done" << std::endl;
Chris@45: }
Chris@45: 
Chris@45: void
Chris@45: FeatureFileIndex::index(QUrl fileUrl)
Chris@45: {
Chris@52:     Triple typeTriple(Uri(fileUrl), m_index->expand("a"), m_index->expand("foaf:Document"));
Chris@45: 
Chris@45:     if (m_index->contains(typeTriple)) {
Chris@45:         return;
Chris@45:     }
Chris@45: 
Chris@45:     Transaction *tx = m_index->startTransaction();
Chris@45:     tx->add(typeTriple);
Chris@45: 
Chris@45:     try {
Chris@45:         BasicStore *b = BasicStore::load(fileUrl);
Chris@45:         Triples ts = b->match
Chris@52:             (Triple(Node(), m_index->expand("a"), m_index->expand("mo:AudioFile")));
Chris@45:         foreach (Triple t, ts) {
Chris@52:             tx->add(Triple(Uri(fileUrl), m_index->expand("foaf:primaryTopic"), t.a));;
Chris@46:             Triples hashts = b->match
Chris@46:                 (Triple(t.a, m_index->expand("foaf:sha1"), Node()));
Chris@46:             foreach (Triple hasht, hashts) {
Chris@46:                 tx->add(hasht);
Chris@46:             }
Chris@45:         }
Chris@46:     } catch (std::exception &e) {
Chris@46:         std::cerr << "Caught exception: \"" << e.what() << "\" while indexing "
Chris@46:                   << Uri(fileUrl) << ", skipping" << std::endl;
Chris@46:     }
Chris@45: 
Chris@48:     tx->commit();
Chris@45:     delete tx;
Chris@44: }
Chris@44: 
Chris@44: 
Chris@44: }
Chris@44: