comparison data/fileio/FileSource.cpp @ 460:93fb1ebff76b

* Add persistent cache file support to FileSource (e.g. for RDF
descriptions)
* Query RDF plugin data in a background thread on startup

 #include <QHttp>
 #include <QFtp>
 #include <QFileInfo>
 #include <QDir>
 #include <QCoreApplication>
+#include <QCryptographicHash>
 #include <QHttpResponseHeader>
 
 #include <iostream>
 #include <cstdlib>
 
-//#define DEBUG_FILE_SOURCE 1
+#define DEBUG_FILE_SOURCE 1
 
 int
 FileSource::m_count = 0;
 
 QMutex

 FileSource::m_remoteLocalMap;
 
 QMutex
 FileSource::m_mapMutex;
 
-FileSource::FileSource(QString fileOrUrl, ProgressReporter *reporter) :
+FileSource::FileSource(QString fileOrUrl, ProgressReporter *reporter,
+                       LocalCacheMode cacheMode) :
     m_url(fileOrUrl),
+    m_cacheMode(cacheMode),
     m_ftp(0),
     m_http(0),
     m_localFile(0),
     m_ok(false),
     m_lastStatus(0),

     m_leaveLocalFile(false),
     m_reporter(reporter),
     m_refCounted(false)
 {
 #ifdef DEBUG_FILE_SOURCE
-    std::cerr << "FileSource::FileSource(" << fileOrUrl.toStdString() << ")" << std::endl;
+    std::cerr << "FileSource::FileSource(" << fileOrUrl.toStdString() << ", " << cacheMode << ")" << std::endl;
 #endif
 
     if (!canHandleScheme(m_url)) {
         std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString().toStdString() << "\"" << std::endl;
         m_errorString = tr("Unsupported scheme in URL");

         emit statusAvailable();
         emit ready();
     }
 }
 
-FileSource::FileSource(QUrl url, ProgressReporter *reporter) :
+FileSource::FileSource(QUrl url, ProgressReporter *reporter,
+                       LocalCacheMode cacheMode) :
     m_url(url),
+    m_cacheMode(cacheMode),
     m_ftp(0),
     m_http(0),
     m_localFile(0),
     m_ok(false),
     m_lastStatus(0),

 }
 
 FileSource::FileSource(const FileSource &rf) :
     QObject(),
     m_url(rf.m_url),
+    m_cacheMode(rf.m_cacheMode),
     m_ftp(0),
     m_http(0),
     m_localFile(0),
     m_ok(rf.m_ok),
     m_lastStatus(rf.m_lastStatus),

         std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString().toStdString() << "\"" << std::endl;
         m_errorString = tr("Unsupported scheme in URL");
         return;
     }
 
-    if (!isRemote()) {
+    if (m_cacheMode == PersistentCache) {
+
         m_localFilename = rf.m_localFilename;
+
+    } else if (!isRemote()) {
+
+        m_localFilename = rf.m_localFilename;
+
     } else {
         QMutexLocker locker(&m_mapMutex);
 #ifdef DEBUG_FILE_SOURCE
         std::cerr << "FileSource::FileSource(copy ctor): ref count is "
                   << m_refCountMap[m_url] << std::endl;

     std::cerr << "FileSource(" << m_url.toString().toStdString() << ")::~FileSource" << std::endl;
 #endif
 
     cleanup();
 
-    if (isRemote() && !m_leaveLocalFile) deleteCacheFile();
+    if (isRemote() && (m_cacheMode == TemporaryCache) && !m_leaveLocalFile) {
+        deleteCacheFile();
+    }
 }
 
 void
 FileSource::init()
 {

         return;
     }
 
     if (createCacheFile()) {
 #ifdef DEBUG_FILE_SOURCE
-        std::cerr << "FileSource::init: Already have this one" << std::endl;
+        std::cerr << "FileSource::init: Already have this one at "
+                  << m_localFilename.toStdString() << std::endl;
 #endif
         m_ok = true;
         if (!QFileInfo(m_localFilename).exists()) {
             m_lastStatus = 404;
         } else {

 
     QString scheme = m_url.scheme().toLower();
 
 #ifdef DEBUG_FILE_SOURCE
     std::cerr << "FileSource::init: Don't have local copy of \""
-              << m_url.toString().toStdString() << "\", retrieving" << std::endl;
+              << m_url.toString().toStdString() << "\", retrieving to "
+              << m_localFilename.toStdString() << std::endl;
 #endif
 
     if (scheme == "http") {
         initHttp();
         std::cerr << "FileSource: initHttp succeeded" << std::endl;

         }
     }
 
     m_fileCreationMutex.lock();
 
+    // We always delete the file here, even in PersistentCache mode,
+    // because this function is also used if retrieval failed (in
+    // which case we want to delete the cache so that subsequent users
+    // won't trust it).  It's up to the calling code to determine
+    // whether we actually want to delete the cache or not, e.g. on
+    // destruction.
+
     if (!QFile(m_localFilename).remove()) {
-#ifdef DEBUG_FILE_SOURCE
         std::cerr << "FileSource::deleteCacheFile: ERROR: Failed to delete file \"" << m_localFilename.toStdString() << "\"" << std::endl;
-#endif
     } else {
 #ifdef DEBUG_FILE_SOURCE
         std::cerr << "FileSource::deleteCacheFile: Deleted cache file \"" << m_localFilename.toStdString() << "\"" << std::endl;
 #endif
         m_localFilename = "";
     }
 
     m_fileCreationMutex.unlock();
 
     m_done = true;
+}
+
+QString
+FileSource::getPersistentCacheDirectory()
+{
+    QDir dir = TempDirectory::getInstance()->getContainingPath();
+
+    QString cacheDirName("cache");
+
+    QFileInfo fi(dir.filePath(cacheDirName));
+
+    if ((fi.exists() && !fi.isDir()) ||
+        (!fi.exists() && !dir.mkdir(cacheDirName))) {
+
+        throw DirectoryCreationFailed(fi.filePath());
+    }
+
+    return fi.filePath();
+}
+
+QString
+FileSource::getPersistentCacheFilePath(QUrl url)
+{
+    QDir dir(getPersistentCacheDirectory());
+
+    QString filename =
+        QString::fromLocal8Bit
+        (QCryptographicHash::hash(url.toString().toLocal8Bit(),
+                                  QCryptographicHash::Sha1).toBase64());
+
+    return dir.filePath(filename);
 }
 
 bool
 FileSource::createCacheFile()
 {
+    if (m_cacheMode == PersistentCache) {
+        m_localFilename = getPersistentCacheFilePath(m_url);
+        if (QFileInfo(m_localFilename).exists()) return true;
+        else return false;
+    }
+
     {
         QMutexLocker locker(&m_mapMutex);
 
 #ifdef DEBUG_FILE_SOURCE
         std::cerr << "FileSource::createCacheFile: refcount is " << m_refCountMap[m_url] << std::endl;