Chris@208: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
Chris@208: 
Chris@208: /*
Chris@208:     Sonic Visualiser
Chris@208:     An audio file viewer and annotation editor.
Chris@208:     Centre for Digital Music, Queen Mary, University of London.
Chris@208:     This file copyright 2007 QMUL.
Chris@208:     
Chris@208:     This program is free software; you can redistribute it and/or
Chris@208:     modify it under the terms of the GNU General Public License as
Chris@208:     published by the Free Software Foundation; either version 2 of the
Chris@208:     License, or (at your option) any later version.  See the file
Chris@208:     COPYING included with this distribution for more information.
Chris@208: */
Chris@208: 
Chris@317: #include "FileSource.h"
Chris@357: 
Chris@208: #include "base/TempDirectory.h"
Chris@208: #include "base/Exceptions.h"
Chris@392: #include "base/ProgressReporter.h"
Chris@502: #include "system/System.h"
Chris@208: 
Chris@762: #include <QNetworkAccessManager>
Chris@762: #include <QNetworkReply>
Chris@208: #include <QFileInfo>
Chris@208: #include <QDir>
Chris@392: #include <QCoreApplication>
Chris@833: #include <QThreadStorage>
Chris@208: 
Chris@208: #include <iostream>
Chris@405: #include <cstdlib>
Chris@208: 
Chris@1067: //#define DEBUG_FILE_SOURCE 1
Chris@327: 
Chris@208: int
Chris@317: FileSource::m_count = 0;
Chris@208: 
Chris@208: QMutex
Chris@317: FileSource::m_fileCreationMutex;
Chris@208: 
Chris@317: FileSource::RemoteRefCountMap
Chris@317: FileSource::m_refCountMap;
Chris@304: 
Chris@317: FileSource::RemoteLocalMap
Chris@317: FileSource::m_remoteLocalMap;
Chris@304: 
Chris@304: QMutex
Chris@317: FileSource::m_mapMutex;
Chris@304: 
Chris@529: #ifdef DEBUG_FILE_SOURCE
Chris@529: static int extantCount = 0;
Chris@1033: static int threadCount = 0;
Chris@529: static std::map<QString, int> urlExtantCountMap;
Chris@1033: static QMutex countMutex;
Chris@529: static void incCount(QString url) {
Chris@1033:     QMutexLocker locker(&countMutex);
Chris@529:     ++extantCount;
Chris@529:     if (urlExtantCountMap.find(url) == urlExtantCountMap.end()) {
Chris@529:         urlExtantCountMap[url] = 1;
Chris@529:     } else {
Chris@529:         ++urlExtantCountMap[url];
Chris@529:     }
Chris@843:     cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << endl;
Chris@529: }
Chris@529: static void decCount(QString url) {
Chris@1033:     QMutexLocker locker(&countMutex);
Chris@529:     --extantCount;
Chris@529:     --urlExtantCountMap[url];
Chris@843:     cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << endl;
Chris@529: }
Chris@1033: void
Chris@1033: FileSource::debugReport()
Chris@1033: {
Chris@1033:     QMutexLocker locker(&countMutex);
Chris@1033:     cerr << "\nFileSource::debugReport: Have " << extantCount << " FileSource object(s) extant across " << threadCount << " thread(s)" << endl;
Chris@1033:     cerr << "URLs by extant count:" << endl;
Chris@1033:     cerr << "Count | URL" << endl;
Chris@1033:     for (std::map<QString, int>::const_iterator i = urlExtantCountMap.begin();
Chris@1033:          i != urlExtantCountMap.end(); ++i) {
Chris@1033:         cerr << i->second << " | " << i->first << endl;
Chris@1033:     }
Chris@1033:     cerr << "FileSource::debugReport done\n" << endl;
Chris@1033: }
Chris@1033: #else
Chris@1033: void FileSource::debugReport() { }
Chris@529: #endif
Chris@529: 
Chris@833: static QThreadStorage<QNetworkAccessManager *> nms;
Chris@762: 
Chris@520: FileSource::FileSource(QString fileOrUrl, ProgressReporter *reporter,
Chris@520:                        QString preferredContentType) :
chris@831:     m_rawFileOrUrl(fileOrUrl),
Chris@614:     m_url(fileOrUrl, QUrl::StrictMode),
Chris@1582:     m_localFile(nullptr),
Chris@1582:     m_reply(nullptr),
Chris@520:     m_preferredContentType(preferredContentType),
Chris@316:     m_ok(false),
Chris@981:     m_cancelled(false),
Chris@316:     m_lastStatus(0),
Chris@706:     m_resource(fileOrUrl.startsWith(':')),
Chris@316:     m_remote(isRemote(fileOrUrl)),
Chris@316:     m_done(false),
Chris@316:     m_leaveLocalFile(false),
Chris@392:     m_reporter(reporter),
Chris@316:     m_refCounted(false)
Chris@316: {
Chris@706:     if (m_resource) { // qrc file
Chris@706:         m_url = QUrl("qrc" + fileOrUrl);
Chris@706:     }
Chris@706: 
Chris@661:     if (m_url.toString() == "") {
Chris@661:         m_url = QUrl(fileOrUrl, QUrl::TolerantMode);
Chris@661:     }
Chris@661:  
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:     cerr << "FileSource::FileSource(" << fileOrUrl << "): url <" << m_url.toString() << ">" << endl;
Chris@529:     incCount(m_url.toString());
Chris@327: #endif
Chris@316: 
Chris@316:     if (!canHandleScheme(m_url)) {
Chris@843:         cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316:         m_errorString = tr("Unsupported scheme in URL");
Chris@316:         return;
Chris@316:     }
Chris@316: 
Chris@357:     init();
Chris@316: 
Chris@614:     if (!isRemote() &&
Chris@614:         !isAvailable()) {
Chris@614: #ifdef DEBUG_FILE_SOURCE
Chris@843:         cerr << "FileSource::FileSource: Failed to open local file with URL \"" << m_url.toString() << "\"; trying again assuming filename was encoded" << endl;
Chris@614: #endif
Chris@762:         m_url = QUrl::fromEncoded(fileOrUrl.toLatin1());
chris@831: #ifdef DEBUG_FILE_SOURCE
Chris@843:         cerr << "FileSource::FileSource: URL is now \"" << m_url.toString() << "\"" << endl;
chris@831: #endif
Chris@614:         init();
Chris@614:     }
Chris@614: 
Chris@316:     if (isRemote() &&
Chris@316:         (fileOrUrl.contains('%') ||
Chris@316:          fileOrUrl.contains("--"))) { // for IDNA
Chris@316: 
Chris@316:         waitForStatus();
Chris@316: 
Chris@316:         if (!isAvailable()) {
Chris@336: 
Chris@316:             // The URL was created on the assumption that the string
Chris@316:             // was human-readable.  Let's try again, this time
Chris@316:             // assuming it was already encoded.
Chris@843:             cerr << "FileSource::FileSource: Failed to retrieve URL \""
Chris@844:                       << fileOrUrl 
Chris@316:                       << "\" as human-readable URL; "
Chris@316:                       << "trying again treating it as encoded URL"
Chris@843:                       << endl;
Chris@336: 
Chris@336:             // even though our cache file doesn't exist (because the
Chris@336:             // resource was 404), we still need to ensure we're no
Chris@336:             // longer associating a filename with this url in the
Chris@336:             // refcount map -- or createCacheFile will think we've
Chris@336:             // already done all the work and no request will be sent
Chris@336:             deleteCacheFile();
Chris@336: 
Chris@762:             m_url = QUrl::fromEncoded(fileOrUrl.toLatin1());
Chris@336: 
Chris@336:             m_ok = false;
Chris@336:             m_done = false;
Chris@336:             m_lastStatus = 0;
Chris@357:             init();
Chris@316:         }
Chris@316:     }
Chris@325: 
Chris@325:     if (!isRemote()) {
Chris@325:         emit statusAvailable();
Chris@325:         emit ready();
Chris@325:     }
Chris@497: 
Chris@504: #ifdef DEBUG_FILE_SOURCE
Chris@843:     cerr << "FileSource::FileSource(string) exiting" << endl;
Chris@504: #endif
Chris@316: }
Chris@316: 
Chris@469: FileSource::FileSource(QUrl url, ProgressReporter *reporter) :
Chris@304:     m_url(url),
Chris@1582:     m_localFile(nullptr),
Chris@1582:     m_reply(nullptr),
Chris@208:     m_ok(false),
Chris@981:     m_cancelled(false),
Chris@210:     m_lastStatus(0),
Chris@706:     m_resource(false),
Chris@316:     m_remote(isRemote(url.toString())),
Chris@208:     m_done(false),
Chris@316:     m_leaveLocalFile(false),
Chris@392:     m_reporter(reporter),
Chris@316:     m_refCounted(false)
Chris@208: {
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:     cerr << "FileSource::FileSource(" << url.toString() << ") [as url]" << endl;
Chris@529:     incCount(m_url.toString());
Chris@327: #endif
Chris@316: 
Chris@316:     if (!canHandleScheme(m_url)) {
Chris@843:         cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316:         m_errorString = tr("Unsupported scheme in URL");
Chris@208:         return;
Chris@208:     }
Chris@208: 
Chris@357:     init();
Chris@497: 
Chris@504: #ifdef DEBUG_FILE_SOURCE
Chris@843:     cerr << "FileSource::FileSource(url) exiting" << endl;
Chris@504: #endif
Chris@316: }
Chris@304: 
Chris@317: FileSource::FileSource(const FileSource &rf) :
Chris@316:     QObject(),
Chris@316:     m_url(rf.m_url),
Chris@1582:     m_localFile(nullptr),
Chris@1582:     m_reply(nullptr),
Chris@316:     m_ok(rf.m_ok),
Chris@981:     m_cancelled(rf.m_cancelled),
Chris@316:     m_lastStatus(rf.m_lastStatus),
Chris@706:     m_resource(rf.m_resource),
Chris@316:     m_remote(rf.m_remote),
Chris@316:     m_done(false),
Chris@316:     m_leaveLocalFile(false),
Chris@392:     m_reporter(rf.m_reporter),
Chris@316:     m_refCounted(false)
Chris@316: {
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:     cerr << "FileSource::FileSource(" << m_url.toString() << ") [copy ctor]" << endl;
Chris@529:     incCount(m_url.toString());
Chris@327: #endif
Chris@304: 
Chris@316:     if (!canHandleScheme(m_url)) {
Chris@843:         cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316:         m_errorString = tr("Unsupported scheme in URL");
Chris@304:         return;
Chris@304:     }
Chris@304: 
Chris@469:     if (!isRemote()) {
Chris@316:         m_localFilename = rf.m_localFilename;
Chris@316:     } else {
Chris@316:         QMutexLocker locker(&m_mapMutex);
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:         cerr << "FileSource::FileSource(copy ctor): ref count is "
Chris@843:                   << m_refCountMap[m_url] << endl;
Chris@327: #endif
Chris@316:         if (m_refCountMap[m_url] > 0) {
Chris@316:             m_refCountMap[m_url]++;
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:             cerr << "raised it to " << m_refCountMap[m_url] << endl;
Chris@327: #endif
Chris@316:             m_localFilename = m_remoteLocalMap[m_url];
Chris@316:             m_refCounted = true;
Chris@316:         } else {
Chris@316:             m_ok = false;
Chris@316:             m_lastStatus = 404;
Chris@316:         }
Chris@316:     }
Chris@316: 
Chris@316:     m_done = true;
Chris@497: 
Chris@504: #ifdef DEBUG_FILE_SOURCE
Chris@843:     cerr << "FileSource::FileSource(" << m_url.toString() << ") [copy ctor]: note: local filename is \"" << m_localFilename << "\"" << endl;
Chris@527: #endif
Chris@527: 
Chris@527: #ifdef DEBUG_FILE_SOURCE
Chris@843:     cerr << "FileSource::FileSource(copy ctor) exiting" << endl;
Chris@504: #endif
Chris@316: }
Chris@316: 
Chris@317: FileSource::~FileSource()
Chris@316: {
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:     cerr << "FileSource(" << m_url.toString() << ")::~FileSource" << endl;
Chris@529:     decCount(m_url.toString());
Chris@327: #endif
Chris@316: 
Chris@316:     cleanup();
Chris@316: 
Chris@469:     if (isRemote() && !m_leaveLocalFile) deleteCacheFile();
Chris@316: }
Chris@316: 
Chris@316: void
Chris@357: FileSource::init()
Chris@316: {
Chris@706:     if (isResource()) {
Chris@355: #ifdef DEBUG_FILE_SOURCE
Chris@843:         cerr << "FileSource::init: Is a resource" << endl;
Chris@706: #endif
Chris@706:         QString resourceFile = m_url.toString();
Chris@706:         resourceFile.replace(QRegExp("^qrc:"), ":");
Chris@706:         
Chris@706:         if (!QFileInfo(resourceFile).exists()) {
Chris@706: #ifdef DEBUG_FILE_SOURCE
Chris@843:             cerr << "FileSource::init: Resource file of this name does not exist, switching to non-resource URL" << endl;
Chris@706: #endif
Chris@706:             m_url = resourceFile;
Chris@706:             m_resource = false;
Chris@706:         }
Chris@706:     }
Chris@706: 
Chris@706:     if (!isRemote() && !isResource()) {
Chris@706: #ifdef DEBUG_FILE_SOURCE
Chris@843:         cerr << "FileSource::init: Not a remote URL" << endl;
Chris@355: #endif
Chris@355:         bool literal = false;
Chris@316:         m_localFilename = m_url.toLocalFile();
chris@831: 
Chris@342:         if (m_localFilename == "") {
Chris@342:             // QUrl may have mishandled the scheme (e.g. in a DOS path)
chris@831:             m_localFilename = m_rawFileOrUrl;
chris@831: #ifdef DEBUG_FILE_SOURCE
Chris@843:             cerr << "FileSource::init: Trying literal local filename \""
Chris@843:                       << m_localFilename << "\"" << endl;
chris@831: #endif
Chris@355:             literal = true;
Chris@342:         }
Chris@439:         m_localFilename = QFileInfo(m_localFilename).absoluteFilePath();
Chris@439: 
Chris@355: #ifdef DEBUG_FILE_SOURCE
Chris@843:         cerr << "FileSource::init: URL translates to absolute filename \""
Chris@706:                   << m_localFilename << "\" (with literal=" << literal << ")"
Chris@843:                   << endl;
Chris@355: #endif
Chris@316:         m_ok = true;
Chris@355:         m_lastStatus = 200;
Chris@355: 
Chris@316:         if (!QFileInfo(m_localFilename).exists()) {
Chris@355:             if (literal) {
Chris@355:                 m_lastStatus = 404;
Chris@355:             } else {
Chris@614: #ifdef DEBUG_FILE_SOURCE
Chris@843:                 cerr << "FileSource::init: Local file of this name does not exist, trying URL as a literal filename" << endl;
Chris@614: #endif
Chris@355:                 // Again, QUrl may have been mistreating us --
Chris@355:                 // e.g. dropping a part that looks like query data
chris@831:                 m_localFilename = m_rawFileOrUrl;
Chris@355:                 literal = true;
Chris@355:                 if (!QFileInfo(m_localFilename).exists()) {
Chris@355:                     m_lastStatus = 404;
Chris@355:                 }
Chris@355:             }
Chris@316:         }
Chris@355: 
Chris@316:         m_done = true;
Chris@316:         return;
Chris@316:     }
Chris@316: 
Chris@316:     if (createCacheFile()) {
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:         cerr << "FileSource::init: Already have this one" << endl;
Chris@327: #endif
Chris@316:         m_ok = true;
Chris@316:         if (!QFileInfo(m_localFilename).exists()) {
Chris@316:             m_lastStatus = 404;
Chris@316:         } else {
Chris@316:             m_lastStatus = 200;
Chris@316:         }
Chris@316:         m_done = true;
Chris@316:         return;
Chris@316:     }
Chris@316: 
Chris@208:     if (m_localFilename == "") return;
Chris@706: 
Chris@208:     m_localFile = new QFile(m_localFilename);
Chris@208:     m_localFile->open(QFile::WriteOnly);
Chris@208: 
Chris@706:     if (isResource()) {
Chris@706: 
Chris@706:         // Absent resource file case was dealt with at the top -- this
Chris@706:         // is the successful case
Chris@706: 
Chris@706:         QString resourceFileName = m_url.toString();
Chris@706:         resourceFileName.replace(QRegExp("^qrc:"), ":");
Chris@706:         QFile resourceFile(resourceFileName);
Chris@706:         resourceFile.open(QFile::ReadOnly);
Chris@706:         QByteArray ba(resourceFile.readAll());
Chris@706:         
Chris@706: #ifdef DEBUG_FILE_SOURCE
Chris@843:         cerr << "Copying " << ba.size() << " bytes from resource file to cache file" << endl;
Chris@706: #endif
Chris@706: 
Chris@706:         qint64 written = m_localFile->write(ba);
Chris@706:         m_localFile->close();
Chris@706:         delete m_localFile;
Chris@1582:         m_localFile = nullptr;
Chris@706: 
Chris@706:         if (written != ba.size()) {
Chris@706: #ifdef DEBUG_FILE_SOURCE
Chris@843:             cerr << "Copy failed (wrote " << written << " bytes)" << endl;
Chris@706: #endif
Chris@706:             m_ok = false;
Chris@706:             return;
Chris@706:         } else {
Chris@706:             m_ok = true;
Chris@706:             m_lastStatus = 200;
Chris@706:             m_done = true;
Chris@706:         }
Chris@706: 
Chris@706:     } else {
Chris@706: 
Chris@706:         QString scheme = m_url.scheme().toLower();
Chris@316: 
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:         cerr << "FileSource::init: Don't have local copy of \""
Chris@843:                   << m_url.toString() << "\", retrieving" << endl;
Chris@327: #endif
Chris@208: 
Chris@762:         if (scheme == "http" || scheme == "https" || scheme == "ftp") {
Chris@762:             initRemote();
Chris@520: #ifdef DEBUG_FILE_SOURCE
Chris@843:             cerr << "FileSource: initRemote returned" << endl;
Chris@520: #endif
Chris@706:         } else {
Chris@706:             m_remote = false;
Chris@706:             m_ok = false;
Chris@706:         }
Chris@208:     }
Chris@208: 
Chris@208:     if (m_ok) {
Chris@316:         
Chris@316:         QMutexLocker locker(&m_mapMutex);
Chris@316: 
Chris@316:         if (m_refCountMap[m_url] > 0) {
Chris@316:             // someone else has been doing the same thing at the same time,
Chris@316:             // but has got there first
Chris@316:             cleanup();
Chris@316:             m_refCountMap[m_url]++;
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:             cerr << "FileSource::init: Another FileSource has got there first, abandoning our download and using theirs" << endl;
Chris@327: #endif
Chris@316:             m_localFilename = m_remoteLocalMap[m_url];
Chris@316:             m_refCounted = true;
Chris@316:             m_ok = true;
Chris@316:             if (!QFileInfo(m_localFilename).exists()) {
Chris@316:                 m_lastStatus = 404;
Chris@316:             }
Chris@316:             m_done = true;
Chris@316:             return;
Chris@316:         }
Chris@304: 
Chris@304:         m_remoteLocalMap[m_url] = m_localFilename;
Chris@304:         m_refCountMap[m_url]++;
Chris@316:         m_refCounted = true;
Chris@304: 
Chris@706:         if (m_reporter && !m_done) {
Chris@392:             m_reporter->setMessage
Chris@392:                 (tr("Downloading %1...").arg(m_url.toString()));
Chris@392:             connect(m_reporter, SIGNAL(cancelled()), this, SLOT(cancelled()));
Chris@357:             connect(this, SIGNAL(progress(int)),
Chris@392:                     m_reporter, SLOT(setProgress(int)));
Chris@316:         }
Chris@208:     }
Chris@208: }
Chris@208: 
Chris@316: void
Chris@762: FileSource::initRemote()
Chris@208: {
Chris@316:     m_ok = true;
Chris@316: 
Chris@762:     QNetworkRequest req;
Chris@762:     req.setUrl(m_url);
Chris@762:     
Chris@762:     if (m_preferredContentType != "") {
Chris@520: #ifdef DEBUG_FILE_SOURCE
Chris@843:         cerr << "FileSource: indicating preferred content type of \""
Chris@843:                   << m_preferredContentType << "\"" << endl;
Chris@520: #endif
Chris@762:         req.setRawHeader
Chris@762:             ("Accept",
Chris@762:              QString("%1, */*").arg(m_preferredContentType).toLatin1());
Chris@520:     }
Chris@316: 
Chris@1033:     { // check we have a QNetworkAccessManager
Chris@1033:         QMutexLocker locker(&m_mapMutex);
Chris@1033:         if (!nms.hasLocalData()) {
Chris@1033: #ifdef DEBUG_FILE_SOURCE
Chris@1033:             ++threadCount;
Chris@1033: #endif
Chris@1033:             nms.setLocalData(new QNetworkAccessManager());
Chris@1033:         }
Chris@1033:     }
Chris@1033: 
Chris@833:     m_reply = nms.localData()->get(req);
Chris@762: 
Chris@762:     connect(m_reply, SIGNAL(readyRead()),
Chris@762:             this, SLOT(readyRead()));
Chris@762:     connect(m_reply, SIGNAL(error(QNetworkReply::NetworkError)),
Chris@762:             this, SLOT(replyFailed(QNetworkReply::NetworkError)));
Chris@762:     connect(m_reply, SIGNAL(finished()),
Chris@762:             this, SLOT(replyFinished()));
Chris@764:     connect(m_reply, SIGNAL(metaDataChanged()),
Chris@764:             this, SLOT(metaDataChanged()));
Chris@762:     connect(m_reply, SIGNAL(downloadProgress(qint64, qint64)),
Chris@762:             this, SLOT(downloadProgress(qint64, qint64)));
Chris@211: }
Chris@211: 
Chris@211: void
Chris@317: FileSource::cleanup()
Chris@211: {
Chris@470:     if (m_done) {
Chris@470:         delete m_localFile; // does not actually delete the file
Chris@1582:         m_localFile = nullptr;
Chris@470:     }
Chris@211:     m_done = true;
Chris@762:     if (m_reply) {
Chris@762:         QNetworkReply *r = m_reply;
Chris@1582:         disconnect(r, nullptr, this, nullptr);
Chris@1582:         m_reply = nullptr;
Chris@913:         // Can only call abort() when there are no errors.
Chris@913:         if (r->error() == QNetworkReply::NoError) {
Chris@913:             r->abort();
Chris@913:         }
Chris@762:         r->deleteLater();
Chris@214:     }
Chris@470:     if (m_localFile) {
Chris@470:         delete m_localFile; // does not actually delete the file
Chris@1582:         m_localFile = nullptr;
Chris@470:     }
Chris@208: }
Chris@208: 
Chris@208: bool
Chris@317: FileSource::isRemote(QString fileOrUrl)
Chris@304: {
Chris@342:     // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@316:     QString scheme = QUrl(fileOrUrl).scheme().toLower();
Chris@342:     if (scheme == "" || scheme == "file" || scheme.length() == 1) return false;
Chris@342:     return true;
Chris@304: }
Chris@304: 
Chris@304: bool
Chris@317: FileSource::canHandleScheme(QUrl url)
Chris@208: {
Chris@342:     // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@208:     QString scheme = url.scheme().toLower();
Chris@762:     return (scheme == "http" || scheme == "https" ||
Chris@762:             scheme == "ftp" || scheme == "file" || scheme == "qrc" ||
Chris@706:             scheme == "" || scheme.length() == 1);
Chris@208: }
Chris@208: 
Chris@210: bool
Chris@317: FileSource::isAvailable()
Chris@210: {
Chris@316:     waitForStatus();
Chris@211:     bool available = true;
Chris@913:     if (!m_ok) {
Chris@913:         available = false;
Chris@913:     } else {
Chris@913:         // http 2xx status codes mean success
Chris@913:         available = (m_lastStatus / 100 == 2);
Chris@913:     }
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@913:     cerr << "FileSource::isAvailable: " << (available ? "yes" : "no") << endl;
Chris@327: #endif
Chris@211:     return available;
Chris@210: }
Chris@210: 
Chris@208: void
Chris@317: FileSource::waitForStatus()
Chris@316: {
Chris@316:     while (m_ok && (!m_done && m_lastStatus == 0)) {
Chris@843: //        cerr << "waitForStatus: processing (last status " << m_lastStatus << ")" << endl;
Chris@392:         QCoreApplication::processEvents();
Chris@316:     }
Chris@316: }
Chris@316: 
Chris@316: void
Chris@317: FileSource::waitForData()
Chris@208: {
Chris@211:     while (m_ok && !m_done) {
Chris@843: //        cerr << "FileSource::waitForData: calling QApplication::processEvents" << endl;
Chris@392:         QCoreApplication::processEvents();
Chris@497:         usleep(10000);
Chris@208:     }
Chris@208: }
Chris@208: 
Chris@316: void
Chris@317: FileSource::setLeaveLocalFile(bool leave)
Chris@316: {
Chris@316:     m_leaveLocalFile = leave;
Chris@316: }
Chris@316: 
Chris@208: bool
Chris@317: FileSource::isOK() const
Chris@208: {
Chris@208:     return m_ok;
Chris@208: }
Chris@208: 
Chris@208: bool
Chris@317: FileSource::isDone() const
Chris@208: {
Chris@208:     return m_done;
Chris@208: }
Chris@208: 
Chris@316: bool
Chris@981: FileSource::wasCancelled() const
Chris@981: {
Chris@981:     return m_cancelled;
Chris@981: }
Chris@981: 
Chris@981: bool
Chris@706: FileSource::isResource() const
Chris@706: {
Chris@706:     return m_resource;
Chris@706: }
Chris@706: 
Chris@706: bool
Chris@317: FileSource::isRemote() const
Chris@316: {
Chris@316:     return m_remote;
Chris@316: }
Chris@316: 
Chris@316: QString
Chris@317: FileSource::getLocation() const
Chris@316: {
Chris@316:     return m_url.toString();
Chris@316: }
Chris@316: 
Chris@208: QString
Chris@317: FileSource::getLocalFilename() const
Chris@208: {
Chris@208:     return m_localFilename;
Chris@208: }
Chris@208: 
Chris@208: QString
Chris@678: FileSource::getBasename() const
Chris@678: {
Chris@678:     return QFileInfo(m_localFilename).fileName();
Chris@678: }
Chris@678: 
Chris@678: QString
Chris@317: FileSource::getContentType() const
Chris@316: {
Chris@316:     return m_contentType;
Chris@316: }
Chris@316: 
Chris@316: QString
Chris@317: FileSource::getExtension() const
Chris@316: {
Chris@316:     if (m_localFilename != "") {
Chris@316:         return QFileInfo(m_localFilename).suffix().toLower();
Chris@316:     } else {
Chris@316:         return QFileInfo(m_url.toLocalFile()).suffix().toLower();
Chris@316:     }
Chris@316: }
Chris@316: 
Chris@316: QString
Chris@317: FileSource::getErrorString() const
Chris@208: {
Chris@208:     return m_errorString;
Chris@208: }
Chris@208: 
Chris@208: void
Chris@762: FileSource::readyRead()
Chris@208: {
Chris@762:     m_localFile->write(m_reply->readAll());
Chris@208: }
Chris@208: 
Chris@208: void
Chris@764: FileSource::metaDataChanged()
Chris@210: {
Chris@520: #ifdef DEBUG_FILE_SOURCE
Chris@843:     cerr << "FileSource::metaDataChanged" << endl;
Chris@520: #endif
Chris@497: 
Chris@762:     if (!m_reply) {
Chris@843:         cerr << "WARNING: FileSource::metaDataChanged() called without a reply object being known to us" << endl;
Chris@762:         return;
Chris@762:     }
Chris@762: 
Chris@913:     // Handle http transfer status codes.
Chris@913: 
Chris@762:     int status =
Chris@762:         m_reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
Chris@762: 
Chris@913:     // If this is a redirection (3xx) code, do the redirect
Chris@762:     if (status / 100 == 3) {
Chris@762:         QString location = m_reply->header
Chris@762:             (QNetworkRequest::LocationHeader).toString();
Chris@496: #ifdef DEBUG_FILE_SOURCE
Chris@843:         cerr << "FileSource::metaDataChanged: redirect to \""
Chris@843:                   << location << "\" received" << endl;
Chris@496: #endif
Chris@496:         if (location != "") {
Chris@496:             QUrl newUrl(location);
Chris@496:             if (newUrl != m_url) {
Chris@497:                 cleanup();
Chris@497:                 deleteCacheFile();
Chris@529: #ifdef DEBUG_FILE_SOURCE
Chris@529:                 decCount(m_url.toString());
Chris@529:                 incCount(newUrl.toString());
Chris@529: #endif
Chris@496:                 m_url = newUrl;
Chris@1582:                 m_localFile = nullptr;
Chris@496:                 m_lastStatus = 0;
Chris@497:                 m_done = false;
Chris@497:                 m_refCounted = false;
Chris@496:                 init();
Chris@496:                 return;
Chris@496:             }
Chris@496:         }
Chris@496:     }
Chris@497: 
Chris@762:     m_lastStatus = status;
Chris@913: 
Chris@913:     // 400 and up are failures, get the error string
Chris@210:     if (m_lastStatus / 100 >= 4) {
Chris@210:         m_errorString = QString("%1 %2")
Chris@762:             .arg(status)
Chris@762:             .arg(m_reply->attribute
Chris@762:                  (QNetworkRequest::HttpReasonPhraseAttribute).toString());
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:         cerr << "FileSource::metaDataChanged: "
Chris@843:                   << m_errorString << endl;
Chris@327: #endif
Chris@211:     } else {
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:         cerr << "FileSource::metaDataChanged: "
Chris@843:                   << m_lastStatus << endl;
Chris@327: #endif
Chris@762:         m_contentType =
Chris@762:             m_reply->header(QNetworkRequest::ContentTypeHeader).toString();
Chris@325:     }
Chris@325:     emit statusAvailable();
Chris@210: }
Chris@210: 
Chris@210: void
Chris@762: FileSource::downloadProgress(qint64 done, qint64 total)
Chris@208: {
Chris@208:     int percent = int((double(done) / double(total)) * 100.0 - 0.1);
Chris@208:     emit progress(percent);
Chris@210: }
Chris@210: 
Chris@210: void
Chris@317: FileSource::cancelled()
Chris@210: {
Chris@210:     m_done = true;
Chris@316:     cleanup();
Chris@316: 
Chris@210:     m_ok = false;
Chris@981:     m_cancelled = true;
Chris@210:     m_errorString = tr("Download cancelled");
Chris@208: }
Chris@208: 
Chris@208: void
Chris@762: FileSource::replyFinished()
Chris@208: {
Chris@327:     emit progress(100);
Chris@327: 
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:     cerr << "FileSource::replyFinished()" << endl;
Chris@327: #endif
Chris@211: 
Chris@211:     if (m_done) return;
Chris@211: 
Chris@913:     QString scheme = m_url.scheme().toLower();
Chris@913:     // For ftp transfers, replyFinished() will be called on success.
Chris@913:     // metaDataChanged() is never called for ftp transfers.
Chris@913:     if (scheme == "ftp") {
Chris@913:         m_lastStatus = 200;  // http ok
Chris@913:     }
Chris@913: 
Chris@762:     bool error = (m_lastStatus / 100 >= 4);
Chris@210: 
Chris@211:     cleanup();
Chris@208: 
Chris@211:     if (!error) {
Chris@208:         QFileInfo fi(m_localFilename);
Chris@208:         if (!fi.exists()) {
Chris@208:             m_errorString = tr("Failed to create local file %1").arg(m_localFilename);
Chris@211:             error = true;
Chris@208:         } else if (fi.size() == 0) {
Chris@208:             m_errorString = tr("File contains no data!");
Chris@211:             error = true;
Chris@208:         }
Chris@208:     }
Chris@211: 
Chris@211:     if (error) {
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:         cerr << "FileSource::done: error is " << error << ", deleting cache file" << endl;
Chris@327: #endif
Chris@316:         deleteCacheFile();
Chris@211:     }
Chris@211: 
Chris@211:     m_ok = !error;
Chris@520:     if (m_localFile) m_localFile->flush();
Chris@211:     m_done = true;
Chris@304:     emit ready();
Chris@211: }
Chris@211: 
Chris@211: void
Chris@763: FileSource::replyFailed(QNetworkReply::NetworkError)
Chris@763: {
Chris@763:     emit progress(100);
Chris@910:     if (!m_reply) {
Chris@910:         cerr << "WARNING: FileSource::replyFailed() called without a reply object being known to us" << endl;
Chris@910:     } else {
Chris@910:         m_errorString = m_reply->errorString();
Chris@910:     }
Chris@763:     m_ok = false;
Chris@763:     m_done = true;
Chris@763:     cleanup();
Chris@763:     emit ready();
Chris@763: }
Chris@763: 
Chris@763: void
Chris@317: FileSource::deleteCacheFile()
Chris@211: {
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:     cerr << "FileSource::deleteCacheFile(\"" << m_localFilename << "\")" << endl;
Chris@327: #endif
Chris@211: 
Chris@211:     cleanup();
Chris@211: 
Chris@316:     if (m_localFilename == "") {
Chris@316:         return;
Chris@316:     }
Chris@211: 
Chris@316:     if (!isRemote()) {
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:         cerr << "not a cache file" << endl;
Chris@327: #endif
Chris@316:         return;
Chris@316:     }
Chris@316: 
Chris@316:     if (m_refCounted) {
Chris@304: 
Chris@304:         QMutexLocker locker(&m_mapMutex);
Chris@316:         m_refCounted = false;
Chris@304: 
Chris@304:         if (m_refCountMap[m_url] > 0) {
Chris@304:             m_refCountMap[m_url]--;
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:             cerr << "reduced ref count to " << m_refCountMap[m_url] << endl;
Chris@327: #endif
Chris@304:             if (m_refCountMap[m_url] > 0) {
Chris@304:                 m_done = true;
Chris@304:                 return;
Chris@304:             }
Chris@304:         }
Chris@304:     }
Chris@304: 
Chris@211:     m_fileCreationMutex.lock();
Chris@211: 
Chris@211:     if (!QFile(m_localFilename).remove()) {
Chris@469: #ifdef DEBUG_FILE_SOURCE
Chris@843:         cerr << "FileSource::deleteCacheFile: ERROR: Failed to delete file \"" << m_localFilename << "\"" << endl;
Chris@469: #endif
Chris@211:     } else {
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:         cerr << "FileSource::deleteCacheFile: Deleted cache file \"" << m_localFilename << "\"" << endl;
Chris@327: #endif
Chris@211:         m_localFilename = "";
Chris@211:     }
Chris@211: 
Chris@211:     m_fileCreationMutex.unlock();
Chris@211: 
Chris@208:     m_done = true;
Chris@208: }
Chris@208: 
Chris@316: bool
Chris@317: FileSource::createCacheFile()
Chris@208: {
Chris@316:     {
Chris@316:         QMutexLocker locker(&m_mapMutex);
Chris@316: 
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:         cerr << "FileSource::createCacheFile: refcount is " << m_refCountMap[m_url] << endl;
Chris@327: #endif
Chris@316: 
Chris@316:         if (m_refCountMap[m_url] > 0) {
Chris@316:             m_refCountMap[m_url]++;
Chris@316:             m_localFilename = m_remoteLocalMap[m_url];
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:             cerr << "raised it to " << m_refCountMap[m_url] << endl;
Chris@327: #endif
Chris@316:             m_refCounted = true;
Chris@316:             return true;
Chris@316:         }
Chris@316:     }
Chris@316: 
Chris@208:     QDir dir;
Chris@208:     try {
Chris@208:         dir = TempDirectory::getInstance()->getSubDirectoryPath("download");
Chris@1465:     } catch (const DirectoryCreationFailed &f) {
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:         cerr << "FileSource::createCacheFile: ERROR: Failed to create temporary directory: " << f.what() << endl;
Chris@327: #endif
Chris@962:         return false;
Chris@208:     }
Chris@208: 
Chris@316:     QString filepart = m_url.path().section('/', -1, -1,
Chris@316:                                             QString::SectionSkipEmpty);
Chris@208: 
Chris@457:     QString extension = "";
Chris@457:     if (filepart.contains('.')) extension = filepart.section('.', -1);
Chris@457: 
Chris@208:     QString base = filepart;
Chris@208:     if (extension != "") {
Chris@208:         base = base.left(base.length() - extension.length() - 1);
Chris@208:     }
Chris@208:     if (base == "") base = "remote";
Chris@208: 
Chris@208:     QString filename;
Chris@208: 
Chris@208:     if (extension == "") {
Chris@208:         filename = base;
Chris@208:     } else {
Chris@208:         filename = QString("%1.%2").arg(base).arg(extension);
Chris@208:     }
Chris@208: 
Chris@208:     QString filepath(dir.filePath(filename));
Chris@208: 
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:     cerr << "FileSource::createCacheFile: URL is \"" << m_url.toString() << "\", dir is \"" << dir.path() << "\", base \"" << base << "\", extension \"" << extension << "\", filebase \"" << filename << "\", filename \"" << filepath << "\"" << endl;
Chris@327: #endif
Chris@208: 
Chris@316:     QMutexLocker fcLocker(&m_fileCreationMutex);
Chris@316: 
Chris@208:     ++m_count;
Chris@208: 
Chris@208:     if (QFileInfo(filepath).exists() ||
Chris@208:         !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208: 
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:         cerr << "FileSource::createCacheFile: Failed to create local file \""
Chris@686:                   << filepath << "\" for URL \""
Chris@843:                   << m_url.toString() << "\" (or file already exists): appending suffix instead" << endl;
Chris@327: #endif
Chris@208: 
Chris@208:         if (extension == "") {
Chris@208:             filename = QString("%1_%2").arg(base).arg(m_count);
Chris@208:         } else {
Chris@208:             filename = QString("%1_%2.%3").arg(base).arg(m_count).arg(extension);
Chris@208:         }
Chris@208:         filepath = dir.filePath(filename);
Chris@208: 
Chris@208:         if (QFileInfo(filepath).exists() ||
Chris@208:             !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208: 
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:             cerr << "FileSource::createCacheFile: ERROR: Failed to create local file \""
Chris@686:                       << filepath << "\" for URL \""
Chris@843:                       << m_url.toString() << "\" (or file already exists)" << endl;
Chris@327: #endif
Chris@208: 
Chris@962:             return false;
Chris@208:         }
Chris@208:     }
Chris@208: 
Chris@327: #ifdef DEBUG_FILE_SOURCE
Chris@843:     cerr << "FileSource::createCacheFile: url "
Chris@686:               << m_url.toString() << " -> local filename "
Chris@843:               << filepath << endl;
Chris@327: #endif
Chris@316:     
Chris@316:     m_localFilename = filepath;
Chris@208: 
Chris@316:     return false;
Chris@208: }
Chris@327: