Chris@208: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
Chris@208: 
Chris@208: /*
Chris@208:     Sonic Visualiser
Chris@208:     An audio file viewer and annotation editor.
Chris@208:     Centre for Digital Music, Queen Mary, University of London.
Chris@208:     This file copyright 2007 QMUL.
Chris@208:     
Chris@208:     This program is free software; you can redistribute it and/or
Chris@208:     modify it under the terms of the GNU General Public License as
Chris@208:     published by the Free Software Foundation; either version 2 of the
Chris@208:     License, or (at your option) any later version.  See the file
Chris@208:     COPYING included with this distribution for more information.
Chris@208: */
Chris@208: 
Chris@208: #include "RemoteFile.h"
Chris@208: #include "base/TempDirectory.h"
Chris@208: #include "base/Exceptions.h"
Chris@208: 
Chris@208: #include <QHttp>
Chris@208: #include <QFtp>
Chris@208: #include <QFileInfo>
Chris@208: #include <QDir>
Chris@208: #include <QApplication>
Chris@208: #include <QProgressDialog>
Chris@210: #include <QHttpResponseHeader>
Chris@208: 
Chris@208: #include <iostream>
Chris@208: 
Chris@208: int
Chris@208: RemoteFile::m_count = 0;
Chris@208: 
Chris@208: QMutex
Chris@208: RemoteFile::m_fileCreationMutex;
Chris@208: 
Chris@304: RemoteFile::RemoteRefCountMap
Chris@304: RemoteFile::m_refCountMap;
Chris@304: 
Chris@304: RemoteFile::RemoteLocalMap
Chris@304: RemoteFile::m_remoteLocalMap;
Chris@304: 
Chris@304: QMutex
Chris@304: RemoteFile::m_mapMutex;
Chris@304: 
Chris@208: RemoteFile::RemoteFile(QUrl url) :
Chris@304:     m_url(url),
Chris@208:     m_ftp(0),
Chris@208:     m_http(0),
Chris@208:     m_localFile(0),
Chris@208:     m_ok(false),
Chris@210:     m_lastStatus(0),
Chris@208:     m_done(false),
Chris@210:     m_progressDialog(0),
Chris@304:     m_progressShowTimer(this),
Chris@304:     m_referenced(false)
Chris@208: {
Chris@208:     if (!canHandleScheme(url)) {
Chris@208:         std::cerr << "RemoteFile::RemoteFile: ERROR: Unsupported scheme in URL \"" << url.toString().toStdString() << "\"" << std::endl;
Chris@208:         return;
Chris@208:     }
Chris@208: 
Chris@304:     QMutexLocker locker(&m_mapMutex);
Chris@304: 
Chris@304:     std::cerr << "RemoteFile::RemoteFile: refcount is " << m_refCountMap[m_url] << std::endl;
Chris@304: 
Chris@304:     if (m_refCountMap[m_url] > 0) {
Chris@304:         m_refCountMap[m_url]++;
Chris@304:         m_localFilename = m_remoteLocalMap[m_url];
Chris@304:         std::cerr << "raising it" << std::endl;
Chris@304:         m_ok = true;
Chris@304:         m_done = true;
Chris@304:         m_referenced = true;
Chris@304:         return;
Chris@304:     }
Chris@304: 
Chris@208:     m_localFilename = createLocalFile(url);
Chris@208:     if (m_localFilename == "") return;
Chris@208:     m_localFile = new QFile(m_localFilename);
Chris@208:     m_localFile->open(QFile::WriteOnly);
Chris@208: 
Chris@208:     QString scheme = url.scheme().toLower();
Chris@208: 
Chris@208:     if (scheme == "http") {
Chris@208: 
Chris@211:         m_ok = true;
Chris@208:         m_http = new QHttp(url.host(), url.port(80));
Chris@208:         connect(m_http, SIGNAL(done(bool)), this, SLOT(done(bool)));
Chris@208:         connect(m_http, SIGNAL(dataReadProgress(int, int)),
Chris@208:                 this, SLOT(dataReadProgress(int, int)));
Chris@210:         connect(m_http, SIGNAL(responseHeaderReceived(const QHttpResponseHeader &)),
Chris@214:                 this, SLOT(httpResponseHeaderReceived(const QHttpResponseHeader &)));
Chris@279: 
Chris@279:         // I don't quite understand this.  url.path() returns a path
Chris@279:         // without percent encoding; for example, spaces appear as
Chris@279:         // literal spaces.  This generally won't work if sent to the
Chris@279:         // server directly.  You can retrieve a correctly encoded URL
Chris@279:         // from QUrl using url.toEncoded(), but that gives you the
Chris@279:         // whole URL; there doesn't seem to be any way to retrieve
Chris@279:         // only an encoded path.  Furthermore there doesn't seem to be
Chris@279:         // any way to convert a retrieved path into an encoded path
Chris@279:         // without explicitly specifying that you don't want the path
Chris@279:         // separators ("/") to be encoded.  (Besides being painful to
Chris@279:         // manage, I don't see how this can work correctly in any case
Chris@279:         // where a percent-encoded "/" is supposed to appear within a
Chris@279:         // path element?)  There also seems to be no way to retrieve
Chris@279:         // the path plus query string, i.e. everything that I need to
Chris@279:         // send to the HTTP server.  And no way for QHttp to take a
Chris@279:         // QUrl argument.  I'm obviously missing something.
Chris@279: 
Chris@279:         // So, two ways to do this: query the bits from the URL,
Chris@279:         // encode them individually, and glue them back together
Chris@279:         // again...
Chris@279:         /*
Chris@279:         QString path = QUrl::toPercentEncoding(url.path(), "/");
Chris@279:         QList<QPair<QString, QString> > query = url.queryItems();
Chris@279:         if (!query.empty()) {
Chris@279:             QStringList q2;
Chris@279:             for (QList<QPair<QString, QString> >::iterator i = query.begin();
Chris@279:                  i != query.end(); ++i) {
Chris@279:                 q2.push_back(QString("%1=%3")
Chris@279:                              .arg(QString(QUrl::toPercentEncoding(i->first)))
Chris@279:                              .arg(QString(QUrl::toPercentEncoding(i->second))));
Chris@279:             }
Chris@279:             path = QString("%1%2%3")
Chris@279:                 .arg(path).arg("?")
Chris@279:                 .arg(q2.join("&"));
Chris@279:         }
Chris@279:         */
Chris@279: 
Chris@279:         // ...or, much simpler but relying on knowledge about the
Chris@279:         // scheme://host/path/path/query etc format of the URL, we can
Chris@279:         // get the whole URL ready-encoded and then split it on "/" as
Chris@279:         // appropriate...
Chris@279:         
Chris@279:         QString path = "/" + QString(url.toEncoded()).section('/', 3);
Chris@279: 
Chris@279:         std::cerr << "RemoteFile: path is \""
Chris@279:                   << path.toStdString() << "\"" << std::endl;
Chris@279: 
Chris@278:         m_http->get(path, m_localFile);
Chris@208: 
Chris@208:     } else if (scheme == "ftp") {
Chris@208: 
Chris@211:         m_ok = true;
Chris@208:         m_ftp = new QFtp;
Chris@208:         connect(m_ftp, SIGNAL(done(bool)), this, SLOT(done(bool)));
Chris@214:         connect(m_ftp, SIGNAL(commandFinished(int, bool)),
Chris@214:                 this, SLOT(ftpCommandFinished(int, bool)));
Chris@208:         connect(m_ftp, SIGNAL(dataTransferProgress(qint64, qint64)),
Chris@208:                 this, SLOT(dataTransferProgress(qint64, qint64)));
Chris@208:         m_ftp->connectToHost(url.host(), url.port(21));
Chris@208: 
Chris@208:         QString username = url.userName();
Chris@208:         if (username == "") {
Chris@208:             username = "anonymous";
Chris@208:         }
Chris@208: 
Chris@208:         QString password = url.password();
Chris@208:         if (password == "") {
Chris@208:             password = QString("%1@%2").arg(getenv("USER")).arg(getenv("HOST"));
Chris@208:         }
Chris@208: 
Chris@214:         m_ftp->login(username, password);
Chris@214: 
Chris@214:         QString dirpath = url.path().section('/', 0, -2);
Chris@214:         QString filename = url.path().section('/', -1);
Chris@214: 
Chris@214:         if (dirpath == "") dirpath = "/";
Chris@214:         m_ftp->cd(dirpath);
Chris@214:         m_ftp->get(filename, m_localFile);
Chris@208:     }
Chris@208: 
Chris@208:     if (m_ok) {
Chris@304: 
Chris@304:         m_remoteLocalMap[m_url] = m_localFilename;
Chris@304:         m_refCountMap[m_url]++;
Chris@304:         m_referenced = true;
Chris@304: 
Chris@208:         m_progressDialog = new QProgressDialog(tr("Downloading %1...").arg(url.toString()), tr("Cancel"), 0, 100);
Chris@210:         m_progressDialog->hide();
Chris@210:         connect(&m_progressShowTimer, SIGNAL(timeout()),
Chris@210:                 this, SLOT(showProgressDialog()));
Chris@210:         connect(m_progressDialog, SIGNAL(canceled()), this, SLOT(cancelled()));
Chris@210:         m_progressShowTimer.setSingleShot(true);
Chris@210:         m_progressShowTimer.start(2000);
Chris@208:     }
Chris@208: }
Chris@208: 
Chris@208: RemoteFile::~RemoteFile()
Chris@208: {
Chris@211:     cleanup();
Chris@211: }
Chris@211: 
Chris@211: void
Chris@211: RemoteFile::cleanup()
Chris@211: {
Chris@211:     m_done = true;
Chris@214:     if (m_http) {
Chris@287:         QHttp *h = m_http;
Chris@214:         m_http = 0;
Chris@287:         h->abort();
Chris@287:         h->deleteLater();
Chris@214:     }
Chris@214:     if (m_ftp) {
Chris@287:         QFtp *f = m_ftp;
Chris@214:         m_ftp = 0;
Chris@287:         f->abort();
Chris@287:         f->deleteLater();
Chris@214:     }
Chris@211:     delete m_progressDialog;
Chris@211:     m_progressDialog = 0;
Chris@304:     delete m_localFile; // does not actually delete the file
Chris@211:     m_localFile = 0;
Chris@208: }
Chris@208: 
Chris@208: bool
Chris@304: RemoteFile::isRemote(QString fileOrUrl)
Chris@304: {
Chris@304:     return (fileOrUrl.startsWith("http:") || fileOrUrl.startsWith("ftp:"));
Chris@304: }
Chris@304: 
Chris@304: bool
Chris@208: RemoteFile::canHandleScheme(QUrl url)
Chris@208: {
Chris@208:     QString scheme = url.scheme().toLower();
Chris@208:     return (scheme == "http" || scheme == "ftp");
Chris@208: }
Chris@208: 
Chris@210: bool
Chris@210: RemoteFile::isAvailable()
Chris@210: {
Chris@211:     while (m_ok && (!m_done && m_lastStatus == 0)) {
Chris@210:         QApplication::processEvents();
Chris@210:     }
Chris@211:     bool available = true;
Chris@211:     if (!m_ok) available = false;
Chris@211:     else available = (m_lastStatus / 100 == 2);
Chris@211:     std::cerr << "RemoteFile::isAvailable: " << (available ? "yes" : "no")
Chris@211:               << std::endl;
Chris@211:     return available;
Chris@210: }
Chris@210: 
Chris@208: void
Chris@208: RemoteFile::wait()
Chris@208: {
Chris@211:     while (m_ok && !m_done) {
Chris@208:         QApplication::processEvents();
Chris@208:     }
Chris@208: }
Chris@208: 
Chris@208: bool
Chris@208: RemoteFile::isOK() const
Chris@208: {
Chris@208:     return m_ok;
Chris@208: }
Chris@208: 
Chris@208: bool
Chris@208: RemoteFile::isDone() const
Chris@208: {
Chris@208:     return m_done;
Chris@208: }
Chris@208: 
Chris@208: QString
Chris@208: RemoteFile::getLocalFilename() const
Chris@208: {
Chris@208:     return m_localFilename;
Chris@208: }
Chris@208: 
Chris@208: QString
Chris@208: RemoteFile::getErrorString() const
Chris@208: {
Chris@208:     return m_errorString;
Chris@208: }
Chris@208: 
Chris@208: void
Chris@208: RemoteFile::dataReadProgress(int done, int total)
Chris@208: {
Chris@208:     dataTransferProgress(done, total);
Chris@208: }
Chris@208: 
Chris@208: void
Chris@214: RemoteFile::httpResponseHeaderReceived(const QHttpResponseHeader &resp)
Chris@210: {
Chris@210:     m_lastStatus = resp.statusCode();
Chris@210:     if (m_lastStatus / 100 >= 4) {
Chris@210:         m_errorString = QString("%1 %2")
Chris@210:             .arg(resp.statusCode()).arg(resp.reasonPhrase());
Chris@211:         std::cerr << "RemoteFile::responseHeaderReceived: "
Chris@211:                   << m_errorString.toStdString() << std::endl;
Chris@211:     } else {
Chris@211:         std::cerr << "RemoteFile::responseHeaderReceived: "
Chris@211:                   << m_lastStatus << std::endl;
Chris@315:         if (resp.hasContentType()) m_contentType = resp.contentType();
Chris@211:     }        
Chris@210: }
Chris@210: 
Chris@210: void
Chris@214: RemoteFile::ftpCommandFinished(int id, bool error)
Chris@214: {
Chris@214:     std::cerr << "RemoteFile::ftpCommandFinished(" << id << ", " << error << ")" << std::endl;
Chris@214: 
Chris@214:     if (!m_ftp) return;
Chris@214: 
Chris@214:     QFtp::Command command = m_ftp->currentCommand();
Chris@214: 
Chris@214:     if (!error) {
Chris@214:         std::cerr << "RemoteFile::ftpCommandFinished: success for command "
Chris@214:                   << command << std::endl;
Chris@214:         return;
Chris@214:     }
Chris@214: 
Chris@214:     if (command == QFtp::ConnectToHost) {
Chris@214:         m_errorString = tr("Failed to connect to FTP server");
Chris@214:     } else if (command == QFtp::Login) {
Chris@214:         m_errorString = tr("Login failed");
Chris@214:     } else if (command == QFtp::Cd) {
Chris@214:         m_errorString = tr("Failed to change to correct directory");
Chris@214:     } else if (command == QFtp::Get) {
Chris@214:         m_errorString = tr("FTP download aborted");
Chris@214:     }
Chris@214: 
Chris@214:     m_lastStatus = 400; // for done()
Chris@214: }
Chris@214: 
Chris@214: void
Chris@208: RemoteFile::dataTransferProgress(qint64 done, qint64 total)
Chris@208: {
Chris@211:     if (!m_progressDialog) return;
Chris@211: 
Chris@208:     int percent = int((double(done) / double(total)) * 100.0 - 0.1);
Chris@208:     emit progress(percent);
Chris@208: 
Chris@211:     if (percent > 0) {
Chris@211:         m_progressDialog->setValue(percent);
Chris@211:         m_progressDialog->show();
Chris@211:     }
Chris@210: }
Chris@210: 
Chris@210: void
Chris@210: RemoteFile::cancelled()
Chris@210: {
Chris@211:     deleteLocalFile();
Chris@210:     m_done = true;
Chris@210:     m_ok = false;
Chris@210:     m_errorString = tr("Download cancelled");
Chris@208: }
Chris@208: 
Chris@208: void
Chris@208: RemoteFile::done(bool error)
Chris@208: {
Chris@214:     std::cerr << "RemoteFile::done(" << error << ")" << std::endl;
Chris@211: 
Chris@211:     if (m_done) return;
Chris@211: 
Chris@208:     emit progress(100);
Chris@210: 
Chris@208:     if (error) {
Chris@208:         if (m_http) {
Chris@208:             m_errorString = m_http->errorString();
Chris@208:         } else if (m_ftp) {
Chris@208:             m_errorString = m_ftp->errorString();
Chris@208:         }
Chris@208:     }
Chris@208: 
Chris@210:     if (m_lastStatus / 100 >= 4) {
Chris@211:         error = true;
Chris@210:     }
Chris@210: 
Chris@211:     cleanup();
Chris@208: 
Chris@211:     if (!error) {
Chris@208:         QFileInfo fi(m_localFilename);
Chris@208:         if (!fi.exists()) {
Chris@208:             m_errorString = tr("Failed to create local file %1").arg(m_localFilename);
Chris@211:             error = true;
Chris@208:         } else if (fi.size() == 0) {
Chris@208:             m_errorString = tr("File contains no data!");
Chris@211:             error = true;
Chris@208:         }
Chris@208:     }
Chris@211: 
Chris@211:     if (error) {
Chris@211:         deleteLocalFile();
Chris@211:     }
Chris@211: 
Chris@211:     m_ok = !error;
Chris@211:     m_done = true;
Chris@304:     emit ready();
Chris@211: }
Chris@211: 
Chris@211: void
Chris@211: RemoteFile::deleteLocalFile()
Chris@211: {
Chris@211: //    std::cerr << "RemoteFile::deleteLocalFile" << std::endl;
Chris@211: 
Chris@211:     cleanup();
Chris@211: 
Chris@211:     if (m_localFilename == "") return;
Chris@211: 
Chris@304:     if (m_referenced) {
Chris@304: 
Chris@304:         QMutexLocker locker(&m_mapMutex);
Chris@304:         m_referenced = false;
Chris@304: 
Chris@304:         if (m_refCountMap[m_url] > 0) {
Chris@304:             m_refCountMap[m_url]--;
Chris@304:             if (m_refCountMap[m_url] > 0) {
Chris@304:                 m_done = true;
Chris@304:                 return;
Chris@304:             }
Chris@304:         }
Chris@304:     }
Chris@304: 
Chris@211:     m_fileCreationMutex.lock();
Chris@211: 
Chris@211:     if (!QFile(m_localFilename).remove()) {
Chris@211:         std::cerr << "RemoteFile::deleteLocalFile: ERROR: Failed to delete file \"" << m_localFilename.toStdString() << "\"" << std::endl;
Chris@211:     } else {
Chris@211:         m_localFilename = "";
Chris@211:     }
Chris@211: 
Chris@211:     m_fileCreationMutex.unlock();
Chris@211: 
Chris@208:     m_done = true;
Chris@208: }
Chris@208: 
Chris@210: void
Chris@210: RemoteFile::showProgressDialog()
Chris@210: {
Chris@210:     if (m_progressDialog) m_progressDialog->show();
Chris@210: }
Chris@210: 
Chris@208: QString
Chris@208: RemoteFile::createLocalFile(QUrl url)
Chris@208: {
Chris@208:     QDir dir;
Chris@208:     try {
Chris@208:         dir = TempDirectory::getInstance()->getSubDirectoryPath("download");
Chris@208:     } catch (DirectoryCreationFailed f) {
Chris@208:         std::cerr << "RemoteFile::createLocalFile: ERROR: Failed to create temporary directory: " << f.what() << std::endl;
Chris@208:         return "";
Chris@208:     }
Chris@208: 
Chris@208:     QString filepart = url.path().section('/', -1, -1,
Chris@208:                                           QString::SectionSkipEmpty);
Chris@208: 
Chris@208:     QString extension = filepart.section('.', -1);
Chris@208:     QString base = filepart;
Chris@208:     if (extension != "") {
Chris@208:         base = base.left(base.length() - extension.length() - 1);
Chris@208:     }
Chris@208:     if (base == "") base = "remote";
Chris@208: 
Chris@208:     QString filename;
Chris@208: 
Chris@208:     if (extension == "") {
Chris@208:         filename = base;
Chris@208:     } else {
Chris@208:         filename = QString("%1.%2").arg(base).arg(extension);
Chris@208:     }
Chris@208: 
Chris@208:     QString filepath(dir.filePath(filename));
Chris@208: 
Chris@208:     std::cerr << "RemoteFile::createLocalFile: URL is \"" << url.toString().toStdString() << "\", dir is \"" << dir.path().toStdString() << "\", base \"" << base.toStdString() << "\", extension \"" << extension.toStdString() << "\", filebase \"" << filename.toStdString() << "\", filename \"" << filepath.toStdString() << "\"" << std::endl;
Chris@208: 
Chris@208:     m_fileCreationMutex.lock();
Chris@208:     ++m_count;
Chris@208: 
Chris@208:     if (QFileInfo(filepath).exists() ||
Chris@208:         !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208: 
Chris@208:         std::cerr << "RemoteFile::createLocalFile: Failed to create local file \""
Chris@208:                   << filepath.toStdString() << "\" for URL \""
Chris@208:                   << url.toString().toStdString() << "\" (or file already exists): appending suffix instead" << std::endl;
Chris@208: 
Chris@208: 
Chris@208:         if (extension == "") {
Chris@208:             filename = QString("%1_%2").arg(base).arg(m_count);
Chris@208:         } else {
Chris@208:             filename = QString("%1_%2.%3").arg(base).arg(m_count).arg(extension);
Chris@208:         }
Chris@208:         filepath = dir.filePath(filename);
Chris@208: 
Chris@208:         if (QFileInfo(filepath).exists() ||
Chris@208:             !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208: 
Chris@208:             std::cerr << "RemoteFile::createLocalFile: ERROR: Failed to create local file \""
Chris@208:                       << filepath.toStdString() << "\" for URL \""
Chris@208:                       << url.toString().toStdString() << "\" (or file already exists)" << std::endl;
Chris@208: 
Chris@208:             m_fileCreationMutex.unlock();
Chris@208:             return "";
Chris@208:         }
Chris@208:     }
Chris@208: 
Chris@208:     m_fileCreationMutex.unlock();
Chris@208: 
Chris@208:     std::cerr << "RemoteFile::createLocalFile: url "
Chris@208:               << url.toString().toStdString() << " -> local filename "
Chris@208:               << filepath.toStdString() << std::endl;
Chris@208: 
Chris@208:     return filepath;
Chris@208: }