annotate data/fileio/FileSource.cpp @ 468:70b333085952

* complete switching the code to use CachedFile -- now to tidy & fix the remaining flaws...
author Chris Cannam
date Mon, 27 Oct 2008 20:29:55 +0000
parents 2019d89ebcf9
children a8a7b8f698c8
rev   line source
Chris@208 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@208 2
Chris@208 3 /*
Chris@208 4 Sonic Visualiser
Chris@208 5 An audio file viewer and annotation editor.
Chris@208 6 Centre for Digital Music, Queen Mary, University of London.
Chris@208 7 This file copyright 2007 QMUL.
Chris@208 8
Chris@208 9 This program is free software; you can redistribute it and/or
Chris@208 10 modify it under the terms of the GNU General Public License as
Chris@208 11 published by the Free Software Foundation; either version 2 of the
Chris@208 12 License, or (at your option) any later version. See the file
Chris@208 13 COPYING included with this distribution for more information.
Chris@208 14 */
Chris@208 15
Chris@317 16 #include "FileSource.h"
Chris@357 17
Chris@208 18 #include "base/TempDirectory.h"
Chris@208 19 #include "base/Exceptions.h"
Chris@392 20 #include "base/ProgressReporter.h"
Chris@208 21
Chris@208 22 #include <QHttp>
Chris@208 23 #include <QFtp>
Chris@208 24 #include <QFileInfo>
Chris@208 25 #include <QDir>
Chris@392 26 #include <QCoreApplication>
Chris@460 27 #include <QCryptographicHash>
Chris@210 28 #include <QHttpResponseHeader>
Chris@208 29
Chris@208 30 #include <iostream>
Chris@405 31 #include <cstdlib>
Chris@208 32
Chris@460 33 #define DEBUG_FILE_SOURCE 1
Chris@327 34
Chris@208 35 int
Chris@317 36 FileSource::m_count = 0;
Chris@208 37
Chris@208 38 QMutex
Chris@317 39 FileSource::m_fileCreationMutex;
Chris@208 40
Chris@317 41 FileSource::RemoteRefCountMap
Chris@317 42 FileSource::m_refCountMap;
Chris@304 43
Chris@317 44 FileSource::RemoteLocalMap
Chris@317 45 FileSource::m_remoteLocalMap;
Chris@304 46
Chris@304 47 QMutex
Chris@317 48 FileSource::m_mapMutex;
Chris@304 49
Chris@460 50 FileSource::FileSource(QString fileOrUrl, ProgressReporter *reporter,
Chris@460 51 LocalCacheMode cacheMode) :
Chris@316 52 m_url(fileOrUrl),
Chris@460 53 m_cacheMode(cacheMode),
Chris@316 54 m_ftp(0),
Chris@316 55 m_http(0),
Chris@316 56 m_localFile(0),
Chris@316 57 m_ok(false),
Chris@316 58 m_lastStatus(0),
Chris@316 59 m_remote(isRemote(fileOrUrl)),
Chris@316 60 m_done(false),
Chris@316 61 m_leaveLocalFile(false),
Chris@392 62 m_reporter(reporter),
Chris@316 63 m_refCounted(false)
Chris@316 64 {
Chris@468 65
Chris@468 66 if (cacheMode == PersistentCache) {
Chris@468 67 std::cerr << "FileSource::FileSource: Persistent cache mode used for \"" << fileOrUrl.toStdString() << "\"" << std::endl;
Chris@468 68 exit(1);
Chris@468 69 }
Chris@468 70
Chris@327 71 #ifdef DEBUG_FILE_SOURCE
Chris@460 72 std::cerr << "FileSource::FileSource(" << fileOrUrl.toStdString() << ", " << cacheMode << ")" << std::endl;
Chris@327 73 #endif
Chris@316 74
Chris@316 75 if (!canHandleScheme(m_url)) {
Chris@317 76 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString().toStdString() << "\"" << std::endl;
Chris@316 77 m_errorString = tr("Unsupported scheme in URL");
Chris@316 78 return;
Chris@316 79 }
Chris@316 80
Chris@357 81 init();
Chris@316 82
Chris@316 83 if (isRemote() &&
Chris@316 84 (fileOrUrl.contains('%') ||
Chris@316 85 fileOrUrl.contains("--"))) { // for IDNA
Chris@316 86
Chris@316 87 waitForStatus();
Chris@316 88
Chris@316 89 if (!isAvailable()) {
Chris@336 90
Chris@316 91 // The URL was created on the assumption that the string
Chris@316 92 // was human-readable. Let's try again, this time
Chris@316 93 // assuming it was already encoded.
Chris@317 94 std::cerr << "FileSource::FileSource: Failed to retrieve URL \""
Chris@316 95 << fileOrUrl.toStdString()
Chris@316 96 << "\" as human-readable URL; "
Chris@316 97 << "trying again treating it as encoded URL"
Chris@316 98 << std::endl;
Chris@336 99
Chris@336 100 // even though our cache file doesn't exist (because the
Chris@336 101 // resource was 404), we still need to ensure we're no
Chris@336 102 // longer associating a filename with this url in the
Chris@336 103 // refcount map -- or createCacheFile will think we've
Chris@336 104 // already done all the work and no request will be sent
Chris@336 105 deleteCacheFile();
Chris@336 106
Chris@316 107 m_url.setEncodedUrl(fileOrUrl.toAscii());
Chris@336 108
Chris@336 109 m_ok = false;
Chris@336 110 m_done = false;
Chris@336 111 m_lastStatus = 0;
Chris@357 112 init();
Chris@316 113 }
Chris@316 114 }
Chris@325 115
Chris@325 116 if (!isRemote()) {
Chris@325 117 emit statusAvailable();
Chris@325 118 emit ready();
Chris@325 119 }
Chris@316 120 }
Chris@316 121
Chris@460 122 FileSource::FileSource(QUrl url, ProgressReporter *reporter,
Chris@460 123 LocalCacheMode cacheMode) :
Chris@304 124 m_url(url),
Chris@460 125 m_cacheMode(cacheMode),
Chris@208 126 m_ftp(0),
Chris@208 127 m_http(0),
Chris@208 128 m_localFile(0),
Chris@208 129 m_ok(false),
Chris@210 130 m_lastStatus(0),
Chris@316 131 m_remote(isRemote(url.toString())),
Chris@208 132 m_done(false),
Chris@316 133 m_leaveLocalFile(false),
Chris@392 134 m_reporter(reporter),
Chris@316 135 m_refCounted(false)
Chris@208 136 {
Chris@468 137
Chris@468 138 if (cacheMode == PersistentCache) {
Chris@468 139 std::cerr << "FileSource::FileSource: Persistent cache mode used for \"" << url.toString().toStdString() << "\"" << std::endl;
Chris@468 140 exit(1);
Chris@468 141 }
Chris@468 142
Chris@327 143 #ifdef DEBUG_FILE_SOURCE
Chris@317 144 std::cerr << "FileSource::FileSource(" << url.toString().toStdString() << ") [as url]" << std::endl;
Chris@327 145 #endif
Chris@316 146
Chris@316 147 if (!canHandleScheme(m_url)) {
Chris@317 148 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString().toStdString() << "\"" << std::endl;
Chris@316 149 m_errorString = tr("Unsupported scheme in URL");
Chris@208 150 return;
Chris@208 151 }
Chris@208 152
Chris@357 153 init();
Chris@316 154 }
Chris@304 155
Chris@317 156 FileSource::FileSource(const FileSource &rf) :
Chris@316 157 QObject(),
Chris@316 158 m_url(rf.m_url),
Chris@460 159 m_cacheMode(rf.m_cacheMode),
Chris@316 160 m_ftp(0),
Chris@316 161 m_http(0),
Chris@316 162 m_localFile(0),
Chris@316 163 m_ok(rf.m_ok),
Chris@316 164 m_lastStatus(rf.m_lastStatus),
Chris@316 165 m_remote(rf.m_remote),
Chris@316 166 m_done(false),
Chris@316 167 m_leaveLocalFile(false),
Chris@392 168 m_reporter(rf.m_reporter),
Chris@316 169 m_refCounted(false)
Chris@316 170 {
Chris@327 171 #ifdef DEBUG_FILE_SOURCE
Chris@317 172 std::cerr << "FileSource::FileSource(" << m_url.toString().toStdString() << ") [copy ctor]" << std::endl;
Chris@327 173 #endif
Chris@304 174
Chris@316 175 if (!canHandleScheme(m_url)) {
Chris@317 176 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString().toStdString() << "\"" << std::endl;
Chris@316 177 m_errorString = tr("Unsupported scheme in URL");
Chris@304 178 return;
Chris@304 179 }
Chris@304 180
Chris@460 181 if (m_cacheMode == PersistentCache) {
Chris@460 182
Chris@316 183 m_localFilename = rf.m_localFilename;
Chris@460 184
Chris@460 185 } else if (!isRemote()) {
Chris@460 186
Chris@460 187 m_localFilename = rf.m_localFilename;
Chris@460 188
Chris@316 189 } else {
Chris@316 190 QMutexLocker locker(&m_mapMutex);
Chris@327 191 #ifdef DEBUG_FILE_SOURCE
Chris@317 192 std::cerr << "FileSource::FileSource(copy ctor): ref count is "
Chris@316 193 << m_refCountMap[m_url] << std::endl;
Chris@327 194 #endif
Chris@316 195 if (m_refCountMap[m_url] > 0) {
Chris@316 196 m_refCountMap[m_url]++;
Chris@327 197 #ifdef DEBUG_FILE_SOURCE
Chris@316 198 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 199 #endif
Chris@316 200 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 201 m_refCounted = true;
Chris@316 202 } else {
Chris@316 203 m_ok = false;
Chris@316 204 m_lastStatus = 404;
Chris@316 205 }
Chris@316 206 }
Chris@316 207
Chris@316 208 m_done = true;
Chris@316 209 }
Chris@316 210
Chris@317 211 FileSource::~FileSource()
Chris@316 212 {
Chris@327 213 #ifdef DEBUG_FILE_SOURCE
Chris@317 214 std::cerr << "FileSource(" << m_url.toString().toStdString() << ")::~FileSource" << std::endl;
Chris@327 215 #endif
Chris@316 216
Chris@316 217 cleanup();
Chris@316 218
Chris@460 219 if (isRemote() && (m_cacheMode == TemporaryCache) && !m_leaveLocalFile) {
Chris@460 220 deleteCacheFile();
Chris@460 221 }
Chris@316 222 }
Chris@316 223
Chris@316 224 void
Chris@357 225 FileSource::init()
Chris@316 226 {
Chris@316 227 if (!isRemote()) {
Chris@355 228 #ifdef DEBUG_FILE_SOURCE
Chris@355 229 std::cerr << "FileSource::init: Not a remote URL" << std::endl;
Chris@355 230 #endif
Chris@355 231 bool literal = false;
Chris@316 232 m_localFilename = m_url.toLocalFile();
Chris@342 233 if (m_localFilename == "") {
Chris@342 234 // QUrl may have mishandled the scheme (e.g. in a DOS path)
Chris@342 235 m_localFilename = m_url.toString();
Chris@355 236 literal = true;
Chris@342 237 }
Chris@439 238 m_localFilename = QFileInfo(m_localFilename).absoluteFilePath();
Chris@439 239
Chris@355 240 #ifdef DEBUG_FILE_SOURCE
Chris@355 241 std::cerr << "FileSource::init: URL translates to local filename \""
Chris@355 242 << m_localFilename.toStdString() << "\"" << std::endl;
Chris@355 243 #endif
Chris@316 244 m_ok = true;
Chris@355 245 m_lastStatus = 200;
Chris@355 246
Chris@316 247 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 248 if (literal) {
Chris@355 249 m_lastStatus = 404;
Chris@355 250 } else {
Chris@355 251 // Again, QUrl may have been mistreating us --
Chris@355 252 // e.g. dropping a part that looks like query data
Chris@355 253 m_localFilename = m_url.toString();
Chris@355 254 literal = true;
Chris@355 255 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 256 m_lastStatus = 404;
Chris@355 257 }
Chris@355 258 }
Chris@316 259 }
Chris@355 260
Chris@316 261 m_done = true;
Chris@316 262 return;
Chris@316 263 }
Chris@316 264
Chris@316 265 if (createCacheFile()) {
Chris@327 266 #ifdef DEBUG_FILE_SOURCE
Chris@460 267 std::cerr << "FileSource::init: Already have this one at "
Chris@460 268 << m_localFilename.toStdString() << std::endl;
Chris@327 269 #endif
Chris@316 270 m_ok = true;
Chris@316 271 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 272 m_lastStatus = 404;
Chris@316 273 } else {
Chris@316 274 m_lastStatus = 200;
Chris@316 275 }
Chris@316 276 m_done = true;
Chris@316 277 return;
Chris@316 278 }
Chris@316 279
Chris@208 280 if (m_localFilename == "") return;
Chris@208 281 m_localFile = new QFile(m_localFilename);
Chris@208 282 m_localFile->open(QFile::WriteOnly);
Chris@208 283
Chris@316 284 QString scheme = m_url.scheme().toLower();
Chris@316 285
Chris@327 286 #ifdef DEBUG_FILE_SOURCE
Chris@317 287 std::cerr << "FileSource::init: Don't have local copy of \""
Chris@460 288 << m_url.toString().toStdString() << "\", retrieving to "
Chris@460 289 << m_localFilename.toStdString() << std::endl;
Chris@327 290 #endif
Chris@208 291
Chris@208 292 if (scheme == "http") {
Chris@316 293 initHttp();
Chris@357 294 std::cerr << "FileSource: initHttp succeeded" << std::endl;
Chris@208 295 } else if (scheme == "ftp") {
Chris@316 296 initFtp();
Chris@316 297 } else {
Chris@316 298 m_remote = false;
Chris@316 299 m_ok = false;
Chris@208 300 }
Chris@208 301
Chris@208 302 if (m_ok) {
Chris@316 303
Chris@316 304 QMutexLocker locker(&m_mapMutex);
Chris@316 305
Chris@316 306 if (m_refCountMap[m_url] > 0) {
Chris@316 307 // someone else has been doing the same thing at the same time,
Chris@316 308 // but has got there first
Chris@316 309 cleanup();
Chris@316 310 m_refCountMap[m_url]++;
Chris@327 311 #ifdef DEBUG_FILE_SOURCE
Chris@317 312 std::cerr << "FileSource::init: Another FileSource has got there first, abandoning our download and using theirs" << std::endl;
Chris@327 313 #endif
Chris@316 314 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 315 m_refCounted = true;
Chris@316 316 m_ok = true;
Chris@316 317 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 318 m_lastStatus = 404;
Chris@316 319 }
Chris@316 320 m_done = true;
Chris@316 321 return;
Chris@316 322 }
Chris@304 323
Chris@304 324 m_remoteLocalMap[m_url] = m_localFilename;
Chris@304 325 m_refCountMap[m_url]++;
Chris@316 326 m_refCounted = true;
Chris@304 327
Chris@392 328 if (m_reporter) {
Chris@392 329 m_reporter->setMessage
Chris@392 330 (tr("Downloading %1...").arg(m_url.toString()));
Chris@392 331 connect(m_reporter, SIGNAL(cancelled()), this, SLOT(cancelled()));
Chris@357 332 connect(this, SIGNAL(progress(int)),
Chris@392 333 m_reporter, SLOT(setProgress(int)));
Chris@316 334 }
Chris@208 335 }
Chris@208 336 }
Chris@208 337
Chris@316 338 void
Chris@317 339 FileSource::initHttp()
Chris@208 340 {
Chris@316 341 m_ok = true;
Chris@316 342 m_http = new QHttp(m_url.host(), m_url.port(80));
Chris@316 343 connect(m_http, SIGNAL(done(bool)), this, SLOT(done(bool)));
Chris@316 344 connect(m_http, SIGNAL(dataReadProgress(int, int)),
Chris@316 345 this, SLOT(dataReadProgress(int, int)));
Chris@316 346 connect(m_http, SIGNAL(responseHeaderReceived(const QHttpResponseHeader &)),
Chris@316 347 this, SLOT(httpResponseHeaderReceived(const QHttpResponseHeader &)));
Chris@316 348
Chris@316 349 // I don't quite understand this. url.path() returns a path
Chris@316 350 // without percent encoding; for example, spaces appear as
Chris@316 351 // literal spaces. This generally won't work if sent to the
Chris@316 352 // server directly. You can retrieve a correctly encoded URL
Chris@316 353 // from QUrl using url.toEncoded(), but that gives you the
Chris@316 354 // whole URL; there doesn't seem to be any way to retrieve
Chris@316 355 // only an encoded path. Furthermore there doesn't seem to be
Chris@316 356 // any way to convert a retrieved path into an encoded path
Chris@316 357 // without explicitly specifying that you don't want the path
Chris@316 358 // separators ("/") to be encoded. (Besides being painful to
Chris@316 359 // manage, I don't see how this can work correctly in any case
Chris@316 360 // where a percent-encoded "/" is supposed to appear within a
Chris@316 361 // path element?) There also seems to be no way to retrieve
Chris@316 362 // the path plus query string, i.e. everything that I need to
Chris@316 363 // send to the HTTP server. And no way for QHttp to take a
Chris@316 364 // QUrl argument. I'm obviously missing something.
Chris@316 365
Chris@316 366 // So, two ways to do this: query the bits from the URL,
Chris@316 367 // encode them individually, and glue them back together
Chris@316 368 // again...
Chris@316 369 /*
Chris@316 370 QString path = QUrl::toPercentEncoding(m_url.path(), "/");
Chris@316 371 QList<QPair<QString, QString> > query = m_url.queryItems();
Chris@316 372 if (!query.empty()) {
Chris@316 373 QStringList q2;
Chris@316 374 for (QList<QPair<QString, QString> >::iterator i = query.begin();
Chris@316 375 i != query.end(); ++i) {
Chris@316 376 q2.push_back(QString("%1=%3")
Chris@316 377 .arg(QString(QUrl::toPercentEncoding(i->first)))
Chris@316 378 .arg(QString(QUrl::toPercentEncoding(i->second))));
Chris@316 379 }
Chris@316 380 path = QString("%1%2%3")
Chris@316 381 .arg(path).arg("?")
Chris@316 382 .arg(q2.join("&"));
Chris@316 383 }
Chris@316 384 */
Chris@316 385
Chris@316 386 // ...or, much simpler but relying on knowledge about the
Chris@316 387 // scheme://host/path/path/query etc format of the URL, we can
Chris@316 388 // get the whole URL ready-encoded and then split it on "/" as
Chris@316 389 // appropriate...
Chris@316 390
Chris@316 391 QString path = "/" + QString(m_url.toEncoded()).section('/', 3);
Chris@316 392
Chris@327 393 #ifdef DEBUG_FILE_SOURCE
Chris@317 394 std::cerr << "FileSource: path is \""
Chris@316 395 << path.toStdString() << "\"" << std::endl;
Chris@327 396 #endif
Chris@316 397
Chris@316 398 m_http->get(path, m_localFile);
Chris@316 399 }
Chris@316 400
Chris@316 401 void
Chris@317 402 FileSource::initFtp()
Chris@316 403 {
Chris@316 404 m_ok = true;
Chris@316 405 m_ftp = new QFtp;
Chris@316 406 connect(m_ftp, SIGNAL(done(bool)), this, SLOT(done(bool)));
Chris@316 407 connect(m_ftp, SIGNAL(commandFinished(int, bool)),
Chris@316 408 this, SLOT(ftpCommandFinished(int, bool)));
Chris@316 409 connect(m_ftp, SIGNAL(dataTransferProgress(qint64, qint64)),
Chris@316 410 this, SLOT(dataTransferProgress(qint64, qint64)));
Chris@316 411 m_ftp->connectToHost(m_url.host(), m_url.port(21));
Chris@316 412
Chris@316 413 QString username = m_url.userName();
Chris@316 414 if (username == "") {
Chris@316 415 username = "anonymous";
Chris@316 416 }
Chris@316 417
Chris@316 418 QString password = m_url.password();
Chris@316 419 if (password == "") {
Chris@316 420 password = QString("%1@%2").arg(getenv("USER")).arg(getenv("HOST"));
Chris@316 421 }
Chris@316 422
Chris@316 423 m_ftp->login(username, password);
Chris@316 424
Chris@316 425 QString dirpath = m_url.path().section('/', 0, -2);
Chris@316 426 QString filename = m_url.path().section('/', -1);
Chris@316 427
Chris@316 428 if (dirpath == "") dirpath = "/";
Chris@316 429 m_ftp->cd(dirpath);
Chris@316 430 m_ftp->get(filename, m_localFile);
Chris@211 431 }
Chris@211 432
Chris@211 433 void
Chris@317 434 FileSource::cleanup()
Chris@211 435 {
Chris@461 436 if (m_done) {
Chris@461 437 delete m_localFile; // does not actually delete the file
Chris@461 438 m_localFile = 0;
Chris@461 439 }
Chris@211 440 m_done = true;
Chris@214 441 if (m_http) {
Chris@287 442 QHttp *h = m_http;
Chris@214 443 m_http = 0;
Chris@287 444 h->abort();
Chris@287 445 h->deleteLater();
Chris@214 446 }
Chris@214 447 if (m_ftp) {
Chris@287 448 QFtp *f = m_ftp;
Chris@214 449 m_ftp = 0;
Chris@287 450 f->abort();
Chris@287 451 f->deleteLater();
Chris@214 452 }
Chris@461 453 if (m_localFile) {
Chris@461 454 delete m_localFile; // does not actually delete the file
Chris@461 455 m_localFile = 0;
Chris@461 456 }
Chris@208 457 }
Chris@208 458
Chris@208 459 bool
Chris@317 460 FileSource::isRemote(QString fileOrUrl)
Chris@304 461 {
Chris@342 462 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@316 463 QString scheme = QUrl(fileOrUrl).scheme().toLower();
Chris@342 464 if (scheme == "" || scheme == "file" || scheme.length() == 1) return false;
Chris@342 465 return true;
Chris@304 466 }
Chris@304 467
Chris@304 468 bool
Chris@317 469 FileSource::canHandleScheme(QUrl url)
Chris@208 470 {
Chris@342 471 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@208 472 QString scheme = url.scheme().toLower();
Chris@316 473 return (scheme == "http" || scheme == "ftp" ||
Chris@342 474 scheme == "file" || scheme == "" || scheme.length() == 1);
Chris@208 475 }
Chris@208 476
Chris@210 477 bool
Chris@317 478 FileSource::isAvailable()
Chris@210 479 {
Chris@316 480 waitForStatus();
Chris@211 481 bool available = true;
Chris@211 482 if (!m_ok) available = false;
Chris@211 483 else available = (m_lastStatus / 100 == 2);
Chris@327 484 #ifdef DEBUG_FILE_SOURCE
Chris@317 485 std::cerr << "FileSource::isAvailable: " << (available ? "yes" : "no")
Chris@211 486 << std::endl;
Chris@327 487 #endif
Chris@211 488 return available;
Chris@210 489 }
Chris@210 490
Chris@208 491 void
Chris@317 492 FileSource::waitForStatus()
Chris@316 493 {
Chris@316 494 while (m_ok && (!m_done && m_lastStatus == 0)) {
Chris@316 495 // std::cerr << "waitForStatus: processing (last status " << m_lastStatus << ")" << std::endl;
Chris@392 496 QCoreApplication::processEvents();
Chris@316 497 }
Chris@316 498 }
Chris@316 499
Chris@316 500 void
Chris@317 501 FileSource::waitForData()
Chris@208 502 {
Chris@211 503 while (m_ok && !m_done) {
Chris@357 504 // std::cerr << "FileSource::waitForData: calling QApplication::processEvents" << std::endl;
Chris@392 505 QCoreApplication::processEvents();
Chris@208 506 }
Chris@208 507 }
Chris@208 508
Chris@316 509 void
Chris@317 510 FileSource::setLeaveLocalFile(bool leave)
Chris@316 511 {
Chris@316 512 m_leaveLocalFile = leave;
Chris@316 513 }
Chris@316 514
Chris@208 515 bool
Chris@317 516 FileSource::isOK() const
Chris@208 517 {
Chris@208 518 return m_ok;
Chris@208 519 }
Chris@208 520
Chris@208 521 bool
Chris@317 522 FileSource::isDone() const
Chris@208 523 {
Chris@208 524 return m_done;
Chris@208 525 }
Chris@208 526
Chris@316 527 bool
Chris@317 528 FileSource::isRemote() const
Chris@316 529 {
Chris@316 530 return m_remote;
Chris@316 531 }
Chris@316 532
Chris@316 533 QString
Chris@317 534 FileSource::getLocation() const
Chris@316 535 {
Chris@316 536 return m_url.toString();
Chris@316 537 }
Chris@316 538
Chris@208 539 QString
Chris@317 540 FileSource::getLocalFilename() const
Chris@208 541 {
Chris@208 542 return m_localFilename;
Chris@208 543 }
Chris@208 544
Chris@208 545 QString
Chris@317 546 FileSource::getContentType() const
Chris@316 547 {
Chris@316 548 return m_contentType;
Chris@316 549 }
Chris@316 550
Chris@316 551 QString
Chris@317 552 FileSource::getExtension() const
Chris@316 553 {
Chris@316 554 if (m_localFilename != "") {
Chris@316 555 return QFileInfo(m_localFilename).suffix().toLower();
Chris@316 556 } else {
Chris@316 557 return QFileInfo(m_url.toLocalFile()).suffix().toLower();
Chris@316 558 }
Chris@316 559 }
Chris@316 560
Chris@316 561 QString
Chris@317 562 FileSource::getErrorString() const
Chris@208 563 {
Chris@208 564 return m_errorString;
Chris@208 565 }
Chris@208 566
Chris@208 567 void
Chris@317 568 FileSource::dataReadProgress(int done, int total)
Chris@208 569 {
Chris@208 570 dataTransferProgress(done, total);
Chris@208 571 }
Chris@208 572
Chris@208 573 void
Chris@317 574 FileSource::httpResponseHeaderReceived(const QHttpResponseHeader &resp)
Chris@210 575 {
Chris@210 576 m_lastStatus = resp.statusCode();
Chris@210 577 if (m_lastStatus / 100 >= 4) {
Chris@210 578 m_errorString = QString("%1 %2")
Chris@210 579 .arg(resp.statusCode()).arg(resp.reasonPhrase());
Chris@327 580 #ifdef DEBUG_FILE_SOURCE
Chris@317 581 std::cerr << "FileSource::responseHeaderReceived: "
Chris@211 582 << m_errorString.toStdString() << std::endl;
Chris@327 583 #endif
Chris@211 584 } else {
Chris@327 585 #ifdef DEBUG_FILE_SOURCE
Chris@317 586 std::cerr << "FileSource::responseHeaderReceived: "
Chris@211 587 << m_lastStatus << std::endl;
Chris@327 588 #endif
Chris@315 589 if (resp.hasContentType()) m_contentType = resp.contentType();
Chris@325 590 }
Chris@325 591 emit statusAvailable();
Chris@210 592 }
Chris@210 593
Chris@210 594 void
Chris@317 595 FileSource::ftpCommandFinished(int id, bool error)
Chris@214 596 {
Chris@327 597 #ifdef DEBUG_FILE_SOURCE
Chris@317 598 std::cerr << "FileSource::ftpCommandFinished(" << id << ", " << error << ")" << std::endl;
Chris@327 599 #endif
Chris@214 600
Chris@214 601 if (!m_ftp) return;
Chris@214 602
Chris@214 603 QFtp::Command command = m_ftp->currentCommand();
Chris@214 604
Chris@214 605 if (!error) {
Chris@327 606 #ifdef DEBUG_FILE_SOURCE
Chris@317 607 std::cerr << "FileSource::ftpCommandFinished: success for command "
Chris@214 608 << command << std::endl;
Chris@327 609 #endif
Chris@214 610 return;
Chris@214 611 }
Chris@214 612
Chris@214 613 if (command == QFtp::ConnectToHost) {
Chris@214 614 m_errorString = tr("Failed to connect to FTP server");
Chris@214 615 } else if (command == QFtp::Login) {
Chris@214 616 m_errorString = tr("Login failed");
Chris@214 617 } else if (command == QFtp::Cd) {
Chris@214 618 m_errorString = tr("Failed to change to correct directory");
Chris@214 619 } else if (command == QFtp::Get) {
Chris@214 620 m_errorString = tr("FTP download aborted");
Chris@214 621 }
Chris@214 622
Chris@214 623 m_lastStatus = 400; // for done()
Chris@214 624 }
Chris@214 625
Chris@214 626 void
Chris@317 627 FileSource::dataTransferProgress(qint64 done, qint64 total)
Chris@208 628 {
Chris@208 629 int percent = int((double(done) / double(total)) * 100.0 - 0.1);
Chris@208 630 emit progress(percent);
Chris@210 631 }
Chris@210 632
Chris@210 633 void
Chris@317 634 FileSource::cancelled()
Chris@210 635 {
Chris@210 636 m_done = true;
Chris@316 637 cleanup();
Chris@316 638
Chris@210 639 m_ok = false;
Chris@210 640 m_errorString = tr("Download cancelled");
Chris@208 641 }
Chris@208 642
Chris@208 643 void
Chris@317 644 FileSource::done(bool error)
Chris@208 645 {
Chris@327 646 emit progress(100);
Chris@327 647
Chris@327 648 #ifdef DEBUG_FILE_SOURCE
Chris@317 649 std::cerr << "FileSource::done(" << error << ")" << std::endl;
Chris@327 650 #endif
Chris@211 651
Chris@211 652 if (m_done) return;
Chris@211 653
Chris@208 654 if (error) {
Chris@208 655 if (m_http) {
Chris@208 656 m_errorString = m_http->errorString();
Chris@208 657 } else if (m_ftp) {
Chris@208 658 m_errorString = m_ftp->errorString();
Chris@208 659 }
Chris@208 660 }
Chris@208 661
Chris@210 662 if (m_lastStatus / 100 >= 4) {
Chris@211 663 error = true;
Chris@210 664 }
Chris@210 665
Chris@211 666 cleanup();
Chris@208 667
Chris@211 668 if (!error) {
Chris@208 669 QFileInfo fi(m_localFilename);
Chris@208 670 if (!fi.exists()) {
Chris@208 671 m_errorString = tr("Failed to create local file %1").arg(m_localFilename);
Chris@211 672 error = true;
Chris@208 673 } else if (fi.size() == 0) {
Chris@208 674 m_errorString = tr("File contains no data!");
Chris@211 675 error = true;
Chris@208 676 }
Chris@208 677 }
Chris@211 678
Chris@211 679 if (error) {
Chris@327 680 #ifdef DEBUG_FILE_SOURCE
Chris@461 681 std::cerr << "FileSource::done: error is " << error << " (\""
Chris@461 682 << m_errorString.toStdString() << "\"), deleting cache file" << std::endl;
Chris@327 683 #endif
Chris@316 684 deleteCacheFile();
Chris@211 685 }
Chris@211 686
Chris@211 687 m_ok = !error;
Chris@211 688 m_done = true;
Chris@304 689 emit ready();
Chris@211 690 }
Chris@211 691
Chris@211 692 void
Chris@317 693 FileSource::deleteCacheFile()
Chris@211 694 {
Chris@327 695 #ifdef DEBUG_FILE_SOURCE
Chris@317 696 std::cerr << "FileSource::deleteCacheFile(\"" << m_localFilename.toStdString() << "\")" << std::endl;
Chris@327 697 #endif
Chris@211 698
Chris@211 699 cleanup();
Chris@211 700
Chris@316 701 if (m_localFilename == "") {
Chris@316 702 return;
Chris@316 703 }
Chris@211 704
Chris@316 705 if (!isRemote()) {
Chris@327 706 #ifdef DEBUG_FILE_SOURCE
Chris@316 707 std::cerr << "not a cache file" << std::endl;
Chris@327 708 #endif
Chris@316 709 return;
Chris@316 710 }
Chris@316 711
Chris@316 712 if (m_refCounted) {
Chris@304 713
Chris@304 714 QMutexLocker locker(&m_mapMutex);
Chris@316 715 m_refCounted = false;
Chris@304 716
Chris@304 717 if (m_refCountMap[m_url] > 0) {
Chris@304 718 m_refCountMap[m_url]--;
Chris@327 719 #ifdef DEBUG_FILE_SOURCE
Chris@316 720 std::cerr << "reduced ref count to " << m_refCountMap[m_url] << std::endl;
Chris@327 721 #endif
Chris@304 722 if (m_refCountMap[m_url] > 0) {
Chris@304 723 m_done = true;
Chris@304 724 return;
Chris@304 725 }
Chris@304 726 }
Chris@304 727 }
Chris@304 728
Chris@211 729 m_fileCreationMutex.lock();
Chris@211 730
Chris@460 731 // We always delete the file here, even in PersistentCache mode,
Chris@460 732 // because this function is also used if retrieval failed (in
Chris@460 733 // which case we want to delete the cache so that subsequent users
Chris@460 734 // won't trust it). It's up to the calling code to determine
Chris@460 735 // whether we actually want to delete the cache or not, e.g. on
Chris@460 736 // destruction.
Chris@460 737
Chris@211 738 if (!QFile(m_localFilename).remove()) {
Chris@317 739 std::cerr << "FileSource::deleteCacheFile: ERROR: Failed to delete file \"" << m_localFilename.toStdString() << "\"" << std::endl;
Chris@211 740 } else {
Chris@327 741 #ifdef DEBUG_FILE_SOURCE
Chris@317 742 std::cerr << "FileSource::deleteCacheFile: Deleted cache file \"" << m_localFilename.toStdString() << "\"" << std::endl;
Chris@327 743 #endif
Chris@211 744 m_localFilename = "";
Chris@211 745 }
Chris@211 746
Chris@211 747 m_fileCreationMutex.unlock();
Chris@211 748
Chris@208 749 m_done = true;
Chris@208 750 }
Chris@208 751
Chris@460 752 QString
Chris@460 753 FileSource::getPersistentCacheDirectory()
Chris@460 754 {
Chris@460 755 QDir dir = TempDirectory::getInstance()->getContainingPath();
Chris@460 756
Chris@460 757 QString cacheDirName("cache");
Chris@460 758
Chris@460 759 QFileInfo fi(dir.filePath(cacheDirName));
Chris@460 760
Chris@460 761 if ((fi.exists() && !fi.isDir()) ||
Chris@460 762 (!fi.exists() && !dir.mkdir(cacheDirName))) {
Chris@460 763
Chris@460 764 throw DirectoryCreationFailed(fi.filePath());
Chris@460 765 }
Chris@460 766
Chris@460 767 return fi.filePath();
Chris@460 768 }
Chris@460 769
Chris@460 770 QString
Chris@460 771 FileSource::getPersistentCacheFilePath(QUrl url)
Chris@460 772 {
Chris@460 773 QDir dir(getPersistentCacheDirectory());
Chris@460 774
Chris@460 775 QString filename =
Chris@460 776 QString::fromLocal8Bit
Chris@460 777 (QCryptographicHash::hash(url.toString().toLocal8Bit(),
Chris@461 778 QCryptographicHash::Sha1).toHex());
Chris@460 779
Chris@460 780 return dir.filePath(filename);
Chris@460 781 }
Chris@460 782
Chris@316 783 bool
Chris@317 784 FileSource::createCacheFile()
Chris@208 785 {
Chris@460 786 if (m_cacheMode == PersistentCache) {
Chris@460 787 m_localFilename = getPersistentCacheFilePath(m_url);
Chris@460 788 if (QFileInfo(m_localFilename).exists()) return true;
Chris@460 789 else return false;
Chris@460 790 }
Chris@460 791
Chris@316 792 {
Chris@316 793 QMutexLocker locker(&m_mapMutex);
Chris@316 794
Chris@327 795 #ifdef DEBUG_FILE_SOURCE
Chris@317 796 std::cerr << "FileSource::createCacheFile: refcount is " << m_refCountMap[m_url] << std::endl;
Chris@327 797 #endif
Chris@316 798
Chris@316 799 if (m_refCountMap[m_url] > 0) {
Chris@316 800 m_refCountMap[m_url]++;
Chris@316 801 m_localFilename = m_remoteLocalMap[m_url];
Chris@327 802 #ifdef DEBUG_FILE_SOURCE
Chris@316 803 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 804 #endif
Chris@316 805 m_refCounted = true;
Chris@316 806 return true;
Chris@316 807 }
Chris@316 808 }
Chris@316 809
Chris@208 810 QDir dir;
Chris@208 811 try {
Chris@208 812 dir = TempDirectory::getInstance()->getSubDirectoryPath("download");
Chris@208 813 } catch (DirectoryCreationFailed f) {
Chris@327 814 #ifdef DEBUG_FILE_SOURCE
Chris@317 815 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create temporary directory: " << f.what() << std::endl;
Chris@327 816 #endif
Chris@208 817 return "";
Chris@208 818 }
Chris@208 819
Chris@316 820 QString filepart = m_url.path().section('/', -1, -1,
Chris@316 821 QString::SectionSkipEmpty);
Chris@208 822
Chris@457 823 QString extension = "";
Chris@457 824 if (filepart.contains('.')) extension = filepart.section('.', -1);
Chris@457 825
Chris@208 826 QString base = filepart;
Chris@208 827 if (extension != "") {
Chris@208 828 base = base.left(base.length() - extension.length() - 1);
Chris@208 829 }
Chris@208 830 if (base == "") base = "remote";
Chris@208 831
Chris@208 832 QString filename;
Chris@208 833
Chris@208 834 if (extension == "") {
Chris@208 835 filename = base;
Chris@208 836 } else {
Chris@208 837 filename = QString("%1.%2").arg(base).arg(extension);
Chris@208 838 }
Chris@208 839
Chris@208 840 QString filepath(dir.filePath(filename));
Chris@208 841
Chris@327 842 #ifdef DEBUG_FILE_SOURCE
Chris@317 843 std::cerr << "FileSource::createCacheFile: URL is \"" << m_url.toString().toStdString() << "\", dir is \"" << dir.path().toStdString() << "\", base \"" << base.toStdString() << "\", extension \"" << extension.toStdString() << "\", filebase \"" << filename.toStdString() << "\", filename \"" << filepath.toStdString() << "\"" << std::endl;
Chris@327 844 #endif
Chris@208 845
Chris@316 846 QMutexLocker fcLocker(&m_fileCreationMutex);
Chris@316 847
Chris@208 848 ++m_count;
Chris@208 849
Chris@208 850 if (QFileInfo(filepath).exists() ||
Chris@208 851 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 852
Chris@327 853 #ifdef DEBUG_FILE_SOURCE
Chris@317 854 std::cerr << "FileSource::createCacheFile: Failed to create local file \""
Chris@208 855 << filepath.toStdString() << "\" for URL \""
Chris@316 856 << m_url.toString().toStdString() << "\" (or file already exists): appending suffix instead" << std::endl;
Chris@327 857 #endif
Chris@208 858
Chris@208 859 if (extension == "") {
Chris@208 860 filename = QString("%1_%2").arg(base).arg(m_count);
Chris@208 861 } else {
Chris@208 862 filename = QString("%1_%2.%3").arg(base).arg(m_count).arg(extension);
Chris@208 863 }
Chris@208 864 filepath = dir.filePath(filename);
Chris@208 865
Chris@208 866 if (QFileInfo(filepath).exists() ||
Chris@208 867 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 868
Chris@327 869 #ifdef DEBUG_FILE_SOURCE
Chris@317 870 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create local file \""
Chris@208 871 << filepath.toStdString() << "\" for URL \""
Chris@316 872 << m_url.toString().toStdString() << "\" (or file already exists)" << std::endl;
Chris@327 873 #endif
Chris@208 874
Chris@208 875 return "";
Chris@208 876 }
Chris@208 877 }
Chris@208 878
Chris@327 879 #ifdef DEBUG_FILE_SOURCE
Chris@317 880 std::cerr << "FileSource::createCacheFile: url "
Chris@316 881 << m_url.toString().toStdString() << " -> local filename "
Chris@316 882 << filepath.toStdString() << std::endl;
Chris@327 883 #endif
Chris@316 884
Chris@316 885 m_localFilename = filepath;
Chris@208 886
Chris@316 887 return false;
Chris@208 888 }
Chris@327 889