annotate data/fileio/FileSource.cpp @ 461:2019d89ebcf9

* Some work on querying and cacheing plugin RDF from a central index
author Chris Cannam
date Fri, 17 Oct 2008 15:26:29 +0000
parents 93fb1ebff76b
children 70b333085952
rev   line source
Chris@208 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@208 2
Chris@208 3 /*
Chris@208 4 Sonic Visualiser
Chris@208 5 An audio file viewer and annotation editor.
Chris@208 6 Centre for Digital Music, Queen Mary, University of London.
Chris@208 7 This file copyright 2007 QMUL.
Chris@208 8
Chris@208 9 This program is free software; you can redistribute it and/or
Chris@208 10 modify it under the terms of the GNU General Public License as
Chris@208 11 published by the Free Software Foundation; either version 2 of the
Chris@208 12 License, or (at your option) any later version. See the file
Chris@208 13 COPYING included with this distribution for more information.
Chris@208 14 */
Chris@208 15
Chris@317 16 #include "FileSource.h"
Chris@357 17
Chris@208 18 #include "base/TempDirectory.h"
Chris@208 19 #include "base/Exceptions.h"
Chris@392 20 #include "base/ProgressReporter.h"
Chris@208 21
Chris@208 22 #include <QHttp>
Chris@208 23 #include <QFtp>
Chris@208 24 #include <QFileInfo>
Chris@208 25 #include <QDir>
Chris@392 26 #include <QCoreApplication>
Chris@460 27 #include <QCryptographicHash>
Chris@210 28 #include <QHttpResponseHeader>
Chris@208 29
Chris@208 30 #include <iostream>
Chris@405 31 #include <cstdlib>
Chris@208 32
Chris@460 33 #define DEBUG_FILE_SOURCE 1
Chris@327 34
Chris@208 35 int
Chris@317 36 FileSource::m_count = 0;
Chris@208 37
Chris@208 38 QMutex
Chris@317 39 FileSource::m_fileCreationMutex;
Chris@208 40
Chris@317 41 FileSource::RemoteRefCountMap
Chris@317 42 FileSource::m_refCountMap;
Chris@304 43
Chris@317 44 FileSource::RemoteLocalMap
Chris@317 45 FileSource::m_remoteLocalMap;
Chris@304 46
Chris@304 47 QMutex
Chris@317 48 FileSource::m_mapMutex;
Chris@304 49
Chris@460 50 FileSource::FileSource(QString fileOrUrl, ProgressReporter *reporter,
Chris@460 51 LocalCacheMode cacheMode) :
Chris@316 52 m_url(fileOrUrl),
Chris@460 53 m_cacheMode(cacheMode),
Chris@316 54 m_ftp(0),
Chris@316 55 m_http(0),
Chris@316 56 m_localFile(0),
Chris@316 57 m_ok(false),
Chris@316 58 m_lastStatus(0),
Chris@316 59 m_remote(isRemote(fileOrUrl)),
Chris@316 60 m_done(false),
Chris@316 61 m_leaveLocalFile(false),
Chris@392 62 m_reporter(reporter),
Chris@316 63 m_refCounted(false)
Chris@316 64 {
Chris@327 65 #ifdef DEBUG_FILE_SOURCE
Chris@460 66 std::cerr << "FileSource::FileSource(" << fileOrUrl.toStdString() << ", " << cacheMode << ")" << std::endl;
Chris@327 67 #endif
Chris@316 68
Chris@316 69 if (!canHandleScheme(m_url)) {
Chris@317 70 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString().toStdString() << "\"" << std::endl;
Chris@316 71 m_errorString = tr("Unsupported scheme in URL");
Chris@316 72 return;
Chris@316 73 }
Chris@316 74
Chris@357 75 init();
Chris@316 76
Chris@316 77 if (isRemote() &&
Chris@316 78 (fileOrUrl.contains('%') ||
Chris@316 79 fileOrUrl.contains("--"))) { // for IDNA
Chris@316 80
Chris@316 81 waitForStatus();
Chris@316 82
Chris@316 83 if (!isAvailable()) {
Chris@336 84
Chris@316 85 // The URL was created on the assumption that the string
Chris@316 86 // was human-readable. Let's try again, this time
Chris@316 87 // assuming it was already encoded.
Chris@317 88 std::cerr << "FileSource::FileSource: Failed to retrieve URL \""
Chris@316 89 << fileOrUrl.toStdString()
Chris@316 90 << "\" as human-readable URL; "
Chris@316 91 << "trying again treating it as encoded URL"
Chris@316 92 << std::endl;
Chris@336 93
Chris@336 94 // even though our cache file doesn't exist (because the
Chris@336 95 // resource was 404), we still need to ensure we're no
Chris@336 96 // longer associating a filename with this url in the
Chris@336 97 // refcount map -- or createCacheFile will think we've
Chris@336 98 // already done all the work and no request will be sent
Chris@336 99 deleteCacheFile();
Chris@336 100
Chris@316 101 m_url.setEncodedUrl(fileOrUrl.toAscii());
Chris@336 102
Chris@336 103 m_ok = false;
Chris@336 104 m_done = false;
Chris@336 105 m_lastStatus = 0;
Chris@357 106 init();
Chris@316 107 }
Chris@316 108 }
Chris@325 109
Chris@325 110 if (!isRemote()) {
Chris@325 111 emit statusAvailable();
Chris@325 112 emit ready();
Chris@325 113 }
Chris@316 114 }
Chris@316 115
Chris@460 116 FileSource::FileSource(QUrl url, ProgressReporter *reporter,
Chris@460 117 LocalCacheMode cacheMode) :
Chris@304 118 m_url(url),
Chris@460 119 m_cacheMode(cacheMode),
Chris@208 120 m_ftp(0),
Chris@208 121 m_http(0),
Chris@208 122 m_localFile(0),
Chris@208 123 m_ok(false),
Chris@210 124 m_lastStatus(0),
Chris@316 125 m_remote(isRemote(url.toString())),
Chris@208 126 m_done(false),
Chris@316 127 m_leaveLocalFile(false),
Chris@392 128 m_reporter(reporter),
Chris@316 129 m_refCounted(false)
Chris@208 130 {
Chris@327 131 #ifdef DEBUG_FILE_SOURCE
Chris@317 132 std::cerr << "FileSource::FileSource(" << url.toString().toStdString() << ") [as url]" << std::endl;
Chris@327 133 #endif
Chris@316 134
Chris@316 135 if (!canHandleScheme(m_url)) {
Chris@317 136 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString().toStdString() << "\"" << std::endl;
Chris@316 137 m_errorString = tr("Unsupported scheme in URL");
Chris@208 138 return;
Chris@208 139 }
Chris@208 140
Chris@357 141 init();
Chris@316 142 }
Chris@304 143
Chris@317 144 FileSource::FileSource(const FileSource &rf) :
Chris@316 145 QObject(),
Chris@316 146 m_url(rf.m_url),
Chris@460 147 m_cacheMode(rf.m_cacheMode),
Chris@316 148 m_ftp(0),
Chris@316 149 m_http(0),
Chris@316 150 m_localFile(0),
Chris@316 151 m_ok(rf.m_ok),
Chris@316 152 m_lastStatus(rf.m_lastStatus),
Chris@316 153 m_remote(rf.m_remote),
Chris@316 154 m_done(false),
Chris@316 155 m_leaveLocalFile(false),
Chris@392 156 m_reporter(rf.m_reporter),
Chris@316 157 m_refCounted(false)
Chris@316 158 {
Chris@327 159 #ifdef DEBUG_FILE_SOURCE
Chris@317 160 std::cerr << "FileSource::FileSource(" << m_url.toString().toStdString() << ") [copy ctor]" << std::endl;
Chris@327 161 #endif
Chris@304 162
Chris@316 163 if (!canHandleScheme(m_url)) {
Chris@317 164 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString().toStdString() << "\"" << std::endl;
Chris@316 165 m_errorString = tr("Unsupported scheme in URL");
Chris@304 166 return;
Chris@304 167 }
Chris@304 168
Chris@460 169 if (m_cacheMode == PersistentCache) {
Chris@460 170
Chris@316 171 m_localFilename = rf.m_localFilename;
Chris@460 172
Chris@460 173 } else if (!isRemote()) {
Chris@460 174
Chris@460 175 m_localFilename = rf.m_localFilename;
Chris@460 176
Chris@316 177 } else {
Chris@316 178 QMutexLocker locker(&m_mapMutex);
Chris@327 179 #ifdef DEBUG_FILE_SOURCE
Chris@317 180 std::cerr << "FileSource::FileSource(copy ctor): ref count is "
Chris@316 181 << m_refCountMap[m_url] << std::endl;
Chris@327 182 #endif
Chris@316 183 if (m_refCountMap[m_url] > 0) {
Chris@316 184 m_refCountMap[m_url]++;
Chris@327 185 #ifdef DEBUG_FILE_SOURCE
Chris@316 186 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 187 #endif
Chris@316 188 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 189 m_refCounted = true;
Chris@316 190 } else {
Chris@316 191 m_ok = false;
Chris@316 192 m_lastStatus = 404;
Chris@316 193 }
Chris@316 194 }
Chris@316 195
Chris@316 196 m_done = true;
Chris@316 197 }
Chris@316 198
Chris@317 199 FileSource::~FileSource()
Chris@316 200 {
Chris@327 201 #ifdef DEBUG_FILE_SOURCE
Chris@317 202 std::cerr << "FileSource(" << m_url.toString().toStdString() << ")::~FileSource" << std::endl;
Chris@327 203 #endif
Chris@316 204
Chris@316 205 cleanup();
Chris@316 206
Chris@460 207 if (isRemote() && (m_cacheMode == TemporaryCache) && !m_leaveLocalFile) {
Chris@460 208 deleteCacheFile();
Chris@460 209 }
Chris@316 210 }
Chris@316 211
Chris@316 212 void
Chris@357 213 FileSource::init()
Chris@316 214 {
Chris@316 215 if (!isRemote()) {
Chris@355 216 #ifdef DEBUG_FILE_SOURCE
Chris@355 217 std::cerr << "FileSource::init: Not a remote URL" << std::endl;
Chris@355 218 #endif
Chris@355 219 bool literal = false;
Chris@316 220 m_localFilename = m_url.toLocalFile();
Chris@342 221 if (m_localFilename == "") {
Chris@342 222 // QUrl may have mishandled the scheme (e.g. in a DOS path)
Chris@342 223 m_localFilename = m_url.toString();
Chris@355 224 literal = true;
Chris@342 225 }
Chris@439 226 m_localFilename = QFileInfo(m_localFilename).absoluteFilePath();
Chris@439 227
Chris@355 228 #ifdef DEBUG_FILE_SOURCE
Chris@355 229 std::cerr << "FileSource::init: URL translates to local filename \""
Chris@355 230 << m_localFilename.toStdString() << "\"" << std::endl;
Chris@355 231 #endif
Chris@316 232 m_ok = true;
Chris@355 233 m_lastStatus = 200;
Chris@355 234
Chris@316 235 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 236 if (literal) {
Chris@355 237 m_lastStatus = 404;
Chris@355 238 } else {
Chris@355 239 // Again, QUrl may have been mistreating us --
Chris@355 240 // e.g. dropping a part that looks like query data
Chris@355 241 m_localFilename = m_url.toString();
Chris@355 242 literal = true;
Chris@355 243 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 244 m_lastStatus = 404;
Chris@355 245 }
Chris@355 246 }
Chris@316 247 }
Chris@355 248
Chris@316 249 m_done = true;
Chris@316 250 return;
Chris@316 251 }
Chris@316 252
Chris@316 253 if (createCacheFile()) {
Chris@327 254 #ifdef DEBUG_FILE_SOURCE
Chris@460 255 std::cerr << "FileSource::init: Already have this one at "
Chris@460 256 << m_localFilename.toStdString() << std::endl;
Chris@327 257 #endif
Chris@316 258 m_ok = true;
Chris@316 259 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 260 m_lastStatus = 404;
Chris@316 261 } else {
Chris@316 262 m_lastStatus = 200;
Chris@316 263 }
Chris@316 264 m_done = true;
Chris@316 265 return;
Chris@316 266 }
Chris@316 267
Chris@208 268 if (m_localFilename == "") return;
Chris@208 269 m_localFile = new QFile(m_localFilename);
Chris@208 270 m_localFile->open(QFile::WriteOnly);
Chris@208 271
Chris@316 272 QString scheme = m_url.scheme().toLower();
Chris@316 273
Chris@327 274 #ifdef DEBUG_FILE_SOURCE
Chris@317 275 std::cerr << "FileSource::init: Don't have local copy of \""
Chris@460 276 << m_url.toString().toStdString() << "\", retrieving to "
Chris@460 277 << m_localFilename.toStdString() << std::endl;
Chris@327 278 #endif
Chris@208 279
Chris@208 280 if (scheme == "http") {
Chris@316 281 initHttp();
Chris@357 282 std::cerr << "FileSource: initHttp succeeded" << std::endl;
Chris@208 283 } else if (scheme == "ftp") {
Chris@316 284 initFtp();
Chris@316 285 } else {
Chris@316 286 m_remote = false;
Chris@316 287 m_ok = false;
Chris@208 288 }
Chris@208 289
Chris@208 290 if (m_ok) {
Chris@316 291
Chris@316 292 QMutexLocker locker(&m_mapMutex);
Chris@316 293
Chris@316 294 if (m_refCountMap[m_url] > 0) {
Chris@316 295 // someone else has been doing the same thing at the same time,
Chris@316 296 // but has got there first
Chris@316 297 cleanup();
Chris@316 298 m_refCountMap[m_url]++;
Chris@327 299 #ifdef DEBUG_FILE_SOURCE
Chris@317 300 std::cerr << "FileSource::init: Another FileSource has got there first, abandoning our download and using theirs" << std::endl;
Chris@327 301 #endif
Chris@316 302 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 303 m_refCounted = true;
Chris@316 304 m_ok = true;
Chris@316 305 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 306 m_lastStatus = 404;
Chris@316 307 }
Chris@316 308 m_done = true;
Chris@316 309 return;
Chris@316 310 }
Chris@304 311
Chris@304 312 m_remoteLocalMap[m_url] = m_localFilename;
Chris@304 313 m_refCountMap[m_url]++;
Chris@316 314 m_refCounted = true;
Chris@304 315
Chris@392 316 if (m_reporter) {
Chris@392 317 m_reporter->setMessage
Chris@392 318 (tr("Downloading %1...").arg(m_url.toString()));
Chris@392 319 connect(m_reporter, SIGNAL(cancelled()), this, SLOT(cancelled()));
Chris@357 320 connect(this, SIGNAL(progress(int)),
Chris@392 321 m_reporter, SLOT(setProgress(int)));
Chris@316 322 }
Chris@208 323 }
Chris@208 324 }
Chris@208 325
Chris@316 326 void
Chris@317 327 FileSource::initHttp()
Chris@208 328 {
Chris@316 329 m_ok = true;
Chris@316 330 m_http = new QHttp(m_url.host(), m_url.port(80));
Chris@316 331 connect(m_http, SIGNAL(done(bool)), this, SLOT(done(bool)));
Chris@316 332 connect(m_http, SIGNAL(dataReadProgress(int, int)),
Chris@316 333 this, SLOT(dataReadProgress(int, int)));
Chris@316 334 connect(m_http, SIGNAL(responseHeaderReceived(const QHttpResponseHeader &)),
Chris@316 335 this, SLOT(httpResponseHeaderReceived(const QHttpResponseHeader &)));
Chris@316 336
Chris@316 337 // I don't quite understand this. url.path() returns a path
Chris@316 338 // without percent encoding; for example, spaces appear as
Chris@316 339 // literal spaces. This generally won't work if sent to the
Chris@316 340 // server directly. You can retrieve a correctly encoded URL
Chris@316 341 // from QUrl using url.toEncoded(), but that gives you the
Chris@316 342 // whole URL; there doesn't seem to be any way to retrieve
Chris@316 343 // only an encoded path. Furthermore there doesn't seem to be
Chris@316 344 // any way to convert a retrieved path into an encoded path
Chris@316 345 // without explicitly specifying that you don't want the path
Chris@316 346 // separators ("/") to be encoded. (Besides being painful to
Chris@316 347 // manage, I don't see how this can work correctly in any case
Chris@316 348 // where a percent-encoded "/" is supposed to appear within a
Chris@316 349 // path element?) There also seems to be no way to retrieve
Chris@316 350 // the path plus query string, i.e. everything that I need to
Chris@316 351 // send to the HTTP server. And no way for QHttp to take a
Chris@316 352 // QUrl argument. I'm obviously missing something.
Chris@316 353
Chris@316 354 // So, two ways to do this: query the bits from the URL,
Chris@316 355 // encode them individually, and glue them back together
Chris@316 356 // again...
Chris@316 357 /*
Chris@316 358 QString path = QUrl::toPercentEncoding(m_url.path(), "/");
Chris@316 359 QList<QPair<QString, QString> > query = m_url.queryItems();
Chris@316 360 if (!query.empty()) {
Chris@316 361 QStringList q2;
Chris@316 362 for (QList<QPair<QString, QString> >::iterator i = query.begin();
Chris@316 363 i != query.end(); ++i) {
Chris@316 364 q2.push_back(QString("%1=%3")
Chris@316 365 .arg(QString(QUrl::toPercentEncoding(i->first)))
Chris@316 366 .arg(QString(QUrl::toPercentEncoding(i->second))));
Chris@316 367 }
Chris@316 368 path = QString("%1%2%3")
Chris@316 369 .arg(path).arg("?")
Chris@316 370 .arg(q2.join("&"));
Chris@316 371 }
Chris@316 372 */
Chris@316 373
Chris@316 374 // ...or, much simpler but relying on knowledge about the
Chris@316 375 // scheme://host/path/path/query etc format of the URL, we can
Chris@316 376 // get the whole URL ready-encoded and then split it on "/" as
Chris@316 377 // appropriate...
Chris@316 378
Chris@316 379 QString path = "/" + QString(m_url.toEncoded()).section('/', 3);
Chris@316 380
Chris@327 381 #ifdef DEBUG_FILE_SOURCE
Chris@317 382 std::cerr << "FileSource: path is \""
Chris@316 383 << path.toStdString() << "\"" << std::endl;
Chris@327 384 #endif
Chris@316 385
Chris@316 386 m_http->get(path, m_localFile);
Chris@316 387 }
Chris@316 388
Chris@316 389 void
Chris@317 390 FileSource::initFtp()
Chris@316 391 {
Chris@316 392 m_ok = true;
Chris@316 393 m_ftp = new QFtp;
Chris@316 394 connect(m_ftp, SIGNAL(done(bool)), this, SLOT(done(bool)));
Chris@316 395 connect(m_ftp, SIGNAL(commandFinished(int, bool)),
Chris@316 396 this, SLOT(ftpCommandFinished(int, bool)));
Chris@316 397 connect(m_ftp, SIGNAL(dataTransferProgress(qint64, qint64)),
Chris@316 398 this, SLOT(dataTransferProgress(qint64, qint64)));
Chris@316 399 m_ftp->connectToHost(m_url.host(), m_url.port(21));
Chris@316 400
Chris@316 401 QString username = m_url.userName();
Chris@316 402 if (username == "") {
Chris@316 403 username = "anonymous";
Chris@316 404 }
Chris@316 405
Chris@316 406 QString password = m_url.password();
Chris@316 407 if (password == "") {
Chris@316 408 password = QString("%1@%2").arg(getenv("USER")).arg(getenv("HOST"));
Chris@316 409 }
Chris@316 410
Chris@316 411 m_ftp->login(username, password);
Chris@316 412
Chris@316 413 QString dirpath = m_url.path().section('/', 0, -2);
Chris@316 414 QString filename = m_url.path().section('/', -1);
Chris@316 415
Chris@316 416 if (dirpath == "") dirpath = "/";
Chris@316 417 m_ftp->cd(dirpath);
Chris@316 418 m_ftp->get(filename, m_localFile);
Chris@211 419 }
Chris@211 420
Chris@211 421 void
Chris@317 422 FileSource::cleanup()
Chris@211 423 {
Chris@461 424 if (m_done) {
Chris@461 425 delete m_localFile; // does not actually delete the file
Chris@461 426 m_localFile = 0;
Chris@461 427 }
Chris@211 428 m_done = true;
Chris@214 429 if (m_http) {
Chris@287 430 QHttp *h = m_http;
Chris@214 431 m_http = 0;
Chris@287 432 h->abort();
Chris@287 433 h->deleteLater();
Chris@214 434 }
Chris@214 435 if (m_ftp) {
Chris@287 436 QFtp *f = m_ftp;
Chris@214 437 m_ftp = 0;
Chris@287 438 f->abort();
Chris@287 439 f->deleteLater();
Chris@214 440 }
Chris@461 441 if (m_localFile) {
Chris@461 442 delete m_localFile; // does not actually delete the file
Chris@461 443 m_localFile = 0;
Chris@461 444 }
Chris@208 445 }
Chris@208 446
Chris@208 447 bool
Chris@317 448 FileSource::isRemote(QString fileOrUrl)
Chris@304 449 {
Chris@342 450 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@316 451 QString scheme = QUrl(fileOrUrl).scheme().toLower();
Chris@342 452 if (scheme == "" || scheme == "file" || scheme.length() == 1) return false;
Chris@342 453 return true;
Chris@304 454 }
Chris@304 455
Chris@304 456 bool
Chris@317 457 FileSource::canHandleScheme(QUrl url)
Chris@208 458 {
Chris@342 459 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@208 460 QString scheme = url.scheme().toLower();
Chris@316 461 return (scheme == "http" || scheme == "ftp" ||
Chris@342 462 scheme == "file" || scheme == "" || scheme.length() == 1);
Chris@208 463 }
Chris@208 464
Chris@210 465 bool
Chris@317 466 FileSource::isAvailable()
Chris@210 467 {
Chris@316 468 waitForStatus();
Chris@211 469 bool available = true;
Chris@211 470 if (!m_ok) available = false;
Chris@211 471 else available = (m_lastStatus / 100 == 2);
Chris@327 472 #ifdef DEBUG_FILE_SOURCE
Chris@317 473 std::cerr << "FileSource::isAvailable: " << (available ? "yes" : "no")
Chris@211 474 << std::endl;
Chris@327 475 #endif
Chris@211 476 return available;
Chris@210 477 }
Chris@210 478
Chris@208 479 void
Chris@317 480 FileSource::waitForStatus()
Chris@316 481 {
Chris@316 482 while (m_ok && (!m_done && m_lastStatus == 0)) {
Chris@316 483 // std::cerr << "waitForStatus: processing (last status " << m_lastStatus << ")" << std::endl;
Chris@392 484 QCoreApplication::processEvents();
Chris@316 485 }
Chris@316 486 }
Chris@316 487
Chris@316 488 void
Chris@317 489 FileSource::waitForData()
Chris@208 490 {
Chris@211 491 while (m_ok && !m_done) {
Chris@357 492 // std::cerr << "FileSource::waitForData: calling QApplication::processEvents" << std::endl;
Chris@392 493 QCoreApplication::processEvents();
Chris@208 494 }
Chris@208 495 }
Chris@208 496
Chris@316 497 void
Chris@317 498 FileSource::setLeaveLocalFile(bool leave)
Chris@316 499 {
Chris@316 500 m_leaveLocalFile = leave;
Chris@316 501 }
Chris@316 502
Chris@208 503 bool
Chris@317 504 FileSource::isOK() const
Chris@208 505 {
Chris@208 506 return m_ok;
Chris@208 507 }
Chris@208 508
Chris@208 509 bool
Chris@317 510 FileSource::isDone() const
Chris@208 511 {
Chris@208 512 return m_done;
Chris@208 513 }
Chris@208 514
Chris@316 515 bool
Chris@317 516 FileSource::isRemote() const
Chris@316 517 {
Chris@316 518 return m_remote;
Chris@316 519 }
Chris@316 520
Chris@316 521 QString
Chris@317 522 FileSource::getLocation() const
Chris@316 523 {
Chris@316 524 return m_url.toString();
Chris@316 525 }
Chris@316 526
Chris@208 527 QString
Chris@317 528 FileSource::getLocalFilename() const
Chris@208 529 {
Chris@208 530 return m_localFilename;
Chris@208 531 }
Chris@208 532
Chris@208 533 QString
Chris@317 534 FileSource::getContentType() const
Chris@316 535 {
Chris@316 536 return m_contentType;
Chris@316 537 }
Chris@316 538
Chris@316 539 QString
Chris@317 540 FileSource::getExtension() const
Chris@316 541 {
Chris@316 542 if (m_localFilename != "") {
Chris@316 543 return QFileInfo(m_localFilename).suffix().toLower();
Chris@316 544 } else {
Chris@316 545 return QFileInfo(m_url.toLocalFile()).suffix().toLower();
Chris@316 546 }
Chris@316 547 }
Chris@316 548
Chris@316 549 QString
Chris@317 550 FileSource::getErrorString() const
Chris@208 551 {
Chris@208 552 return m_errorString;
Chris@208 553 }
Chris@208 554
Chris@208 555 void
Chris@317 556 FileSource::dataReadProgress(int done, int total)
Chris@208 557 {
Chris@208 558 dataTransferProgress(done, total);
Chris@208 559 }
Chris@208 560
Chris@208 561 void
Chris@317 562 FileSource::httpResponseHeaderReceived(const QHttpResponseHeader &resp)
Chris@210 563 {
Chris@210 564 m_lastStatus = resp.statusCode();
Chris@210 565 if (m_lastStatus / 100 >= 4) {
Chris@210 566 m_errorString = QString("%1 %2")
Chris@210 567 .arg(resp.statusCode()).arg(resp.reasonPhrase());
Chris@327 568 #ifdef DEBUG_FILE_SOURCE
Chris@317 569 std::cerr << "FileSource::responseHeaderReceived: "
Chris@211 570 << m_errorString.toStdString() << std::endl;
Chris@327 571 #endif
Chris@211 572 } else {
Chris@327 573 #ifdef DEBUG_FILE_SOURCE
Chris@317 574 std::cerr << "FileSource::responseHeaderReceived: "
Chris@211 575 << m_lastStatus << std::endl;
Chris@327 576 #endif
Chris@315 577 if (resp.hasContentType()) m_contentType = resp.contentType();
Chris@325 578 }
Chris@325 579 emit statusAvailable();
Chris@210 580 }
Chris@210 581
Chris@210 582 void
Chris@317 583 FileSource::ftpCommandFinished(int id, bool error)
Chris@214 584 {
Chris@327 585 #ifdef DEBUG_FILE_SOURCE
Chris@317 586 std::cerr << "FileSource::ftpCommandFinished(" << id << ", " << error << ")" << std::endl;
Chris@327 587 #endif
Chris@214 588
Chris@214 589 if (!m_ftp) return;
Chris@214 590
Chris@214 591 QFtp::Command command = m_ftp->currentCommand();
Chris@214 592
Chris@214 593 if (!error) {
Chris@327 594 #ifdef DEBUG_FILE_SOURCE
Chris@317 595 std::cerr << "FileSource::ftpCommandFinished: success for command "
Chris@214 596 << command << std::endl;
Chris@327 597 #endif
Chris@214 598 return;
Chris@214 599 }
Chris@214 600
Chris@214 601 if (command == QFtp::ConnectToHost) {
Chris@214 602 m_errorString = tr("Failed to connect to FTP server");
Chris@214 603 } else if (command == QFtp::Login) {
Chris@214 604 m_errorString = tr("Login failed");
Chris@214 605 } else if (command == QFtp::Cd) {
Chris@214 606 m_errorString = tr("Failed to change to correct directory");
Chris@214 607 } else if (command == QFtp::Get) {
Chris@214 608 m_errorString = tr("FTP download aborted");
Chris@214 609 }
Chris@214 610
Chris@214 611 m_lastStatus = 400; // for done()
Chris@214 612 }
Chris@214 613
Chris@214 614 void
Chris@317 615 FileSource::dataTransferProgress(qint64 done, qint64 total)
Chris@208 616 {
Chris@208 617 int percent = int((double(done) / double(total)) * 100.0 - 0.1);
Chris@208 618 emit progress(percent);
Chris@210 619 }
Chris@210 620
Chris@210 621 void
Chris@317 622 FileSource::cancelled()
Chris@210 623 {
Chris@210 624 m_done = true;
Chris@316 625 cleanup();
Chris@316 626
Chris@210 627 m_ok = false;
Chris@210 628 m_errorString = tr("Download cancelled");
Chris@208 629 }
Chris@208 630
Chris@208 631 void
Chris@317 632 FileSource::done(bool error)
Chris@208 633 {
Chris@327 634 emit progress(100);
Chris@327 635
Chris@327 636 #ifdef DEBUG_FILE_SOURCE
Chris@317 637 std::cerr << "FileSource::done(" << error << ")" << std::endl;
Chris@327 638 #endif
Chris@211 639
Chris@211 640 if (m_done) return;
Chris@211 641
Chris@208 642 if (error) {
Chris@208 643 if (m_http) {
Chris@208 644 m_errorString = m_http->errorString();
Chris@208 645 } else if (m_ftp) {
Chris@208 646 m_errorString = m_ftp->errorString();
Chris@208 647 }
Chris@208 648 }
Chris@208 649
Chris@210 650 if (m_lastStatus / 100 >= 4) {
Chris@211 651 error = true;
Chris@210 652 }
Chris@210 653
Chris@211 654 cleanup();
Chris@208 655
Chris@211 656 if (!error) {
Chris@208 657 QFileInfo fi(m_localFilename);
Chris@208 658 if (!fi.exists()) {
Chris@208 659 m_errorString = tr("Failed to create local file %1").arg(m_localFilename);
Chris@211 660 error = true;
Chris@208 661 } else if (fi.size() == 0) {
Chris@208 662 m_errorString = tr("File contains no data!");
Chris@211 663 error = true;
Chris@208 664 }
Chris@208 665 }
Chris@211 666
Chris@211 667 if (error) {
Chris@327 668 #ifdef DEBUG_FILE_SOURCE
Chris@461 669 std::cerr << "FileSource::done: error is " << error << " (\""
Chris@461 670 << m_errorString.toStdString() << "\"), deleting cache file" << std::endl;
Chris@327 671 #endif
Chris@316 672 deleteCacheFile();
Chris@211 673 }
Chris@211 674
Chris@211 675 m_ok = !error;
Chris@211 676 m_done = true;
Chris@304 677 emit ready();
Chris@211 678 }
Chris@211 679
Chris@211 680 void
Chris@317 681 FileSource::deleteCacheFile()
Chris@211 682 {
Chris@327 683 #ifdef DEBUG_FILE_SOURCE
Chris@317 684 std::cerr << "FileSource::deleteCacheFile(\"" << m_localFilename.toStdString() << "\")" << std::endl;
Chris@327 685 #endif
Chris@211 686
Chris@211 687 cleanup();
Chris@211 688
Chris@316 689 if (m_localFilename == "") {
Chris@316 690 return;
Chris@316 691 }
Chris@211 692
Chris@316 693 if (!isRemote()) {
Chris@327 694 #ifdef DEBUG_FILE_SOURCE
Chris@316 695 std::cerr << "not a cache file" << std::endl;
Chris@327 696 #endif
Chris@316 697 return;
Chris@316 698 }
Chris@316 699
Chris@316 700 if (m_refCounted) {
Chris@304 701
Chris@304 702 QMutexLocker locker(&m_mapMutex);
Chris@316 703 m_refCounted = false;
Chris@304 704
Chris@304 705 if (m_refCountMap[m_url] > 0) {
Chris@304 706 m_refCountMap[m_url]--;
Chris@327 707 #ifdef DEBUG_FILE_SOURCE
Chris@316 708 std::cerr << "reduced ref count to " << m_refCountMap[m_url] << std::endl;
Chris@327 709 #endif
Chris@304 710 if (m_refCountMap[m_url] > 0) {
Chris@304 711 m_done = true;
Chris@304 712 return;
Chris@304 713 }
Chris@304 714 }
Chris@304 715 }
Chris@304 716
Chris@211 717 m_fileCreationMutex.lock();
Chris@211 718
Chris@460 719 // We always delete the file here, even in PersistentCache mode,
Chris@460 720 // because this function is also used if retrieval failed (in
Chris@460 721 // which case we want to delete the cache so that subsequent users
Chris@460 722 // won't trust it). It's up to the calling code to determine
Chris@460 723 // whether we actually want to delete the cache or not, e.g. on
Chris@460 724 // destruction.
Chris@460 725
Chris@211 726 if (!QFile(m_localFilename).remove()) {
Chris@317 727 std::cerr << "FileSource::deleteCacheFile: ERROR: Failed to delete file \"" << m_localFilename.toStdString() << "\"" << std::endl;
Chris@211 728 } else {
Chris@327 729 #ifdef DEBUG_FILE_SOURCE
Chris@317 730 std::cerr << "FileSource::deleteCacheFile: Deleted cache file \"" << m_localFilename.toStdString() << "\"" << std::endl;
Chris@327 731 #endif
Chris@211 732 m_localFilename = "";
Chris@211 733 }
Chris@211 734
Chris@211 735 m_fileCreationMutex.unlock();
Chris@211 736
Chris@208 737 m_done = true;
Chris@208 738 }
Chris@208 739
Chris@460 740 QString
Chris@460 741 FileSource::getPersistentCacheDirectory()
Chris@460 742 {
Chris@460 743 QDir dir = TempDirectory::getInstance()->getContainingPath();
Chris@460 744
Chris@460 745 QString cacheDirName("cache");
Chris@460 746
Chris@460 747 QFileInfo fi(dir.filePath(cacheDirName));
Chris@460 748
Chris@460 749 if ((fi.exists() && !fi.isDir()) ||
Chris@460 750 (!fi.exists() && !dir.mkdir(cacheDirName))) {
Chris@460 751
Chris@460 752 throw DirectoryCreationFailed(fi.filePath());
Chris@460 753 }
Chris@460 754
Chris@460 755 return fi.filePath();
Chris@460 756 }
Chris@460 757
Chris@460 758 QString
Chris@460 759 FileSource::getPersistentCacheFilePath(QUrl url)
Chris@460 760 {
Chris@460 761 QDir dir(getPersistentCacheDirectory());
Chris@460 762
Chris@460 763 QString filename =
Chris@460 764 QString::fromLocal8Bit
Chris@460 765 (QCryptographicHash::hash(url.toString().toLocal8Bit(),
Chris@461 766 QCryptographicHash::Sha1).toHex());
Chris@460 767
Chris@460 768 return dir.filePath(filename);
Chris@460 769 }
Chris@460 770
Chris@316 771 bool
Chris@317 772 FileSource::createCacheFile()
Chris@208 773 {
Chris@460 774 if (m_cacheMode == PersistentCache) {
Chris@460 775 m_localFilename = getPersistentCacheFilePath(m_url);
Chris@460 776 if (QFileInfo(m_localFilename).exists()) return true;
Chris@460 777 else return false;
Chris@460 778 }
Chris@460 779
Chris@316 780 {
Chris@316 781 QMutexLocker locker(&m_mapMutex);
Chris@316 782
Chris@327 783 #ifdef DEBUG_FILE_SOURCE
Chris@317 784 std::cerr << "FileSource::createCacheFile: refcount is " << m_refCountMap[m_url] << std::endl;
Chris@327 785 #endif
Chris@316 786
Chris@316 787 if (m_refCountMap[m_url] > 0) {
Chris@316 788 m_refCountMap[m_url]++;
Chris@316 789 m_localFilename = m_remoteLocalMap[m_url];
Chris@327 790 #ifdef DEBUG_FILE_SOURCE
Chris@316 791 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 792 #endif
Chris@316 793 m_refCounted = true;
Chris@316 794 return true;
Chris@316 795 }
Chris@316 796 }
Chris@316 797
Chris@208 798 QDir dir;
Chris@208 799 try {
Chris@208 800 dir = TempDirectory::getInstance()->getSubDirectoryPath("download");
Chris@208 801 } catch (DirectoryCreationFailed f) {
Chris@327 802 #ifdef DEBUG_FILE_SOURCE
Chris@317 803 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create temporary directory: " << f.what() << std::endl;
Chris@327 804 #endif
Chris@208 805 return "";
Chris@208 806 }
Chris@208 807
Chris@316 808 QString filepart = m_url.path().section('/', -1, -1,
Chris@316 809 QString::SectionSkipEmpty);
Chris@208 810
Chris@457 811 QString extension = "";
Chris@457 812 if (filepart.contains('.')) extension = filepart.section('.', -1);
Chris@457 813
Chris@208 814 QString base = filepart;
Chris@208 815 if (extension != "") {
Chris@208 816 base = base.left(base.length() - extension.length() - 1);
Chris@208 817 }
Chris@208 818 if (base == "") base = "remote";
Chris@208 819
Chris@208 820 QString filename;
Chris@208 821
Chris@208 822 if (extension == "") {
Chris@208 823 filename = base;
Chris@208 824 } else {
Chris@208 825 filename = QString("%1.%2").arg(base).arg(extension);
Chris@208 826 }
Chris@208 827
Chris@208 828 QString filepath(dir.filePath(filename));
Chris@208 829
Chris@327 830 #ifdef DEBUG_FILE_SOURCE
Chris@317 831 std::cerr << "FileSource::createCacheFile: URL is \"" << m_url.toString().toStdString() << "\", dir is \"" << dir.path().toStdString() << "\", base \"" << base.toStdString() << "\", extension \"" << extension.toStdString() << "\", filebase \"" << filename.toStdString() << "\", filename \"" << filepath.toStdString() << "\"" << std::endl;
Chris@327 832 #endif
Chris@208 833
Chris@316 834 QMutexLocker fcLocker(&m_fileCreationMutex);
Chris@316 835
Chris@208 836 ++m_count;
Chris@208 837
Chris@208 838 if (QFileInfo(filepath).exists() ||
Chris@208 839 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 840
Chris@327 841 #ifdef DEBUG_FILE_SOURCE
Chris@317 842 std::cerr << "FileSource::createCacheFile: Failed to create local file \""
Chris@208 843 << filepath.toStdString() << "\" for URL \""
Chris@316 844 << m_url.toString().toStdString() << "\" (or file already exists): appending suffix instead" << std::endl;
Chris@327 845 #endif
Chris@208 846
Chris@208 847 if (extension == "") {
Chris@208 848 filename = QString("%1_%2").arg(base).arg(m_count);
Chris@208 849 } else {
Chris@208 850 filename = QString("%1_%2.%3").arg(base).arg(m_count).arg(extension);
Chris@208 851 }
Chris@208 852 filepath = dir.filePath(filename);
Chris@208 853
Chris@208 854 if (QFileInfo(filepath).exists() ||
Chris@208 855 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 856
Chris@327 857 #ifdef DEBUG_FILE_SOURCE
Chris@317 858 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create local file \""
Chris@208 859 << filepath.toStdString() << "\" for URL \""
Chris@316 860 << m_url.toString().toStdString() << "\" (or file already exists)" << std::endl;
Chris@327 861 #endif
Chris@208 862
Chris@208 863 return "";
Chris@208 864 }
Chris@208 865 }
Chris@208 866
Chris@327 867 #ifdef DEBUG_FILE_SOURCE
Chris@317 868 std::cerr << "FileSource::createCacheFile: url "
Chris@316 869 << m_url.toString().toStdString() << " -> local filename "
Chris@316 870 << filepath.toStdString() << std::endl;
Chris@327 871 #endif
Chris@316 872
Chris@316 873 m_localFilename = filepath;
Chris@208 874
Chris@316 875 return false;
Chris@208 876 }
Chris@327 877