annotate data/fileio/FileSource.cpp @ 1028:abe13fe10ed6

RealTime frame-conversion test
author Chris Cannam
date Wed, 14 Jan 2015 14:58:38 +0000
parents 60e2927b1475
children 3a48b22fed48
rev   line source
Chris@208 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@208 2
Chris@208 3 /*
Chris@208 4 Sonic Visualiser
Chris@208 5 An audio file viewer and annotation editor.
Chris@208 6 Centre for Digital Music, Queen Mary, University of London.
Chris@208 7 This file copyright 2007 QMUL.
Chris@208 8
Chris@208 9 This program is free software; you can redistribute it and/or
Chris@208 10 modify it under the terms of the GNU General Public License as
Chris@208 11 published by the Free Software Foundation; either version 2 of the
Chris@208 12 License, or (at your option) any later version. See the file
Chris@208 13 COPYING included with this distribution for more information.
Chris@208 14 */
Chris@208 15
Chris@317 16 #include "FileSource.h"
Chris@357 17
Chris@208 18 #include "base/TempDirectory.h"
Chris@208 19 #include "base/Exceptions.h"
Chris@392 20 #include "base/ProgressReporter.h"
Chris@502 21 #include "system/System.h"
Chris@208 22
Chris@762 23 #include <QNetworkAccessManager>
Chris@762 24 #include <QNetworkReply>
Chris@208 25 #include <QFileInfo>
Chris@208 26 #include <QDir>
Chris@392 27 #include <QCoreApplication>
Chris@833 28 #include <QThreadStorage>
Chris@208 29
Chris@208 30 #include <iostream>
Chris@405 31 #include <cstdlib>
Chris@208 32
Chris@608 33 #include <unistd.h>
Chris@608 34
Chris@763 35 //#define DEBUG_FILE_SOURCE 1
Chris@327 36
Chris@208 37 int
Chris@317 38 FileSource::m_count = 0;
Chris@208 39
Chris@208 40 QMutex
Chris@317 41 FileSource::m_fileCreationMutex;
Chris@208 42
Chris@317 43 FileSource::RemoteRefCountMap
Chris@317 44 FileSource::m_refCountMap;
Chris@304 45
Chris@317 46 FileSource::RemoteLocalMap
Chris@317 47 FileSource::m_remoteLocalMap;
Chris@304 48
Chris@304 49 QMutex
Chris@317 50 FileSource::m_mapMutex;
Chris@304 51
Chris@529 52 #ifdef DEBUG_FILE_SOURCE
Chris@529 53 static int extantCount = 0;
Chris@529 54 static std::map<QString, int> urlExtantCountMap;
Chris@529 55 static void incCount(QString url) {
Chris@529 56 ++extantCount;
Chris@529 57 if (urlExtantCountMap.find(url) == urlExtantCountMap.end()) {
Chris@529 58 urlExtantCountMap[url] = 1;
Chris@529 59 } else {
Chris@529 60 ++urlExtantCountMap[url];
Chris@529 61 }
Chris@843 62 cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << endl;
Chris@529 63 }
Chris@529 64 static void decCount(QString url) {
Chris@529 65 --extantCount;
Chris@529 66 --urlExtantCountMap[url];
Chris@843 67 cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << endl;
Chris@529 68 }
Chris@529 69 #endif
Chris@529 70
Chris@833 71 static QThreadStorage<QNetworkAccessManager *> nms;
Chris@762 72
Chris@520 73 FileSource::FileSource(QString fileOrUrl, ProgressReporter *reporter,
Chris@520 74 QString preferredContentType) :
chris@831 75 m_rawFileOrUrl(fileOrUrl),
Chris@614 76 m_url(fileOrUrl, QUrl::StrictMode),
Chris@316 77 m_localFile(0),
Chris@762 78 m_reply(0),
Chris@520 79 m_preferredContentType(preferredContentType),
Chris@316 80 m_ok(false),
Chris@981 81 m_cancelled(false),
Chris@316 82 m_lastStatus(0),
Chris@706 83 m_resource(fileOrUrl.startsWith(':')),
Chris@316 84 m_remote(isRemote(fileOrUrl)),
Chris@316 85 m_done(false),
Chris@316 86 m_leaveLocalFile(false),
Chris@392 87 m_reporter(reporter),
Chris@316 88 m_refCounted(false)
Chris@316 89 {
Chris@706 90 if (m_resource) { // qrc file
Chris@706 91 m_url = QUrl("qrc" + fileOrUrl);
Chris@706 92 }
Chris@706 93
Chris@661 94 if (m_url.toString() == "") {
Chris@661 95 m_url = QUrl(fileOrUrl, QUrl::TolerantMode);
Chris@661 96 }
Chris@661 97
Chris@327 98 #ifdef DEBUG_FILE_SOURCE
Chris@843 99 cerr << "FileSource::FileSource(" << fileOrUrl << "): url <" << m_url.toString() << ">" << endl;
Chris@529 100 incCount(m_url.toString());
Chris@327 101 #endif
Chris@316 102
Chris@316 103 if (!canHandleScheme(m_url)) {
Chris@843 104 cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316 105 m_errorString = tr("Unsupported scheme in URL");
Chris@316 106 return;
Chris@316 107 }
Chris@316 108
Chris@357 109 init();
Chris@316 110
Chris@614 111 if (!isRemote() &&
Chris@614 112 !isAvailable()) {
Chris@614 113 #ifdef DEBUG_FILE_SOURCE
Chris@843 114 cerr << "FileSource::FileSource: Failed to open local file with URL \"" << m_url.toString() << "\"; trying again assuming filename was encoded" << endl;
Chris@614 115 #endif
Chris@762 116 m_url = QUrl::fromEncoded(fileOrUrl.toLatin1());
chris@831 117 #ifdef DEBUG_FILE_SOURCE
Chris@843 118 cerr << "FileSource::FileSource: URL is now \"" << m_url.toString() << "\"" << endl;
chris@831 119 #endif
Chris@614 120 init();
Chris@614 121 }
Chris@614 122
Chris@316 123 if (isRemote() &&
Chris@316 124 (fileOrUrl.contains('%') ||
Chris@316 125 fileOrUrl.contains("--"))) { // for IDNA
Chris@316 126
Chris@316 127 waitForStatus();
Chris@316 128
Chris@316 129 if (!isAvailable()) {
Chris@336 130
Chris@316 131 // The URL was created on the assumption that the string
Chris@316 132 // was human-readable. Let's try again, this time
Chris@316 133 // assuming it was already encoded.
Chris@843 134 cerr << "FileSource::FileSource: Failed to retrieve URL \""
Chris@844 135 << fileOrUrl
Chris@316 136 << "\" as human-readable URL; "
Chris@316 137 << "trying again treating it as encoded URL"
Chris@843 138 << endl;
Chris@336 139
Chris@336 140 // even though our cache file doesn't exist (because the
Chris@336 141 // resource was 404), we still need to ensure we're no
Chris@336 142 // longer associating a filename with this url in the
Chris@336 143 // refcount map -- or createCacheFile will think we've
Chris@336 144 // already done all the work and no request will be sent
Chris@336 145 deleteCacheFile();
Chris@336 146
Chris@762 147 m_url = QUrl::fromEncoded(fileOrUrl.toLatin1());
Chris@336 148
Chris@336 149 m_ok = false;
Chris@336 150 m_done = false;
Chris@336 151 m_lastStatus = 0;
Chris@357 152 init();
Chris@316 153 }
Chris@316 154 }
Chris@325 155
Chris@325 156 if (!isRemote()) {
Chris@325 157 emit statusAvailable();
Chris@325 158 emit ready();
Chris@325 159 }
Chris@497 160
Chris@504 161 #ifdef DEBUG_FILE_SOURCE
Chris@843 162 cerr << "FileSource::FileSource(string) exiting" << endl;
Chris@504 163 #endif
Chris@316 164 }
Chris@316 165
Chris@469 166 FileSource::FileSource(QUrl url, ProgressReporter *reporter) :
Chris@304 167 m_url(url),
Chris@208 168 m_localFile(0),
Chris@762 169 m_reply(0),
Chris@208 170 m_ok(false),
Chris@981 171 m_cancelled(false),
Chris@210 172 m_lastStatus(0),
Chris@706 173 m_resource(false),
Chris@316 174 m_remote(isRemote(url.toString())),
Chris@208 175 m_done(false),
Chris@316 176 m_leaveLocalFile(false),
Chris@392 177 m_reporter(reporter),
Chris@316 178 m_refCounted(false)
Chris@208 179 {
Chris@327 180 #ifdef DEBUG_FILE_SOURCE
Chris@843 181 cerr << "FileSource::FileSource(" << url.toString() << ") [as url]" << endl;
Chris@529 182 incCount(m_url.toString());
Chris@327 183 #endif
Chris@316 184
Chris@316 185 if (!canHandleScheme(m_url)) {
Chris@843 186 cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316 187 m_errorString = tr("Unsupported scheme in URL");
Chris@208 188 return;
Chris@208 189 }
Chris@208 190
Chris@357 191 init();
Chris@497 192
Chris@504 193 #ifdef DEBUG_FILE_SOURCE
Chris@843 194 cerr << "FileSource::FileSource(url) exiting" << endl;
Chris@504 195 #endif
Chris@316 196 }
Chris@304 197
Chris@317 198 FileSource::FileSource(const FileSource &rf) :
Chris@316 199 QObject(),
Chris@316 200 m_url(rf.m_url),
Chris@316 201 m_localFile(0),
Chris@762 202 m_reply(0),
Chris@316 203 m_ok(rf.m_ok),
Chris@981 204 m_cancelled(rf.m_cancelled),
Chris@316 205 m_lastStatus(rf.m_lastStatus),
Chris@706 206 m_resource(rf.m_resource),
Chris@316 207 m_remote(rf.m_remote),
Chris@316 208 m_done(false),
Chris@316 209 m_leaveLocalFile(false),
Chris@392 210 m_reporter(rf.m_reporter),
Chris@316 211 m_refCounted(false)
Chris@316 212 {
Chris@327 213 #ifdef DEBUG_FILE_SOURCE
Chris@843 214 cerr << "FileSource::FileSource(" << m_url.toString() << ") [copy ctor]" << endl;
Chris@529 215 incCount(m_url.toString());
Chris@327 216 #endif
Chris@304 217
Chris@316 218 if (!canHandleScheme(m_url)) {
Chris@843 219 cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316 220 m_errorString = tr("Unsupported scheme in URL");
Chris@304 221 return;
Chris@304 222 }
Chris@304 223
Chris@469 224 if (!isRemote()) {
Chris@316 225 m_localFilename = rf.m_localFilename;
Chris@316 226 } else {
Chris@316 227 QMutexLocker locker(&m_mapMutex);
Chris@327 228 #ifdef DEBUG_FILE_SOURCE
Chris@843 229 cerr << "FileSource::FileSource(copy ctor): ref count is "
Chris@843 230 << m_refCountMap[m_url] << endl;
Chris@327 231 #endif
Chris@316 232 if (m_refCountMap[m_url] > 0) {
Chris@316 233 m_refCountMap[m_url]++;
Chris@327 234 #ifdef DEBUG_FILE_SOURCE
Chris@843 235 cerr << "raised it to " << m_refCountMap[m_url] << endl;
Chris@327 236 #endif
Chris@316 237 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 238 m_refCounted = true;
Chris@316 239 } else {
Chris@316 240 m_ok = false;
Chris@316 241 m_lastStatus = 404;
Chris@316 242 }
Chris@316 243 }
Chris@316 244
Chris@316 245 m_done = true;
Chris@497 246
Chris@504 247 #ifdef DEBUG_FILE_SOURCE
Chris@843 248 cerr << "FileSource::FileSource(" << m_url.toString() << ") [copy ctor]: note: local filename is \"" << m_localFilename << "\"" << endl;
Chris@527 249 #endif
Chris@527 250
Chris@527 251 #ifdef DEBUG_FILE_SOURCE
Chris@843 252 cerr << "FileSource::FileSource(copy ctor) exiting" << endl;
Chris@504 253 #endif
Chris@316 254 }
Chris@316 255
Chris@317 256 FileSource::~FileSource()
Chris@316 257 {
Chris@327 258 #ifdef DEBUG_FILE_SOURCE
Chris@843 259 cerr << "FileSource(" << m_url.toString() << ")::~FileSource" << endl;
Chris@529 260 decCount(m_url.toString());
Chris@327 261 #endif
Chris@316 262
Chris@316 263 cleanup();
Chris@316 264
Chris@469 265 if (isRemote() && !m_leaveLocalFile) deleteCacheFile();
Chris@316 266 }
Chris@316 267
Chris@316 268 void
Chris@357 269 FileSource::init()
Chris@316 270 {
Chris@833 271 { // check we have a QNetworkAccessManager
Chris@833 272 QMutexLocker locker(&m_mapMutex);
Chris@833 273 if (!nms.hasLocalData()) {
Chris@833 274 nms.setLocalData(new QNetworkAccessManager());
Chris@833 275 }
Chris@833 276 }
Chris@833 277
Chris@706 278 if (isResource()) {
Chris@355 279 #ifdef DEBUG_FILE_SOURCE
Chris@843 280 cerr << "FileSource::init: Is a resource" << endl;
Chris@706 281 #endif
Chris@706 282 QString resourceFile = m_url.toString();
Chris@706 283 resourceFile.replace(QRegExp("^qrc:"), ":");
Chris@706 284
Chris@706 285 if (!QFileInfo(resourceFile).exists()) {
Chris@706 286 #ifdef DEBUG_FILE_SOURCE
Chris@843 287 cerr << "FileSource::init: Resource file of this name does not exist, switching to non-resource URL" << endl;
Chris@706 288 #endif
Chris@706 289 m_url = resourceFile;
Chris@706 290 m_resource = false;
Chris@706 291 }
Chris@706 292 }
Chris@706 293
Chris@706 294 if (!isRemote() && !isResource()) {
Chris@706 295 #ifdef DEBUG_FILE_SOURCE
Chris@843 296 cerr << "FileSource::init: Not a remote URL" << endl;
Chris@355 297 #endif
Chris@355 298 bool literal = false;
Chris@316 299 m_localFilename = m_url.toLocalFile();
chris@831 300
Chris@342 301 if (m_localFilename == "") {
Chris@342 302 // QUrl may have mishandled the scheme (e.g. in a DOS path)
chris@831 303 m_localFilename = m_rawFileOrUrl;
chris@831 304 #ifdef DEBUG_FILE_SOURCE
Chris@843 305 cerr << "FileSource::init: Trying literal local filename \""
Chris@843 306 << m_localFilename << "\"" << endl;
chris@831 307 #endif
Chris@355 308 literal = true;
Chris@342 309 }
Chris@439 310 m_localFilename = QFileInfo(m_localFilename).absoluteFilePath();
Chris@439 311
Chris@355 312 #ifdef DEBUG_FILE_SOURCE
Chris@843 313 cerr << "FileSource::init: URL translates to absolute filename \""
Chris@706 314 << m_localFilename << "\" (with literal=" << literal << ")"
Chris@843 315 << endl;
Chris@355 316 #endif
Chris@316 317 m_ok = true;
Chris@355 318 m_lastStatus = 200;
Chris@355 319
Chris@316 320 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 321 if (literal) {
Chris@355 322 m_lastStatus = 404;
Chris@355 323 } else {
Chris@614 324 #ifdef DEBUG_FILE_SOURCE
Chris@843 325 cerr << "FileSource::init: Local file of this name does not exist, trying URL as a literal filename" << endl;
Chris@614 326 #endif
Chris@355 327 // Again, QUrl may have been mistreating us --
Chris@355 328 // e.g. dropping a part that looks like query data
chris@831 329 m_localFilename = m_rawFileOrUrl;
Chris@355 330 literal = true;
Chris@355 331 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 332 m_lastStatus = 404;
Chris@355 333 }
Chris@355 334 }
Chris@316 335 }
Chris@355 336
Chris@316 337 m_done = true;
Chris@316 338 return;
Chris@316 339 }
Chris@316 340
Chris@316 341 if (createCacheFile()) {
Chris@327 342 #ifdef DEBUG_FILE_SOURCE
Chris@843 343 cerr << "FileSource::init: Already have this one" << endl;
Chris@327 344 #endif
Chris@316 345 m_ok = true;
Chris@316 346 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 347 m_lastStatus = 404;
Chris@316 348 } else {
Chris@316 349 m_lastStatus = 200;
Chris@316 350 }
Chris@316 351 m_done = true;
Chris@316 352 return;
Chris@316 353 }
Chris@316 354
Chris@208 355 if (m_localFilename == "") return;
Chris@706 356
Chris@208 357 m_localFile = new QFile(m_localFilename);
Chris@208 358 m_localFile->open(QFile::WriteOnly);
Chris@208 359
Chris@706 360 if (isResource()) {
Chris@706 361
Chris@706 362 // Absent resource file case was dealt with at the top -- this
Chris@706 363 // is the successful case
Chris@706 364
Chris@706 365 QString resourceFileName = m_url.toString();
Chris@706 366 resourceFileName.replace(QRegExp("^qrc:"), ":");
Chris@706 367 QFile resourceFile(resourceFileName);
Chris@706 368 resourceFile.open(QFile::ReadOnly);
Chris@706 369 QByteArray ba(resourceFile.readAll());
Chris@706 370
Chris@706 371 #ifdef DEBUG_FILE_SOURCE
Chris@843 372 cerr << "Copying " << ba.size() << " bytes from resource file to cache file" << endl;
Chris@706 373 #endif
Chris@706 374
Chris@706 375 qint64 written = m_localFile->write(ba);
Chris@706 376 m_localFile->close();
Chris@706 377 delete m_localFile;
Chris@706 378 m_localFile = 0;
Chris@706 379
Chris@706 380 if (written != ba.size()) {
Chris@706 381 #ifdef DEBUG_FILE_SOURCE
Chris@843 382 cerr << "Copy failed (wrote " << written << " bytes)" << endl;
Chris@706 383 #endif
Chris@706 384 m_ok = false;
Chris@706 385 return;
Chris@706 386 } else {
Chris@706 387 m_ok = true;
Chris@706 388 m_lastStatus = 200;
Chris@706 389 m_done = true;
Chris@706 390 }
Chris@706 391
Chris@706 392 } else {
Chris@706 393
Chris@706 394 QString scheme = m_url.scheme().toLower();
Chris@316 395
Chris@327 396 #ifdef DEBUG_FILE_SOURCE
Chris@843 397 cerr << "FileSource::init: Don't have local copy of \""
Chris@843 398 << m_url.toString() << "\", retrieving" << endl;
Chris@327 399 #endif
Chris@208 400
Chris@762 401 if (scheme == "http" || scheme == "https" || scheme == "ftp") {
Chris@762 402 initRemote();
Chris@520 403 #ifdef DEBUG_FILE_SOURCE
Chris@843 404 cerr << "FileSource: initRemote returned" << endl;
Chris@520 405 #endif
Chris@706 406 } else {
Chris@706 407 m_remote = false;
Chris@706 408 m_ok = false;
Chris@706 409 }
Chris@208 410 }
Chris@208 411
Chris@208 412 if (m_ok) {
Chris@316 413
Chris@316 414 QMutexLocker locker(&m_mapMutex);
Chris@316 415
Chris@316 416 if (m_refCountMap[m_url] > 0) {
Chris@316 417 // someone else has been doing the same thing at the same time,
Chris@316 418 // but has got there first
Chris@316 419 cleanup();
Chris@316 420 m_refCountMap[m_url]++;
Chris@327 421 #ifdef DEBUG_FILE_SOURCE
Chris@843 422 cerr << "FileSource::init: Another FileSource has got there first, abandoning our download and using theirs" << endl;
Chris@327 423 #endif
Chris@316 424 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 425 m_refCounted = true;
Chris@316 426 m_ok = true;
Chris@316 427 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 428 m_lastStatus = 404;
Chris@316 429 }
Chris@316 430 m_done = true;
Chris@316 431 return;
Chris@316 432 }
Chris@304 433
Chris@304 434 m_remoteLocalMap[m_url] = m_localFilename;
Chris@304 435 m_refCountMap[m_url]++;
Chris@316 436 m_refCounted = true;
Chris@304 437
Chris@706 438 if (m_reporter && !m_done) {
Chris@392 439 m_reporter->setMessage
Chris@392 440 (tr("Downloading %1...").arg(m_url.toString()));
Chris@392 441 connect(m_reporter, SIGNAL(cancelled()), this, SLOT(cancelled()));
Chris@357 442 connect(this, SIGNAL(progress(int)),
Chris@392 443 m_reporter, SLOT(setProgress(int)));
Chris@316 444 }
Chris@208 445 }
Chris@208 446 }
Chris@208 447
Chris@316 448 void
Chris@762 449 FileSource::initRemote()
Chris@208 450 {
Chris@316 451 m_ok = true;
Chris@316 452
Chris@762 453 QNetworkRequest req;
Chris@762 454 req.setUrl(m_url);
Chris@762 455
Chris@762 456 if (m_preferredContentType != "") {
Chris@520 457 #ifdef DEBUG_FILE_SOURCE
Chris@843 458 cerr << "FileSource: indicating preferred content type of \""
Chris@843 459 << m_preferredContentType << "\"" << endl;
Chris@520 460 #endif
Chris@762 461 req.setRawHeader
Chris@762 462 ("Accept",
Chris@762 463 QString("%1, */*").arg(m_preferredContentType).toLatin1());
Chris@520 464 }
Chris@316 465
Chris@833 466 m_reply = nms.localData()->get(req);
Chris@762 467
Chris@762 468 connect(m_reply, SIGNAL(readyRead()),
Chris@762 469 this, SLOT(readyRead()));
Chris@762 470 connect(m_reply, SIGNAL(error(QNetworkReply::NetworkError)),
Chris@762 471 this, SLOT(replyFailed(QNetworkReply::NetworkError)));
Chris@762 472 connect(m_reply, SIGNAL(finished()),
Chris@762 473 this, SLOT(replyFinished()));
Chris@764 474 connect(m_reply, SIGNAL(metaDataChanged()),
Chris@764 475 this, SLOT(metaDataChanged()));
Chris@762 476 connect(m_reply, SIGNAL(downloadProgress(qint64, qint64)),
Chris@762 477 this, SLOT(downloadProgress(qint64, qint64)));
Chris@211 478 }
Chris@211 479
Chris@211 480 void
Chris@317 481 FileSource::cleanup()
Chris@211 482 {
Chris@470 483 if (m_done) {
Chris@470 484 delete m_localFile; // does not actually delete the file
Chris@470 485 m_localFile = 0;
Chris@470 486 }
Chris@211 487 m_done = true;
Chris@762 488 if (m_reply) {
Chris@762 489 QNetworkReply *r = m_reply;
Chris@993 490 disconnect(r, 0, this, 0);
Chris@762 491 m_reply = 0;
Chris@913 492 // Can only call abort() when there are no errors.
Chris@913 493 if (r->error() == QNetworkReply::NoError) {
Chris@913 494 r->abort();
Chris@913 495 }
Chris@762 496 r->deleteLater();
Chris@214 497 }
Chris@470 498 if (m_localFile) {
Chris@470 499 delete m_localFile; // does not actually delete the file
Chris@470 500 m_localFile = 0;
Chris@470 501 }
Chris@208 502 }
Chris@208 503
Chris@208 504 bool
Chris@317 505 FileSource::isRemote(QString fileOrUrl)
Chris@304 506 {
Chris@342 507 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@316 508 QString scheme = QUrl(fileOrUrl).scheme().toLower();
Chris@342 509 if (scheme == "" || scheme == "file" || scheme.length() == 1) return false;
Chris@342 510 return true;
Chris@304 511 }
Chris@304 512
Chris@304 513 bool
Chris@317 514 FileSource::canHandleScheme(QUrl url)
Chris@208 515 {
Chris@342 516 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@208 517 QString scheme = url.scheme().toLower();
Chris@762 518 return (scheme == "http" || scheme == "https" ||
Chris@762 519 scheme == "ftp" || scheme == "file" || scheme == "qrc" ||
Chris@706 520 scheme == "" || scheme.length() == 1);
Chris@208 521 }
Chris@208 522
Chris@210 523 bool
Chris@317 524 FileSource::isAvailable()
Chris@210 525 {
Chris@316 526 waitForStatus();
Chris@211 527 bool available = true;
Chris@913 528 if (!m_ok) {
Chris@913 529 available = false;
Chris@913 530 } else {
Chris@913 531 // http 2xx status codes mean success
Chris@913 532 available = (m_lastStatus / 100 == 2);
Chris@913 533 }
Chris@327 534 #ifdef DEBUG_FILE_SOURCE
Chris@913 535 cerr << "FileSource::isAvailable: " << (available ? "yes" : "no") << endl;
Chris@327 536 #endif
Chris@211 537 return available;
Chris@210 538 }
Chris@210 539
Chris@208 540 void
Chris@317 541 FileSource::waitForStatus()
Chris@316 542 {
Chris@316 543 while (m_ok && (!m_done && m_lastStatus == 0)) {
Chris@843 544 // cerr << "waitForStatus: processing (last status " << m_lastStatus << ")" << endl;
Chris@392 545 QCoreApplication::processEvents();
Chris@316 546 }
Chris@316 547 }
Chris@316 548
Chris@316 549 void
Chris@317 550 FileSource::waitForData()
Chris@208 551 {
Chris@211 552 while (m_ok && !m_done) {
Chris@843 553 // cerr << "FileSource::waitForData: calling QApplication::processEvents" << endl;
Chris@392 554 QCoreApplication::processEvents();
Chris@497 555 usleep(10000);
Chris@208 556 }
Chris@208 557 }
Chris@208 558
Chris@316 559 void
Chris@317 560 FileSource::setLeaveLocalFile(bool leave)
Chris@316 561 {
Chris@316 562 m_leaveLocalFile = leave;
Chris@316 563 }
Chris@316 564
Chris@208 565 bool
Chris@317 566 FileSource::isOK() const
Chris@208 567 {
Chris@208 568 return m_ok;
Chris@208 569 }
Chris@208 570
Chris@208 571 bool
Chris@317 572 FileSource::isDone() const
Chris@208 573 {
Chris@208 574 return m_done;
Chris@208 575 }
Chris@208 576
Chris@316 577 bool
Chris@981 578 FileSource::wasCancelled() const
Chris@981 579 {
Chris@981 580 return m_cancelled;
Chris@981 581 }
Chris@981 582
Chris@981 583 bool
Chris@706 584 FileSource::isResource() const
Chris@706 585 {
Chris@706 586 return m_resource;
Chris@706 587 }
Chris@706 588
Chris@706 589 bool
Chris@317 590 FileSource::isRemote() const
Chris@316 591 {
Chris@316 592 return m_remote;
Chris@316 593 }
Chris@316 594
Chris@316 595 QString
Chris@317 596 FileSource::getLocation() const
Chris@316 597 {
Chris@316 598 return m_url.toString();
Chris@316 599 }
Chris@316 600
Chris@208 601 QString
Chris@317 602 FileSource::getLocalFilename() const
Chris@208 603 {
Chris@208 604 return m_localFilename;
Chris@208 605 }
Chris@208 606
Chris@208 607 QString
Chris@678 608 FileSource::getBasename() const
Chris@678 609 {
Chris@678 610 return QFileInfo(m_localFilename).fileName();
Chris@678 611 }
Chris@678 612
Chris@678 613 QString
Chris@317 614 FileSource::getContentType() const
Chris@316 615 {
Chris@316 616 return m_contentType;
Chris@316 617 }
Chris@316 618
Chris@316 619 QString
Chris@317 620 FileSource::getExtension() const
Chris@316 621 {
Chris@316 622 if (m_localFilename != "") {
Chris@316 623 return QFileInfo(m_localFilename).suffix().toLower();
Chris@316 624 } else {
Chris@316 625 return QFileInfo(m_url.toLocalFile()).suffix().toLower();
Chris@316 626 }
Chris@316 627 }
Chris@316 628
Chris@316 629 QString
Chris@317 630 FileSource::getErrorString() const
Chris@208 631 {
Chris@208 632 return m_errorString;
Chris@208 633 }
Chris@208 634
Chris@208 635 void
Chris@762 636 FileSource::readyRead()
Chris@208 637 {
Chris@762 638 m_localFile->write(m_reply->readAll());
Chris@208 639 }
Chris@208 640
Chris@208 641 void
Chris@764 642 FileSource::metaDataChanged()
Chris@210 643 {
Chris@520 644 #ifdef DEBUG_FILE_SOURCE
Chris@843 645 cerr << "FileSource::metaDataChanged" << endl;
Chris@520 646 #endif
Chris@497 647
Chris@762 648 if (!m_reply) {
Chris@843 649 cerr << "WARNING: FileSource::metaDataChanged() called without a reply object being known to us" << endl;
Chris@762 650 return;
Chris@762 651 }
Chris@762 652
Chris@913 653 // Handle http transfer status codes.
Chris@913 654
Chris@762 655 int status =
Chris@762 656 m_reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
Chris@762 657
Chris@913 658 // If this is a redirection (3xx) code, do the redirect
Chris@762 659 if (status / 100 == 3) {
Chris@762 660 QString location = m_reply->header
Chris@762 661 (QNetworkRequest::LocationHeader).toString();
Chris@496 662 #ifdef DEBUG_FILE_SOURCE
Chris@843 663 cerr << "FileSource::metaDataChanged: redirect to \""
Chris@843 664 << location << "\" received" << endl;
Chris@496 665 #endif
Chris@496 666 if (location != "") {
Chris@496 667 QUrl newUrl(location);
Chris@496 668 if (newUrl != m_url) {
Chris@497 669 cleanup();
Chris@497 670 deleteCacheFile();
Chris@529 671 #ifdef DEBUG_FILE_SOURCE
Chris@529 672 decCount(m_url.toString());
Chris@529 673 incCount(newUrl.toString());
Chris@529 674 #endif
Chris@496 675 m_url = newUrl;
Chris@497 676 m_localFile = 0;
Chris@496 677 m_lastStatus = 0;
Chris@497 678 m_done = false;
Chris@497 679 m_refCounted = false;
Chris@496 680 init();
Chris@496 681 return;
Chris@496 682 }
Chris@496 683 }
Chris@496 684 }
Chris@497 685
Chris@762 686 m_lastStatus = status;
Chris@913 687
Chris@913 688 // 400 and up are failures, get the error string
Chris@210 689 if (m_lastStatus / 100 >= 4) {
Chris@210 690 m_errorString = QString("%1 %2")
Chris@762 691 .arg(status)
Chris@762 692 .arg(m_reply->attribute
Chris@762 693 (QNetworkRequest::HttpReasonPhraseAttribute).toString());
Chris@327 694 #ifdef DEBUG_FILE_SOURCE
Chris@843 695 cerr << "FileSource::metaDataChanged: "
Chris@843 696 << m_errorString << endl;
Chris@327 697 #endif
Chris@211 698 } else {
Chris@327 699 #ifdef DEBUG_FILE_SOURCE
Chris@843 700 cerr << "FileSource::metaDataChanged: "
Chris@843 701 << m_lastStatus << endl;
Chris@327 702 #endif
Chris@762 703 m_contentType =
Chris@762 704 m_reply->header(QNetworkRequest::ContentTypeHeader).toString();
Chris@325 705 }
Chris@325 706 emit statusAvailable();
Chris@210 707 }
Chris@210 708
Chris@210 709 void
Chris@762 710 FileSource::downloadProgress(qint64 done, qint64 total)
Chris@208 711 {
Chris@208 712 int percent = int((double(done) / double(total)) * 100.0 - 0.1);
Chris@208 713 emit progress(percent);
Chris@210 714 }
Chris@210 715
Chris@210 716 void
Chris@317 717 FileSource::cancelled()
Chris@210 718 {
Chris@210 719 m_done = true;
Chris@316 720 cleanup();
Chris@316 721
Chris@210 722 m_ok = false;
Chris@981 723 m_cancelled = true;
Chris@210 724 m_errorString = tr("Download cancelled");
Chris@208 725 }
Chris@208 726
Chris@208 727 void
Chris@762 728 FileSource::replyFinished()
Chris@208 729 {
Chris@327 730 emit progress(100);
Chris@327 731
Chris@327 732 #ifdef DEBUG_FILE_SOURCE
Chris@843 733 cerr << "FileSource::replyFinished()" << endl;
Chris@327 734 #endif
Chris@211 735
Chris@211 736 if (m_done) return;
Chris@211 737
Chris@913 738 QString scheme = m_url.scheme().toLower();
Chris@913 739 // For ftp transfers, replyFinished() will be called on success.
Chris@913 740 // metaDataChanged() is never called for ftp transfers.
Chris@913 741 if (scheme == "ftp") {
Chris@913 742 m_lastStatus = 200; // http ok
Chris@913 743 }
Chris@913 744
Chris@762 745 bool error = (m_lastStatus / 100 >= 4);
Chris@210 746
Chris@211 747 cleanup();
Chris@208 748
Chris@211 749 if (!error) {
Chris@208 750 QFileInfo fi(m_localFilename);
Chris@208 751 if (!fi.exists()) {
Chris@208 752 m_errorString = tr("Failed to create local file %1").arg(m_localFilename);
Chris@211 753 error = true;
Chris@208 754 } else if (fi.size() == 0) {
Chris@208 755 m_errorString = tr("File contains no data!");
Chris@211 756 error = true;
Chris@208 757 }
Chris@208 758 }
Chris@211 759
Chris@211 760 if (error) {
Chris@327 761 #ifdef DEBUG_FILE_SOURCE
Chris@843 762 cerr << "FileSource::done: error is " << error << ", deleting cache file" << endl;
Chris@327 763 #endif
Chris@316 764 deleteCacheFile();
Chris@211 765 }
Chris@211 766
Chris@211 767 m_ok = !error;
Chris@520 768 if (m_localFile) m_localFile->flush();
Chris@211 769 m_done = true;
Chris@304 770 emit ready();
Chris@211 771 }
Chris@211 772
Chris@211 773 void
Chris@763 774 FileSource::replyFailed(QNetworkReply::NetworkError)
Chris@763 775 {
Chris@763 776 emit progress(100);
Chris@910 777 if (!m_reply) {
Chris@910 778 cerr << "WARNING: FileSource::replyFailed() called without a reply object being known to us" << endl;
Chris@910 779 } else {
Chris@910 780 m_errorString = m_reply->errorString();
Chris@910 781 }
Chris@763 782 m_ok = false;
Chris@763 783 m_done = true;
Chris@763 784 cleanup();
Chris@763 785 emit ready();
Chris@763 786 }
Chris@763 787
Chris@763 788 void
Chris@317 789 FileSource::deleteCacheFile()
Chris@211 790 {
Chris@327 791 #ifdef DEBUG_FILE_SOURCE
Chris@843 792 cerr << "FileSource::deleteCacheFile(\"" << m_localFilename << "\")" << endl;
Chris@327 793 #endif
Chris@211 794
Chris@211 795 cleanup();
Chris@211 796
Chris@316 797 if (m_localFilename == "") {
Chris@316 798 return;
Chris@316 799 }
Chris@211 800
Chris@316 801 if (!isRemote()) {
Chris@327 802 #ifdef DEBUG_FILE_SOURCE
Chris@843 803 cerr << "not a cache file" << endl;
Chris@327 804 #endif
Chris@316 805 return;
Chris@316 806 }
Chris@316 807
Chris@316 808 if (m_refCounted) {
Chris@304 809
Chris@304 810 QMutexLocker locker(&m_mapMutex);
Chris@316 811 m_refCounted = false;
Chris@304 812
Chris@304 813 if (m_refCountMap[m_url] > 0) {
Chris@304 814 m_refCountMap[m_url]--;
Chris@327 815 #ifdef DEBUG_FILE_SOURCE
Chris@843 816 cerr << "reduced ref count to " << m_refCountMap[m_url] << endl;
Chris@327 817 #endif
Chris@304 818 if (m_refCountMap[m_url] > 0) {
Chris@304 819 m_done = true;
Chris@304 820 return;
Chris@304 821 }
Chris@304 822 }
Chris@304 823 }
Chris@304 824
Chris@211 825 m_fileCreationMutex.lock();
Chris@211 826
Chris@211 827 if (!QFile(m_localFilename).remove()) {
Chris@469 828 #ifdef DEBUG_FILE_SOURCE
Chris@843 829 cerr << "FileSource::deleteCacheFile: ERROR: Failed to delete file \"" << m_localFilename << "\"" << endl;
Chris@469 830 #endif
Chris@211 831 } else {
Chris@327 832 #ifdef DEBUG_FILE_SOURCE
Chris@843 833 cerr << "FileSource::deleteCacheFile: Deleted cache file \"" << m_localFilename << "\"" << endl;
Chris@327 834 #endif
Chris@211 835 m_localFilename = "";
Chris@211 836 }
Chris@211 837
Chris@211 838 m_fileCreationMutex.unlock();
Chris@211 839
Chris@208 840 m_done = true;
Chris@208 841 }
Chris@208 842
Chris@316 843 bool
Chris@317 844 FileSource::createCacheFile()
Chris@208 845 {
Chris@316 846 {
Chris@316 847 QMutexLocker locker(&m_mapMutex);
Chris@316 848
Chris@327 849 #ifdef DEBUG_FILE_SOURCE
Chris@843 850 cerr << "FileSource::createCacheFile: refcount is " << m_refCountMap[m_url] << endl;
Chris@327 851 #endif
Chris@316 852
Chris@316 853 if (m_refCountMap[m_url] > 0) {
Chris@316 854 m_refCountMap[m_url]++;
Chris@316 855 m_localFilename = m_remoteLocalMap[m_url];
Chris@327 856 #ifdef DEBUG_FILE_SOURCE
Chris@843 857 cerr << "raised it to " << m_refCountMap[m_url] << endl;
Chris@327 858 #endif
Chris@316 859 m_refCounted = true;
Chris@316 860 return true;
Chris@316 861 }
Chris@316 862 }
Chris@316 863
Chris@208 864 QDir dir;
Chris@208 865 try {
Chris@208 866 dir = TempDirectory::getInstance()->getSubDirectoryPath("download");
Chris@208 867 } catch (DirectoryCreationFailed f) {
Chris@327 868 #ifdef DEBUG_FILE_SOURCE
Chris@843 869 cerr << "FileSource::createCacheFile: ERROR: Failed to create temporary directory: " << f.what() << endl;
Chris@327 870 #endif
Chris@962 871 return false;
Chris@208 872 }
Chris@208 873
Chris@316 874 QString filepart = m_url.path().section('/', -1, -1,
Chris@316 875 QString::SectionSkipEmpty);
Chris@208 876
Chris@457 877 QString extension = "";
Chris@457 878 if (filepart.contains('.')) extension = filepart.section('.', -1);
Chris@457 879
Chris@208 880 QString base = filepart;
Chris@208 881 if (extension != "") {
Chris@208 882 base = base.left(base.length() - extension.length() - 1);
Chris@208 883 }
Chris@208 884 if (base == "") base = "remote";
Chris@208 885
Chris@208 886 QString filename;
Chris@208 887
Chris@208 888 if (extension == "") {
Chris@208 889 filename = base;
Chris@208 890 } else {
Chris@208 891 filename = QString("%1.%2").arg(base).arg(extension);
Chris@208 892 }
Chris@208 893
Chris@208 894 QString filepath(dir.filePath(filename));
Chris@208 895
Chris@327 896 #ifdef DEBUG_FILE_SOURCE
Chris@843 897 cerr << "FileSource::createCacheFile: URL is \"" << m_url.toString() << "\", dir is \"" << dir.path() << "\", base \"" << base << "\", extension \"" << extension << "\", filebase \"" << filename << "\", filename \"" << filepath << "\"" << endl;
Chris@327 898 #endif
Chris@208 899
Chris@316 900 QMutexLocker fcLocker(&m_fileCreationMutex);
Chris@316 901
Chris@208 902 ++m_count;
Chris@208 903
Chris@208 904 if (QFileInfo(filepath).exists() ||
Chris@208 905 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 906
Chris@327 907 #ifdef DEBUG_FILE_SOURCE
Chris@843 908 cerr << "FileSource::createCacheFile: Failed to create local file \""
Chris@686 909 << filepath << "\" for URL \""
Chris@843 910 << m_url.toString() << "\" (or file already exists): appending suffix instead" << endl;
Chris@327 911 #endif
Chris@208 912
Chris@208 913 if (extension == "") {
Chris@208 914 filename = QString("%1_%2").arg(base).arg(m_count);
Chris@208 915 } else {
Chris@208 916 filename = QString("%1_%2.%3").arg(base).arg(m_count).arg(extension);
Chris@208 917 }
Chris@208 918 filepath = dir.filePath(filename);
Chris@208 919
Chris@208 920 if (QFileInfo(filepath).exists() ||
Chris@208 921 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 922
Chris@327 923 #ifdef DEBUG_FILE_SOURCE
Chris@843 924 cerr << "FileSource::createCacheFile: ERROR: Failed to create local file \""
Chris@686 925 << filepath << "\" for URL \""
Chris@843 926 << m_url.toString() << "\" (or file already exists)" << endl;
Chris@327 927 #endif
Chris@208 928
Chris@962 929 return false;
Chris@208 930 }
Chris@208 931 }
Chris@208 932
Chris@327 933 #ifdef DEBUG_FILE_SOURCE
Chris@843 934 cerr << "FileSource::createCacheFile: url "
Chris@686 935 << m_url.toString() << " -> local filename "
Chris@843 936 << filepath << endl;
Chris@327 937 #endif
Chris@316 938
Chris@316 939 m_localFilename = filepath;
Chris@208 940
Chris@316 941 return false;
Chris@208 942 }
Chris@327 943