annotate data/fileio/FileSource.cpp @ 1443:cdb316c6a151 streaming-csv-writer

Fix typo of old name
author Lucas Thompson <dev@lucas.im>
date Tue, 17 Apr 2018 10:03:50 +0100
parents 794b64e62e58
children cee1be4fb8c1
rev   line source
Chris@208 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@208 2
Chris@208 3 /*
Chris@208 4 Sonic Visualiser
Chris@208 5 An audio file viewer and annotation editor.
Chris@208 6 Centre for Digital Music, Queen Mary, University of London.
Chris@208 7 This file copyright 2007 QMUL.
Chris@208 8
Chris@208 9 This program is free software; you can redistribute it and/or
Chris@208 10 modify it under the terms of the GNU General Public License as
Chris@208 11 published by the Free Software Foundation; either version 2 of the
Chris@208 12 License, or (at your option) any later version. See the file
Chris@208 13 COPYING included with this distribution for more information.
Chris@208 14 */
Chris@208 15
Chris@317 16 #include "FileSource.h"
Chris@357 17
Chris@208 18 #include "base/TempDirectory.h"
Chris@208 19 #include "base/Exceptions.h"
Chris@392 20 #include "base/ProgressReporter.h"
Chris@502 21 #include "system/System.h"
Chris@208 22
Chris@762 23 #include <QNetworkAccessManager>
Chris@762 24 #include <QNetworkReply>
Chris@208 25 #include <QFileInfo>
Chris@208 26 #include <QDir>
Chris@392 27 #include <QCoreApplication>
Chris@833 28 #include <QThreadStorage>
Chris@208 29
Chris@208 30 #include <iostream>
Chris@405 31 #include <cstdlib>
Chris@208 32
Chris@1067 33 //#define DEBUG_FILE_SOURCE 1
Chris@327 34
Chris@208 35 int
Chris@317 36 FileSource::m_count = 0;
Chris@208 37
Chris@208 38 QMutex
Chris@317 39 FileSource::m_fileCreationMutex;
Chris@208 40
Chris@317 41 FileSource::RemoteRefCountMap
Chris@317 42 FileSource::m_refCountMap;
Chris@304 43
Chris@317 44 FileSource::RemoteLocalMap
Chris@317 45 FileSource::m_remoteLocalMap;
Chris@304 46
Chris@304 47 QMutex
Chris@317 48 FileSource::m_mapMutex;
Chris@304 49
Chris@529 50 #ifdef DEBUG_FILE_SOURCE
Chris@529 51 static int extantCount = 0;
Chris@1033 52 static int threadCount = 0;
Chris@529 53 static std::map<QString, int> urlExtantCountMap;
Chris@1033 54 static QMutex countMutex;
Chris@529 55 static void incCount(QString url) {
Chris@1033 56 QMutexLocker locker(&countMutex);
Chris@529 57 ++extantCount;
Chris@529 58 if (urlExtantCountMap.find(url) == urlExtantCountMap.end()) {
Chris@529 59 urlExtantCountMap[url] = 1;
Chris@529 60 } else {
Chris@529 61 ++urlExtantCountMap[url];
Chris@529 62 }
Chris@843 63 cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << endl;
Chris@529 64 }
Chris@529 65 static void decCount(QString url) {
Chris@1033 66 QMutexLocker locker(&countMutex);
Chris@529 67 --extantCount;
Chris@529 68 --urlExtantCountMap[url];
Chris@843 69 cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << endl;
Chris@529 70 }
Chris@1033 71 void
Chris@1033 72 FileSource::debugReport()
Chris@1033 73 {
Chris@1033 74 QMutexLocker locker(&countMutex);
Chris@1033 75 cerr << "\nFileSource::debugReport: Have " << extantCount << " FileSource object(s) extant across " << threadCount << " thread(s)" << endl;
Chris@1033 76 cerr << "URLs by extant count:" << endl;
Chris@1033 77 cerr << "Count | URL" << endl;
Chris@1033 78 for (std::map<QString, int>::const_iterator i = urlExtantCountMap.begin();
Chris@1033 79 i != urlExtantCountMap.end(); ++i) {
Chris@1033 80 cerr << i->second << " | " << i->first << endl;
Chris@1033 81 }
Chris@1033 82 cerr << "FileSource::debugReport done\n" << endl;
Chris@1033 83 }
Chris@1033 84 #else
Chris@1033 85 void FileSource::debugReport() { }
Chris@529 86 #endif
Chris@529 87
Chris@833 88 static QThreadStorage<QNetworkAccessManager *> nms;
Chris@762 89
Chris@520 90 FileSource::FileSource(QString fileOrUrl, ProgressReporter *reporter,
Chris@520 91 QString preferredContentType) :
chris@831 92 m_rawFileOrUrl(fileOrUrl),
Chris@614 93 m_url(fileOrUrl, QUrl::StrictMode),
Chris@316 94 m_localFile(0),
Chris@762 95 m_reply(0),
Chris@520 96 m_preferredContentType(preferredContentType),
Chris@316 97 m_ok(false),
Chris@981 98 m_cancelled(false),
Chris@316 99 m_lastStatus(0),
Chris@706 100 m_resource(fileOrUrl.startsWith(':')),
Chris@316 101 m_remote(isRemote(fileOrUrl)),
Chris@316 102 m_done(false),
Chris@316 103 m_leaveLocalFile(false),
Chris@392 104 m_reporter(reporter),
Chris@316 105 m_refCounted(false)
Chris@316 106 {
Chris@706 107 if (m_resource) { // qrc file
Chris@706 108 m_url = QUrl("qrc" + fileOrUrl);
Chris@706 109 }
Chris@706 110
Chris@661 111 if (m_url.toString() == "") {
Chris@661 112 m_url = QUrl(fileOrUrl, QUrl::TolerantMode);
Chris@661 113 }
Chris@661 114
Chris@327 115 #ifdef DEBUG_FILE_SOURCE
Chris@843 116 cerr << "FileSource::FileSource(" << fileOrUrl << "): url <" << m_url.toString() << ">" << endl;
Chris@529 117 incCount(m_url.toString());
Chris@327 118 #endif
Chris@316 119
Chris@316 120 if (!canHandleScheme(m_url)) {
Chris@843 121 cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316 122 m_errorString = tr("Unsupported scheme in URL");
Chris@316 123 return;
Chris@316 124 }
Chris@316 125
Chris@357 126 init();
Chris@316 127
Chris@614 128 if (!isRemote() &&
Chris@614 129 !isAvailable()) {
Chris@614 130 #ifdef DEBUG_FILE_SOURCE
Chris@843 131 cerr << "FileSource::FileSource: Failed to open local file with URL \"" << m_url.toString() << "\"; trying again assuming filename was encoded" << endl;
Chris@614 132 #endif
Chris@762 133 m_url = QUrl::fromEncoded(fileOrUrl.toLatin1());
chris@831 134 #ifdef DEBUG_FILE_SOURCE
Chris@843 135 cerr << "FileSource::FileSource: URL is now \"" << m_url.toString() << "\"" << endl;
chris@831 136 #endif
Chris@614 137 init();
Chris@614 138 }
Chris@614 139
Chris@316 140 if (isRemote() &&
Chris@316 141 (fileOrUrl.contains('%') ||
Chris@316 142 fileOrUrl.contains("--"))) { // for IDNA
Chris@316 143
Chris@316 144 waitForStatus();
Chris@316 145
Chris@316 146 if (!isAvailable()) {
Chris@336 147
Chris@316 148 // The URL was created on the assumption that the string
Chris@316 149 // was human-readable. Let's try again, this time
Chris@316 150 // assuming it was already encoded.
Chris@843 151 cerr << "FileSource::FileSource: Failed to retrieve URL \""
Chris@844 152 << fileOrUrl
Chris@316 153 << "\" as human-readable URL; "
Chris@316 154 << "trying again treating it as encoded URL"
Chris@843 155 << endl;
Chris@336 156
Chris@336 157 // even though our cache file doesn't exist (because the
Chris@336 158 // resource was 404), we still need to ensure we're no
Chris@336 159 // longer associating a filename with this url in the
Chris@336 160 // refcount map -- or createCacheFile will think we've
Chris@336 161 // already done all the work and no request will be sent
Chris@336 162 deleteCacheFile();
Chris@336 163
Chris@762 164 m_url = QUrl::fromEncoded(fileOrUrl.toLatin1());
Chris@336 165
Chris@336 166 m_ok = false;
Chris@336 167 m_done = false;
Chris@336 168 m_lastStatus = 0;
Chris@357 169 init();
Chris@316 170 }
Chris@316 171 }
Chris@325 172
Chris@325 173 if (!isRemote()) {
Chris@325 174 emit statusAvailable();
Chris@325 175 emit ready();
Chris@325 176 }
Chris@497 177
Chris@504 178 #ifdef DEBUG_FILE_SOURCE
Chris@843 179 cerr << "FileSource::FileSource(string) exiting" << endl;
Chris@504 180 #endif
Chris@316 181 }
Chris@316 182
Chris@469 183 FileSource::FileSource(QUrl url, ProgressReporter *reporter) :
Chris@304 184 m_url(url),
Chris@208 185 m_localFile(0),
Chris@762 186 m_reply(0),
Chris@208 187 m_ok(false),
Chris@981 188 m_cancelled(false),
Chris@210 189 m_lastStatus(0),
Chris@706 190 m_resource(false),
Chris@316 191 m_remote(isRemote(url.toString())),
Chris@208 192 m_done(false),
Chris@316 193 m_leaveLocalFile(false),
Chris@392 194 m_reporter(reporter),
Chris@316 195 m_refCounted(false)
Chris@208 196 {
Chris@327 197 #ifdef DEBUG_FILE_SOURCE
Chris@843 198 cerr << "FileSource::FileSource(" << url.toString() << ") [as url]" << endl;
Chris@529 199 incCount(m_url.toString());
Chris@327 200 #endif
Chris@316 201
Chris@316 202 if (!canHandleScheme(m_url)) {
Chris@843 203 cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316 204 m_errorString = tr("Unsupported scheme in URL");
Chris@208 205 return;
Chris@208 206 }
Chris@208 207
Chris@357 208 init();
Chris@497 209
Chris@504 210 #ifdef DEBUG_FILE_SOURCE
Chris@843 211 cerr << "FileSource::FileSource(url) exiting" << endl;
Chris@504 212 #endif
Chris@316 213 }
Chris@304 214
Chris@317 215 FileSource::FileSource(const FileSource &rf) :
Chris@316 216 QObject(),
Chris@316 217 m_url(rf.m_url),
Chris@316 218 m_localFile(0),
Chris@762 219 m_reply(0),
Chris@316 220 m_ok(rf.m_ok),
Chris@981 221 m_cancelled(rf.m_cancelled),
Chris@316 222 m_lastStatus(rf.m_lastStatus),
Chris@706 223 m_resource(rf.m_resource),
Chris@316 224 m_remote(rf.m_remote),
Chris@316 225 m_done(false),
Chris@316 226 m_leaveLocalFile(false),
Chris@392 227 m_reporter(rf.m_reporter),
Chris@316 228 m_refCounted(false)
Chris@316 229 {
Chris@327 230 #ifdef DEBUG_FILE_SOURCE
Chris@843 231 cerr << "FileSource::FileSource(" << m_url.toString() << ") [copy ctor]" << endl;
Chris@529 232 incCount(m_url.toString());
Chris@327 233 #endif
Chris@304 234
Chris@316 235 if (!canHandleScheme(m_url)) {
Chris@843 236 cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316 237 m_errorString = tr("Unsupported scheme in URL");
Chris@304 238 return;
Chris@304 239 }
Chris@304 240
Chris@469 241 if (!isRemote()) {
Chris@316 242 m_localFilename = rf.m_localFilename;
Chris@316 243 } else {
Chris@316 244 QMutexLocker locker(&m_mapMutex);
Chris@327 245 #ifdef DEBUG_FILE_SOURCE
Chris@843 246 cerr << "FileSource::FileSource(copy ctor): ref count is "
Chris@843 247 << m_refCountMap[m_url] << endl;
Chris@327 248 #endif
Chris@316 249 if (m_refCountMap[m_url] > 0) {
Chris@316 250 m_refCountMap[m_url]++;
Chris@327 251 #ifdef DEBUG_FILE_SOURCE
Chris@843 252 cerr << "raised it to " << m_refCountMap[m_url] << endl;
Chris@327 253 #endif
Chris@316 254 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 255 m_refCounted = true;
Chris@316 256 } else {
Chris@316 257 m_ok = false;
Chris@316 258 m_lastStatus = 404;
Chris@316 259 }
Chris@316 260 }
Chris@316 261
Chris@316 262 m_done = true;
Chris@497 263
Chris@504 264 #ifdef DEBUG_FILE_SOURCE
Chris@843 265 cerr << "FileSource::FileSource(" << m_url.toString() << ") [copy ctor]: note: local filename is \"" << m_localFilename << "\"" << endl;
Chris@527 266 #endif
Chris@527 267
Chris@527 268 #ifdef DEBUG_FILE_SOURCE
Chris@843 269 cerr << "FileSource::FileSource(copy ctor) exiting" << endl;
Chris@504 270 #endif
Chris@316 271 }
Chris@316 272
Chris@317 273 FileSource::~FileSource()
Chris@316 274 {
Chris@327 275 #ifdef DEBUG_FILE_SOURCE
Chris@843 276 cerr << "FileSource(" << m_url.toString() << ")::~FileSource" << endl;
Chris@529 277 decCount(m_url.toString());
Chris@327 278 #endif
Chris@316 279
Chris@316 280 cleanup();
Chris@316 281
Chris@469 282 if (isRemote() && !m_leaveLocalFile) deleteCacheFile();
Chris@316 283 }
Chris@316 284
Chris@316 285 void
Chris@357 286 FileSource::init()
Chris@316 287 {
Chris@706 288 if (isResource()) {
Chris@355 289 #ifdef DEBUG_FILE_SOURCE
Chris@843 290 cerr << "FileSource::init: Is a resource" << endl;
Chris@706 291 #endif
Chris@706 292 QString resourceFile = m_url.toString();
Chris@706 293 resourceFile.replace(QRegExp("^qrc:"), ":");
Chris@706 294
Chris@706 295 if (!QFileInfo(resourceFile).exists()) {
Chris@706 296 #ifdef DEBUG_FILE_SOURCE
Chris@843 297 cerr << "FileSource::init: Resource file of this name does not exist, switching to non-resource URL" << endl;
Chris@706 298 #endif
Chris@706 299 m_url = resourceFile;
Chris@706 300 m_resource = false;
Chris@706 301 }
Chris@706 302 }
Chris@706 303
Chris@706 304 if (!isRemote() && !isResource()) {
Chris@706 305 #ifdef DEBUG_FILE_SOURCE
Chris@843 306 cerr << "FileSource::init: Not a remote URL" << endl;
Chris@355 307 #endif
Chris@355 308 bool literal = false;
Chris@316 309 m_localFilename = m_url.toLocalFile();
chris@831 310
Chris@342 311 if (m_localFilename == "") {
Chris@342 312 // QUrl may have mishandled the scheme (e.g. in a DOS path)
chris@831 313 m_localFilename = m_rawFileOrUrl;
chris@831 314 #ifdef DEBUG_FILE_SOURCE
Chris@843 315 cerr << "FileSource::init: Trying literal local filename \""
Chris@843 316 << m_localFilename << "\"" << endl;
chris@831 317 #endif
Chris@355 318 literal = true;
Chris@342 319 }
Chris@439 320 m_localFilename = QFileInfo(m_localFilename).absoluteFilePath();
Chris@439 321
Chris@355 322 #ifdef DEBUG_FILE_SOURCE
Chris@843 323 cerr << "FileSource::init: URL translates to absolute filename \""
Chris@706 324 << m_localFilename << "\" (with literal=" << literal << ")"
Chris@843 325 << endl;
Chris@355 326 #endif
Chris@316 327 m_ok = true;
Chris@355 328 m_lastStatus = 200;
Chris@355 329
Chris@316 330 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 331 if (literal) {
Chris@355 332 m_lastStatus = 404;
Chris@355 333 } else {
Chris@614 334 #ifdef DEBUG_FILE_SOURCE
Chris@843 335 cerr << "FileSource::init: Local file of this name does not exist, trying URL as a literal filename" << endl;
Chris@614 336 #endif
Chris@355 337 // Again, QUrl may have been mistreating us --
Chris@355 338 // e.g. dropping a part that looks like query data
chris@831 339 m_localFilename = m_rawFileOrUrl;
Chris@355 340 literal = true;
Chris@355 341 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 342 m_lastStatus = 404;
Chris@355 343 }
Chris@355 344 }
Chris@316 345 }
Chris@355 346
Chris@316 347 m_done = true;
Chris@316 348 return;
Chris@316 349 }
Chris@316 350
Chris@316 351 if (createCacheFile()) {
Chris@327 352 #ifdef DEBUG_FILE_SOURCE
Chris@843 353 cerr << "FileSource::init: Already have this one" << endl;
Chris@327 354 #endif
Chris@316 355 m_ok = true;
Chris@316 356 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 357 m_lastStatus = 404;
Chris@316 358 } else {
Chris@316 359 m_lastStatus = 200;
Chris@316 360 }
Chris@316 361 m_done = true;
Chris@316 362 return;
Chris@316 363 }
Chris@316 364
Chris@208 365 if (m_localFilename == "") return;
Chris@706 366
Chris@208 367 m_localFile = new QFile(m_localFilename);
Chris@208 368 m_localFile->open(QFile::WriteOnly);
Chris@208 369
Chris@706 370 if (isResource()) {
Chris@706 371
Chris@706 372 // Absent resource file case was dealt with at the top -- this
Chris@706 373 // is the successful case
Chris@706 374
Chris@706 375 QString resourceFileName = m_url.toString();
Chris@706 376 resourceFileName.replace(QRegExp("^qrc:"), ":");
Chris@706 377 QFile resourceFile(resourceFileName);
Chris@706 378 resourceFile.open(QFile::ReadOnly);
Chris@706 379 QByteArray ba(resourceFile.readAll());
Chris@706 380
Chris@706 381 #ifdef DEBUG_FILE_SOURCE
Chris@843 382 cerr << "Copying " << ba.size() << " bytes from resource file to cache file" << endl;
Chris@706 383 #endif
Chris@706 384
Chris@706 385 qint64 written = m_localFile->write(ba);
Chris@706 386 m_localFile->close();
Chris@706 387 delete m_localFile;
Chris@706 388 m_localFile = 0;
Chris@706 389
Chris@706 390 if (written != ba.size()) {
Chris@706 391 #ifdef DEBUG_FILE_SOURCE
Chris@843 392 cerr << "Copy failed (wrote " << written << " bytes)" << endl;
Chris@706 393 #endif
Chris@706 394 m_ok = false;
Chris@706 395 return;
Chris@706 396 } else {
Chris@706 397 m_ok = true;
Chris@706 398 m_lastStatus = 200;
Chris@706 399 m_done = true;
Chris@706 400 }
Chris@706 401
Chris@706 402 } else {
Chris@706 403
Chris@706 404 QString scheme = m_url.scheme().toLower();
Chris@316 405
Chris@327 406 #ifdef DEBUG_FILE_SOURCE
Chris@843 407 cerr << "FileSource::init: Don't have local copy of \""
Chris@843 408 << m_url.toString() << "\", retrieving" << endl;
Chris@327 409 #endif
Chris@208 410
Chris@762 411 if (scheme == "http" || scheme == "https" || scheme == "ftp") {
Chris@762 412 initRemote();
Chris@520 413 #ifdef DEBUG_FILE_SOURCE
Chris@843 414 cerr << "FileSource: initRemote returned" << endl;
Chris@520 415 #endif
Chris@706 416 } else {
Chris@706 417 m_remote = false;
Chris@706 418 m_ok = false;
Chris@706 419 }
Chris@208 420 }
Chris@208 421
Chris@208 422 if (m_ok) {
Chris@316 423
Chris@316 424 QMutexLocker locker(&m_mapMutex);
Chris@316 425
Chris@316 426 if (m_refCountMap[m_url] > 0) {
Chris@316 427 // someone else has been doing the same thing at the same time,
Chris@316 428 // but has got there first
Chris@316 429 cleanup();
Chris@316 430 m_refCountMap[m_url]++;
Chris@327 431 #ifdef DEBUG_FILE_SOURCE
Chris@843 432 cerr << "FileSource::init: Another FileSource has got there first, abandoning our download and using theirs" << endl;
Chris@327 433 #endif
Chris@316 434 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 435 m_refCounted = true;
Chris@316 436 m_ok = true;
Chris@316 437 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 438 m_lastStatus = 404;
Chris@316 439 }
Chris@316 440 m_done = true;
Chris@316 441 return;
Chris@316 442 }
Chris@304 443
Chris@304 444 m_remoteLocalMap[m_url] = m_localFilename;
Chris@304 445 m_refCountMap[m_url]++;
Chris@316 446 m_refCounted = true;
Chris@304 447
Chris@706 448 if (m_reporter && !m_done) {
Chris@392 449 m_reporter->setMessage
Chris@392 450 (tr("Downloading %1...").arg(m_url.toString()));
Chris@392 451 connect(m_reporter, SIGNAL(cancelled()), this, SLOT(cancelled()));
Chris@357 452 connect(this, SIGNAL(progress(int)),
Chris@392 453 m_reporter, SLOT(setProgress(int)));
Chris@316 454 }
Chris@208 455 }
Chris@208 456 }
Chris@208 457
Chris@316 458 void
Chris@762 459 FileSource::initRemote()
Chris@208 460 {
Chris@316 461 m_ok = true;
Chris@316 462
Chris@762 463 QNetworkRequest req;
Chris@762 464 req.setUrl(m_url);
Chris@762 465
Chris@762 466 if (m_preferredContentType != "") {
Chris@520 467 #ifdef DEBUG_FILE_SOURCE
Chris@843 468 cerr << "FileSource: indicating preferred content type of \""
Chris@843 469 << m_preferredContentType << "\"" << endl;
Chris@520 470 #endif
Chris@762 471 req.setRawHeader
Chris@762 472 ("Accept",
Chris@762 473 QString("%1, */*").arg(m_preferredContentType).toLatin1());
Chris@520 474 }
Chris@316 475
Chris@1033 476 { // check we have a QNetworkAccessManager
Chris@1033 477 QMutexLocker locker(&m_mapMutex);
Chris@1033 478 if (!nms.hasLocalData()) {
Chris@1033 479 #ifdef DEBUG_FILE_SOURCE
Chris@1033 480 ++threadCount;
Chris@1033 481 #endif
Chris@1033 482 nms.setLocalData(new QNetworkAccessManager());
Chris@1033 483 }
Chris@1033 484 }
Chris@1033 485
Chris@833 486 m_reply = nms.localData()->get(req);
Chris@762 487
Chris@762 488 connect(m_reply, SIGNAL(readyRead()),
Chris@762 489 this, SLOT(readyRead()));
Chris@762 490 connect(m_reply, SIGNAL(error(QNetworkReply::NetworkError)),
Chris@762 491 this, SLOT(replyFailed(QNetworkReply::NetworkError)));
Chris@762 492 connect(m_reply, SIGNAL(finished()),
Chris@762 493 this, SLOT(replyFinished()));
Chris@764 494 connect(m_reply, SIGNAL(metaDataChanged()),
Chris@764 495 this, SLOT(metaDataChanged()));
Chris@762 496 connect(m_reply, SIGNAL(downloadProgress(qint64, qint64)),
Chris@762 497 this, SLOT(downloadProgress(qint64, qint64)));
Chris@211 498 }
Chris@211 499
Chris@211 500 void
Chris@317 501 FileSource::cleanup()
Chris@211 502 {
Chris@470 503 if (m_done) {
Chris@470 504 delete m_localFile; // does not actually delete the file
Chris@470 505 m_localFile = 0;
Chris@470 506 }
Chris@211 507 m_done = true;
Chris@762 508 if (m_reply) {
Chris@762 509 QNetworkReply *r = m_reply;
Chris@993 510 disconnect(r, 0, this, 0);
Chris@762 511 m_reply = 0;
Chris@913 512 // Can only call abort() when there are no errors.
Chris@913 513 if (r->error() == QNetworkReply::NoError) {
Chris@913 514 r->abort();
Chris@913 515 }
Chris@762 516 r->deleteLater();
Chris@214 517 }
Chris@470 518 if (m_localFile) {
Chris@470 519 delete m_localFile; // does not actually delete the file
Chris@470 520 m_localFile = 0;
Chris@470 521 }
Chris@208 522 }
Chris@208 523
Chris@208 524 bool
Chris@317 525 FileSource::isRemote(QString fileOrUrl)
Chris@304 526 {
Chris@342 527 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@316 528 QString scheme = QUrl(fileOrUrl).scheme().toLower();
Chris@342 529 if (scheme == "" || scheme == "file" || scheme.length() == 1) return false;
Chris@342 530 return true;
Chris@304 531 }
Chris@304 532
Chris@304 533 bool
Chris@317 534 FileSource::canHandleScheme(QUrl url)
Chris@208 535 {
Chris@342 536 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@208 537 QString scheme = url.scheme().toLower();
Chris@762 538 return (scheme == "http" || scheme == "https" ||
Chris@762 539 scheme == "ftp" || scheme == "file" || scheme == "qrc" ||
Chris@706 540 scheme == "" || scheme.length() == 1);
Chris@208 541 }
Chris@208 542
Chris@210 543 bool
Chris@317 544 FileSource::isAvailable()
Chris@210 545 {
Chris@316 546 waitForStatus();
Chris@211 547 bool available = true;
Chris@913 548 if (!m_ok) {
Chris@913 549 available = false;
Chris@913 550 } else {
Chris@913 551 // http 2xx status codes mean success
Chris@913 552 available = (m_lastStatus / 100 == 2);
Chris@913 553 }
Chris@327 554 #ifdef DEBUG_FILE_SOURCE
Chris@913 555 cerr << "FileSource::isAvailable: " << (available ? "yes" : "no") << endl;
Chris@327 556 #endif
Chris@211 557 return available;
Chris@210 558 }
Chris@210 559
Chris@208 560 void
Chris@317 561 FileSource::waitForStatus()
Chris@316 562 {
Chris@316 563 while (m_ok && (!m_done && m_lastStatus == 0)) {
Chris@843 564 // cerr << "waitForStatus: processing (last status " << m_lastStatus << ")" << endl;
Chris@392 565 QCoreApplication::processEvents();
Chris@316 566 }
Chris@316 567 }
Chris@316 568
Chris@316 569 void
Chris@317 570 FileSource::waitForData()
Chris@208 571 {
Chris@211 572 while (m_ok && !m_done) {
Chris@843 573 // cerr << "FileSource::waitForData: calling QApplication::processEvents" << endl;
Chris@392 574 QCoreApplication::processEvents();
Chris@497 575 usleep(10000);
Chris@208 576 }
Chris@208 577 }
Chris@208 578
Chris@316 579 void
Chris@317 580 FileSource::setLeaveLocalFile(bool leave)
Chris@316 581 {
Chris@316 582 m_leaveLocalFile = leave;
Chris@316 583 }
Chris@316 584
Chris@208 585 bool
Chris@317 586 FileSource::isOK() const
Chris@208 587 {
Chris@208 588 return m_ok;
Chris@208 589 }
Chris@208 590
Chris@208 591 bool
Chris@317 592 FileSource::isDone() const
Chris@208 593 {
Chris@208 594 return m_done;
Chris@208 595 }
Chris@208 596
Chris@316 597 bool
Chris@981 598 FileSource::wasCancelled() const
Chris@981 599 {
Chris@981 600 return m_cancelled;
Chris@981 601 }
Chris@981 602
Chris@981 603 bool
Chris@706 604 FileSource::isResource() const
Chris@706 605 {
Chris@706 606 return m_resource;
Chris@706 607 }
Chris@706 608
Chris@706 609 bool
Chris@317 610 FileSource::isRemote() const
Chris@316 611 {
Chris@316 612 return m_remote;
Chris@316 613 }
Chris@316 614
Chris@316 615 QString
Chris@317 616 FileSource::getLocation() const
Chris@316 617 {
Chris@316 618 return m_url.toString();
Chris@316 619 }
Chris@316 620
Chris@208 621 QString
Chris@317 622 FileSource::getLocalFilename() const
Chris@208 623 {
Chris@208 624 return m_localFilename;
Chris@208 625 }
Chris@208 626
Chris@208 627 QString
Chris@678 628 FileSource::getBasename() const
Chris@678 629 {
Chris@678 630 return QFileInfo(m_localFilename).fileName();
Chris@678 631 }
Chris@678 632
Chris@678 633 QString
Chris@317 634 FileSource::getContentType() const
Chris@316 635 {
Chris@316 636 return m_contentType;
Chris@316 637 }
Chris@316 638
Chris@316 639 QString
Chris@317 640 FileSource::getExtension() const
Chris@316 641 {
Chris@316 642 if (m_localFilename != "") {
Chris@316 643 return QFileInfo(m_localFilename).suffix().toLower();
Chris@316 644 } else {
Chris@316 645 return QFileInfo(m_url.toLocalFile()).suffix().toLower();
Chris@316 646 }
Chris@316 647 }
Chris@316 648
Chris@316 649 QString
Chris@317 650 FileSource::getErrorString() const
Chris@208 651 {
Chris@208 652 return m_errorString;
Chris@208 653 }
Chris@208 654
Chris@208 655 void
Chris@762 656 FileSource::readyRead()
Chris@208 657 {
Chris@762 658 m_localFile->write(m_reply->readAll());
Chris@208 659 }
Chris@208 660
Chris@208 661 void
Chris@764 662 FileSource::metaDataChanged()
Chris@210 663 {
Chris@520 664 #ifdef DEBUG_FILE_SOURCE
Chris@843 665 cerr << "FileSource::metaDataChanged" << endl;
Chris@520 666 #endif
Chris@497 667
Chris@762 668 if (!m_reply) {
Chris@843 669 cerr << "WARNING: FileSource::metaDataChanged() called without a reply object being known to us" << endl;
Chris@762 670 return;
Chris@762 671 }
Chris@762 672
Chris@913 673 // Handle http transfer status codes.
Chris@913 674
Chris@762 675 int status =
Chris@762 676 m_reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
Chris@762 677
Chris@913 678 // If this is a redirection (3xx) code, do the redirect
Chris@762 679 if (status / 100 == 3) {
Chris@762 680 QString location = m_reply->header
Chris@762 681 (QNetworkRequest::LocationHeader).toString();
Chris@496 682 #ifdef DEBUG_FILE_SOURCE
Chris@843 683 cerr << "FileSource::metaDataChanged: redirect to \""
Chris@843 684 << location << "\" received" << endl;
Chris@496 685 #endif
Chris@496 686 if (location != "") {
Chris@496 687 QUrl newUrl(location);
Chris@496 688 if (newUrl != m_url) {
Chris@497 689 cleanup();
Chris@497 690 deleteCacheFile();
Chris@529 691 #ifdef DEBUG_FILE_SOURCE
Chris@529 692 decCount(m_url.toString());
Chris@529 693 incCount(newUrl.toString());
Chris@529 694 #endif
Chris@496 695 m_url = newUrl;
Chris@497 696 m_localFile = 0;
Chris@496 697 m_lastStatus = 0;
Chris@497 698 m_done = false;
Chris@497 699 m_refCounted = false;
Chris@496 700 init();
Chris@496 701 return;
Chris@496 702 }
Chris@496 703 }
Chris@496 704 }
Chris@497 705
Chris@762 706 m_lastStatus = status;
Chris@913 707
Chris@913 708 // 400 and up are failures, get the error string
Chris@210 709 if (m_lastStatus / 100 >= 4) {
Chris@210 710 m_errorString = QString("%1 %2")
Chris@762 711 .arg(status)
Chris@762 712 .arg(m_reply->attribute
Chris@762 713 (QNetworkRequest::HttpReasonPhraseAttribute).toString());
Chris@327 714 #ifdef DEBUG_FILE_SOURCE
Chris@843 715 cerr << "FileSource::metaDataChanged: "
Chris@843 716 << m_errorString << endl;
Chris@327 717 #endif
Chris@211 718 } else {
Chris@327 719 #ifdef DEBUG_FILE_SOURCE
Chris@843 720 cerr << "FileSource::metaDataChanged: "
Chris@843 721 << m_lastStatus << endl;
Chris@327 722 #endif
Chris@762 723 m_contentType =
Chris@762 724 m_reply->header(QNetworkRequest::ContentTypeHeader).toString();
Chris@325 725 }
Chris@325 726 emit statusAvailable();
Chris@210 727 }
Chris@210 728
Chris@210 729 void
Chris@762 730 FileSource::downloadProgress(qint64 done, qint64 total)
Chris@208 731 {
Chris@208 732 int percent = int((double(done) / double(total)) * 100.0 - 0.1);
Chris@208 733 emit progress(percent);
Chris@210 734 }
Chris@210 735
Chris@210 736 void
Chris@317 737 FileSource::cancelled()
Chris@210 738 {
Chris@210 739 m_done = true;
Chris@316 740 cleanup();
Chris@316 741
Chris@210 742 m_ok = false;
Chris@981 743 m_cancelled = true;
Chris@210 744 m_errorString = tr("Download cancelled");
Chris@208 745 }
Chris@208 746
Chris@208 747 void
Chris@762 748 FileSource::replyFinished()
Chris@208 749 {
Chris@327 750 emit progress(100);
Chris@327 751
Chris@327 752 #ifdef DEBUG_FILE_SOURCE
Chris@843 753 cerr << "FileSource::replyFinished()" << endl;
Chris@327 754 #endif
Chris@211 755
Chris@211 756 if (m_done) return;
Chris@211 757
Chris@913 758 QString scheme = m_url.scheme().toLower();
Chris@913 759 // For ftp transfers, replyFinished() will be called on success.
Chris@913 760 // metaDataChanged() is never called for ftp transfers.
Chris@913 761 if (scheme == "ftp") {
Chris@913 762 m_lastStatus = 200; // http ok
Chris@913 763 }
Chris@913 764
Chris@762 765 bool error = (m_lastStatus / 100 >= 4);
Chris@210 766
Chris@211 767 cleanup();
Chris@208 768
Chris@211 769 if (!error) {
Chris@208 770 QFileInfo fi(m_localFilename);
Chris@208 771 if (!fi.exists()) {
Chris@208 772 m_errorString = tr("Failed to create local file %1").arg(m_localFilename);
Chris@211 773 error = true;
Chris@208 774 } else if (fi.size() == 0) {
Chris@208 775 m_errorString = tr("File contains no data!");
Chris@211 776 error = true;
Chris@208 777 }
Chris@208 778 }
Chris@211 779
Chris@211 780 if (error) {
Chris@327 781 #ifdef DEBUG_FILE_SOURCE
Chris@843 782 cerr << "FileSource::done: error is " << error << ", deleting cache file" << endl;
Chris@327 783 #endif
Chris@316 784 deleteCacheFile();
Chris@211 785 }
Chris@211 786
Chris@211 787 m_ok = !error;
Chris@520 788 if (m_localFile) m_localFile->flush();
Chris@211 789 m_done = true;
Chris@304 790 emit ready();
Chris@211 791 }
Chris@211 792
Chris@211 793 void
Chris@763 794 FileSource::replyFailed(QNetworkReply::NetworkError)
Chris@763 795 {
Chris@763 796 emit progress(100);
Chris@910 797 if (!m_reply) {
Chris@910 798 cerr << "WARNING: FileSource::replyFailed() called without a reply object being known to us" << endl;
Chris@910 799 } else {
Chris@910 800 m_errorString = m_reply->errorString();
Chris@910 801 }
Chris@763 802 m_ok = false;
Chris@763 803 m_done = true;
Chris@763 804 cleanup();
Chris@763 805 emit ready();
Chris@763 806 }
Chris@763 807
Chris@763 808 void
Chris@317 809 FileSource::deleteCacheFile()
Chris@211 810 {
Chris@327 811 #ifdef DEBUG_FILE_SOURCE
Chris@843 812 cerr << "FileSource::deleteCacheFile(\"" << m_localFilename << "\")" << endl;
Chris@327 813 #endif
Chris@211 814
Chris@211 815 cleanup();
Chris@211 816
Chris@316 817 if (m_localFilename == "") {
Chris@316 818 return;
Chris@316 819 }
Chris@211 820
Chris@316 821 if (!isRemote()) {
Chris@327 822 #ifdef DEBUG_FILE_SOURCE
Chris@843 823 cerr << "not a cache file" << endl;
Chris@327 824 #endif
Chris@316 825 return;
Chris@316 826 }
Chris@316 827
Chris@316 828 if (m_refCounted) {
Chris@304 829
Chris@304 830 QMutexLocker locker(&m_mapMutex);
Chris@316 831 m_refCounted = false;
Chris@304 832
Chris@304 833 if (m_refCountMap[m_url] > 0) {
Chris@304 834 m_refCountMap[m_url]--;
Chris@327 835 #ifdef DEBUG_FILE_SOURCE
Chris@843 836 cerr << "reduced ref count to " << m_refCountMap[m_url] << endl;
Chris@327 837 #endif
Chris@304 838 if (m_refCountMap[m_url] > 0) {
Chris@304 839 m_done = true;
Chris@304 840 return;
Chris@304 841 }
Chris@304 842 }
Chris@304 843 }
Chris@304 844
Chris@211 845 m_fileCreationMutex.lock();
Chris@211 846
Chris@211 847 if (!QFile(m_localFilename).remove()) {
Chris@469 848 #ifdef DEBUG_FILE_SOURCE
Chris@843 849 cerr << "FileSource::deleteCacheFile: ERROR: Failed to delete file \"" << m_localFilename << "\"" << endl;
Chris@469 850 #endif
Chris@211 851 } else {
Chris@327 852 #ifdef DEBUG_FILE_SOURCE
Chris@843 853 cerr << "FileSource::deleteCacheFile: Deleted cache file \"" << m_localFilename << "\"" << endl;
Chris@327 854 #endif
Chris@211 855 m_localFilename = "";
Chris@211 856 }
Chris@211 857
Chris@211 858 m_fileCreationMutex.unlock();
Chris@211 859
Chris@208 860 m_done = true;
Chris@208 861 }
Chris@208 862
Chris@316 863 bool
Chris@317 864 FileSource::createCacheFile()
Chris@208 865 {
Chris@316 866 {
Chris@316 867 QMutexLocker locker(&m_mapMutex);
Chris@316 868
Chris@327 869 #ifdef DEBUG_FILE_SOURCE
Chris@843 870 cerr << "FileSource::createCacheFile: refcount is " << m_refCountMap[m_url] << endl;
Chris@327 871 #endif
Chris@316 872
Chris@316 873 if (m_refCountMap[m_url] > 0) {
Chris@316 874 m_refCountMap[m_url]++;
Chris@316 875 m_localFilename = m_remoteLocalMap[m_url];
Chris@327 876 #ifdef DEBUG_FILE_SOURCE
Chris@843 877 cerr << "raised it to " << m_refCountMap[m_url] << endl;
Chris@327 878 #endif
Chris@316 879 m_refCounted = true;
Chris@316 880 return true;
Chris@316 881 }
Chris@316 882 }
Chris@316 883
Chris@208 884 QDir dir;
Chris@208 885 try {
Chris@208 886 dir = TempDirectory::getInstance()->getSubDirectoryPath("download");
Chris@208 887 } catch (DirectoryCreationFailed f) {
Chris@327 888 #ifdef DEBUG_FILE_SOURCE
Chris@843 889 cerr << "FileSource::createCacheFile: ERROR: Failed to create temporary directory: " << f.what() << endl;
Chris@327 890 #endif
Chris@962 891 return false;
Chris@208 892 }
Chris@208 893
Chris@316 894 QString filepart = m_url.path().section('/', -1, -1,
Chris@316 895 QString::SectionSkipEmpty);
Chris@208 896
Chris@457 897 QString extension = "";
Chris@457 898 if (filepart.contains('.')) extension = filepart.section('.', -1);
Chris@457 899
Chris@208 900 QString base = filepart;
Chris@208 901 if (extension != "") {
Chris@208 902 base = base.left(base.length() - extension.length() - 1);
Chris@208 903 }
Chris@208 904 if (base == "") base = "remote";
Chris@208 905
Chris@208 906 QString filename;
Chris@208 907
Chris@208 908 if (extension == "") {
Chris@208 909 filename = base;
Chris@208 910 } else {
Chris@208 911 filename = QString("%1.%2").arg(base).arg(extension);
Chris@208 912 }
Chris@208 913
Chris@208 914 QString filepath(dir.filePath(filename));
Chris@208 915
Chris@327 916 #ifdef DEBUG_FILE_SOURCE
Chris@843 917 cerr << "FileSource::createCacheFile: URL is \"" << m_url.toString() << "\", dir is \"" << dir.path() << "\", base \"" << base << "\", extension \"" << extension << "\", filebase \"" << filename << "\", filename \"" << filepath << "\"" << endl;
Chris@327 918 #endif
Chris@208 919
Chris@316 920 QMutexLocker fcLocker(&m_fileCreationMutex);
Chris@316 921
Chris@208 922 ++m_count;
Chris@208 923
Chris@208 924 if (QFileInfo(filepath).exists() ||
Chris@208 925 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 926
Chris@327 927 #ifdef DEBUG_FILE_SOURCE
Chris@843 928 cerr << "FileSource::createCacheFile: Failed to create local file \""
Chris@686 929 << filepath << "\" for URL \""
Chris@843 930 << m_url.toString() << "\" (or file already exists): appending suffix instead" << endl;
Chris@327 931 #endif
Chris@208 932
Chris@208 933 if (extension == "") {
Chris@208 934 filename = QString("%1_%2").arg(base).arg(m_count);
Chris@208 935 } else {
Chris@208 936 filename = QString("%1_%2.%3").arg(base).arg(m_count).arg(extension);
Chris@208 937 }
Chris@208 938 filepath = dir.filePath(filename);
Chris@208 939
Chris@208 940 if (QFileInfo(filepath).exists() ||
Chris@208 941 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 942
Chris@327 943 #ifdef DEBUG_FILE_SOURCE
Chris@843 944 cerr << "FileSource::createCacheFile: ERROR: Failed to create local file \""
Chris@686 945 << filepath << "\" for URL \""
Chris@843 946 << m_url.toString() << "\" (or file already exists)" << endl;
Chris@327 947 #endif
Chris@208 948
Chris@962 949 return false;
Chris@208 950 }
Chris@208 951 }
Chris@208 952
Chris@327 953 #ifdef DEBUG_FILE_SOURCE
Chris@843 954 cerr << "FileSource::createCacheFile: url "
Chris@686 955 << m_url.toString() << " -> local filename "
Chris@843 956 << filepath << endl;
Chris@327 957 #endif
Chris@316 958
Chris@316 959 m_localFilename = filepath;
Chris@208 960
Chris@316 961 return false;
Chris@208 962 }
Chris@327 963