annotate data/fileio/FileSource.cpp @ 1104:393134235fa0 simple-fft-model

Two-channel test
author Chris Cannam
date Tue, 16 Jun 2015 20:32:44 +0100
parents 24c48323c662
children 6b847a59d908
rev   line source
Chris@208 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@208 2
Chris@208 3 /*
Chris@208 4 Sonic Visualiser
Chris@208 5 An audio file viewer and annotation editor.
Chris@208 6 Centre for Digital Music, Queen Mary, University of London.
Chris@208 7 This file copyright 2007 QMUL.
Chris@208 8
Chris@208 9 This program is free software; you can redistribute it and/or
Chris@208 10 modify it under the terms of the GNU General Public License as
Chris@208 11 published by the Free Software Foundation; either version 2 of the
Chris@208 12 License, or (at your option) any later version. See the file
Chris@208 13 COPYING included with this distribution for more information.
Chris@208 14 */
Chris@208 15
Chris@317 16 #include "FileSource.h"
Chris@357 17
Chris@208 18 #include "base/TempDirectory.h"
Chris@208 19 #include "base/Exceptions.h"
Chris@392 20 #include "base/ProgressReporter.h"
Chris@502 21 #include "system/System.h"
Chris@208 22
Chris@762 23 #include <QNetworkAccessManager>
Chris@762 24 #include <QNetworkReply>
Chris@208 25 #include <QFileInfo>
Chris@208 26 #include <QDir>
Chris@392 27 #include <QCoreApplication>
Chris@833 28 #include <QThreadStorage>
Chris@208 29
Chris@208 30 #include <iostream>
Chris@405 31 #include <cstdlib>
Chris@208 32
Chris@608 33 #include <unistd.h>
Chris@608 34
Chris@1067 35 //#define DEBUG_FILE_SOURCE 1
Chris@327 36
Chris@208 37 int
Chris@317 38 FileSource::m_count = 0;
Chris@208 39
Chris@208 40 QMutex
Chris@317 41 FileSource::m_fileCreationMutex;
Chris@208 42
Chris@317 43 FileSource::RemoteRefCountMap
Chris@317 44 FileSource::m_refCountMap;
Chris@304 45
Chris@317 46 FileSource::RemoteLocalMap
Chris@317 47 FileSource::m_remoteLocalMap;
Chris@304 48
Chris@304 49 QMutex
Chris@317 50 FileSource::m_mapMutex;
Chris@304 51
Chris@529 52 #ifdef DEBUG_FILE_SOURCE
Chris@529 53 static int extantCount = 0;
Chris@1033 54 static int threadCount = 0;
Chris@529 55 static std::map<QString, int> urlExtantCountMap;
Chris@1033 56 static QMutex countMutex;
Chris@529 57 static void incCount(QString url) {
Chris@1033 58 QMutexLocker locker(&countMutex);
Chris@529 59 ++extantCount;
Chris@529 60 if (urlExtantCountMap.find(url) == urlExtantCountMap.end()) {
Chris@529 61 urlExtantCountMap[url] = 1;
Chris@529 62 } else {
Chris@529 63 ++urlExtantCountMap[url];
Chris@529 64 }
Chris@843 65 cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << endl;
Chris@529 66 }
Chris@529 67 static void decCount(QString url) {
Chris@1033 68 QMutexLocker locker(&countMutex);
Chris@529 69 --extantCount;
Chris@529 70 --urlExtantCountMap[url];
Chris@843 71 cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << endl;
Chris@529 72 }
Chris@1033 73 void
Chris@1033 74 FileSource::debugReport()
Chris@1033 75 {
Chris@1033 76 QMutexLocker locker(&countMutex);
Chris@1033 77 cerr << "\nFileSource::debugReport: Have " << extantCount << " FileSource object(s) extant across " << threadCount << " thread(s)" << endl;
Chris@1033 78 cerr << "URLs by extant count:" << endl;
Chris@1033 79 cerr << "Count | URL" << endl;
Chris@1033 80 for (std::map<QString, int>::const_iterator i = urlExtantCountMap.begin();
Chris@1033 81 i != urlExtantCountMap.end(); ++i) {
Chris@1033 82 cerr << i->second << " | " << i->first << endl;
Chris@1033 83 }
Chris@1033 84 cerr << "FileSource::debugReport done\n" << endl;
Chris@1033 85 }
Chris@1033 86 #else
Chris@1033 87 void FileSource::debugReport() { }
Chris@529 88 #endif
Chris@529 89
Chris@833 90 static QThreadStorage<QNetworkAccessManager *> nms;
Chris@762 91
Chris@520 92 FileSource::FileSource(QString fileOrUrl, ProgressReporter *reporter,
Chris@520 93 QString preferredContentType) :
chris@831 94 m_rawFileOrUrl(fileOrUrl),
Chris@614 95 m_url(fileOrUrl, QUrl::StrictMode),
Chris@316 96 m_localFile(0),
Chris@762 97 m_reply(0),
Chris@520 98 m_preferredContentType(preferredContentType),
Chris@316 99 m_ok(false),
Chris@981 100 m_cancelled(false),
Chris@316 101 m_lastStatus(0),
Chris@706 102 m_resource(fileOrUrl.startsWith(':')),
Chris@316 103 m_remote(isRemote(fileOrUrl)),
Chris@316 104 m_done(false),
Chris@316 105 m_leaveLocalFile(false),
Chris@392 106 m_reporter(reporter),
Chris@316 107 m_refCounted(false)
Chris@316 108 {
Chris@706 109 if (m_resource) { // qrc file
Chris@706 110 m_url = QUrl("qrc" + fileOrUrl);
Chris@706 111 }
Chris@706 112
Chris@661 113 if (m_url.toString() == "") {
Chris@661 114 m_url = QUrl(fileOrUrl, QUrl::TolerantMode);
Chris@661 115 }
Chris@661 116
Chris@327 117 #ifdef DEBUG_FILE_SOURCE
Chris@843 118 cerr << "FileSource::FileSource(" << fileOrUrl << "): url <" << m_url.toString() << ">" << endl;
Chris@529 119 incCount(m_url.toString());
Chris@327 120 #endif
Chris@316 121
Chris@316 122 if (!canHandleScheme(m_url)) {
Chris@843 123 cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316 124 m_errorString = tr("Unsupported scheme in URL");
Chris@316 125 return;
Chris@316 126 }
Chris@316 127
Chris@357 128 init();
Chris@316 129
Chris@614 130 if (!isRemote() &&
Chris@614 131 !isAvailable()) {
Chris@614 132 #ifdef DEBUG_FILE_SOURCE
Chris@843 133 cerr << "FileSource::FileSource: Failed to open local file with URL \"" << m_url.toString() << "\"; trying again assuming filename was encoded" << endl;
Chris@614 134 #endif
Chris@762 135 m_url = QUrl::fromEncoded(fileOrUrl.toLatin1());
chris@831 136 #ifdef DEBUG_FILE_SOURCE
Chris@843 137 cerr << "FileSource::FileSource: URL is now \"" << m_url.toString() << "\"" << endl;
chris@831 138 #endif
Chris@614 139 init();
Chris@614 140 }
Chris@614 141
Chris@316 142 if (isRemote() &&
Chris@316 143 (fileOrUrl.contains('%') ||
Chris@316 144 fileOrUrl.contains("--"))) { // for IDNA
Chris@316 145
Chris@316 146 waitForStatus();
Chris@316 147
Chris@316 148 if (!isAvailable()) {
Chris@336 149
Chris@316 150 // The URL was created on the assumption that the string
Chris@316 151 // was human-readable. Let's try again, this time
Chris@316 152 // assuming it was already encoded.
Chris@843 153 cerr << "FileSource::FileSource: Failed to retrieve URL \""
Chris@844 154 << fileOrUrl
Chris@316 155 << "\" as human-readable URL; "
Chris@316 156 << "trying again treating it as encoded URL"
Chris@843 157 << endl;
Chris@336 158
Chris@336 159 // even though our cache file doesn't exist (because the
Chris@336 160 // resource was 404), we still need to ensure we're no
Chris@336 161 // longer associating a filename with this url in the
Chris@336 162 // refcount map -- or createCacheFile will think we've
Chris@336 163 // already done all the work and no request will be sent
Chris@336 164 deleteCacheFile();
Chris@336 165
Chris@762 166 m_url = QUrl::fromEncoded(fileOrUrl.toLatin1());
Chris@336 167
Chris@336 168 m_ok = false;
Chris@336 169 m_done = false;
Chris@336 170 m_lastStatus = 0;
Chris@357 171 init();
Chris@316 172 }
Chris@316 173 }
Chris@325 174
Chris@325 175 if (!isRemote()) {
Chris@325 176 emit statusAvailable();
Chris@325 177 emit ready();
Chris@325 178 }
Chris@497 179
Chris@504 180 #ifdef DEBUG_FILE_SOURCE
Chris@843 181 cerr << "FileSource::FileSource(string) exiting" << endl;
Chris@504 182 #endif
Chris@316 183 }
Chris@316 184
Chris@469 185 FileSource::FileSource(QUrl url, ProgressReporter *reporter) :
Chris@304 186 m_url(url),
Chris@208 187 m_localFile(0),
Chris@762 188 m_reply(0),
Chris@208 189 m_ok(false),
Chris@981 190 m_cancelled(false),
Chris@210 191 m_lastStatus(0),
Chris@706 192 m_resource(false),
Chris@316 193 m_remote(isRemote(url.toString())),
Chris@208 194 m_done(false),
Chris@316 195 m_leaveLocalFile(false),
Chris@392 196 m_reporter(reporter),
Chris@316 197 m_refCounted(false)
Chris@208 198 {
Chris@327 199 #ifdef DEBUG_FILE_SOURCE
Chris@843 200 cerr << "FileSource::FileSource(" << url.toString() << ") [as url]" << endl;
Chris@529 201 incCount(m_url.toString());
Chris@327 202 #endif
Chris@316 203
Chris@316 204 if (!canHandleScheme(m_url)) {
Chris@843 205 cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316 206 m_errorString = tr("Unsupported scheme in URL");
Chris@208 207 return;
Chris@208 208 }
Chris@208 209
Chris@357 210 init();
Chris@497 211
Chris@504 212 #ifdef DEBUG_FILE_SOURCE
Chris@843 213 cerr << "FileSource::FileSource(url) exiting" << endl;
Chris@504 214 #endif
Chris@316 215 }
Chris@304 216
Chris@317 217 FileSource::FileSource(const FileSource &rf) :
Chris@316 218 QObject(),
Chris@316 219 m_url(rf.m_url),
Chris@316 220 m_localFile(0),
Chris@762 221 m_reply(0),
Chris@316 222 m_ok(rf.m_ok),
Chris@981 223 m_cancelled(rf.m_cancelled),
Chris@316 224 m_lastStatus(rf.m_lastStatus),
Chris@706 225 m_resource(rf.m_resource),
Chris@316 226 m_remote(rf.m_remote),
Chris@316 227 m_done(false),
Chris@316 228 m_leaveLocalFile(false),
Chris@392 229 m_reporter(rf.m_reporter),
Chris@316 230 m_refCounted(false)
Chris@316 231 {
Chris@327 232 #ifdef DEBUG_FILE_SOURCE
Chris@843 233 cerr << "FileSource::FileSource(" << m_url.toString() << ") [copy ctor]" << endl;
Chris@529 234 incCount(m_url.toString());
Chris@327 235 #endif
Chris@304 236
Chris@316 237 if (!canHandleScheme(m_url)) {
Chris@843 238 cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316 239 m_errorString = tr("Unsupported scheme in URL");
Chris@304 240 return;
Chris@304 241 }
Chris@304 242
Chris@469 243 if (!isRemote()) {
Chris@316 244 m_localFilename = rf.m_localFilename;
Chris@316 245 } else {
Chris@316 246 QMutexLocker locker(&m_mapMutex);
Chris@327 247 #ifdef DEBUG_FILE_SOURCE
Chris@843 248 cerr << "FileSource::FileSource(copy ctor): ref count is "
Chris@843 249 << m_refCountMap[m_url] << endl;
Chris@327 250 #endif
Chris@316 251 if (m_refCountMap[m_url] > 0) {
Chris@316 252 m_refCountMap[m_url]++;
Chris@327 253 #ifdef DEBUG_FILE_SOURCE
Chris@843 254 cerr << "raised it to " << m_refCountMap[m_url] << endl;
Chris@327 255 #endif
Chris@316 256 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 257 m_refCounted = true;
Chris@316 258 } else {
Chris@316 259 m_ok = false;
Chris@316 260 m_lastStatus = 404;
Chris@316 261 }
Chris@316 262 }
Chris@316 263
Chris@316 264 m_done = true;
Chris@497 265
Chris@504 266 #ifdef DEBUG_FILE_SOURCE
Chris@843 267 cerr << "FileSource::FileSource(" << m_url.toString() << ") [copy ctor]: note: local filename is \"" << m_localFilename << "\"" << endl;
Chris@527 268 #endif
Chris@527 269
Chris@527 270 #ifdef DEBUG_FILE_SOURCE
Chris@843 271 cerr << "FileSource::FileSource(copy ctor) exiting" << endl;
Chris@504 272 #endif
Chris@316 273 }
Chris@316 274
Chris@317 275 FileSource::~FileSource()
Chris@316 276 {
Chris@327 277 #ifdef DEBUG_FILE_SOURCE
Chris@843 278 cerr << "FileSource(" << m_url.toString() << ")::~FileSource" << endl;
Chris@529 279 decCount(m_url.toString());
Chris@327 280 #endif
Chris@316 281
Chris@316 282 cleanup();
Chris@316 283
Chris@469 284 if (isRemote() && !m_leaveLocalFile) deleteCacheFile();
Chris@316 285 }
Chris@316 286
Chris@316 287 void
Chris@357 288 FileSource::init()
Chris@316 289 {
Chris@706 290 if (isResource()) {
Chris@355 291 #ifdef DEBUG_FILE_SOURCE
Chris@843 292 cerr << "FileSource::init: Is a resource" << endl;
Chris@706 293 #endif
Chris@706 294 QString resourceFile = m_url.toString();
Chris@706 295 resourceFile.replace(QRegExp("^qrc:"), ":");
Chris@706 296
Chris@706 297 if (!QFileInfo(resourceFile).exists()) {
Chris@706 298 #ifdef DEBUG_FILE_SOURCE
Chris@843 299 cerr << "FileSource::init: Resource file of this name does not exist, switching to non-resource URL" << endl;
Chris@706 300 #endif
Chris@706 301 m_url = resourceFile;
Chris@706 302 m_resource = false;
Chris@706 303 }
Chris@706 304 }
Chris@706 305
Chris@706 306 if (!isRemote() && !isResource()) {
Chris@706 307 #ifdef DEBUG_FILE_SOURCE
Chris@843 308 cerr << "FileSource::init: Not a remote URL" << endl;
Chris@355 309 #endif
Chris@355 310 bool literal = false;
Chris@316 311 m_localFilename = m_url.toLocalFile();
chris@831 312
Chris@342 313 if (m_localFilename == "") {
Chris@342 314 // QUrl may have mishandled the scheme (e.g. in a DOS path)
chris@831 315 m_localFilename = m_rawFileOrUrl;
chris@831 316 #ifdef DEBUG_FILE_SOURCE
Chris@843 317 cerr << "FileSource::init: Trying literal local filename \""
Chris@843 318 << m_localFilename << "\"" << endl;
chris@831 319 #endif
Chris@355 320 literal = true;
Chris@342 321 }
Chris@439 322 m_localFilename = QFileInfo(m_localFilename).absoluteFilePath();
Chris@439 323
Chris@355 324 #ifdef DEBUG_FILE_SOURCE
Chris@843 325 cerr << "FileSource::init: URL translates to absolute filename \""
Chris@706 326 << m_localFilename << "\" (with literal=" << literal << ")"
Chris@843 327 << endl;
Chris@355 328 #endif
Chris@316 329 m_ok = true;
Chris@355 330 m_lastStatus = 200;
Chris@355 331
Chris@316 332 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 333 if (literal) {
Chris@355 334 m_lastStatus = 404;
Chris@355 335 } else {
Chris@614 336 #ifdef DEBUG_FILE_SOURCE
Chris@843 337 cerr << "FileSource::init: Local file of this name does not exist, trying URL as a literal filename" << endl;
Chris@614 338 #endif
Chris@355 339 // Again, QUrl may have been mistreating us --
Chris@355 340 // e.g. dropping a part that looks like query data
chris@831 341 m_localFilename = m_rawFileOrUrl;
Chris@355 342 literal = true;
Chris@355 343 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 344 m_lastStatus = 404;
Chris@355 345 }
Chris@355 346 }
Chris@316 347 }
Chris@355 348
Chris@316 349 m_done = true;
Chris@316 350 return;
Chris@316 351 }
Chris@316 352
Chris@316 353 if (createCacheFile()) {
Chris@327 354 #ifdef DEBUG_FILE_SOURCE
Chris@843 355 cerr << "FileSource::init: Already have this one" << endl;
Chris@327 356 #endif
Chris@316 357 m_ok = true;
Chris@316 358 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 359 m_lastStatus = 404;
Chris@316 360 } else {
Chris@316 361 m_lastStatus = 200;
Chris@316 362 }
Chris@316 363 m_done = true;
Chris@316 364 return;
Chris@316 365 }
Chris@316 366
Chris@208 367 if (m_localFilename == "") return;
Chris@706 368
Chris@208 369 m_localFile = new QFile(m_localFilename);
Chris@208 370 m_localFile->open(QFile::WriteOnly);
Chris@208 371
Chris@706 372 if (isResource()) {
Chris@706 373
Chris@706 374 // Absent resource file case was dealt with at the top -- this
Chris@706 375 // is the successful case
Chris@706 376
Chris@706 377 QString resourceFileName = m_url.toString();
Chris@706 378 resourceFileName.replace(QRegExp("^qrc:"), ":");
Chris@706 379 QFile resourceFile(resourceFileName);
Chris@706 380 resourceFile.open(QFile::ReadOnly);
Chris@706 381 QByteArray ba(resourceFile.readAll());
Chris@706 382
Chris@706 383 #ifdef DEBUG_FILE_SOURCE
Chris@843 384 cerr << "Copying " << ba.size() << " bytes from resource file to cache file" << endl;
Chris@706 385 #endif
Chris@706 386
Chris@706 387 qint64 written = m_localFile->write(ba);
Chris@706 388 m_localFile->close();
Chris@706 389 delete m_localFile;
Chris@706 390 m_localFile = 0;
Chris@706 391
Chris@706 392 if (written != ba.size()) {
Chris@706 393 #ifdef DEBUG_FILE_SOURCE
Chris@843 394 cerr << "Copy failed (wrote " << written << " bytes)" << endl;
Chris@706 395 #endif
Chris@706 396 m_ok = false;
Chris@706 397 return;
Chris@706 398 } else {
Chris@706 399 m_ok = true;
Chris@706 400 m_lastStatus = 200;
Chris@706 401 m_done = true;
Chris@706 402 }
Chris@706 403
Chris@706 404 } else {
Chris@706 405
Chris@706 406 QString scheme = m_url.scheme().toLower();
Chris@316 407
Chris@327 408 #ifdef DEBUG_FILE_SOURCE
Chris@843 409 cerr << "FileSource::init: Don't have local copy of \""
Chris@843 410 << m_url.toString() << "\", retrieving" << endl;
Chris@327 411 #endif
Chris@208 412
Chris@762 413 if (scheme == "http" || scheme == "https" || scheme == "ftp") {
Chris@762 414 initRemote();
Chris@520 415 #ifdef DEBUG_FILE_SOURCE
Chris@843 416 cerr << "FileSource: initRemote returned" << endl;
Chris@520 417 #endif
Chris@706 418 } else {
Chris@706 419 m_remote = false;
Chris@706 420 m_ok = false;
Chris@706 421 }
Chris@208 422 }
Chris@208 423
Chris@208 424 if (m_ok) {
Chris@316 425
Chris@316 426 QMutexLocker locker(&m_mapMutex);
Chris@316 427
Chris@316 428 if (m_refCountMap[m_url] > 0) {
Chris@316 429 // someone else has been doing the same thing at the same time,
Chris@316 430 // but has got there first
Chris@316 431 cleanup();
Chris@316 432 m_refCountMap[m_url]++;
Chris@327 433 #ifdef DEBUG_FILE_SOURCE
Chris@843 434 cerr << "FileSource::init: Another FileSource has got there first, abandoning our download and using theirs" << endl;
Chris@327 435 #endif
Chris@316 436 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 437 m_refCounted = true;
Chris@316 438 m_ok = true;
Chris@316 439 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 440 m_lastStatus = 404;
Chris@316 441 }
Chris@316 442 m_done = true;
Chris@316 443 return;
Chris@316 444 }
Chris@304 445
Chris@304 446 m_remoteLocalMap[m_url] = m_localFilename;
Chris@304 447 m_refCountMap[m_url]++;
Chris@316 448 m_refCounted = true;
Chris@304 449
Chris@706 450 if (m_reporter && !m_done) {
Chris@392 451 m_reporter->setMessage
Chris@392 452 (tr("Downloading %1...").arg(m_url.toString()));
Chris@392 453 connect(m_reporter, SIGNAL(cancelled()), this, SLOT(cancelled()));
Chris@357 454 connect(this, SIGNAL(progress(int)),
Chris@392 455 m_reporter, SLOT(setProgress(int)));
Chris@316 456 }
Chris@208 457 }
Chris@208 458 }
Chris@208 459
Chris@316 460 void
Chris@762 461 FileSource::initRemote()
Chris@208 462 {
Chris@316 463 m_ok = true;
Chris@316 464
Chris@762 465 QNetworkRequest req;
Chris@762 466 req.setUrl(m_url);
Chris@762 467
Chris@762 468 if (m_preferredContentType != "") {
Chris@520 469 #ifdef DEBUG_FILE_SOURCE
Chris@843 470 cerr << "FileSource: indicating preferred content type of \""
Chris@843 471 << m_preferredContentType << "\"" << endl;
Chris@520 472 #endif
Chris@762 473 req.setRawHeader
Chris@762 474 ("Accept",
Chris@762 475 QString("%1, */*").arg(m_preferredContentType).toLatin1());
Chris@520 476 }
Chris@316 477
Chris@1033 478 { // check we have a QNetworkAccessManager
Chris@1033 479 QMutexLocker locker(&m_mapMutex);
Chris@1033 480 if (!nms.hasLocalData()) {
Chris@1033 481 #ifdef DEBUG_FILE_SOURCE
Chris@1033 482 ++threadCount;
Chris@1033 483 #endif
Chris@1033 484 nms.setLocalData(new QNetworkAccessManager());
Chris@1033 485 }
Chris@1033 486 }
Chris@1033 487
Chris@833 488 m_reply = nms.localData()->get(req);
Chris@762 489
Chris@762 490 connect(m_reply, SIGNAL(readyRead()),
Chris@762 491 this, SLOT(readyRead()));
Chris@762 492 connect(m_reply, SIGNAL(error(QNetworkReply::NetworkError)),
Chris@762 493 this, SLOT(replyFailed(QNetworkReply::NetworkError)));
Chris@762 494 connect(m_reply, SIGNAL(finished()),
Chris@762 495 this, SLOT(replyFinished()));
Chris@764 496 connect(m_reply, SIGNAL(metaDataChanged()),
Chris@764 497 this, SLOT(metaDataChanged()));
Chris@762 498 connect(m_reply, SIGNAL(downloadProgress(qint64, qint64)),
Chris@762 499 this, SLOT(downloadProgress(qint64, qint64)));
Chris@211 500 }
Chris@211 501
Chris@211 502 void
Chris@317 503 FileSource::cleanup()
Chris@211 504 {
Chris@470 505 if (m_done) {
Chris@470 506 delete m_localFile; // does not actually delete the file
Chris@470 507 m_localFile = 0;
Chris@470 508 }
Chris@211 509 m_done = true;
Chris@762 510 if (m_reply) {
Chris@762 511 QNetworkReply *r = m_reply;
Chris@993 512 disconnect(r, 0, this, 0);
Chris@762 513 m_reply = 0;
Chris@913 514 // Can only call abort() when there are no errors.
Chris@913 515 if (r->error() == QNetworkReply::NoError) {
Chris@913 516 r->abort();
Chris@913 517 }
Chris@762 518 r->deleteLater();
Chris@214 519 }
Chris@470 520 if (m_localFile) {
Chris@470 521 delete m_localFile; // does not actually delete the file
Chris@470 522 m_localFile = 0;
Chris@470 523 }
Chris@208 524 }
Chris@208 525
Chris@208 526 bool
Chris@317 527 FileSource::isRemote(QString fileOrUrl)
Chris@304 528 {
Chris@342 529 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@316 530 QString scheme = QUrl(fileOrUrl).scheme().toLower();
Chris@342 531 if (scheme == "" || scheme == "file" || scheme.length() == 1) return false;
Chris@342 532 return true;
Chris@304 533 }
Chris@304 534
Chris@304 535 bool
Chris@317 536 FileSource::canHandleScheme(QUrl url)
Chris@208 537 {
Chris@342 538 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@208 539 QString scheme = url.scheme().toLower();
Chris@762 540 return (scheme == "http" || scheme == "https" ||
Chris@762 541 scheme == "ftp" || scheme == "file" || scheme == "qrc" ||
Chris@706 542 scheme == "" || scheme.length() == 1);
Chris@208 543 }
Chris@208 544
Chris@210 545 bool
Chris@317 546 FileSource::isAvailable()
Chris@210 547 {
Chris@316 548 waitForStatus();
Chris@211 549 bool available = true;
Chris@913 550 if (!m_ok) {
Chris@913 551 available = false;
Chris@913 552 } else {
Chris@913 553 // http 2xx status codes mean success
Chris@913 554 available = (m_lastStatus / 100 == 2);
Chris@913 555 }
Chris@327 556 #ifdef DEBUG_FILE_SOURCE
Chris@913 557 cerr << "FileSource::isAvailable: " << (available ? "yes" : "no") << endl;
Chris@327 558 #endif
Chris@211 559 return available;
Chris@210 560 }
Chris@210 561
Chris@208 562 void
Chris@317 563 FileSource::waitForStatus()
Chris@316 564 {
Chris@316 565 while (m_ok && (!m_done && m_lastStatus == 0)) {
Chris@843 566 // cerr << "waitForStatus: processing (last status " << m_lastStatus << ")" << endl;
Chris@392 567 QCoreApplication::processEvents();
Chris@316 568 }
Chris@316 569 }
Chris@316 570
Chris@316 571 void
Chris@317 572 FileSource::waitForData()
Chris@208 573 {
Chris@211 574 while (m_ok && !m_done) {
Chris@843 575 // cerr << "FileSource::waitForData: calling QApplication::processEvents" << endl;
Chris@392 576 QCoreApplication::processEvents();
Chris@497 577 usleep(10000);
Chris@208 578 }
Chris@208 579 }
Chris@208 580
Chris@316 581 void
Chris@317 582 FileSource::setLeaveLocalFile(bool leave)
Chris@316 583 {
Chris@316 584 m_leaveLocalFile = leave;
Chris@316 585 }
Chris@316 586
Chris@208 587 bool
Chris@317 588 FileSource::isOK() const
Chris@208 589 {
Chris@208 590 return m_ok;
Chris@208 591 }
Chris@208 592
Chris@208 593 bool
Chris@317 594 FileSource::isDone() const
Chris@208 595 {
Chris@208 596 return m_done;
Chris@208 597 }
Chris@208 598
Chris@316 599 bool
Chris@981 600 FileSource::wasCancelled() const
Chris@981 601 {
Chris@981 602 return m_cancelled;
Chris@981 603 }
Chris@981 604
Chris@981 605 bool
Chris@706 606 FileSource::isResource() const
Chris@706 607 {
Chris@706 608 return m_resource;
Chris@706 609 }
Chris@706 610
Chris@706 611 bool
Chris@317 612 FileSource::isRemote() const
Chris@316 613 {
Chris@316 614 return m_remote;
Chris@316 615 }
Chris@316 616
Chris@316 617 QString
Chris@317 618 FileSource::getLocation() const
Chris@316 619 {
Chris@316 620 return m_url.toString();
Chris@316 621 }
Chris@316 622
Chris@208 623 QString
Chris@317 624 FileSource::getLocalFilename() const
Chris@208 625 {
Chris@208 626 return m_localFilename;
Chris@208 627 }
Chris@208 628
Chris@208 629 QString
Chris@678 630 FileSource::getBasename() const
Chris@678 631 {
Chris@678 632 return QFileInfo(m_localFilename).fileName();
Chris@678 633 }
Chris@678 634
Chris@678 635 QString
Chris@317 636 FileSource::getContentType() const
Chris@316 637 {
Chris@316 638 return m_contentType;
Chris@316 639 }
Chris@316 640
Chris@316 641 QString
Chris@317 642 FileSource::getExtension() const
Chris@316 643 {
Chris@316 644 if (m_localFilename != "") {
Chris@316 645 return QFileInfo(m_localFilename).suffix().toLower();
Chris@316 646 } else {
Chris@316 647 return QFileInfo(m_url.toLocalFile()).suffix().toLower();
Chris@316 648 }
Chris@316 649 }
Chris@316 650
Chris@316 651 QString
Chris@317 652 FileSource::getErrorString() const
Chris@208 653 {
Chris@208 654 return m_errorString;
Chris@208 655 }
Chris@208 656
Chris@208 657 void
Chris@762 658 FileSource::readyRead()
Chris@208 659 {
Chris@762 660 m_localFile->write(m_reply->readAll());
Chris@208 661 }
Chris@208 662
Chris@208 663 void
Chris@764 664 FileSource::metaDataChanged()
Chris@210 665 {
Chris@520 666 #ifdef DEBUG_FILE_SOURCE
Chris@843 667 cerr << "FileSource::metaDataChanged" << endl;
Chris@520 668 #endif
Chris@497 669
Chris@762 670 if (!m_reply) {
Chris@843 671 cerr << "WARNING: FileSource::metaDataChanged() called without a reply object being known to us" << endl;
Chris@762 672 return;
Chris@762 673 }
Chris@762 674
Chris@913 675 // Handle http transfer status codes.
Chris@913 676
Chris@762 677 int status =
Chris@762 678 m_reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
Chris@762 679
Chris@913 680 // If this is a redirection (3xx) code, do the redirect
Chris@762 681 if (status / 100 == 3) {
Chris@762 682 QString location = m_reply->header
Chris@762 683 (QNetworkRequest::LocationHeader).toString();
Chris@496 684 #ifdef DEBUG_FILE_SOURCE
Chris@843 685 cerr << "FileSource::metaDataChanged: redirect to \""
Chris@843 686 << location << "\" received" << endl;
Chris@496 687 #endif
Chris@496 688 if (location != "") {
Chris@496 689 QUrl newUrl(location);
Chris@496 690 if (newUrl != m_url) {
Chris@497 691 cleanup();
Chris@497 692 deleteCacheFile();
Chris@529 693 #ifdef DEBUG_FILE_SOURCE
Chris@529 694 decCount(m_url.toString());
Chris@529 695 incCount(newUrl.toString());
Chris@529 696 #endif
Chris@496 697 m_url = newUrl;
Chris@497 698 m_localFile = 0;
Chris@496 699 m_lastStatus = 0;
Chris@497 700 m_done = false;
Chris@497 701 m_refCounted = false;
Chris@496 702 init();
Chris@496 703 return;
Chris@496 704 }
Chris@496 705 }
Chris@496 706 }
Chris@497 707
Chris@762 708 m_lastStatus = status;
Chris@913 709
Chris@913 710 // 400 and up are failures, get the error string
Chris@210 711 if (m_lastStatus / 100 >= 4) {
Chris@210 712 m_errorString = QString("%1 %2")
Chris@762 713 .arg(status)
Chris@762 714 .arg(m_reply->attribute
Chris@762 715 (QNetworkRequest::HttpReasonPhraseAttribute).toString());
Chris@327 716 #ifdef DEBUG_FILE_SOURCE
Chris@843 717 cerr << "FileSource::metaDataChanged: "
Chris@843 718 << m_errorString << endl;
Chris@327 719 #endif
Chris@211 720 } else {
Chris@327 721 #ifdef DEBUG_FILE_SOURCE
Chris@843 722 cerr << "FileSource::metaDataChanged: "
Chris@843 723 << m_lastStatus << endl;
Chris@327 724 #endif
Chris@762 725 m_contentType =
Chris@762 726 m_reply->header(QNetworkRequest::ContentTypeHeader).toString();
Chris@325 727 }
Chris@325 728 emit statusAvailable();
Chris@210 729 }
Chris@210 730
Chris@210 731 void
Chris@762 732 FileSource::downloadProgress(qint64 done, qint64 total)
Chris@208 733 {
Chris@208 734 int percent = int((double(done) / double(total)) * 100.0 - 0.1);
Chris@208 735 emit progress(percent);
Chris@210 736 }
Chris@210 737
Chris@210 738 void
Chris@317 739 FileSource::cancelled()
Chris@210 740 {
Chris@210 741 m_done = true;
Chris@316 742 cleanup();
Chris@316 743
Chris@210 744 m_ok = false;
Chris@981 745 m_cancelled = true;
Chris@210 746 m_errorString = tr("Download cancelled");
Chris@208 747 }
Chris@208 748
Chris@208 749 void
Chris@762 750 FileSource::replyFinished()
Chris@208 751 {
Chris@327 752 emit progress(100);
Chris@327 753
Chris@327 754 #ifdef DEBUG_FILE_SOURCE
Chris@843 755 cerr << "FileSource::replyFinished()" << endl;
Chris@327 756 #endif
Chris@211 757
Chris@211 758 if (m_done) return;
Chris@211 759
Chris@913 760 QString scheme = m_url.scheme().toLower();
Chris@913 761 // For ftp transfers, replyFinished() will be called on success.
Chris@913 762 // metaDataChanged() is never called for ftp transfers.
Chris@913 763 if (scheme == "ftp") {
Chris@913 764 m_lastStatus = 200; // http ok
Chris@913 765 }
Chris@913 766
Chris@762 767 bool error = (m_lastStatus / 100 >= 4);
Chris@210 768
Chris@211 769 cleanup();
Chris@208 770
Chris@211 771 if (!error) {
Chris@208 772 QFileInfo fi(m_localFilename);
Chris@208 773 if (!fi.exists()) {
Chris@208 774 m_errorString = tr("Failed to create local file %1").arg(m_localFilename);
Chris@211 775 error = true;
Chris@208 776 } else if (fi.size() == 0) {
Chris@208 777 m_errorString = tr("File contains no data!");
Chris@211 778 error = true;
Chris@208 779 }
Chris@208 780 }
Chris@211 781
Chris@211 782 if (error) {
Chris@327 783 #ifdef DEBUG_FILE_SOURCE
Chris@843 784 cerr << "FileSource::done: error is " << error << ", deleting cache file" << endl;
Chris@327 785 #endif
Chris@316 786 deleteCacheFile();
Chris@211 787 }
Chris@211 788
Chris@211 789 m_ok = !error;
Chris@520 790 if (m_localFile) m_localFile->flush();
Chris@211 791 m_done = true;
Chris@304 792 emit ready();
Chris@211 793 }
Chris@211 794
Chris@211 795 void
Chris@763 796 FileSource::replyFailed(QNetworkReply::NetworkError)
Chris@763 797 {
Chris@763 798 emit progress(100);
Chris@910 799 if (!m_reply) {
Chris@910 800 cerr << "WARNING: FileSource::replyFailed() called without a reply object being known to us" << endl;
Chris@910 801 } else {
Chris@910 802 m_errorString = m_reply->errorString();
Chris@910 803 }
Chris@763 804 m_ok = false;
Chris@763 805 m_done = true;
Chris@763 806 cleanup();
Chris@763 807 emit ready();
Chris@763 808 }
Chris@763 809
Chris@763 810 void
Chris@317 811 FileSource::deleteCacheFile()
Chris@211 812 {
Chris@327 813 #ifdef DEBUG_FILE_SOURCE
Chris@843 814 cerr << "FileSource::deleteCacheFile(\"" << m_localFilename << "\")" << endl;
Chris@327 815 #endif
Chris@211 816
Chris@211 817 cleanup();
Chris@211 818
Chris@316 819 if (m_localFilename == "") {
Chris@316 820 return;
Chris@316 821 }
Chris@211 822
Chris@316 823 if (!isRemote()) {
Chris@327 824 #ifdef DEBUG_FILE_SOURCE
Chris@843 825 cerr << "not a cache file" << endl;
Chris@327 826 #endif
Chris@316 827 return;
Chris@316 828 }
Chris@316 829
Chris@316 830 if (m_refCounted) {
Chris@304 831
Chris@304 832 QMutexLocker locker(&m_mapMutex);
Chris@316 833 m_refCounted = false;
Chris@304 834
Chris@304 835 if (m_refCountMap[m_url] > 0) {
Chris@304 836 m_refCountMap[m_url]--;
Chris@327 837 #ifdef DEBUG_FILE_SOURCE
Chris@843 838 cerr << "reduced ref count to " << m_refCountMap[m_url] << endl;
Chris@327 839 #endif
Chris@304 840 if (m_refCountMap[m_url] > 0) {
Chris@304 841 m_done = true;
Chris@304 842 return;
Chris@304 843 }
Chris@304 844 }
Chris@304 845 }
Chris@304 846
Chris@211 847 m_fileCreationMutex.lock();
Chris@211 848
Chris@211 849 if (!QFile(m_localFilename).remove()) {
Chris@469 850 #ifdef DEBUG_FILE_SOURCE
Chris@843 851 cerr << "FileSource::deleteCacheFile: ERROR: Failed to delete file \"" << m_localFilename << "\"" << endl;
Chris@469 852 #endif
Chris@211 853 } else {
Chris@327 854 #ifdef DEBUG_FILE_SOURCE
Chris@843 855 cerr << "FileSource::deleteCacheFile: Deleted cache file \"" << m_localFilename << "\"" << endl;
Chris@327 856 #endif
Chris@211 857 m_localFilename = "";
Chris@211 858 }
Chris@211 859
Chris@211 860 m_fileCreationMutex.unlock();
Chris@211 861
Chris@208 862 m_done = true;
Chris@208 863 }
Chris@208 864
Chris@316 865 bool
Chris@317 866 FileSource::createCacheFile()
Chris@208 867 {
Chris@316 868 {
Chris@316 869 QMutexLocker locker(&m_mapMutex);
Chris@316 870
Chris@327 871 #ifdef DEBUG_FILE_SOURCE
Chris@843 872 cerr << "FileSource::createCacheFile: refcount is " << m_refCountMap[m_url] << endl;
Chris@327 873 #endif
Chris@316 874
Chris@316 875 if (m_refCountMap[m_url] > 0) {
Chris@316 876 m_refCountMap[m_url]++;
Chris@316 877 m_localFilename = m_remoteLocalMap[m_url];
Chris@327 878 #ifdef DEBUG_FILE_SOURCE
Chris@843 879 cerr << "raised it to " << m_refCountMap[m_url] << endl;
Chris@327 880 #endif
Chris@316 881 m_refCounted = true;
Chris@316 882 return true;
Chris@316 883 }
Chris@316 884 }
Chris@316 885
Chris@208 886 QDir dir;
Chris@208 887 try {
Chris@208 888 dir = TempDirectory::getInstance()->getSubDirectoryPath("download");
Chris@208 889 } catch (DirectoryCreationFailed f) {
Chris@327 890 #ifdef DEBUG_FILE_SOURCE
Chris@843 891 cerr << "FileSource::createCacheFile: ERROR: Failed to create temporary directory: " << f.what() << endl;
Chris@327 892 #endif
Chris@962 893 return false;
Chris@208 894 }
Chris@208 895
Chris@316 896 QString filepart = m_url.path().section('/', -1, -1,
Chris@316 897 QString::SectionSkipEmpty);
Chris@208 898
Chris@457 899 QString extension = "";
Chris@457 900 if (filepart.contains('.')) extension = filepart.section('.', -1);
Chris@457 901
Chris@208 902 QString base = filepart;
Chris@208 903 if (extension != "") {
Chris@208 904 base = base.left(base.length() - extension.length() - 1);
Chris@208 905 }
Chris@208 906 if (base == "") base = "remote";
Chris@208 907
Chris@208 908 QString filename;
Chris@208 909
Chris@208 910 if (extension == "") {
Chris@208 911 filename = base;
Chris@208 912 } else {
Chris@208 913 filename = QString("%1.%2").arg(base).arg(extension);
Chris@208 914 }
Chris@208 915
Chris@208 916 QString filepath(dir.filePath(filename));
Chris@208 917
Chris@327 918 #ifdef DEBUG_FILE_SOURCE
Chris@843 919 cerr << "FileSource::createCacheFile: URL is \"" << m_url.toString() << "\", dir is \"" << dir.path() << "\", base \"" << base << "\", extension \"" << extension << "\", filebase \"" << filename << "\", filename \"" << filepath << "\"" << endl;
Chris@327 920 #endif
Chris@208 921
Chris@316 922 QMutexLocker fcLocker(&m_fileCreationMutex);
Chris@316 923
Chris@208 924 ++m_count;
Chris@208 925
Chris@208 926 if (QFileInfo(filepath).exists() ||
Chris@208 927 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 928
Chris@327 929 #ifdef DEBUG_FILE_SOURCE
Chris@843 930 cerr << "FileSource::createCacheFile: Failed to create local file \""
Chris@686 931 << filepath << "\" for URL \""
Chris@843 932 << m_url.toString() << "\" (or file already exists): appending suffix instead" << endl;
Chris@327 933 #endif
Chris@208 934
Chris@208 935 if (extension == "") {
Chris@208 936 filename = QString("%1_%2").arg(base).arg(m_count);
Chris@208 937 } else {
Chris@208 938 filename = QString("%1_%2.%3").arg(base).arg(m_count).arg(extension);
Chris@208 939 }
Chris@208 940 filepath = dir.filePath(filename);
Chris@208 941
Chris@208 942 if (QFileInfo(filepath).exists() ||
Chris@208 943 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 944
Chris@327 945 #ifdef DEBUG_FILE_SOURCE
Chris@843 946 cerr << "FileSource::createCacheFile: ERROR: Failed to create local file \""
Chris@686 947 << filepath << "\" for URL \""
Chris@843 948 << m_url.toString() << "\" (or file already exists)" << endl;
Chris@327 949 #endif
Chris@208 950
Chris@962 951 return false;
Chris@208 952 }
Chris@208 953 }
Chris@208 954
Chris@327 955 #ifdef DEBUG_FILE_SOURCE
Chris@843 956 cerr << "FileSource::createCacheFile: url "
Chris@686 957 << m_url.toString() << " -> local filename "
Chris@843 958 << filepath << endl;
Chris@327 959 #endif
Chris@316 960
Chris@316 961 m_localFilename = filepath;
Chris@208 962
Chris@316 963 return false;
Chris@208 964 }
Chris@327 965