annotate data/fileio/FileSource.cpp @ 875:3e6ed8a8577b tonioni

Use a sparse time-value model only for outputs with fixed bin count of 1, not for those with unknown bin count. (Precursor to using more than one model for outputs with unknown bin count)
author Chris Cannam
date Tue, 28 Jan 2014 18:52:22 +0000
parents f5cd33909744
children 55e552b4f1b7
rev   line source
Chris@208 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@208 2
Chris@208 3 /*
Chris@208 4 Sonic Visualiser
Chris@208 5 An audio file viewer and annotation editor.
Chris@208 6 Centre for Digital Music, Queen Mary, University of London.
Chris@208 7 This file copyright 2007 QMUL.
Chris@208 8
Chris@208 9 This program is free software; you can redistribute it and/or
Chris@208 10 modify it under the terms of the GNU General Public License as
Chris@208 11 published by the Free Software Foundation; either version 2 of the
Chris@208 12 License, or (at your option) any later version. See the file
Chris@208 13 COPYING included with this distribution for more information.
Chris@208 14 */
Chris@208 15
Chris@317 16 #include "FileSource.h"
Chris@357 17
Chris@208 18 #include "base/TempDirectory.h"
Chris@208 19 #include "base/Exceptions.h"
Chris@392 20 #include "base/ProgressReporter.h"
Chris@502 21 #include "system/System.h"
Chris@208 22
Chris@762 23 #include <QNetworkAccessManager>
Chris@762 24 #include <QNetworkReply>
Chris@208 25 #include <QFileInfo>
Chris@208 26 #include <QDir>
Chris@392 27 #include <QCoreApplication>
Chris@833 28 #include <QThreadStorage>
Chris@208 29
Chris@208 30 #include <iostream>
Chris@405 31 #include <cstdlib>
Chris@208 32
Chris@608 33 #include <unistd.h>
Chris@608 34
Chris@763 35 //#define DEBUG_FILE_SOURCE 1
Chris@327 36
Chris@208 37 int
Chris@317 38 FileSource::m_count = 0;
Chris@208 39
Chris@208 40 QMutex
Chris@317 41 FileSource::m_fileCreationMutex;
Chris@208 42
Chris@317 43 FileSource::RemoteRefCountMap
Chris@317 44 FileSource::m_refCountMap;
Chris@304 45
Chris@317 46 FileSource::RemoteLocalMap
Chris@317 47 FileSource::m_remoteLocalMap;
Chris@304 48
Chris@304 49 QMutex
Chris@317 50 FileSource::m_mapMutex;
Chris@304 51
Chris@529 52 #ifdef DEBUG_FILE_SOURCE
Chris@529 53 static int extantCount = 0;
Chris@529 54 static std::map<QString, int> urlExtantCountMap;
Chris@529 55 static void incCount(QString url) {
Chris@529 56 ++extantCount;
Chris@529 57 if (urlExtantCountMap.find(url) == urlExtantCountMap.end()) {
Chris@529 58 urlExtantCountMap[url] = 1;
Chris@529 59 } else {
Chris@529 60 ++urlExtantCountMap[url];
Chris@529 61 }
Chris@843 62 cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << endl;
Chris@529 63 }
Chris@529 64 static void decCount(QString url) {
Chris@529 65 --extantCount;
Chris@529 66 --urlExtantCountMap[url];
Chris@843 67 cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << endl;
Chris@529 68 }
Chris@529 69 #endif
Chris@529 70
Chris@833 71 static QThreadStorage<QNetworkAccessManager *> nms;
Chris@762 72
Chris@520 73 FileSource::FileSource(QString fileOrUrl, ProgressReporter *reporter,
Chris@520 74 QString preferredContentType) :
chris@831 75 m_rawFileOrUrl(fileOrUrl),
Chris@614 76 m_url(fileOrUrl, QUrl::StrictMode),
Chris@316 77 m_localFile(0),
Chris@762 78 m_reply(0),
Chris@520 79 m_preferredContentType(preferredContentType),
Chris@316 80 m_ok(false),
Chris@316 81 m_lastStatus(0),
Chris@706 82 m_resource(fileOrUrl.startsWith(':')),
Chris@316 83 m_remote(isRemote(fileOrUrl)),
Chris@316 84 m_done(false),
Chris@316 85 m_leaveLocalFile(false),
Chris@392 86 m_reporter(reporter),
Chris@316 87 m_refCounted(false)
Chris@316 88 {
Chris@706 89 if (m_resource) { // qrc file
Chris@706 90 m_url = QUrl("qrc" + fileOrUrl);
Chris@706 91 }
Chris@706 92
Chris@661 93 if (m_url.toString() == "") {
Chris@661 94 m_url = QUrl(fileOrUrl, QUrl::TolerantMode);
Chris@661 95 }
Chris@661 96
Chris@327 97 #ifdef DEBUG_FILE_SOURCE
Chris@843 98 cerr << "FileSource::FileSource(" << fileOrUrl << "): url <" << m_url.toString() << ">" << endl;
Chris@529 99 incCount(m_url.toString());
Chris@327 100 #endif
Chris@316 101
Chris@316 102 if (!canHandleScheme(m_url)) {
Chris@843 103 cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316 104 m_errorString = tr("Unsupported scheme in URL");
Chris@316 105 return;
Chris@316 106 }
Chris@316 107
Chris@357 108 init();
Chris@316 109
Chris@614 110 if (!isRemote() &&
Chris@614 111 !isAvailable()) {
Chris@614 112 #ifdef DEBUG_FILE_SOURCE
Chris@843 113 cerr << "FileSource::FileSource: Failed to open local file with URL \"" << m_url.toString() << "\"; trying again assuming filename was encoded" << endl;
Chris@614 114 #endif
Chris@762 115 m_url = QUrl::fromEncoded(fileOrUrl.toLatin1());
chris@831 116 #ifdef DEBUG_FILE_SOURCE
Chris@843 117 cerr << "FileSource::FileSource: URL is now \"" << m_url.toString() << "\"" << endl;
chris@831 118 #endif
Chris@614 119 init();
Chris@614 120 }
Chris@614 121
Chris@316 122 if (isRemote() &&
Chris@316 123 (fileOrUrl.contains('%') ||
Chris@316 124 fileOrUrl.contains("--"))) { // for IDNA
Chris@316 125
Chris@316 126 waitForStatus();
Chris@316 127
Chris@316 128 if (!isAvailable()) {
Chris@336 129
Chris@316 130 // The URL was created on the assumption that the string
Chris@316 131 // was human-readable. Let's try again, this time
Chris@316 132 // assuming it was already encoded.
Chris@843 133 cerr << "FileSource::FileSource: Failed to retrieve URL \""
Chris@844 134 << fileOrUrl
Chris@316 135 << "\" as human-readable URL; "
Chris@316 136 << "trying again treating it as encoded URL"
Chris@843 137 << endl;
Chris@336 138
Chris@336 139 // even though our cache file doesn't exist (because the
Chris@336 140 // resource was 404), we still need to ensure we're no
Chris@336 141 // longer associating a filename with this url in the
Chris@336 142 // refcount map -- or createCacheFile will think we've
Chris@336 143 // already done all the work and no request will be sent
Chris@336 144 deleteCacheFile();
Chris@336 145
Chris@762 146 m_url = QUrl::fromEncoded(fileOrUrl.toLatin1());
Chris@336 147
Chris@336 148 m_ok = false;
Chris@336 149 m_done = false;
Chris@336 150 m_lastStatus = 0;
Chris@357 151 init();
Chris@316 152 }
Chris@316 153 }
Chris@325 154
Chris@325 155 if (!isRemote()) {
Chris@325 156 emit statusAvailable();
Chris@325 157 emit ready();
Chris@325 158 }
Chris@497 159
Chris@504 160 #ifdef DEBUG_FILE_SOURCE
Chris@843 161 cerr << "FileSource::FileSource(string) exiting" << endl;
Chris@504 162 #endif
Chris@316 163 }
Chris@316 164
Chris@469 165 FileSource::FileSource(QUrl url, ProgressReporter *reporter) :
Chris@304 166 m_url(url),
Chris@208 167 m_localFile(0),
Chris@762 168 m_reply(0),
Chris@208 169 m_ok(false),
Chris@210 170 m_lastStatus(0),
Chris@706 171 m_resource(false),
Chris@316 172 m_remote(isRemote(url.toString())),
Chris@208 173 m_done(false),
Chris@316 174 m_leaveLocalFile(false),
Chris@392 175 m_reporter(reporter),
Chris@316 176 m_refCounted(false)
Chris@208 177 {
Chris@327 178 #ifdef DEBUG_FILE_SOURCE
Chris@843 179 cerr << "FileSource::FileSource(" << url.toString() << ") [as url]" << endl;
Chris@529 180 incCount(m_url.toString());
Chris@327 181 #endif
Chris@316 182
Chris@316 183 if (!canHandleScheme(m_url)) {
Chris@843 184 cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316 185 m_errorString = tr("Unsupported scheme in URL");
Chris@208 186 return;
Chris@208 187 }
Chris@208 188
Chris@357 189 init();
Chris@497 190
Chris@504 191 #ifdef DEBUG_FILE_SOURCE
Chris@843 192 cerr << "FileSource::FileSource(url) exiting" << endl;
Chris@504 193 #endif
Chris@316 194 }
Chris@304 195
Chris@317 196 FileSource::FileSource(const FileSource &rf) :
Chris@316 197 QObject(),
Chris@316 198 m_url(rf.m_url),
Chris@316 199 m_localFile(0),
Chris@762 200 m_reply(0),
Chris@316 201 m_ok(rf.m_ok),
Chris@316 202 m_lastStatus(rf.m_lastStatus),
Chris@706 203 m_resource(rf.m_resource),
Chris@316 204 m_remote(rf.m_remote),
Chris@316 205 m_done(false),
Chris@316 206 m_leaveLocalFile(false),
Chris@392 207 m_reporter(rf.m_reporter),
Chris@316 208 m_refCounted(false)
Chris@316 209 {
Chris@327 210 #ifdef DEBUG_FILE_SOURCE
Chris@843 211 cerr << "FileSource::FileSource(" << m_url.toString() << ") [copy ctor]" << endl;
Chris@529 212 incCount(m_url.toString());
Chris@327 213 #endif
Chris@304 214
Chris@316 215 if (!canHandleScheme(m_url)) {
Chris@843 216 cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316 217 m_errorString = tr("Unsupported scheme in URL");
Chris@304 218 return;
Chris@304 219 }
Chris@304 220
Chris@469 221 if (!isRemote()) {
Chris@316 222 m_localFilename = rf.m_localFilename;
Chris@316 223 } else {
Chris@316 224 QMutexLocker locker(&m_mapMutex);
Chris@327 225 #ifdef DEBUG_FILE_SOURCE
Chris@843 226 cerr << "FileSource::FileSource(copy ctor): ref count is "
Chris@843 227 << m_refCountMap[m_url] << endl;
Chris@327 228 #endif
Chris@316 229 if (m_refCountMap[m_url] > 0) {
Chris@316 230 m_refCountMap[m_url]++;
Chris@327 231 #ifdef DEBUG_FILE_SOURCE
Chris@843 232 cerr << "raised it to " << m_refCountMap[m_url] << endl;
Chris@327 233 #endif
Chris@316 234 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 235 m_refCounted = true;
Chris@316 236 } else {
Chris@316 237 m_ok = false;
Chris@316 238 m_lastStatus = 404;
Chris@316 239 }
Chris@316 240 }
Chris@316 241
Chris@316 242 m_done = true;
Chris@497 243
Chris@504 244 #ifdef DEBUG_FILE_SOURCE
Chris@843 245 cerr << "FileSource::FileSource(" << m_url.toString() << ") [copy ctor]: note: local filename is \"" << m_localFilename << "\"" << endl;
Chris@527 246 #endif
Chris@527 247
Chris@527 248 #ifdef DEBUG_FILE_SOURCE
Chris@843 249 cerr << "FileSource::FileSource(copy ctor) exiting" << endl;
Chris@504 250 #endif
Chris@316 251 }
Chris@316 252
Chris@317 253 FileSource::~FileSource()
Chris@316 254 {
Chris@327 255 #ifdef DEBUG_FILE_SOURCE
Chris@843 256 cerr << "FileSource(" << m_url.toString() << ")::~FileSource" << endl;
Chris@529 257 decCount(m_url.toString());
Chris@327 258 #endif
Chris@316 259
Chris@316 260 cleanup();
Chris@316 261
Chris@469 262 if (isRemote() && !m_leaveLocalFile) deleteCacheFile();
Chris@316 263 }
Chris@316 264
Chris@316 265 void
Chris@357 266 FileSource::init()
Chris@316 267 {
Chris@833 268 { // check we have a QNetworkAccessManager
Chris@833 269 QMutexLocker locker(&m_mapMutex);
Chris@833 270 if (!nms.hasLocalData()) {
Chris@833 271 nms.setLocalData(new QNetworkAccessManager());
Chris@833 272 }
Chris@833 273 }
Chris@833 274
Chris@706 275 if (isResource()) {
Chris@355 276 #ifdef DEBUG_FILE_SOURCE
Chris@843 277 cerr << "FileSource::init: Is a resource" << endl;
Chris@706 278 #endif
Chris@706 279 QString resourceFile = m_url.toString();
Chris@706 280 resourceFile.replace(QRegExp("^qrc:"), ":");
Chris@706 281
Chris@706 282 if (!QFileInfo(resourceFile).exists()) {
Chris@706 283 #ifdef DEBUG_FILE_SOURCE
Chris@843 284 cerr << "FileSource::init: Resource file of this name does not exist, switching to non-resource URL" << endl;
Chris@706 285 #endif
Chris@706 286 m_url = resourceFile;
Chris@706 287 m_resource = false;
Chris@706 288 }
Chris@706 289 }
Chris@706 290
Chris@706 291 if (!isRemote() && !isResource()) {
Chris@706 292 #ifdef DEBUG_FILE_SOURCE
Chris@843 293 cerr << "FileSource::init: Not a remote URL" << endl;
Chris@355 294 #endif
Chris@355 295 bool literal = false;
Chris@316 296 m_localFilename = m_url.toLocalFile();
chris@831 297
Chris@342 298 if (m_localFilename == "") {
Chris@342 299 // QUrl may have mishandled the scheme (e.g. in a DOS path)
chris@831 300 m_localFilename = m_rawFileOrUrl;
chris@831 301 #ifdef DEBUG_FILE_SOURCE
Chris@843 302 cerr << "FileSource::init: Trying literal local filename \""
Chris@843 303 << m_localFilename << "\"" << endl;
chris@831 304 #endif
Chris@355 305 literal = true;
Chris@342 306 }
Chris@439 307 m_localFilename = QFileInfo(m_localFilename).absoluteFilePath();
Chris@439 308
Chris@355 309 #ifdef DEBUG_FILE_SOURCE
Chris@843 310 cerr << "FileSource::init: URL translates to absolute filename \""
Chris@706 311 << m_localFilename << "\" (with literal=" << literal << ")"
Chris@843 312 << endl;
Chris@355 313 #endif
Chris@316 314 m_ok = true;
Chris@355 315 m_lastStatus = 200;
Chris@355 316
Chris@316 317 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 318 if (literal) {
Chris@355 319 m_lastStatus = 404;
Chris@355 320 } else {
Chris@614 321 #ifdef DEBUG_FILE_SOURCE
Chris@843 322 cerr << "FileSource::init: Local file of this name does not exist, trying URL as a literal filename" << endl;
Chris@614 323 #endif
Chris@355 324 // Again, QUrl may have been mistreating us --
Chris@355 325 // e.g. dropping a part that looks like query data
chris@831 326 m_localFilename = m_rawFileOrUrl;
Chris@355 327 literal = true;
Chris@355 328 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 329 m_lastStatus = 404;
Chris@355 330 }
Chris@355 331 }
Chris@316 332 }
Chris@355 333
Chris@316 334 m_done = true;
Chris@316 335 return;
Chris@316 336 }
Chris@316 337
Chris@316 338 if (createCacheFile()) {
Chris@327 339 #ifdef DEBUG_FILE_SOURCE
Chris@843 340 cerr << "FileSource::init: Already have this one" << endl;
Chris@327 341 #endif
Chris@316 342 m_ok = true;
Chris@316 343 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 344 m_lastStatus = 404;
Chris@316 345 } else {
Chris@316 346 m_lastStatus = 200;
Chris@316 347 }
Chris@316 348 m_done = true;
Chris@316 349 return;
Chris@316 350 }
Chris@316 351
Chris@208 352 if (m_localFilename == "") return;
Chris@706 353
Chris@208 354 m_localFile = new QFile(m_localFilename);
Chris@208 355 m_localFile->open(QFile::WriteOnly);
Chris@208 356
Chris@706 357 if (isResource()) {
Chris@706 358
Chris@706 359 // Absent resource file case was dealt with at the top -- this
Chris@706 360 // is the successful case
Chris@706 361
Chris@706 362 QString resourceFileName = m_url.toString();
Chris@706 363 resourceFileName.replace(QRegExp("^qrc:"), ":");
Chris@706 364 QFile resourceFile(resourceFileName);
Chris@706 365 resourceFile.open(QFile::ReadOnly);
Chris@706 366 QByteArray ba(resourceFile.readAll());
Chris@706 367
Chris@706 368 #ifdef DEBUG_FILE_SOURCE
Chris@843 369 cerr << "Copying " << ba.size() << " bytes from resource file to cache file" << endl;
Chris@706 370 #endif
Chris@706 371
Chris@706 372 qint64 written = m_localFile->write(ba);
Chris@706 373 m_localFile->close();
Chris@706 374 delete m_localFile;
Chris@706 375 m_localFile = 0;
Chris@706 376
Chris@706 377 if (written != ba.size()) {
Chris@706 378 #ifdef DEBUG_FILE_SOURCE
Chris@843 379 cerr << "Copy failed (wrote " << written << " bytes)" << endl;
Chris@706 380 #endif
Chris@706 381 m_ok = false;
Chris@706 382 return;
Chris@706 383 } else {
Chris@706 384 m_ok = true;
Chris@706 385 m_lastStatus = 200;
Chris@706 386 m_done = true;
Chris@706 387 }
Chris@706 388
Chris@706 389 } else {
Chris@706 390
Chris@706 391 QString scheme = m_url.scheme().toLower();
Chris@316 392
Chris@327 393 #ifdef DEBUG_FILE_SOURCE
Chris@843 394 cerr << "FileSource::init: Don't have local copy of \""
Chris@843 395 << m_url.toString() << "\", retrieving" << endl;
Chris@327 396 #endif
Chris@208 397
Chris@762 398 if (scheme == "http" || scheme == "https" || scheme == "ftp") {
Chris@762 399 initRemote();
Chris@520 400 #ifdef DEBUG_FILE_SOURCE
Chris@843 401 cerr << "FileSource: initRemote returned" << endl;
Chris@520 402 #endif
Chris@706 403 } else {
Chris@706 404 m_remote = false;
Chris@706 405 m_ok = false;
Chris@706 406 }
Chris@208 407 }
Chris@208 408
Chris@208 409 if (m_ok) {
Chris@316 410
Chris@316 411 QMutexLocker locker(&m_mapMutex);
Chris@316 412
Chris@316 413 if (m_refCountMap[m_url] > 0) {
Chris@316 414 // someone else has been doing the same thing at the same time,
Chris@316 415 // but has got there first
Chris@316 416 cleanup();
Chris@316 417 m_refCountMap[m_url]++;
Chris@327 418 #ifdef DEBUG_FILE_SOURCE
Chris@843 419 cerr << "FileSource::init: Another FileSource has got there first, abandoning our download and using theirs" << endl;
Chris@327 420 #endif
Chris@316 421 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 422 m_refCounted = true;
Chris@316 423 m_ok = true;
Chris@316 424 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 425 m_lastStatus = 404;
Chris@316 426 }
Chris@316 427 m_done = true;
Chris@316 428 return;
Chris@316 429 }
Chris@304 430
Chris@304 431 m_remoteLocalMap[m_url] = m_localFilename;
Chris@304 432 m_refCountMap[m_url]++;
Chris@316 433 m_refCounted = true;
Chris@304 434
Chris@706 435 if (m_reporter && !m_done) {
Chris@392 436 m_reporter->setMessage
Chris@392 437 (tr("Downloading %1...").arg(m_url.toString()));
Chris@392 438 connect(m_reporter, SIGNAL(cancelled()), this, SLOT(cancelled()));
Chris@357 439 connect(this, SIGNAL(progress(int)),
Chris@392 440 m_reporter, SLOT(setProgress(int)));
Chris@316 441 }
Chris@208 442 }
Chris@208 443 }
Chris@208 444
Chris@316 445 void
Chris@762 446 FileSource::initRemote()
Chris@208 447 {
Chris@316 448 m_ok = true;
Chris@316 449
Chris@762 450 QNetworkRequest req;
Chris@762 451 req.setUrl(m_url);
Chris@762 452
Chris@762 453 if (m_preferredContentType != "") {
Chris@520 454 #ifdef DEBUG_FILE_SOURCE
Chris@843 455 cerr << "FileSource: indicating preferred content type of \""
Chris@843 456 << m_preferredContentType << "\"" << endl;
Chris@520 457 #endif
Chris@762 458 req.setRawHeader
Chris@762 459 ("Accept",
Chris@762 460 QString("%1, */*").arg(m_preferredContentType).toLatin1());
Chris@520 461 }
Chris@316 462
Chris@833 463 m_reply = nms.localData()->get(req);
Chris@762 464
Chris@762 465 connect(m_reply, SIGNAL(readyRead()),
Chris@762 466 this, SLOT(readyRead()));
Chris@762 467 connect(m_reply, SIGNAL(error(QNetworkReply::NetworkError)),
Chris@762 468 this, SLOT(replyFailed(QNetworkReply::NetworkError)));
Chris@762 469 connect(m_reply, SIGNAL(finished()),
Chris@762 470 this, SLOT(replyFinished()));
Chris@764 471 connect(m_reply, SIGNAL(metaDataChanged()),
Chris@764 472 this, SLOT(metaDataChanged()));
Chris@762 473 connect(m_reply, SIGNAL(downloadProgress(qint64, qint64)),
Chris@762 474 this, SLOT(downloadProgress(qint64, qint64)));
Chris@211 475 }
Chris@211 476
Chris@211 477 void
Chris@317 478 FileSource::cleanup()
Chris@211 479 {
Chris@470 480 if (m_done) {
Chris@470 481 delete m_localFile; // does not actually delete the file
Chris@470 482 m_localFile = 0;
Chris@470 483 }
Chris@211 484 m_done = true;
Chris@762 485 if (m_reply) {
Chris@762 486 QNetworkReply *r = m_reply;
Chris@762 487 m_reply = 0;
Chris@762 488 r->abort();
Chris@762 489 r->deleteLater();
Chris@214 490 }
Chris@470 491 if (m_localFile) {
Chris@470 492 delete m_localFile; // does not actually delete the file
Chris@470 493 m_localFile = 0;
Chris@470 494 }
Chris@208 495 }
Chris@208 496
Chris@208 497 bool
Chris@317 498 FileSource::isRemote(QString fileOrUrl)
Chris@304 499 {
Chris@342 500 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@316 501 QString scheme = QUrl(fileOrUrl).scheme().toLower();
Chris@342 502 if (scheme == "" || scheme == "file" || scheme.length() == 1) return false;
Chris@342 503 return true;
Chris@304 504 }
Chris@304 505
Chris@304 506 bool
Chris@317 507 FileSource::canHandleScheme(QUrl url)
Chris@208 508 {
Chris@342 509 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@208 510 QString scheme = url.scheme().toLower();
Chris@762 511 return (scheme == "http" || scheme == "https" ||
Chris@762 512 scheme == "ftp" || scheme == "file" || scheme == "qrc" ||
Chris@706 513 scheme == "" || scheme.length() == 1);
Chris@208 514 }
Chris@208 515
Chris@210 516 bool
Chris@317 517 FileSource::isAvailable()
Chris@210 518 {
Chris@316 519 waitForStatus();
Chris@211 520 bool available = true;
Chris@211 521 if (!m_ok) available = false;
Chris@211 522 else available = (m_lastStatus / 100 == 2);
Chris@327 523 #ifdef DEBUG_FILE_SOURCE
Chris@843 524 cerr << "FileSource::isAvailable: " << (available ? "yes" : "no")
Chris@843 525 << endl;
Chris@327 526 #endif
Chris@211 527 return available;
Chris@210 528 }
Chris@210 529
Chris@208 530 void
Chris@317 531 FileSource::waitForStatus()
Chris@316 532 {
Chris@316 533 while (m_ok && (!m_done && m_lastStatus == 0)) {
Chris@843 534 // cerr << "waitForStatus: processing (last status " << m_lastStatus << ")" << endl;
Chris@392 535 QCoreApplication::processEvents();
Chris@316 536 }
Chris@316 537 }
Chris@316 538
Chris@316 539 void
Chris@317 540 FileSource::waitForData()
Chris@208 541 {
Chris@211 542 while (m_ok && !m_done) {
Chris@843 543 // cerr << "FileSource::waitForData: calling QApplication::processEvents" << endl;
Chris@392 544 QCoreApplication::processEvents();
Chris@497 545 usleep(10000);
Chris@208 546 }
Chris@208 547 }
Chris@208 548
Chris@316 549 void
Chris@317 550 FileSource::setLeaveLocalFile(bool leave)
Chris@316 551 {
Chris@316 552 m_leaveLocalFile = leave;
Chris@316 553 }
Chris@316 554
Chris@208 555 bool
Chris@317 556 FileSource::isOK() const
Chris@208 557 {
Chris@208 558 return m_ok;
Chris@208 559 }
Chris@208 560
Chris@208 561 bool
Chris@317 562 FileSource::isDone() const
Chris@208 563 {
Chris@208 564 return m_done;
Chris@208 565 }
Chris@208 566
Chris@316 567 bool
Chris@706 568 FileSource::isResource() const
Chris@706 569 {
Chris@706 570 return m_resource;
Chris@706 571 }
Chris@706 572
Chris@706 573 bool
Chris@317 574 FileSource::isRemote() const
Chris@316 575 {
Chris@316 576 return m_remote;
Chris@316 577 }
Chris@316 578
Chris@316 579 QString
Chris@317 580 FileSource::getLocation() const
Chris@316 581 {
Chris@316 582 return m_url.toString();
Chris@316 583 }
Chris@316 584
Chris@208 585 QString
Chris@317 586 FileSource::getLocalFilename() const
Chris@208 587 {
Chris@208 588 return m_localFilename;
Chris@208 589 }
Chris@208 590
Chris@208 591 QString
Chris@678 592 FileSource::getBasename() const
Chris@678 593 {
Chris@678 594 return QFileInfo(m_localFilename).fileName();
Chris@678 595 }
Chris@678 596
Chris@678 597 QString
Chris@317 598 FileSource::getContentType() const
Chris@316 599 {
Chris@316 600 return m_contentType;
Chris@316 601 }
Chris@316 602
Chris@316 603 QString
Chris@317 604 FileSource::getExtension() const
Chris@316 605 {
Chris@316 606 if (m_localFilename != "") {
Chris@316 607 return QFileInfo(m_localFilename).suffix().toLower();
Chris@316 608 } else {
Chris@316 609 return QFileInfo(m_url.toLocalFile()).suffix().toLower();
Chris@316 610 }
Chris@316 611 }
Chris@316 612
Chris@316 613 QString
Chris@317 614 FileSource::getErrorString() const
Chris@208 615 {
Chris@208 616 return m_errorString;
Chris@208 617 }
Chris@208 618
Chris@208 619 void
Chris@762 620 FileSource::readyRead()
Chris@208 621 {
Chris@762 622 m_localFile->write(m_reply->readAll());
Chris@208 623 }
Chris@208 624
Chris@208 625 void
Chris@764 626 FileSource::metaDataChanged()
Chris@210 627 {
Chris@520 628 #ifdef DEBUG_FILE_SOURCE
Chris@843 629 cerr << "FileSource::metaDataChanged" << endl;
Chris@520 630 #endif
Chris@497 631
Chris@762 632 if (!m_reply) {
Chris@843 633 cerr << "WARNING: FileSource::metaDataChanged() called without a reply object being known to us" << endl;
Chris@762 634 return;
Chris@762 635 }
Chris@762 636
Chris@762 637 int status =
Chris@762 638 m_reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
Chris@762 639
Chris@762 640 if (status / 100 == 3) {
Chris@762 641 QString location = m_reply->header
Chris@762 642 (QNetworkRequest::LocationHeader).toString();
Chris@496 643 #ifdef DEBUG_FILE_SOURCE
Chris@843 644 cerr << "FileSource::metaDataChanged: redirect to \""
Chris@843 645 << location << "\" received" << endl;
Chris@496 646 #endif
Chris@496 647 if (location != "") {
Chris@496 648 QUrl newUrl(location);
Chris@496 649 if (newUrl != m_url) {
Chris@497 650 cleanup();
Chris@497 651 deleteCacheFile();
Chris@529 652 #ifdef DEBUG_FILE_SOURCE
Chris@529 653 decCount(m_url.toString());
Chris@529 654 incCount(newUrl.toString());
Chris@529 655 #endif
Chris@496 656 m_url = newUrl;
Chris@497 657 m_localFile = 0;
Chris@496 658 m_lastStatus = 0;
Chris@497 659 m_done = false;
Chris@497 660 m_refCounted = false;
Chris@496 661 init();
Chris@496 662 return;
Chris@496 663 }
Chris@496 664 }
Chris@496 665 }
Chris@497 666
Chris@762 667 m_lastStatus = status;
Chris@210 668 if (m_lastStatus / 100 >= 4) {
Chris@210 669 m_errorString = QString("%1 %2")
Chris@762 670 .arg(status)
Chris@762 671 .arg(m_reply->attribute
Chris@762 672 (QNetworkRequest::HttpReasonPhraseAttribute).toString());
Chris@327 673 #ifdef DEBUG_FILE_SOURCE
Chris@843 674 cerr << "FileSource::metaDataChanged: "
Chris@843 675 << m_errorString << endl;
Chris@327 676 #endif
Chris@211 677 } else {
Chris@327 678 #ifdef DEBUG_FILE_SOURCE
Chris@843 679 cerr << "FileSource::metaDataChanged: "
Chris@843 680 << m_lastStatus << endl;
Chris@327 681 #endif
Chris@762 682 m_contentType =
Chris@762 683 m_reply->header(QNetworkRequest::ContentTypeHeader).toString();
Chris@325 684 }
Chris@325 685 emit statusAvailable();
Chris@210 686 }
Chris@210 687
Chris@210 688 void
Chris@762 689 FileSource::downloadProgress(qint64 done, qint64 total)
Chris@208 690 {
Chris@208 691 int percent = int((double(done) / double(total)) * 100.0 - 0.1);
Chris@208 692 emit progress(percent);
Chris@210 693 }
Chris@210 694
Chris@210 695 void
Chris@317 696 FileSource::cancelled()
Chris@210 697 {
Chris@210 698 m_done = true;
Chris@316 699 cleanup();
Chris@316 700
Chris@210 701 m_ok = false;
Chris@210 702 m_errorString = tr("Download cancelled");
Chris@208 703 }
Chris@208 704
Chris@208 705 void
Chris@762 706 FileSource::replyFinished()
Chris@208 707 {
Chris@327 708 emit progress(100);
Chris@327 709
Chris@327 710 #ifdef DEBUG_FILE_SOURCE
Chris@843 711 cerr << "FileSource::replyFinished()" << endl;
Chris@327 712 #endif
Chris@211 713
Chris@211 714 if (m_done) return;
Chris@211 715
Chris@762 716 bool error = (m_lastStatus / 100 >= 4);
Chris@210 717
Chris@211 718 cleanup();
Chris@208 719
Chris@211 720 if (!error) {
Chris@208 721 QFileInfo fi(m_localFilename);
Chris@208 722 if (!fi.exists()) {
Chris@208 723 m_errorString = tr("Failed to create local file %1").arg(m_localFilename);
Chris@211 724 error = true;
Chris@208 725 } else if (fi.size() == 0) {
Chris@208 726 m_errorString = tr("File contains no data!");
Chris@211 727 error = true;
Chris@208 728 }
Chris@208 729 }
Chris@211 730
Chris@211 731 if (error) {
Chris@327 732 #ifdef DEBUG_FILE_SOURCE
Chris@843 733 cerr << "FileSource::done: error is " << error << ", deleting cache file" << endl;
Chris@327 734 #endif
Chris@316 735 deleteCacheFile();
Chris@211 736 }
Chris@211 737
Chris@211 738 m_ok = !error;
Chris@520 739 if (m_localFile) m_localFile->flush();
Chris@211 740 m_done = true;
Chris@304 741 emit ready();
Chris@211 742 }
Chris@211 743
Chris@211 744 void
Chris@763 745 FileSource::replyFailed(QNetworkReply::NetworkError)
Chris@763 746 {
Chris@763 747 emit progress(100);
Chris@763 748 m_errorString = m_reply->errorString();
Chris@763 749 m_ok = false;
Chris@763 750 m_done = true;
Chris@763 751 cleanup();
Chris@763 752 emit ready();
Chris@763 753 }
Chris@763 754
Chris@763 755 void
Chris@317 756 FileSource::deleteCacheFile()
Chris@211 757 {
Chris@327 758 #ifdef DEBUG_FILE_SOURCE
Chris@843 759 cerr << "FileSource::deleteCacheFile(\"" << m_localFilename << "\")" << endl;
Chris@327 760 #endif
Chris@211 761
Chris@211 762 cleanup();
Chris@211 763
Chris@316 764 if (m_localFilename == "") {
Chris@316 765 return;
Chris@316 766 }
Chris@211 767
Chris@316 768 if (!isRemote()) {
Chris@327 769 #ifdef DEBUG_FILE_SOURCE
Chris@843 770 cerr << "not a cache file" << endl;
Chris@327 771 #endif
Chris@316 772 return;
Chris@316 773 }
Chris@316 774
Chris@316 775 if (m_refCounted) {
Chris@304 776
Chris@304 777 QMutexLocker locker(&m_mapMutex);
Chris@316 778 m_refCounted = false;
Chris@304 779
Chris@304 780 if (m_refCountMap[m_url] > 0) {
Chris@304 781 m_refCountMap[m_url]--;
Chris@327 782 #ifdef DEBUG_FILE_SOURCE
Chris@843 783 cerr << "reduced ref count to " << m_refCountMap[m_url] << endl;
Chris@327 784 #endif
Chris@304 785 if (m_refCountMap[m_url] > 0) {
Chris@304 786 m_done = true;
Chris@304 787 return;
Chris@304 788 }
Chris@304 789 }
Chris@304 790 }
Chris@304 791
Chris@211 792 m_fileCreationMutex.lock();
Chris@211 793
Chris@211 794 if (!QFile(m_localFilename).remove()) {
Chris@469 795 #ifdef DEBUG_FILE_SOURCE
Chris@843 796 cerr << "FileSource::deleteCacheFile: ERROR: Failed to delete file \"" << m_localFilename << "\"" << endl;
Chris@469 797 #endif
Chris@211 798 } else {
Chris@327 799 #ifdef DEBUG_FILE_SOURCE
Chris@843 800 cerr << "FileSource::deleteCacheFile: Deleted cache file \"" << m_localFilename << "\"" << endl;
Chris@327 801 #endif
Chris@211 802 m_localFilename = "";
Chris@211 803 }
Chris@211 804
Chris@211 805 m_fileCreationMutex.unlock();
Chris@211 806
Chris@208 807 m_done = true;
Chris@208 808 }
Chris@208 809
Chris@316 810 bool
Chris@317 811 FileSource::createCacheFile()
Chris@208 812 {
Chris@316 813 {
Chris@316 814 QMutexLocker locker(&m_mapMutex);
Chris@316 815
Chris@327 816 #ifdef DEBUG_FILE_SOURCE
Chris@843 817 cerr << "FileSource::createCacheFile: refcount is " << m_refCountMap[m_url] << endl;
Chris@327 818 #endif
Chris@316 819
Chris@316 820 if (m_refCountMap[m_url] > 0) {
Chris@316 821 m_refCountMap[m_url]++;
Chris@316 822 m_localFilename = m_remoteLocalMap[m_url];
Chris@327 823 #ifdef DEBUG_FILE_SOURCE
Chris@843 824 cerr << "raised it to " << m_refCountMap[m_url] << endl;
Chris@327 825 #endif
Chris@316 826 m_refCounted = true;
Chris@316 827 return true;
Chris@316 828 }
Chris@316 829 }
Chris@316 830
Chris@208 831 QDir dir;
Chris@208 832 try {
Chris@208 833 dir = TempDirectory::getInstance()->getSubDirectoryPath("download");
Chris@208 834 } catch (DirectoryCreationFailed f) {
Chris@327 835 #ifdef DEBUG_FILE_SOURCE
Chris@843 836 cerr << "FileSource::createCacheFile: ERROR: Failed to create temporary directory: " << f.what() << endl;
Chris@327 837 #endif
Chris@208 838 return "";
Chris@208 839 }
Chris@208 840
Chris@316 841 QString filepart = m_url.path().section('/', -1, -1,
Chris@316 842 QString::SectionSkipEmpty);
Chris@208 843
Chris@457 844 QString extension = "";
Chris@457 845 if (filepart.contains('.')) extension = filepart.section('.', -1);
Chris@457 846
Chris@208 847 QString base = filepart;
Chris@208 848 if (extension != "") {
Chris@208 849 base = base.left(base.length() - extension.length() - 1);
Chris@208 850 }
Chris@208 851 if (base == "") base = "remote";
Chris@208 852
Chris@208 853 QString filename;
Chris@208 854
Chris@208 855 if (extension == "") {
Chris@208 856 filename = base;
Chris@208 857 } else {
Chris@208 858 filename = QString("%1.%2").arg(base).arg(extension);
Chris@208 859 }
Chris@208 860
Chris@208 861 QString filepath(dir.filePath(filename));
Chris@208 862
Chris@327 863 #ifdef DEBUG_FILE_SOURCE
Chris@843 864 cerr << "FileSource::createCacheFile: URL is \"" << m_url.toString() << "\", dir is \"" << dir.path() << "\", base \"" << base << "\", extension \"" << extension << "\", filebase \"" << filename << "\", filename \"" << filepath << "\"" << endl;
Chris@327 865 #endif
Chris@208 866
Chris@316 867 QMutexLocker fcLocker(&m_fileCreationMutex);
Chris@316 868
Chris@208 869 ++m_count;
Chris@208 870
Chris@208 871 if (QFileInfo(filepath).exists() ||
Chris@208 872 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 873
Chris@327 874 #ifdef DEBUG_FILE_SOURCE
Chris@843 875 cerr << "FileSource::createCacheFile: Failed to create local file \""
Chris@686 876 << filepath << "\" for URL \""
Chris@843 877 << m_url.toString() << "\" (or file already exists): appending suffix instead" << endl;
Chris@327 878 #endif
Chris@208 879
Chris@208 880 if (extension == "") {
Chris@208 881 filename = QString("%1_%2").arg(base).arg(m_count);
Chris@208 882 } else {
Chris@208 883 filename = QString("%1_%2.%3").arg(base).arg(m_count).arg(extension);
Chris@208 884 }
Chris@208 885 filepath = dir.filePath(filename);
Chris@208 886
Chris@208 887 if (QFileInfo(filepath).exists() ||
Chris@208 888 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 889
Chris@327 890 #ifdef DEBUG_FILE_SOURCE
Chris@843 891 cerr << "FileSource::createCacheFile: ERROR: Failed to create local file \""
Chris@686 892 << filepath << "\" for URL \""
Chris@843 893 << m_url.toString() << "\" (or file already exists)" << endl;
Chris@327 894 #endif
Chris@208 895
Chris@208 896 return "";
Chris@208 897 }
Chris@208 898 }
Chris@208 899
Chris@327 900 #ifdef DEBUG_FILE_SOURCE
Chris@843 901 cerr << "FileSource::createCacheFile: url "
Chris@686 902 << m_url.toString() << " -> local filename "
Chris@843 903 << filepath << endl;
Chris@327 904 #endif
Chris@316 905
Chris@316 906 m_localFilename = filepath;
Chris@208 907
Chris@316 908 return false;
Chris@208 909 }
Chris@327 910