annotate data/fileio/FileSource.cpp @ 833:3cc81dbc31bb

Use thread-local storage for network managers; otherwise we get "Cannot create children for a parent that is in a different thread" from nm.get()
author Chris Cannam
date Fri, 19 Jul 2013 13:47:11 +0100
parents 95d4a59295b7
children fcee7e040ab4
rev   line source
Chris@208 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@208 2
Chris@208 3 /*
Chris@208 4 Sonic Visualiser
Chris@208 5 An audio file viewer and annotation editor.
Chris@208 6 Centre for Digital Music, Queen Mary, University of London.
Chris@208 7 This file copyright 2007 QMUL.
Chris@208 8
Chris@208 9 This program is free software; you can redistribute it and/or
Chris@208 10 modify it under the terms of the GNU General Public License as
Chris@208 11 published by the Free Software Foundation; either version 2 of the
Chris@208 12 License, or (at your option) any later version. See the file
Chris@208 13 COPYING included with this distribution for more information.
Chris@208 14 */
Chris@208 15
Chris@317 16 #include "FileSource.h"
Chris@357 17
Chris@208 18 #include "base/TempDirectory.h"
Chris@208 19 #include "base/Exceptions.h"
Chris@392 20 #include "base/ProgressReporter.h"
Chris@502 21 #include "system/System.h"
Chris@208 22
Chris@762 23 #include <QNetworkAccessManager>
Chris@762 24 #include <QNetworkReply>
Chris@208 25 #include <QFileInfo>
Chris@208 26 #include <QDir>
Chris@392 27 #include <QCoreApplication>
Chris@833 28 #include <QThreadStorage>
Chris@208 29
Chris@208 30 #include <iostream>
Chris@405 31 #include <cstdlib>
Chris@208 32
Chris@608 33 #include <unistd.h>
Chris@608 34
Chris@763 35 //#define DEBUG_FILE_SOURCE 1
Chris@327 36
Chris@208 37 int
Chris@317 38 FileSource::m_count = 0;
Chris@208 39
Chris@208 40 QMutex
Chris@317 41 FileSource::m_fileCreationMutex;
Chris@208 42
Chris@317 43 FileSource::RemoteRefCountMap
Chris@317 44 FileSource::m_refCountMap;
Chris@304 45
Chris@317 46 FileSource::RemoteLocalMap
Chris@317 47 FileSource::m_remoteLocalMap;
Chris@304 48
Chris@304 49 QMutex
Chris@317 50 FileSource::m_mapMutex;
Chris@304 51
Chris@529 52 #ifdef DEBUG_FILE_SOURCE
Chris@529 53 static int extantCount = 0;
Chris@529 54 static std::map<QString, int> urlExtantCountMap;
Chris@529 55 static void incCount(QString url) {
Chris@529 56 ++extantCount;
Chris@529 57 if (urlExtantCountMap.find(url) == urlExtantCountMap.end()) {
Chris@529 58 urlExtantCountMap[url] = 1;
Chris@529 59 } else {
Chris@529 60 ++urlExtantCountMap[url];
Chris@529 61 }
Chris@529 62 std::cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << std::endl;
Chris@529 63 }
Chris@529 64 static void decCount(QString url) {
Chris@529 65 --extantCount;
Chris@529 66 --urlExtantCountMap[url];
Chris@529 67 std::cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << std::endl;
Chris@529 68 }
Chris@529 69 #endif
Chris@529 70
Chris@833 71 static QThreadStorage<QNetworkAccessManager *> nms;
Chris@762 72
Chris@520 73 FileSource::FileSource(QString fileOrUrl, ProgressReporter *reporter,
Chris@520 74 QString preferredContentType) :
Chris@614 75 m_url(fileOrUrl, QUrl::StrictMode),
Chris@316 76 m_localFile(0),
Chris@762 77 m_reply(0),
Chris@520 78 m_preferredContentType(preferredContentType),
Chris@316 79 m_ok(false),
Chris@316 80 m_lastStatus(0),
Chris@706 81 m_resource(fileOrUrl.startsWith(':')),
Chris@316 82 m_remote(isRemote(fileOrUrl)),
Chris@316 83 m_done(false),
Chris@316 84 m_leaveLocalFile(false),
Chris@392 85 m_reporter(reporter),
Chris@316 86 m_refCounted(false)
Chris@316 87 {
Chris@706 88 if (m_resource) { // qrc file
Chris@706 89 m_url = QUrl("qrc" + fileOrUrl);
Chris@706 90 }
Chris@706 91
Chris@661 92 if (m_url.toString() == "") {
Chris@661 93 m_url = QUrl(fileOrUrl, QUrl::TolerantMode);
Chris@661 94 }
Chris@661 95
Chris@327 96 #ifdef DEBUG_FILE_SOURCE
Chris@706 97 std::cerr << "FileSource::FileSource(" << fileOrUrl << "): url <" << m_url.toString() << ">" << std::endl;
Chris@529 98 incCount(m_url.toString());
Chris@327 99 #endif
Chris@316 100
Chris@316 101 if (!canHandleScheme(m_url)) {
Chris@762 102 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << std::endl;
Chris@316 103 m_errorString = tr("Unsupported scheme in URL");
Chris@316 104 return;
Chris@316 105 }
Chris@316 106
Chris@357 107 init();
Chris@316 108
Chris@614 109 if (!isRemote() &&
Chris@614 110 !isAvailable()) {
Chris@614 111 #ifdef DEBUG_FILE_SOURCE
Chris@706 112 std::cerr << "FileSource::FileSource: Failed to open local file with URL \"" << m_url.toString() << "\"; trying again assuming filename was encoded" << std::endl;
Chris@614 113 #endif
Chris@762 114 m_url = QUrl::fromEncoded(fileOrUrl.toLatin1());
Chris@614 115 init();
Chris@614 116 }
Chris@614 117
Chris@316 118 if (isRemote() &&
Chris@316 119 (fileOrUrl.contains('%') ||
Chris@316 120 fileOrUrl.contains("--"))) { // for IDNA
Chris@316 121
Chris@316 122 waitForStatus();
Chris@316 123
Chris@316 124 if (!isAvailable()) {
Chris@336 125
Chris@316 126 // The URL was created on the assumption that the string
Chris@316 127 // was human-readable. Let's try again, this time
Chris@316 128 // assuming it was already encoded.
Chris@317 129 std::cerr << "FileSource::FileSource: Failed to retrieve URL \""
Chris@316 130 << fileOrUrl.toStdString()
Chris@316 131 << "\" as human-readable URL; "
Chris@316 132 << "trying again treating it as encoded URL"
Chris@316 133 << std::endl;
Chris@336 134
Chris@336 135 // even though our cache file doesn't exist (because the
Chris@336 136 // resource was 404), we still need to ensure we're no
Chris@336 137 // longer associating a filename with this url in the
Chris@336 138 // refcount map -- or createCacheFile will think we've
Chris@336 139 // already done all the work and no request will be sent
Chris@336 140 deleteCacheFile();
Chris@336 141
Chris@762 142 m_url = QUrl::fromEncoded(fileOrUrl.toLatin1());
Chris@336 143
Chris@336 144 m_ok = false;
Chris@336 145 m_done = false;
Chris@336 146 m_lastStatus = 0;
Chris@357 147 init();
Chris@316 148 }
Chris@316 149 }
Chris@325 150
Chris@325 151 if (!isRemote()) {
Chris@325 152 emit statusAvailable();
Chris@325 153 emit ready();
Chris@325 154 }
Chris@497 155
Chris@504 156 #ifdef DEBUG_FILE_SOURCE
Chris@762 157 std::cerr << "FileSource::FileSource(string) exiting" << std::endl;
Chris@504 158 #endif
Chris@316 159 }
Chris@316 160
Chris@469 161 FileSource::FileSource(QUrl url, ProgressReporter *reporter) :
Chris@304 162 m_url(url),
Chris@208 163 m_localFile(0),
Chris@762 164 m_reply(0),
Chris@208 165 m_ok(false),
Chris@210 166 m_lastStatus(0),
Chris@706 167 m_resource(false),
Chris@316 168 m_remote(isRemote(url.toString())),
Chris@208 169 m_done(false),
Chris@316 170 m_leaveLocalFile(false),
Chris@392 171 m_reporter(reporter),
Chris@316 172 m_refCounted(false)
Chris@208 173 {
Chris@327 174 #ifdef DEBUG_FILE_SOURCE
Chris@762 175 std::cerr << "FileSource::FileSource(" << url.toString() << ") [as url]" << std::endl;
Chris@529 176 incCount(m_url.toString());
Chris@327 177 #endif
Chris@316 178
Chris@316 179 if (!canHandleScheme(m_url)) {
Chris@762 180 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << std::endl;
Chris@316 181 m_errorString = tr("Unsupported scheme in URL");
Chris@208 182 return;
Chris@208 183 }
Chris@208 184
Chris@357 185 init();
Chris@497 186
Chris@504 187 #ifdef DEBUG_FILE_SOURCE
Chris@762 188 std::cerr << "FileSource::FileSource(url) exiting" << std::endl;
Chris@504 189 #endif
Chris@316 190 }
Chris@304 191
Chris@317 192 FileSource::FileSource(const FileSource &rf) :
Chris@316 193 QObject(),
Chris@316 194 m_url(rf.m_url),
Chris@316 195 m_localFile(0),
Chris@762 196 m_reply(0),
Chris@316 197 m_ok(rf.m_ok),
Chris@316 198 m_lastStatus(rf.m_lastStatus),
Chris@706 199 m_resource(rf.m_resource),
Chris@316 200 m_remote(rf.m_remote),
Chris@316 201 m_done(false),
Chris@316 202 m_leaveLocalFile(false),
Chris@392 203 m_reporter(rf.m_reporter),
Chris@316 204 m_refCounted(false)
Chris@316 205 {
Chris@327 206 #ifdef DEBUG_FILE_SOURCE
Chris@762 207 std::cerr << "FileSource::FileSource(" << m_url.toString() << ") [copy ctor]" << std::endl;
Chris@529 208 incCount(m_url.toString());
Chris@327 209 #endif
Chris@304 210
Chris@316 211 if (!canHandleScheme(m_url)) {
Chris@762 212 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << std::endl;
Chris@316 213 m_errorString = tr("Unsupported scheme in URL");
Chris@304 214 return;
Chris@304 215 }
Chris@304 216
Chris@469 217 if (!isRemote()) {
Chris@316 218 m_localFilename = rf.m_localFilename;
Chris@316 219 } else {
Chris@316 220 QMutexLocker locker(&m_mapMutex);
Chris@327 221 #ifdef DEBUG_FILE_SOURCE
Chris@762 222 std::cerr << "FileSource::FileSource(copy ctor): ref count is "
Chris@762 223 << m_refCountMap[m_url] << std::endl;
Chris@327 224 #endif
Chris@316 225 if (m_refCountMap[m_url] > 0) {
Chris@316 226 m_refCountMap[m_url]++;
Chris@327 227 #ifdef DEBUG_FILE_SOURCE
Chris@316 228 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 229 #endif
Chris@316 230 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 231 m_refCounted = true;
Chris@316 232 } else {
Chris@316 233 m_ok = false;
Chris@316 234 m_lastStatus = 404;
Chris@316 235 }
Chris@316 236 }
Chris@316 237
Chris@316 238 m_done = true;
Chris@497 239
Chris@504 240 #ifdef DEBUG_FILE_SOURCE
Chris@762 241 std::cerr << "FileSource::FileSource(" << m_url.toString() << ") [copy ctor]: note: local filename is \"" << m_localFilename << "\"" << std::endl;
Chris@527 242 #endif
Chris@527 243
Chris@527 244 #ifdef DEBUG_FILE_SOURCE
Chris@762 245 std::cerr << "FileSource::FileSource(copy ctor) exiting" << std::endl;
Chris@504 246 #endif
Chris@316 247 }
Chris@316 248
Chris@317 249 FileSource::~FileSource()
Chris@316 250 {
Chris@327 251 #ifdef DEBUG_FILE_SOURCE
Chris@686 252 std::cerr << "FileSource(" << m_url.toString() << ")::~FileSource" << std::endl;
Chris@529 253 decCount(m_url.toString());
Chris@327 254 #endif
Chris@316 255
Chris@316 256 cleanup();
Chris@316 257
Chris@469 258 if (isRemote() && !m_leaveLocalFile) deleteCacheFile();
Chris@316 259 }
Chris@316 260
Chris@316 261 void
Chris@357 262 FileSource::init()
Chris@316 263 {
Chris@833 264 { // check we have a QNetworkAccessManager
Chris@833 265 QMutexLocker locker(&m_mapMutex);
Chris@833 266 if (!nms.hasLocalData()) {
Chris@833 267 nms.setLocalData(new QNetworkAccessManager());
Chris@833 268 }
Chris@833 269 }
Chris@833 270
Chris@706 271 if (isResource()) {
Chris@355 272 #ifdef DEBUG_FILE_SOURCE
Chris@706 273 std::cerr << "FileSource::init: Is a resource" << std::endl;
Chris@706 274 #endif
Chris@706 275 QString resourceFile = m_url.toString();
Chris@706 276 resourceFile.replace(QRegExp("^qrc:"), ":");
Chris@706 277
Chris@706 278 if (!QFileInfo(resourceFile).exists()) {
Chris@706 279 #ifdef DEBUG_FILE_SOURCE
Chris@706 280 std::cerr << "FileSource::init: Resource file of this name does not exist, switching to non-resource URL" << std::endl;
Chris@706 281 #endif
Chris@706 282 m_url = resourceFile;
Chris@706 283 m_resource = false;
Chris@706 284 }
Chris@706 285 }
Chris@706 286
Chris@706 287 if (!isRemote() && !isResource()) {
Chris@706 288 #ifdef DEBUG_FILE_SOURCE
Chris@706 289 std::cerr << "FileSource::init: Not a remote URL" << std::endl;
Chris@355 290 #endif
Chris@355 291 bool literal = false;
Chris@316 292 m_localFilename = m_url.toLocalFile();
Chris@342 293 if (m_localFilename == "") {
Chris@342 294 // QUrl may have mishandled the scheme (e.g. in a DOS path)
Chris@342 295 m_localFilename = m_url.toString();
Chris@355 296 literal = true;
Chris@342 297 }
Chris@439 298 m_localFilename = QFileInfo(m_localFilename).absoluteFilePath();
Chris@439 299
Chris@355 300 #ifdef DEBUG_FILE_SOURCE
Chris@706 301 std::cerr << "FileSource::init: URL translates to local filename \""
Chris@706 302 << m_localFilename << "\" (with literal=" << literal << ")"
Chris@706 303 << std::endl;
Chris@355 304 #endif
Chris@316 305 m_ok = true;
Chris@355 306 m_lastStatus = 200;
Chris@355 307
Chris@316 308 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 309 if (literal) {
Chris@355 310 m_lastStatus = 404;
Chris@355 311 } else {
Chris@614 312 #ifdef DEBUG_FILE_SOURCE
Chris@706 313 std::cerr << "FileSource::init: Local file of this name does not exist, trying URL as a literal filename" << std::endl;
Chris@614 314 #endif
Chris@355 315 // Again, QUrl may have been mistreating us --
Chris@355 316 // e.g. dropping a part that looks like query data
Chris@355 317 m_localFilename = m_url.toString();
Chris@355 318 literal = true;
Chris@355 319 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 320 m_lastStatus = 404;
Chris@355 321 }
Chris@355 322 }
Chris@316 323 }
Chris@355 324
Chris@316 325 m_done = true;
Chris@316 326 return;
Chris@316 327 }
Chris@316 328
Chris@316 329 if (createCacheFile()) {
Chris@327 330 #ifdef DEBUG_FILE_SOURCE
Chris@706 331 std::cerr << "FileSource::init: Already have this one" << std::endl;
Chris@327 332 #endif
Chris@316 333 m_ok = true;
Chris@316 334 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 335 m_lastStatus = 404;
Chris@316 336 } else {
Chris@316 337 m_lastStatus = 200;
Chris@316 338 }
Chris@316 339 m_done = true;
Chris@316 340 return;
Chris@316 341 }
Chris@316 342
Chris@208 343 if (m_localFilename == "") return;
Chris@706 344
Chris@208 345 m_localFile = new QFile(m_localFilename);
Chris@208 346 m_localFile->open(QFile::WriteOnly);
Chris@208 347
Chris@706 348 if (isResource()) {
Chris@706 349
Chris@706 350 // Absent resource file case was dealt with at the top -- this
Chris@706 351 // is the successful case
Chris@706 352
Chris@706 353 QString resourceFileName = m_url.toString();
Chris@706 354 resourceFileName.replace(QRegExp("^qrc:"), ":");
Chris@706 355 QFile resourceFile(resourceFileName);
Chris@706 356 resourceFile.open(QFile::ReadOnly);
Chris@706 357 QByteArray ba(resourceFile.readAll());
Chris@706 358
Chris@706 359 #ifdef DEBUG_FILE_SOURCE
Chris@706 360 std::cerr << "Copying " << ba.size() << " bytes from resource file to cache file" << std::endl;
Chris@706 361 #endif
Chris@706 362
Chris@706 363 qint64 written = m_localFile->write(ba);
Chris@706 364 m_localFile->close();
Chris@706 365 delete m_localFile;
Chris@706 366 m_localFile = 0;
Chris@706 367
Chris@706 368 if (written != ba.size()) {
Chris@706 369 #ifdef DEBUG_FILE_SOURCE
Chris@706 370 std::cerr << "Copy failed (wrote " << written << " bytes)" << std::endl;
Chris@706 371 #endif
Chris@706 372 m_ok = false;
Chris@706 373 return;
Chris@706 374 } else {
Chris@706 375 m_ok = true;
Chris@706 376 m_lastStatus = 200;
Chris@706 377 m_done = true;
Chris@706 378 }
Chris@706 379
Chris@706 380 } else {
Chris@706 381
Chris@706 382 QString scheme = m_url.scheme().toLower();
Chris@316 383
Chris@327 384 #ifdef DEBUG_FILE_SOURCE
Chris@706 385 std::cerr << "FileSource::init: Don't have local copy of \""
Chris@706 386 << m_url.toString() << "\", retrieving" << std::endl;
Chris@327 387 #endif
Chris@208 388
Chris@762 389 if (scheme == "http" || scheme == "https" || scheme == "ftp") {
Chris@762 390 initRemote();
Chris@520 391 #ifdef DEBUG_FILE_SOURCE
Chris@762 392 std::cerr << "FileSource: initRemote returned" << std::endl;
Chris@520 393 #endif
Chris@706 394 } else {
Chris@706 395 m_remote = false;
Chris@706 396 m_ok = false;
Chris@706 397 }
Chris@208 398 }
Chris@208 399
Chris@208 400 if (m_ok) {
Chris@316 401
Chris@316 402 QMutexLocker locker(&m_mapMutex);
Chris@316 403
Chris@316 404 if (m_refCountMap[m_url] > 0) {
Chris@316 405 // someone else has been doing the same thing at the same time,
Chris@316 406 // but has got there first
Chris@316 407 cleanup();
Chris@316 408 m_refCountMap[m_url]++;
Chris@327 409 #ifdef DEBUG_FILE_SOURCE
Chris@706 410 std::cerr << "FileSource::init: Another FileSource has got there first, abandoning our download and using theirs" << std::endl;
Chris@327 411 #endif
Chris@316 412 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 413 m_refCounted = true;
Chris@316 414 m_ok = true;
Chris@316 415 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 416 m_lastStatus = 404;
Chris@316 417 }
Chris@316 418 m_done = true;
Chris@316 419 return;
Chris@316 420 }
Chris@304 421
Chris@304 422 m_remoteLocalMap[m_url] = m_localFilename;
Chris@304 423 m_refCountMap[m_url]++;
Chris@316 424 m_refCounted = true;
Chris@304 425
Chris@706 426 if (m_reporter && !m_done) {
Chris@392 427 m_reporter->setMessage
Chris@392 428 (tr("Downloading %1...").arg(m_url.toString()));
Chris@392 429 connect(m_reporter, SIGNAL(cancelled()), this, SLOT(cancelled()));
Chris@357 430 connect(this, SIGNAL(progress(int)),
Chris@392 431 m_reporter, SLOT(setProgress(int)));
Chris@316 432 }
Chris@208 433 }
Chris@208 434 }
Chris@208 435
Chris@316 436 void
Chris@762 437 FileSource::initRemote()
Chris@208 438 {
Chris@316 439 m_ok = true;
Chris@316 440
Chris@762 441 QNetworkRequest req;
Chris@762 442 req.setUrl(m_url);
Chris@762 443
Chris@762 444 if (m_preferredContentType != "") {
Chris@520 445 #ifdef DEBUG_FILE_SOURCE
Chris@520 446 std::cerr << "FileSource: indicating preferred content type of \""
Chris@686 447 << m_preferredContentType << "\"" << std::endl;
Chris@520 448 #endif
Chris@762 449 req.setRawHeader
Chris@762 450 ("Accept",
Chris@762 451 QString("%1, */*").arg(m_preferredContentType).toLatin1());
Chris@520 452 }
Chris@316 453
Chris@833 454 m_reply = nms.localData()->get(req);
Chris@762 455
Chris@762 456 connect(m_reply, SIGNAL(readyRead()),
Chris@762 457 this, SLOT(readyRead()));
Chris@762 458 connect(m_reply, SIGNAL(error(QNetworkReply::NetworkError)),
Chris@762 459 this, SLOT(replyFailed(QNetworkReply::NetworkError)));
Chris@762 460 connect(m_reply, SIGNAL(finished()),
Chris@762 461 this, SLOT(replyFinished()));
Chris@764 462 connect(m_reply, SIGNAL(metaDataChanged()),
Chris@764 463 this, SLOT(metaDataChanged()));
Chris@762 464 connect(m_reply, SIGNAL(downloadProgress(qint64, qint64)),
Chris@762 465 this, SLOT(downloadProgress(qint64, qint64)));
Chris@211 466 }
Chris@211 467
Chris@211 468 void
Chris@317 469 FileSource::cleanup()
Chris@211 470 {
Chris@470 471 if (m_done) {
Chris@470 472 delete m_localFile; // does not actually delete the file
Chris@470 473 m_localFile = 0;
Chris@470 474 }
Chris@211 475 m_done = true;
Chris@762 476 if (m_reply) {
Chris@762 477 QNetworkReply *r = m_reply;
Chris@762 478 m_reply = 0;
Chris@762 479 r->abort();
Chris@762 480 r->deleteLater();
Chris@214 481 }
Chris@470 482 if (m_localFile) {
Chris@470 483 delete m_localFile; // does not actually delete the file
Chris@470 484 m_localFile = 0;
Chris@470 485 }
Chris@208 486 }
Chris@208 487
Chris@208 488 bool
Chris@317 489 FileSource::isRemote(QString fileOrUrl)
Chris@304 490 {
Chris@342 491 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@316 492 QString scheme = QUrl(fileOrUrl).scheme().toLower();
Chris@342 493 if (scheme == "" || scheme == "file" || scheme.length() == 1) return false;
Chris@342 494 return true;
Chris@304 495 }
Chris@304 496
Chris@304 497 bool
Chris@317 498 FileSource::canHandleScheme(QUrl url)
Chris@208 499 {
Chris@342 500 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@208 501 QString scheme = url.scheme().toLower();
Chris@762 502 return (scheme == "http" || scheme == "https" ||
Chris@762 503 scheme == "ftp" || scheme == "file" || scheme == "qrc" ||
Chris@706 504 scheme == "" || scheme.length() == 1);
Chris@208 505 }
Chris@208 506
Chris@210 507 bool
Chris@317 508 FileSource::isAvailable()
Chris@210 509 {
Chris@316 510 waitForStatus();
Chris@211 511 bool available = true;
Chris@211 512 if (!m_ok) available = false;
Chris@211 513 else available = (m_lastStatus / 100 == 2);
Chris@327 514 #ifdef DEBUG_FILE_SOURCE
Chris@762 515 std::cerr << "FileSource::isAvailable: " << (available ? "yes" : "no")
Chris@762 516 << std::endl;
Chris@327 517 #endif
Chris@211 518 return available;
Chris@210 519 }
Chris@210 520
Chris@208 521 void
Chris@317 522 FileSource::waitForStatus()
Chris@316 523 {
Chris@316 524 while (m_ok && (!m_done && m_lastStatus == 0)) {
Chris@316 525 // std::cerr << "waitForStatus: processing (last status " << m_lastStatus << ")" << std::endl;
Chris@392 526 QCoreApplication::processEvents();
Chris@316 527 }
Chris@316 528 }
Chris@316 529
Chris@316 530 void
Chris@317 531 FileSource::waitForData()
Chris@208 532 {
Chris@211 533 while (m_ok && !m_done) {
Chris@762 534 // std::cerr << "FileSource::waitForData: calling QApplication::processEvents" << std::endl;
Chris@392 535 QCoreApplication::processEvents();
Chris@497 536 usleep(10000);
Chris@208 537 }
Chris@208 538 }
Chris@208 539
Chris@316 540 void
Chris@317 541 FileSource::setLeaveLocalFile(bool leave)
Chris@316 542 {
Chris@316 543 m_leaveLocalFile = leave;
Chris@316 544 }
Chris@316 545
Chris@208 546 bool
Chris@317 547 FileSource::isOK() const
Chris@208 548 {
Chris@208 549 return m_ok;
Chris@208 550 }
Chris@208 551
Chris@208 552 bool
Chris@317 553 FileSource::isDone() const
Chris@208 554 {
Chris@208 555 return m_done;
Chris@208 556 }
Chris@208 557
Chris@316 558 bool
Chris@706 559 FileSource::isResource() const
Chris@706 560 {
Chris@706 561 return m_resource;
Chris@706 562 }
Chris@706 563
Chris@706 564 bool
Chris@317 565 FileSource::isRemote() const
Chris@316 566 {
Chris@316 567 return m_remote;
Chris@316 568 }
Chris@316 569
Chris@316 570 QString
Chris@317 571 FileSource::getLocation() const
Chris@316 572 {
Chris@316 573 return m_url.toString();
Chris@316 574 }
Chris@316 575
Chris@208 576 QString
Chris@317 577 FileSource::getLocalFilename() const
Chris@208 578 {
Chris@208 579 return m_localFilename;
Chris@208 580 }
Chris@208 581
Chris@208 582 QString
Chris@678 583 FileSource::getBasename() const
Chris@678 584 {
Chris@678 585 return QFileInfo(m_localFilename).fileName();
Chris@678 586 }
Chris@678 587
Chris@678 588 QString
Chris@317 589 FileSource::getContentType() const
Chris@316 590 {
Chris@316 591 return m_contentType;
Chris@316 592 }
Chris@316 593
Chris@316 594 QString
Chris@317 595 FileSource::getExtension() const
Chris@316 596 {
Chris@316 597 if (m_localFilename != "") {
Chris@316 598 return QFileInfo(m_localFilename).suffix().toLower();
Chris@316 599 } else {
Chris@316 600 return QFileInfo(m_url.toLocalFile()).suffix().toLower();
Chris@316 601 }
Chris@316 602 }
Chris@316 603
Chris@316 604 QString
Chris@317 605 FileSource::getErrorString() const
Chris@208 606 {
Chris@208 607 return m_errorString;
Chris@208 608 }
Chris@208 609
Chris@208 610 void
Chris@762 611 FileSource::readyRead()
Chris@208 612 {
Chris@762 613 m_localFile->write(m_reply->readAll());
Chris@208 614 }
Chris@208 615
Chris@208 616 void
Chris@764 617 FileSource::metaDataChanged()
Chris@210 618 {
Chris@520 619 #ifdef DEBUG_FILE_SOURCE
Chris@764 620 std::cerr << "FileSource::metaDataChanged" << std::endl;
Chris@520 621 #endif
Chris@497 622
Chris@762 623 if (!m_reply) {
Chris@764 624 std::cerr << "WARNING: FileSource::metaDataChanged() called without a reply object being known to us" << std::endl;
Chris@762 625 return;
Chris@762 626 }
Chris@762 627
Chris@762 628 int status =
Chris@762 629 m_reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
Chris@762 630
Chris@762 631 if (status / 100 == 3) {
Chris@762 632 QString location = m_reply->header
Chris@762 633 (QNetworkRequest::LocationHeader).toString();
Chris@496 634 #ifdef DEBUG_FILE_SOURCE
Chris@764 635 std::cerr << "FileSource::metaDataChanged: redirect to \""
Chris@762 636 << location << "\" received" << std::endl;
Chris@496 637 #endif
Chris@496 638 if (location != "") {
Chris@496 639 QUrl newUrl(location);
Chris@496 640 if (newUrl != m_url) {
Chris@497 641 cleanup();
Chris@497 642 deleteCacheFile();
Chris@529 643 #ifdef DEBUG_FILE_SOURCE
Chris@529 644 decCount(m_url.toString());
Chris@529 645 incCount(newUrl.toString());
Chris@529 646 #endif
Chris@496 647 m_url = newUrl;
Chris@497 648 m_localFile = 0;
Chris@496 649 m_lastStatus = 0;
Chris@497 650 m_done = false;
Chris@497 651 m_refCounted = false;
Chris@496 652 init();
Chris@496 653 return;
Chris@496 654 }
Chris@496 655 }
Chris@496 656 }
Chris@497 657
Chris@762 658 m_lastStatus = status;
Chris@210 659 if (m_lastStatus / 100 >= 4) {
Chris@210 660 m_errorString = QString("%1 %2")
Chris@762 661 .arg(status)
Chris@762 662 .arg(m_reply->attribute
Chris@762 663 (QNetworkRequest::HttpReasonPhraseAttribute).toString());
Chris@327 664 #ifdef DEBUG_FILE_SOURCE
Chris@764 665 std::cerr << "FileSource::metaDataChanged: "
Chris@762 666 << m_errorString << std::endl;
Chris@327 667 #endif
Chris@211 668 } else {
Chris@327 669 #ifdef DEBUG_FILE_SOURCE
Chris@764 670 std::cerr << "FileSource::metaDataChanged: "
Chris@762 671 << m_lastStatus << std::endl;
Chris@327 672 #endif
Chris@762 673 m_contentType =
Chris@762 674 m_reply->header(QNetworkRequest::ContentTypeHeader).toString();
Chris@325 675 }
Chris@325 676 emit statusAvailable();
Chris@210 677 }
Chris@210 678
Chris@210 679 void
Chris@762 680 FileSource::downloadProgress(qint64 done, qint64 total)
Chris@208 681 {
Chris@208 682 int percent = int((double(done) / double(total)) * 100.0 - 0.1);
Chris@208 683 emit progress(percent);
Chris@210 684 }
Chris@210 685
Chris@210 686 void
Chris@317 687 FileSource::cancelled()
Chris@210 688 {
Chris@210 689 m_done = true;
Chris@316 690 cleanup();
Chris@316 691
Chris@210 692 m_ok = false;
Chris@210 693 m_errorString = tr("Download cancelled");
Chris@208 694 }
Chris@208 695
Chris@208 696 void
Chris@762 697 FileSource::replyFinished()
Chris@208 698 {
Chris@327 699 emit progress(100);
Chris@327 700
Chris@327 701 #ifdef DEBUG_FILE_SOURCE
Chris@762 702 std::cerr << "FileSource::replyFinished()" << std::endl;
Chris@327 703 #endif
Chris@211 704
Chris@211 705 if (m_done) return;
Chris@211 706
Chris@762 707 bool error = (m_lastStatus / 100 >= 4);
Chris@210 708
Chris@211 709 cleanup();
Chris@208 710
Chris@211 711 if (!error) {
Chris@208 712 QFileInfo fi(m_localFilename);
Chris@208 713 if (!fi.exists()) {
Chris@208 714 m_errorString = tr("Failed to create local file %1").arg(m_localFilename);
Chris@211 715 error = true;
Chris@208 716 } else if (fi.size() == 0) {
Chris@208 717 m_errorString = tr("File contains no data!");
Chris@211 718 error = true;
Chris@208 719 }
Chris@208 720 }
Chris@211 721
Chris@211 722 if (error) {
Chris@327 723 #ifdef DEBUG_FILE_SOURCE
Chris@469 724 std::cerr << "FileSource::done: error is " << error << ", deleting cache file" << std::endl;
Chris@327 725 #endif
Chris@316 726 deleteCacheFile();
Chris@211 727 }
Chris@211 728
Chris@211 729 m_ok = !error;
Chris@520 730 if (m_localFile) m_localFile->flush();
Chris@211 731 m_done = true;
Chris@304 732 emit ready();
Chris@211 733 }
Chris@211 734
Chris@211 735 void
Chris@763 736 FileSource::replyFailed(QNetworkReply::NetworkError)
Chris@763 737 {
Chris@763 738 emit progress(100);
Chris@763 739 m_errorString = m_reply->errorString();
Chris@763 740 m_ok = false;
Chris@763 741 m_done = true;
Chris@763 742 cleanup();
Chris@763 743 emit ready();
Chris@763 744 }
Chris@763 745
Chris@763 746 void
Chris@317 747 FileSource::deleteCacheFile()
Chris@211 748 {
Chris@327 749 #ifdef DEBUG_FILE_SOURCE
Chris@762 750 std::cerr << "FileSource::deleteCacheFile(\"" << m_localFilename << "\")" << std::endl;
Chris@327 751 #endif
Chris@211 752
Chris@211 753 cleanup();
Chris@211 754
Chris@316 755 if (m_localFilename == "") {
Chris@316 756 return;
Chris@316 757 }
Chris@211 758
Chris@316 759 if (!isRemote()) {
Chris@327 760 #ifdef DEBUG_FILE_SOURCE
Chris@316 761 std::cerr << "not a cache file" << std::endl;
Chris@327 762 #endif
Chris@316 763 return;
Chris@316 764 }
Chris@316 765
Chris@316 766 if (m_refCounted) {
Chris@304 767
Chris@304 768 QMutexLocker locker(&m_mapMutex);
Chris@316 769 m_refCounted = false;
Chris@304 770
Chris@304 771 if (m_refCountMap[m_url] > 0) {
Chris@304 772 m_refCountMap[m_url]--;
Chris@327 773 #ifdef DEBUG_FILE_SOURCE
Chris@316 774 std::cerr << "reduced ref count to " << m_refCountMap[m_url] << std::endl;
Chris@327 775 #endif
Chris@304 776 if (m_refCountMap[m_url] > 0) {
Chris@304 777 m_done = true;
Chris@304 778 return;
Chris@304 779 }
Chris@304 780 }
Chris@304 781 }
Chris@304 782
Chris@211 783 m_fileCreationMutex.lock();
Chris@211 784
Chris@211 785 if (!QFile(m_localFilename).remove()) {
Chris@469 786 #ifdef DEBUG_FILE_SOURCE
Chris@686 787 std::cerr << "FileSource::deleteCacheFile: ERROR: Failed to delete file \"" << m_localFilename << "\"" << std::endl;
Chris@469 788 #endif
Chris@211 789 } else {
Chris@327 790 #ifdef DEBUG_FILE_SOURCE
Chris@762 791 std::cerr << "FileSource::deleteCacheFile: Deleted cache file \"" << m_localFilename << "\"" << std::endl;
Chris@327 792 #endif
Chris@211 793 m_localFilename = "";
Chris@211 794 }
Chris@211 795
Chris@211 796 m_fileCreationMutex.unlock();
Chris@211 797
Chris@208 798 m_done = true;
Chris@208 799 }
Chris@208 800
Chris@316 801 bool
Chris@317 802 FileSource::createCacheFile()
Chris@208 803 {
Chris@316 804 {
Chris@316 805 QMutexLocker locker(&m_mapMutex);
Chris@316 806
Chris@327 807 #ifdef DEBUG_FILE_SOURCE
Chris@762 808 std::cerr << "FileSource::createCacheFile: refcount is " << m_refCountMap[m_url] << std::endl;
Chris@327 809 #endif
Chris@316 810
Chris@316 811 if (m_refCountMap[m_url] > 0) {
Chris@316 812 m_refCountMap[m_url]++;
Chris@316 813 m_localFilename = m_remoteLocalMap[m_url];
Chris@327 814 #ifdef DEBUG_FILE_SOURCE
Chris@316 815 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 816 #endif
Chris@316 817 m_refCounted = true;
Chris@316 818 return true;
Chris@316 819 }
Chris@316 820 }
Chris@316 821
Chris@208 822 QDir dir;
Chris@208 823 try {
Chris@208 824 dir = TempDirectory::getInstance()->getSubDirectoryPath("download");
Chris@208 825 } catch (DirectoryCreationFailed f) {
Chris@327 826 #ifdef DEBUG_FILE_SOURCE
Chris@317 827 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create temporary directory: " << f.what() << std::endl;
Chris@327 828 #endif
Chris@208 829 return "";
Chris@208 830 }
Chris@208 831
Chris@316 832 QString filepart = m_url.path().section('/', -1, -1,
Chris@316 833 QString::SectionSkipEmpty);
Chris@208 834
Chris@457 835 QString extension = "";
Chris@457 836 if (filepart.contains('.')) extension = filepart.section('.', -1);
Chris@457 837
Chris@208 838 QString base = filepart;
Chris@208 839 if (extension != "") {
Chris@208 840 base = base.left(base.length() - extension.length() - 1);
Chris@208 841 }
Chris@208 842 if (base == "") base = "remote";
Chris@208 843
Chris@208 844 QString filename;
Chris@208 845
Chris@208 846 if (extension == "") {
Chris@208 847 filename = base;
Chris@208 848 } else {
Chris@208 849 filename = QString("%1.%2").arg(base).arg(extension);
Chris@208 850 }
Chris@208 851
Chris@208 852 QString filepath(dir.filePath(filename));
Chris@208 853
Chris@327 854 #ifdef DEBUG_FILE_SOURCE
Chris@706 855 std::cerr << "FileSource::createCacheFile: URL is \"" << m_url.toString() << "\", dir is \"" << dir.path() << "\", base \"" << base << "\", extension \"" << extension << "\", filebase \"" << filename << "\", filename \"" << filepath << "\"" << std::endl;
Chris@327 856 #endif
Chris@208 857
Chris@316 858 QMutexLocker fcLocker(&m_fileCreationMutex);
Chris@316 859
Chris@208 860 ++m_count;
Chris@208 861
Chris@208 862 if (QFileInfo(filepath).exists() ||
Chris@208 863 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 864
Chris@327 865 #ifdef DEBUG_FILE_SOURCE
Chris@317 866 std::cerr << "FileSource::createCacheFile: Failed to create local file \""
Chris@686 867 << filepath << "\" for URL \""
Chris@686 868 << m_url.toString() << "\" (or file already exists): appending suffix instead" << std::endl;
Chris@327 869 #endif
Chris@208 870
Chris@208 871 if (extension == "") {
Chris@208 872 filename = QString("%1_%2").arg(base).arg(m_count);
Chris@208 873 } else {
Chris@208 874 filename = QString("%1_%2.%3").arg(base).arg(m_count).arg(extension);
Chris@208 875 }
Chris@208 876 filepath = dir.filePath(filename);
Chris@208 877
Chris@208 878 if (QFileInfo(filepath).exists() ||
Chris@208 879 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 880
Chris@327 881 #ifdef DEBUG_FILE_SOURCE
Chris@317 882 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create local file \""
Chris@686 883 << filepath << "\" for URL \""
Chris@686 884 << m_url.toString() << "\" (or file already exists)" << std::endl;
Chris@327 885 #endif
Chris@208 886
Chris@208 887 return "";
Chris@208 888 }
Chris@208 889 }
Chris@208 890
Chris@327 891 #ifdef DEBUG_FILE_SOURCE
Chris@706 892 std::cerr << "FileSource::createCacheFile: url "
Chris@686 893 << m_url.toString() << " -> local filename "
Chris@706 894 << filepath << std::endl;
Chris@327 895 #endif
Chris@316 896
Chris@316 897 m_localFilename = filepath;
Chris@208 898
Chris@316 899 return false;
Chris@208 900 }
Chris@327 901