annotate data/fileio/FileSource.cpp @ 832:a1bb2989013d

Merge
author Chris Cannam <chris.cannam@eecs.qmul.ac.uk>
date Wed, 24 Jul 2013 18:24:55 +0100
parents 12aff5a181bc
children fcee7e040ab4
rev   line source
Chris@208 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@208 2
Chris@208 3 /*
Chris@208 4 Sonic Visualiser
Chris@208 5 An audio file viewer and annotation editor.
Chris@208 6 Centre for Digital Music, Queen Mary, University of London.
Chris@208 7 This file copyright 2007 QMUL.
Chris@208 8
Chris@208 9 This program is free software; you can redistribute it and/or
Chris@208 10 modify it under the terms of the GNU General Public License as
Chris@208 11 published by the Free Software Foundation; either version 2 of the
Chris@208 12 License, or (at your option) any later version. See the file
Chris@208 13 COPYING included with this distribution for more information.
Chris@208 14 */
Chris@208 15
Chris@317 16 #include "FileSource.h"
Chris@357 17
Chris@208 18 #include "base/TempDirectory.h"
Chris@208 19 #include "base/Exceptions.h"
Chris@392 20 #include "base/ProgressReporter.h"
Chris@502 21 #include "system/System.h"
Chris@208 22
Chris@762 23 #include <QNetworkAccessManager>
Chris@762 24 #include <QNetworkReply>
Chris@208 25 #include <QFileInfo>
Chris@208 26 #include <QDir>
Chris@392 27 #include <QCoreApplication>
Chris@208 28
Chris@208 29 #include <iostream>
Chris@405 30 #include <cstdlib>
Chris@208 31
Chris@608 32 #include <unistd.h>
Chris@608 33
Chris@763 34 //#define DEBUG_FILE_SOURCE 1
Chris@327 35
Chris@208 36 int
Chris@317 37 FileSource::m_count = 0;
Chris@208 38
Chris@208 39 QMutex
Chris@317 40 FileSource::m_fileCreationMutex;
Chris@208 41
Chris@317 42 FileSource::RemoteRefCountMap
Chris@317 43 FileSource::m_refCountMap;
Chris@304 44
Chris@317 45 FileSource::RemoteLocalMap
Chris@317 46 FileSource::m_remoteLocalMap;
Chris@304 47
Chris@304 48 QMutex
Chris@317 49 FileSource::m_mapMutex;
Chris@304 50
Chris@529 51 #ifdef DEBUG_FILE_SOURCE
Chris@529 52 static int extantCount = 0;
Chris@529 53 static std::map<QString, int> urlExtantCountMap;
Chris@529 54 static void incCount(QString url) {
Chris@529 55 ++extantCount;
Chris@529 56 if (urlExtantCountMap.find(url) == urlExtantCountMap.end()) {
Chris@529 57 urlExtantCountMap[url] = 1;
Chris@529 58 } else {
Chris@529 59 ++urlExtantCountMap[url];
Chris@529 60 }
Chris@529 61 std::cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << std::endl;
Chris@529 62 }
Chris@529 63 static void decCount(QString url) {
Chris@529 64 --extantCount;
Chris@529 65 --urlExtantCountMap[url];
Chris@529 66 std::cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << std::endl;
Chris@529 67 }
Chris@529 68 #endif
Chris@529 69
Chris@762 70 static QNetworkAccessManager nm;
Chris@762 71
Chris@520 72 FileSource::FileSource(QString fileOrUrl, ProgressReporter *reporter,
Chris@520 73 QString preferredContentType) :
chris@831 74 m_rawFileOrUrl(fileOrUrl),
Chris@614 75 m_url(fileOrUrl, QUrl::StrictMode),
Chris@316 76 m_localFile(0),
Chris@762 77 m_reply(0),
Chris@520 78 m_preferredContentType(preferredContentType),
Chris@316 79 m_ok(false),
Chris@316 80 m_lastStatus(0),
Chris@706 81 m_resource(fileOrUrl.startsWith(':')),
Chris@316 82 m_remote(isRemote(fileOrUrl)),
Chris@316 83 m_done(false),
Chris@316 84 m_leaveLocalFile(false),
Chris@392 85 m_reporter(reporter),
Chris@316 86 m_refCounted(false)
Chris@316 87 {
Chris@706 88 if (m_resource) { // qrc file
Chris@706 89 m_url = QUrl("qrc" + fileOrUrl);
Chris@706 90 }
Chris@706 91
Chris@661 92 if (m_url.toString() == "") {
Chris@661 93 m_url = QUrl(fileOrUrl, QUrl::TolerantMode);
Chris@661 94 }
Chris@661 95
Chris@327 96 #ifdef DEBUG_FILE_SOURCE
Chris@706 97 std::cerr << "FileSource::FileSource(" << fileOrUrl << "): url <" << m_url.toString() << ">" << std::endl;
Chris@529 98 incCount(m_url.toString());
Chris@327 99 #endif
Chris@316 100
Chris@316 101 if (!canHandleScheme(m_url)) {
Chris@762 102 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << std::endl;
Chris@316 103 m_errorString = tr("Unsupported scheme in URL");
Chris@316 104 return;
Chris@316 105 }
Chris@316 106
Chris@357 107 init();
Chris@316 108
Chris@614 109 if (!isRemote() &&
Chris@614 110 !isAvailable()) {
Chris@614 111 #ifdef DEBUG_FILE_SOURCE
Chris@706 112 std::cerr << "FileSource::FileSource: Failed to open local file with URL \"" << m_url.toString() << "\"; trying again assuming filename was encoded" << std::endl;
Chris@614 113 #endif
Chris@762 114 m_url = QUrl::fromEncoded(fileOrUrl.toLatin1());
chris@831 115 #ifdef DEBUG_FILE_SOURCE
chris@831 116 std::cerr << "FileSource::FileSource: URL is now \"" << m_url.toString() << "\"" << std::endl;
chris@831 117 #endif
Chris@614 118 init();
Chris@614 119 }
Chris@614 120
Chris@316 121 if (isRemote() &&
Chris@316 122 (fileOrUrl.contains('%') ||
Chris@316 123 fileOrUrl.contains("--"))) { // for IDNA
Chris@316 124
Chris@316 125 waitForStatus();
Chris@316 126
Chris@316 127 if (!isAvailable()) {
Chris@336 128
Chris@316 129 // The URL was created on the assumption that the string
Chris@316 130 // was human-readable. Let's try again, this time
Chris@316 131 // assuming it was already encoded.
Chris@317 132 std::cerr << "FileSource::FileSource: Failed to retrieve URL \""
Chris@316 133 << fileOrUrl.toStdString()
Chris@316 134 << "\" as human-readable URL; "
Chris@316 135 << "trying again treating it as encoded URL"
Chris@316 136 << std::endl;
Chris@336 137
Chris@336 138 // even though our cache file doesn't exist (because the
Chris@336 139 // resource was 404), we still need to ensure we're no
Chris@336 140 // longer associating a filename with this url in the
Chris@336 141 // refcount map -- or createCacheFile will think we've
Chris@336 142 // already done all the work and no request will be sent
Chris@336 143 deleteCacheFile();
Chris@336 144
Chris@762 145 m_url = QUrl::fromEncoded(fileOrUrl.toLatin1());
Chris@336 146
Chris@336 147 m_ok = false;
Chris@336 148 m_done = false;
Chris@336 149 m_lastStatus = 0;
Chris@357 150 init();
Chris@316 151 }
Chris@316 152 }
Chris@325 153
Chris@325 154 if (!isRemote()) {
Chris@325 155 emit statusAvailable();
Chris@325 156 emit ready();
Chris@325 157 }
Chris@497 158
Chris@504 159 #ifdef DEBUG_FILE_SOURCE
Chris@762 160 std::cerr << "FileSource::FileSource(string) exiting" << std::endl;
Chris@504 161 #endif
Chris@316 162 }
Chris@316 163
Chris@469 164 FileSource::FileSource(QUrl url, ProgressReporter *reporter) :
Chris@304 165 m_url(url),
Chris@208 166 m_localFile(0),
Chris@762 167 m_reply(0),
Chris@208 168 m_ok(false),
Chris@210 169 m_lastStatus(0),
Chris@706 170 m_resource(false),
Chris@316 171 m_remote(isRemote(url.toString())),
Chris@208 172 m_done(false),
Chris@316 173 m_leaveLocalFile(false),
Chris@392 174 m_reporter(reporter),
Chris@316 175 m_refCounted(false)
Chris@208 176 {
Chris@327 177 #ifdef DEBUG_FILE_SOURCE
Chris@762 178 std::cerr << "FileSource::FileSource(" << url.toString() << ") [as url]" << std::endl;
Chris@529 179 incCount(m_url.toString());
Chris@327 180 #endif
Chris@316 181
Chris@316 182 if (!canHandleScheme(m_url)) {
Chris@762 183 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << std::endl;
Chris@316 184 m_errorString = tr("Unsupported scheme in URL");
Chris@208 185 return;
Chris@208 186 }
Chris@208 187
Chris@357 188 init();
Chris@497 189
Chris@504 190 #ifdef DEBUG_FILE_SOURCE
Chris@762 191 std::cerr << "FileSource::FileSource(url) exiting" << std::endl;
Chris@504 192 #endif
Chris@316 193 }
Chris@304 194
Chris@317 195 FileSource::FileSource(const FileSource &rf) :
Chris@316 196 QObject(),
Chris@316 197 m_url(rf.m_url),
Chris@316 198 m_localFile(0),
Chris@762 199 m_reply(0),
Chris@316 200 m_ok(rf.m_ok),
Chris@316 201 m_lastStatus(rf.m_lastStatus),
Chris@706 202 m_resource(rf.m_resource),
Chris@316 203 m_remote(rf.m_remote),
Chris@316 204 m_done(false),
Chris@316 205 m_leaveLocalFile(false),
Chris@392 206 m_reporter(rf.m_reporter),
Chris@316 207 m_refCounted(false)
Chris@316 208 {
Chris@327 209 #ifdef DEBUG_FILE_SOURCE
Chris@762 210 std::cerr << "FileSource::FileSource(" << m_url.toString() << ") [copy ctor]" << std::endl;
Chris@529 211 incCount(m_url.toString());
Chris@327 212 #endif
Chris@304 213
Chris@316 214 if (!canHandleScheme(m_url)) {
Chris@762 215 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << std::endl;
Chris@316 216 m_errorString = tr("Unsupported scheme in URL");
Chris@304 217 return;
Chris@304 218 }
Chris@304 219
Chris@469 220 if (!isRemote()) {
Chris@316 221 m_localFilename = rf.m_localFilename;
Chris@316 222 } else {
Chris@316 223 QMutexLocker locker(&m_mapMutex);
Chris@327 224 #ifdef DEBUG_FILE_SOURCE
Chris@762 225 std::cerr << "FileSource::FileSource(copy ctor): ref count is "
Chris@762 226 << m_refCountMap[m_url] << std::endl;
Chris@327 227 #endif
Chris@316 228 if (m_refCountMap[m_url] > 0) {
Chris@316 229 m_refCountMap[m_url]++;
Chris@327 230 #ifdef DEBUG_FILE_SOURCE
Chris@316 231 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 232 #endif
Chris@316 233 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 234 m_refCounted = true;
Chris@316 235 } else {
Chris@316 236 m_ok = false;
Chris@316 237 m_lastStatus = 404;
Chris@316 238 }
Chris@316 239 }
Chris@316 240
Chris@316 241 m_done = true;
Chris@497 242
Chris@504 243 #ifdef DEBUG_FILE_SOURCE
Chris@762 244 std::cerr << "FileSource::FileSource(" << m_url.toString() << ") [copy ctor]: note: local filename is \"" << m_localFilename << "\"" << std::endl;
Chris@527 245 #endif
Chris@527 246
Chris@527 247 #ifdef DEBUG_FILE_SOURCE
Chris@762 248 std::cerr << "FileSource::FileSource(copy ctor) exiting" << std::endl;
Chris@504 249 #endif
Chris@316 250 }
Chris@316 251
Chris@317 252 FileSource::~FileSource()
Chris@316 253 {
Chris@327 254 #ifdef DEBUG_FILE_SOURCE
Chris@686 255 std::cerr << "FileSource(" << m_url.toString() << ")::~FileSource" << std::endl;
Chris@529 256 decCount(m_url.toString());
Chris@327 257 #endif
Chris@316 258
Chris@316 259 cleanup();
Chris@316 260
Chris@469 261 if (isRemote() && !m_leaveLocalFile) deleteCacheFile();
Chris@316 262 }
Chris@316 263
Chris@316 264 void
Chris@357 265 FileSource::init()
Chris@316 266 {
Chris@706 267 if (isResource()) {
Chris@355 268 #ifdef DEBUG_FILE_SOURCE
Chris@706 269 std::cerr << "FileSource::init: Is a resource" << std::endl;
Chris@706 270 #endif
Chris@706 271 QString resourceFile = m_url.toString();
Chris@706 272 resourceFile.replace(QRegExp("^qrc:"), ":");
Chris@706 273
Chris@706 274 if (!QFileInfo(resourceFile).exists()) {
Chris@706 275 #ifdef DEBUG_FILE_SOURCE
Chris@706 276 std::cerr << "FileSource::init: Resource file of this name does not exist, switching to non-resource URL" << std::endl;
Chris@706 277 #endif
Chris@706 278 m_url = resourceFile;
Chris@706 279 m_resource = false;
Chris@706 280 }
Chris@706 281 }
Chris@706 282
Chris@706 283 if (!isRemote() && !isResource()) {
Chris@706 284 #ifdef DEBUG_FILE_SOURCE
Chris@706 285 std::cerr << "FileSource::init: Not a remote URL" << std::endl;
Chris@355 286 #endif
Chris@355 287 bool literal = false;
Chris@316 288 m_localFilename = m_url.toLocalFile();
chris@831 289
Chris@342 290 if (m_localFilename == "") {
Chris@342 291 // QUrl may have mishandled the scheme (e.g. in a DOS path)
chris@831 292 m_localFilename = m_rawFileOrUrl;
chris@831 293 #ifdef DEBUG_FILE_SOURCE
chris@831 294 std::cerr << "FileSource::init: Trying literal local filename \""
chris@831 295 << m_localFilename << "\"" << std::endl;
chris@831 296 #endif
Chris@355 297 literal = true;
Chris@342 298 }
Chris@439 299 m_localFilename = QFileInfo(m_localFilename).absoluteFilePath();
Chris@439 300
Chris@355 301 #ifdef DEBUG_FILE_SOURCE
chris@831 302 std::cerr << "FileSource::init: URL translates to absolute filename \""
Chris@706 303 << m_localFilename << "\" (with literal=" << literal << ")"
Chris@706 304 << std::endl;
Chris@355 305 #endif
Chris@316 306 m_ok = true;
Chris@355 307 m_lastStatus = 200;
Chris@355 308
Chris@316 309 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 310 if (literal) {
Chris@355 311 m_lastStatus = 404;
Chris@355 312 } else {
Chris@614 313 #ifdef DEBUG_FILE_SOURCE
Chris@706 314 std::cerr << "FileSource::init: Local file of this name does not exist, trying URL as a literal filename" << std::endl;
Chris@614 315 #endif
Chris@355 316 // Again, QUrl may have been mistreating us --
Chris@355 317 // e.g. dropping a part that looks like query data
chris@831 318 m_localFilename = m_rawFileOrUrl;
Chris@355 319 literal = true;
Chris@355 320 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 321 m_lastStatus = 404;
Chris@355 322 }
Chris@355 323 }
Chris@316 324 }
Chris@355 325
Chris@316 326 m_done = true;
Chris@316 327 return;
Chris@316 328 }
Chris@316 329
Chris@316 330 if (createCacheFile()) {
Chris@327 331 #ifdef DEBUG_FILE_SOURCE
Chris@706 332 std::cerr << "FileSource::init: Already have this one" << std::endl;
Chris@327 333 #endif
Chris@316 334 m_ok = true;
Chris@316 335 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 336 m_lastStatus = 404;
Chris@316 337 } else {
Chris@316 338 m_lastStatus = 200;
Chris@316 339 }
Chris@316 340 m_done = true;
Chris@316 341 return;
Chris@316 342 }
Chris@316 343
Chris@208 344 if (m_localFilename == "") return;
Chris@706 345
Chris@208 346 m_localFile = new QFile(m_localFilename);
Chris@208 347 m_localFile->open(QFile::WriteOnly);
Chris@208 348
Chris@706 349 if (isResource()) {
Chris@706 350
Chris@706 351 // Absent resource file case was dealt with at the top -- this
Chris@706 352 // is the successful case
Chris@706 353
Chris@706 354 QString resourceFileName = m_url.toString();
Chris@706 355 resourceFileName.replace(QRegExp("^qrc:"), ":");
Chris@706 356 QFile resourceFile(resourceFileName);
Chris@706 357 resourceFile.open(QFile::ReadOnly);
Chris@706 358 QByteArray ba(resourceFile.readAll());
Chris@706 359
Chris@706 360 #ifdef DEBUG_FILE_SOURCE
Chris@706 361 std::cerr << "Copying " << ba.size() << " bytes from resource file to cache file" << std::endl;
Chris@706 362 #endif
Chris@706 363
Chris@706 364 qint64 written = m_localFile->write(ba);
Chris@706 365 m_localFile->close();
Chris@706 366 delete m_localFile;
Chris@706 367 m_localFile = 0;
Chris@706 368
Chris@706 369 if (written != ba.size()) {
Chris@706 370 #ifdef DEBUG_FILE_SOURCE
Chris@706 371 std::cerr << "Copy failed (wrote " << written << " bytes)" << std::endl;
Chris@706 372 #endif
Chris@706 373 m_ok = false;
Chris@706 374 return;
Chris@706 375 } else {
Chris@706 376 m_ok = true;
Chris@706 377 m_lastStatus = 200;
Chris@706 378 m_done = true;
Chris@706 379 }
Chris@706 380
Chris@706 381 } else {
Chris@706 382
Chris@706 383 QString scheme = m_url.scheme().toLower();
Chris@316 384
Chris@327 385 #ifdef DEBUG_FILE_SOURCE
Chris@706 386 std::cerr << "FileSource::init: Don't have local copy of \""
Chris@706 387 << m_url.toString() << "\", retrieving" << std::endl;
Chris@327 388 #endif
Chris@208 389
Chris@762 390 if (scheme == "http" || scheme == "https" || scheme == "ftp") {
Chris@762 391 initRemote();
Chris@520 392 #ifdef DEBUG_FILE_SOURCE
Chris@762 393 std::cerr << "FileSource: initRemote returned" << std::endl;
Chris@520 394 #endif
Chris@706 395 } else {
Chris@706 396 m_remote = false;
Chris@706 397 m_ok = false;
Chris@706 398 }
Chris@208 399 }
Chris@208 400
Chris@208 401 if (m_ok) {
Chris@316 402
Chris@316 403 QMutexLocker locker(&m_mapMutex);
Chris@316 404
Chris@316 405 if (m_refCountMap[m_url] > 0) {
Chris@316 406 // someone else has been doing the same thing at the same time,
Chris@316 407 // but has got there first
Chris@316 408 cleanup();
Chris@316 409 m_refCountMap[m_url]++;
Chris@327 410 #ifdef DEBUG_FILE_SOURCE
Chris@706 411 std::cerr << "FileSource::init: Another FileSource has got there first, abandoning our download and using theirs" << std::endl;
Chris@327 412 #endif
Chris@316 413 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 414 m_refCounted = true;
Chris@316 415 m_ok = true;
Chris@316 416 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 417 m_lastStatus = 404;
Chris@316 418 }
Chris@316 419 m_done = true;
Chris@316 420 return;
Chris@316 421 }
Chris@304 422
Chris@304 423 m_remoteLocalMap[m_url] = m_localFilename;
Chris@304 424 m_refCountMap[m_url]++;
Chris@316 425 m_refCounted = true;
Chris@304 426
Chris@706 427 if (m_reporter && !m_done) {
Chris@392 428 m_reporter->setMessage
Chris@392 429 (tr("Downloading %1...").arg(m_url.toString()));
Chris@392 430 connect(m_reporter, SIGNAL(cancelled()), this, SLOT(cancelled()));
Chris@357 431 connect(this, SIGNAL(progress(int)),
Chris@392 432 m_reporter, SLOT(setProgress(int)));
Chris@316 433 }
Chris@208 434 }
Chris@208 435 }
Chris@208 436
Chris@316 437 void
Chris@762 438 FileSource::initRemote()
Chris@208 439 {
Chris@316 440 m_ok = true;
Chris@316 441
Chris@762 442 QNetworkRequest req;
Chris@762 443 req.setUrl(m_url);
Chris@762 444
Chris@762 445 if (m_preferredContentType != "") {
Chris@520 446 #ifdef DEBUG_FILE_SOURCE
Chris@520 447 std::cerr << "FileSource: indicating preferred content type of \""
Chris@686 448 << m_preferredContentType << "\"" << std::endl;
Chris@520 449 #endif
Chris@762 450 req.setRawHeader
Chris@762 451 ("Accept",
Chris@762 452 QString("%1, */*").arg(m_preferredContentType).toLatin1());
Chris@520 453 }
Chris@316 454
Chris@762 455 m_reply = nm.get(req);
Chris@762 456
Chris@762 457 connect(m_reply, SIGNAL(readyRead()),
Chris@762 458 this, SLOT(readyRead()));
Chris@762 459 connect(m_reply, SIGNAL(error(QNetworkReply::NetworkError)),
Chris@762 460 this, SLOT(replyFailed(QNetworkReply::NetworkError)));
Chris@762 461 connect(m_reply, SIGNAL(finished()),
Chris@762 462 this, SLOT(replyFinished()));
Chris@764 463 connect(m_reply, SIGNAL(metaDataChanged()),
Chris@764 464 this, SLOT(metaDataChanged()));
Chris@762 465 connect(m_reply, SIGNAL(downloadProgress(qint64, qint64)),
Chris@762 466 this, SLOT(downloadProgress(qint64, qint64)));
Chris@211 467 }
Chris@211 468
Chris@211 469 void
Chris@317 470 FileSource::cleanup()
Chris@211 471 {
Chris@470 472 if (m_done) {
Chris@470 473 delete m_localFile; // does not actually delete the file
Chris@470 474 m_localFile = 0;
Chris@470 475 }
Chris@211 476 m_done = true;
Chris@762 477 if (m_reply) {
Chris@762 478 QNetworkReply *r = m_reply;
Chris@762 479 m_reply = 0;
Chris@762 480 r->abort();
Chris@762 481 r->deleteLater();
Chris@214 482 }
Chris@470 483 if (m_localFile) {
Chris@470 484 delete m_localFile; // does not actually delete the file
Chris@470 485 m_localFile = 0;
Chris@470 486 }
Chris@208 487 }
Chris@208 488
Chris@208 489 bool
Chris@317 490 FileSource::isRemote(QString fileOrUrl)
Chris@304 491 {
Chris@342 492 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@316 493 QString scheme = QUrl(fileOrUrl).scheme().toLower();
Chris@342 494 if (scheme == "" || scheme == "file" || scheme.length() == 1) return false;
Chris@342 495 return true;
Chris@304 496 }
Chris@304 497
Chris@304 498 bool
Chris@317 499 FileSource::canHandleScheme(QUrl url)
Chris@208 500 {
Chris@342 501 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@208 502 QString scheme = url.scheme().toLower();
Chris@762 503 return (scheme == "http" || scheme == "https" ||
Chris@762 504 scheme == "ftp" || scheme == "file" || scheme == "qrc" ||
Chris@706 505 scheme == "" || scheme.length() == 1);
Chris@208 506 }
Chris@208 507
Chris@210 508 bool
Chris@317 509 FileSource::isAvailable()
Chris@210 510 {
Chris@316 511 waitForStatus();
Chris@211 512 bool available = true;
Chris@211 513 if (!m_ok) available = false;
Chris@211 514 else available = (m_lastStatus / 100 == 2);
Chris@327 515 #ifdef DEBUG_FILE_SOURCE
Chris@762 516 std::cerr << "FileSource::isAvailable: " << (available ? "yes" : "no")
Chris@762 517 << std::endl;
Chris@327 518 #endif
Chris@211 519 return available;
Chris@210 520 }
Chris@210 521
Chris@208 522 void
Chris@317 523 FileSource::waitForStatus()
Chris@316 524 {
Chris@316 525 while (m_ok && (!m_done && m_lastStatus == 0)) {
Chris@316 526 // std::cerr << "waitForStatus: processing (last status " << m_lastStatus << ")" << std::endl;
Chris@392 527 QCoreApplication::processEvents();
Chris@316 528 }
Chris@316 529 }
Chris@316 530
Chris@316 531 void
Chris@317 532 FileSource::waitForData()
Chris@208 533 {
Chris@211 534 while (m_ok && !m_done) {
Chris@762 535 // std::cerr << "FileSource::waitForData: calling QApplication::processEvents" << std::endl;
Chris@392 536 QCoreApplication::processEvents();
Chris@497 537 usleep(10000);
Chris@208 538 }
Chris@208 539 }
Chris@208 540
Chris@316 541 void
Chris@317 542 FileSource::setLeaveLocalFile(bool leave)
Chris@316 543 {
Chris@316 544 m_leaveLocalFile = leave;
Chris@316 545 }
Chris@316 546
Chris@208 547 bool
Chris@317 548 FileSource::isOK() const
Chris@208 549 {
Chris@208 550 return m_ok;
Chris@208 551 }
Chris@208 552
Chris@208 553 bool
Chris@317 554 FileSource::isDone() const
Chris@208 555 {
Chris@208 556 return m_done;
Chris@208 557 }
Chris@208 558
Chris@316 559 bool
Chris@706 560 FileSource::isResource() const
Chris@706 561 {
Chris@706 562 return m_resource;
Chris@706 563 }
Chris@706 564
Chris@706 565 bool
Chris@317 566 FileSource::isRemote() const
Chris@316 567 {
Chris@316 568 return m_remote;
Chris@316 569 }
Chris@316 570
Chris@316 571 QString
Chris@317 572 FileSource::getLocation() const
Chris@316 573 {
Chris@316 574 return m_url.toString();
Chris@316 575 }
Chris@316 576
Chris@208 577 QString
Chris@317 578 FileSource::getLocalFilename() const
Chris@208 579 {
Chris@208 580 return m_localFilename;
Chris@208 581 }
Chris@208 582
Chris@208 583 QString
Chris@678 584 FileSource::getBasename() const
Chris@678 585 {
Chris@678 586 return QFileInfo(m_localFilename).fileName();
Chris@678 587 }
Chris@678 588
Chris@678 589 QString
Chris@317 590 FileSource::getContentType() const
Chris@316 591 {
Chris@316 592 return m_contentType;
Chris@316 593 }
Chris@316 594
Chris@316 595 QString
Chris@317 596 FileSource::getExtension() const
Chris@316 597 {
Chris@316 598 if (m_localFilename != "") {
Chris@316 599 return QFileInfo(m_localFilename).suffix().toLower();
Chris@316 600 } else {
Chris@316 601 return QFileInfo(m_url.toLocalFile()).suffix().toLower();
Chris@316 602 }
Chris@316 603 }
Chris@316 604
Chris@316 605 QString
Chris@317 606 FileSource::getErrorString() const
Chris@208 607 {
Chris@208 608 return m_errorString;
Chris@208 609 }
Chris@208 610
Chris@208 611 void
Chris@762 612 FileSource::readyRead()
Chris@208 613 {
Chris@762 614 m_localFile->write(m_reply->readAll());
Chris@208 615 }
Chris@208 616
Chris@208 617 void
Chris@764 618 FileSource::metaDataChanged()
Chris@210 619 {
Chris@520 620 #ifdef DEBUG_FILE_SOURCE
Chris@764 621 std::cerr << "FileSource::metaDataChanged" << std::endl;
Chris@520 622 #endif
Chris@497 623
Chris@762 624 if (!m_reply) {
Chris@764 625 std::cerr << "WARNING: FileSource::metaDataChanged() called without a reply object being known to us" << std::endl;
Chris@762 626 return;
Chris@762 627 }
Chris@762 628
Chris@762 629 int status =
Chris@762 630 m_reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
Chris@762 631
Chris@762 632 if (status / 100 == 3) {
Chris@762 633 QString location = m_reply->header
Chris@762 634 (QNetworkRequest::LocationHeader).toString();
Chris@496 635 #ifdef DEBUG_FILE_SOURCE
Chris@764 636 std::cerr << "FileSource::metaDataChanged: redirect to \""
Chris@762 637 << location << "\" received" << std::endl;
Chris@496 638 #endif
Chris@496 639 if (location != "") {
Chris@496 640 QUrl newUrl(location);
Chris@496 641 if (newUrl != m_url) {
Chris@497 642 cleanup();
Chris@497 643 deleteCacheFile();
Chris@529 644 #ifdef DEBUG_FILE_SOURCE
Chris@529 645 decCount(m_url.toString());
Chris@529 646 incCount(newUrl.toString());
Chris@529 647 #endif
Chris@496 648 m_url = newUrl;
Chris@497 649 m_localFile = 0;
Chris@496 650 m_lastStatus = 0;
Chris@497 651 m_done = false;
Chris@497 652 m_refCounted = false;
Chris@496 653 init();
Chris@496 654 return;
Chris@496 655 }
Chris@496 656 }
Chris@496 657 }
Chris@497 658
Chris@762 659 m_lastStatus = status;
Chris@210 660 if (m_lastStatus / 100 >= 4) {
Chris@210 661 m_errorString = QString("%1 %2")
Chris@762 662 .arg(status)
Chris@762 663 .arg(m_reply->attribute
Chris@762 664 (QNetworkRequest::HttpReasonPhraseAttribute).toString());
Chris@327 665 #ifdef DEBUG_FILE_SOURCE
Chris@764 666 std::cerr << "FileSource::metaDataChanged: "
Chris@762 667 << m_errorString << std::endl;
Chris@327 668 #endif
Chris@211 669 } else {
Chris@327 670 #ifdef DEBUG_FILE_SOURCE
Chris@764 671 std::cerr << "FileSource::metaDataChanged: "
Chris@762 672 << m_lastStatus << std::endl;
Chris@327 673 #endif
Chris@762 674 m_contentType =
Chris@762 675 m_reply->header(QNetworkRequest::ContentTypeHeader).toString();
Chris@325 676 }
Chris@325 677 emit statusAvailable();
Chris@210 678 }
Chris@210 679
Chris@210 680 void
Chris@762 681 FileSource::downloadProgress(qint64 done, qint64 total)
Chris@208 682 {
Chris@208 683 int percent = int((double(done) / double(total)) * 100.0 - 0.1);
Chris@208 684 emit progress(percent);
Chris@210 685 }
Chris@210 686
Chris@210 687 void
Chris@317 688 FileSource::cancelled()
Chris@210 689 {
Chris@210 690 m_done = true;
Chris@316 691 cleanup();
Chris@316 692
Chris@210 693 m_ok = false;
Chris@210 694 m_errorString = tr("Download cancelled");
Chris@208 695 }
Chris@208 696
Chris@208 697 void
Chris@762 698 FileSource::replyFinished()
Chris@208 699 {
Chris@327 700 emit progress(100);
Chris@327 701
Chris@327 702 #ifdef DEBUG_FILE_SOURCE
Chris@762 703 std::cerr << "FileSource::replyFinished()" << std::endl;
Chris@327 704 #endif
Chris@211 705
Chris@211 706 if (m_done) return;
Chris@211 707
Chris@762 708 bool error = (m_lastStatus / 100 >= 4);
Chris@210 709
Chris@211 710 cleanup();
Chris@208 711
Chris@211 712 if (!error) {
Chris@208 713 QFileInfo fi(m_localFilename);
Chris@208 714 if (!fi.exists()) {
Chris@208 715 m_errorString = tr("Failed to create local file %1").arg(m_localFilename);
Chris@211 716 error = true;
Chris@208 717 } else if (fi.size() == 0) {
Chris@208 718 m_errorString = tr("File contains no data!");
Chris@211 719 error = true;
Chris@208 720 }
Chris@208 721 }
Chris@211 722
Chris@211 723 if (error) {
Chris@327 724 #ifdef DEBUG_FILE_SOURCE
Chris@469 725 std::cerr << "FileSource::done: error is " << error << ", deleting cache file" << std::endl;
Chris@327 726 #endif
Chris@316 727 deleteCacheFile();
Chris@211 728 }
Chris@211 729
Chris@211 730 m_ok = !error;
Chris@520 731 if (m_localFile) m_localFile->flush();
Chris@211 732 m_done = true;
Chris@304 733 emit ready();
Chris@211 734 }
Chris@211 735
Chris@211 736 void
Chris@763 737 FileSource::replyFailed(QNetworkReply::NetworkError)
Chris@763 738 {
Chris@763 739 emit progress(100);
Chris@763 740 m_errorString = m_reply->errorString();
Chris@763 741 m_ok = false;
Chris@763 742 m_done = true;
Chris@763 743 cleanup();
Chris@763 744 emit ready();
Chris@763 745 }
Chris@763 746
Chris@763 747 void
Chris@317 748 FileSource::deleteCacheFile()
Chris@211 749 {
Chris@327 750 #ifdef DEBUG_FILE_SOURCE
Chris@762 751 std::cerr << "FileSource::deleteCacheFile(\"" << m_localFilename << "\")" << std::endl;
Chris@327 752 #endif
Chris@211 753
Chris@211 754 cleanup();
Chris@211 755
Chris@316 756 if (m_localFilename == "") {
Chris@316 757 return;
Chris@316 758 }
Chris@211 759
Chris@316 760 if (!isRemote()) {
Chris@327 761 #ifdef DEBUG_FILE_SOURCE
Chris@316 762 std::cerr << "not a cache file" << std::endl;
Chris@327 763 #endif
Chris@316 764 return;
Chris@316 765 }
Chris@316 766
Chris@316 767 if (m_refCounted) {
Chris@304 768
Chris@304 769 QMutexLocker locker(&m_mapMutex);
Chris@316 770 m_refCounted = false;
Chris@304 771
Chris@304 772 if (m_refCountMap[m_url] > 0) {
Chris@304 773 m_refCountMap[m_url]--;
Chris@327 774 #ifdef DEBUG_FILE_SOURCE
Chris@316 775 std::cerr << "reduced ref count to " << m_refCountMap[m_url] << std::endl;
Chris@327 776 #endif
Chris@304 777 if (m_refCountMap[m_url] > 0) {
Chris@304 778 m_done = true;
Chris@304 779 return;
Chris@304 780 }
Chris@304 781 }
Chris@304 782 }
Chris@304 783
Chris@211 784 m_fileCreationMutex.lock();
Chris@211 785
Chris@211 786 if (!QFile(m_localFilename).remove()) {
Chris@469 787 #ifdef DEBUG_FILE_SOURCE
Chris@686 788 std::cerr << "FileSource::deleteCacheFile: ERROR: Failed to delete file \"" << m_localFilename << "\"" << std::endl;
Chris@469 789 #endif
Chris@211 790 } else {
Chris@327 791 #ifdef DEBUG_FILE_SOURCE
Chris@762 792 std::cerr << "FileSource::deleteCacheFile: Deleted cache file \"" << m_localFilename << "\"" << std::endl;
Chris@327 793 #endif
Chris@211 794 m_localFilename = "";
Chris@211 795 }
Chris@211 796
Chris@211 797 m_fileCreationMutex.unlock();
Chris@211 798
Chris@208 799 m_done = true;
Chris@208 800 }
Chris@208 801
Chris@316 802 bool
Chris@317 803 FileSource::createCacheFile()
Chris@208 804 {
Chris@316 805 {
Chris@316 806 QMutexLocker locker(&m_mapMutex);
Chris@316 807
Chris@327 808 #ifdef DEBUG_FILE_SOURCE
Chris@762 809 std::cerr << "FileSource::createCacheFile: refcount is " << m_refCountMap[m_url] << std::endl;
Chris@327 810 #endif
Chris@316 811
Chris@316 812 if (m_refCountMap[m_url] > 0) {
Chris@316 813 m_refCountMap[m_url]++;
Chris@316 814 m_localFilename = m_remoteLocalMap[m_url];
Chris@327 815 #ifdef DEBUG_FILE_SOURCE
Chris@316 816 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 817 #endif
Chris@316 818 m_refCounted = true;
Chris@316 819 return true;
Chris@316 820 }
Chris@316 821 }
Chris@316 822
Chris@208 823 QDir dir;
Chris@208 824 try {
Chris@208 825 dir = TempDirectory::getInstance()->getSubDirectoryPath("download");
Chris@208 826 } catch (DirectoryCreationFailed f) {
Chris@327 827 #ifdef DEBUG_FILE_SOURCE
Chris@317 828 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create temporary directory: " << f.what() << std::endl;
Chris@327 829 #endif
Chris@208 830 return "";
Chris@208 831 }
Chris@208 832
Chris@316 833 QString filepart = m_url.path().section('/', -1, -1,
Chris@316 834 QString::SectionSkipEmpty);
Chris@208 835
Chris@457 836 QString extension = "";
Chris@457 837 if (filepart.contains('.')) extension = filepart.section('.', -1);
Chris@457 838
Chris@208 839 QString base = filepart;
Chris@208 840 if (extension != "") {
Chris@208 841 base = base.left(base.length() - extension.length() - 1);
Chris@208 842 }
Chris@208 843 if (base == "") base = "remote";
Chris@208 844
Chris@208 845 QString filename;
Chris@208 846
Chris@208 847 if (extension == "") {
Chris@208 848 filename = base;
Chris@208 849 } else {
Chris@208 850 filename = QString("%1.%2").arg(base).arg(extension);
Chris@208 851 }
Chris@208 852
Chris@208 853 QString filepath(dir.filePath(filename));
Chris@208 854
Chris@327 855 #ifdef DEBUG_FILE_SOURCE
Chris@706 856 std::cerr << "FileSource::createCacheFile: URL is \"" << m_url.toString() << "\", dir is \"" << dir.path() << "\", base \"" << base << "\", extension \"" << extension << "\", filebase \"" << filename << "\", filename \"" << filepath << "\"" << std::endl;
Chris@327 857 #endif
Chris@208 858
Chris@316 859 QMutexLocker fcLocker(&m_fileCreationMutex);
Chris@316 860
Chris@208 861 ++m_count;
Chris@208 862
Chris@208 863 if (QFileInfo(filepath).exists() ||
Chris@208 864 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 865
Chris@327 866 #ifdef DEBUG_FILE_SOURCE
Chris@317 867 std::cerr << "FileSource::createCacheFile: Failed to create local file \""
Chris@686 868 << filepath << "\" for URL \""
Chris@686 869 << m_url.toString() << "\" (or file already exists): appending suffix instead" << std::endl;
Chris@327 870 #endif
Chris@208 871
Chris@208 872 if (extension == "") {
Chris@208 873 filename = QString("%1_%2").arg(base).arg(m_count);
Chris@208 874 } else {
Chris@208 875 filename = QString("%1_%2.%3").arg(base).arg(m_count).arg(extension);
Chris@208 876 }
Chris@208 877 filepath = dir.filePath(filename);
Chris@208 878
Chris@208 879 if (QFileInfo(filepath).exists() ||
Chris@208 880 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 881
Chris@327 882 #ifdef DEBUG_FILE_SOURCE
Chris@317 883 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create local file \""
Chris@686 884 << filepath << "\" for URL \""
Chris@686 885 << m_url.toString() << "\" (or file already exists)" << std::endl;
Chris@327 886 #endif
Chris@208 887
Chris@208 888 return "";
Chris@208 889 }
Chris@208 890 }
Chris@208 891
Chris@327 892 #ifdef DEBUG_FILE_SOURCE
Chris@706 893 std::cerr << "FileSource::createCacheFile: url "
Chris@686 894 << m_url.toString() << " -> local filename "
Chris@706 895 << filepath << std::endl;
Chris@327 896 #endif
Chris@316 897
Chris@316 898 m_localFilename = filepath;
Chris@208 899
Chris@316 900 return false;
Chris@208 901 }
Chris@327 902