annotate data/fileio/FileSource.cpp @ 823:f0558e69a074

Rename Resampling- to DecodingWavFileReader, and use it whenever we have an audio file that is not quickly seekable using libsndfile. Avoids very slow performance when analysing ogg files.
author Chris Cannam
date Wed, 17 Jul 2013 15:40:01 +0100
parents 95d4a59295b7
children 12aff5a181bc 3cc81dbc31bb
rev   line source
Chris@208 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@208 2
Chris@208 3 /*
Chris@208 4 Sonic Visualiser
Chris@208 5 An audio file viewer and annotation editor.
Chris@208 6 Centre for Digital Music, Queen Mary, University of London.
Chris@208 7 This file copyright 2007 QMUL.
Chris@208 8
Chris@208 9 This program is free software; you can redistribute it and/or
Chris@208 10 modify it under the terms of the GNU General Public License as
Chris@208 11 published by the Free Software Foundation; either version 2 of the
Chris@208 12 License, or (at your option) any later version. See the file
Chris@208 13 COPYING included with this distribution for more information.
Chris@208 14 */
Chris@208 15
Chris@317 16 #include "FileSource.h"
Chris@357 17
Chris@208 18 #include "base/TempDirectory.h"
Chris@208 19 #include "base/Exceptions.h"
Chris@392 20 #include "base/ProgressReporter.h"
Chris@502 21 #include "system/System.h"
Chris@208 22
Chris@762 23 #include <QNetworkAccessManager>
Chris@762 24 #include <QNetworkReply>
Chris@208 25 #include <QFileInfo>
Chris@208 26 #include <QDir>
Chris@392 27 #include <QCoreApplication>
Chris@208 28
Chris@208 29 #include <iostream>
Chris@405 30 #include <cstdlib>
Chris@208 31
Chris@608 32 #include <unistd.h>
Chris@608 33
Chris@763 34 //#define DEBUG_FILE_SOURCE 1
Chris@327 35
Chris@208 36 int
Chris@317 37 FileSource::m_count = 0;
Chris@208 38
Chris@208 39 QMutex
Chris@317 40 FileSource::m_fileCreationMutex;
Chris@208 41
Chris@317 42 FileSource::RemoteRefCountMap
Chris@317 43 FileSource::m_refCountMap;
Chris@304 44
Chris@317 45 FileSource::RemoteLocalMap
Chris@317 46 FileSource::m_remoteLocalMap;
Chris@304 47
Chris@304 48 QMutex
Chris@317 49 FileSource::m_mapMutex;
Chris@304 50
Chris@529 51 #ifdef DEBUG_FILE_SOURCE
Chris@529 52 static int extantCount = 0;
Chris@529 53 static std::map<QString, int> urlExtantCountMap;
Chris@529 54 static void incCount(QString url) {
Chris@529 55 ++extantCount;
Chris@529 56 if (urlExtantCountMap.find(url) == urlExtantCountMap.end()) {
Chris@529 57 urlExtantCountMap[url] = 1;
Chris@529 58 } else {
Chris@529 59 ++urlExtantCountMap[url];
Chris@529 60 }
Chris@529 61 std::cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << std::endl;
Chris@529 62 }
Chris@529 63 static void decCount(QString url) {
Chris@529 64 --extantCount;
Chris@529 65 --urlExtantCountMap[url];
Chris@529 66 std::cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << std::endl;
Chris@529 67 }
Chris@529 68 #endif
Chris@529 69
Chris@762 70 static QNetworkAccessManager nm;
Chris@762 71
Chris@520 72 FileSource::FileSource(QString fileOrUrl, ProgressReporter *reporter,
Chris@520 73 QString preferredContentType) :
Chris@614 74 m_url(fileOrUrl, QUrl::StrictMode),
Chris@316 75 m_localFile(0),
Chris@762 76 m_reply(0),
Chris@520 77 m_preferredContentType(preferredContentType),
Chris@316 78 m_ok(false),
Chris@316 79 m_lastStatus(0),
Chris@706 80 m_resource(fileOrUrl.startsWith(':')),
Chris@316 81 m_remote(isRemote(fileOrUrl)),
Chris@316 82 m_done(false),
Chris@316 83 m_leaveLocalFile(false),
Chris@392 84 m_reporter(reporter),
Chris@316 85 m_refCounted(false)
Chris@316 86 {
Chris@706 87 if (m_resource) { // qrc file
Chris@706 88 m_url = QUrl("qrc" + fileOrUrl);
Chris@706 89 }
Chris@706 90
Chris@661 91 if (m_url.toString() == "") {
Chris@661 92 m_url = QUrl(fileOrUrl, QUrl::TolerantMode);
Chris@661 93 }
Chris@661 94
Chris@327 95 #ifdef DEBUG_FILE_SOURCE
Chris@706 96 std::cerr << "FileSource::FileSource(" << fileOrUrl << "): url <" << m_url.toString() << ">" << std::endl;
Chris@529 97 incCount(m_url.toString());
Chris@327 98 #endif
Chris@316 99
Chris@316 100 if (!canHandleScheme(m_url)) {
Chris@762 101 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << std::endl;
Chris@316 102 m_errorString = tr("Unsupported scheme in URL");
Chris@316 103 return;
Chris@316 104 }
Chris@316 105
Chris@357 106 init();
Chris@316 107
Chris@614 108 if (!isRemote() &&
Chris@614 109 !isAvailable()) {
Chris@614 110 #ifdef DEBUG_FILE_SOURCE
Chris@706 111 std::cerr << "FileSource::FileSource: Failed to open local file with URL \"" << m_url.toString() << "\"; trying again assuming filename was encoded" << std::endl;
Chris@614 112 #endif
Chris@762 113 m_url = QUrl::fromEncoded(fileOrUrl.toLatin1());
Chris@614 114 init();
Chris@614 115 }
Chris@614 116
Chris@316 117 if (isRemote() &&
Chris@316 118 (fileOrUrl.contains('%') ||
Chris@316 119 fileOrUrl.contains("--"))) { // for IDNA
Chris@316 120
Chris@316 121 waitForStatus();
Chris@316 122
Chris@316 123 if (!isAvailable()) {
Chris@336 124
Chris@316 125 // The URL was created on the assumption that the string
Chris@316 126 // was human-readable. Let's try again, this time
Chris@316 127 // assuming it was already encoded.
Chris@317 128 std::cerr << "FileSource::FileSource: Failed to retrieve URL \""
Chris@316 129 << fileOrUrl.toStdString()
Chris@316 130 << "\" as human-readable URL; "
Chris@316 131 << "trying again treating it as encoded URL"
Chris@316 132 << std::endl;
Chris@336 133
Chris@336 134 // even though our cache file doesn't exist (because the
Chris@336 135 // resource was 404), we still need to ensure we're no
Chris@336 136 // longer associating a filename with this url in the
Chris@336 137 // refcount map -- or createCacheFile will think we've
Chris@336 138 // already done all the work and no request will be sent
Chris@336 139 deleteCacheFile();
Chris@336 140
Chris@762 141 m_url = QUrl::fromEncoded(fileOrUrl.toLatin1());
Chris@336 142
Chris@336 143 m_ok = false;
Chris@336 144 m_done = false;
Chris@336 145 m_lastStatus = 0;
Chris@357 146 init();
Chris@316 147 }
Chris@316 148 }
Chris@325 149
Chris@325 150 if (!isRemote()) {
Chris@325 151 emit statusAvailable();
Chris@325 152 emit ready();
Chris@325 153 }
Chris@497 154
Chris@504 155 #ifdef DEBUG_FILE_SOURCE
Chris@762 156 std::cerr << "FileSource::FileSource(string) exiting" << std::endl;
Chris@504 157 #endif
Chris@316 158 }
Chris@316 159
Chris@469 160 FileSource::FileSource(QUrl url, ProgressReporter *reporter) :
Chris@304 161 m_url(url),
Chris@208 162 m_localFile(0),
Chris@762 163 m_reply(0),
Chris@208 164 m_ok(false),
Chris@210 165 m_lastStatus(0),
Chris@706 166 m_resource(false),
Chris@316 167 m_remote(isRemote(url.toString())),
Chris@208 168 m_done(false),
Chris@316 169 m_leaveLocalFile(false),
Chris@392 170 m_reporter(reporter),
Chris@316 171 m_refCounted(false)
Chris@208 172 {
Chris@327 173 #ifdef DEBUG_FILE_SOURCE
Chris@762 174 std::cerr << "FileSource::FileSource(" << url.toString() << ") [as url]" << std::endl;
Chris@529 175 incCount(m_url.toString());
Chris@327 176 #endif
Chris@316 177
Chris@316 178 if (!canHandleScheme(m_url)) {
Chris@762 179 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << std::endl;
Chris@316 180 m_errorString = tr("Unsupported scheme in URL");
Chris@208 181 return;
Chris@208 182 }
Chris@208 183
Chris@357 184 init();
Chris@497 185
Chris@504 186 #ifdef DEBUG_FILE_SOURCE
Chris@762 187 std::cerr << "FileSource::FileSource(url) exiting" << std::endl;
Chris@504 188 #endif
Chris@316 189 }
Chris@304 190
Chris@317 191 FileSource::FileSource(const FileSource &rf) :
Chris@316 192 QObject(),
Chris@316 193 m_url(rf.m_url),
Chris@316 194 m_localFile(0),
Chris@762 195 m_reply(0),
Chris@316 196 m_ok(rf.m_ok),
Chris@316 197 m_lastStatus(rf.m_lastStatus),
Chris@706 198 m_resource(rf.m_resource),
Chris@316 199 m_remote(rf.m_remote),
Chris@316 200 m_done(false),
Chris@316 201 m_leaveLocalFile(false),
Chris@392 202 m_reporter(rf.m_reporter),
Chris@316 203 m_refCounted(false)
Chris@316 204 {
Chris@327 205 #ifdef DEBUG_FILE_SOURCE
Chris@762 206 std::cerr << "FileSource::FileSource(" << m_url.toString() << ") [copy ctor]" << std::endl;
Chris@529 207 incCount(m_url.toString());
Chris@327 208 #endif
Chris@304 209
Chris@316 210 if (!canHandleScheme(m_url)) {
Chris@762 211 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << std::endl;
Chris@316 212 m_errorString = tr("Unsupported scheme in URL");
Chris@304 213 return;
Chris@304 214 }
Chris@304 215
Chris@469 216 if (!isRemote()) {
Chris@316 217 m_localFilename = rf.m_localFilename;
Chris@316 218 } else {
Chris@316 219 QMutexLocker locker(&m_mapMutex);
Chris@327 220 #ifdef DEBUG_FILE_SOURCE
Chris@762 221 std::cerr << "FileSource::FileSource(copy ctor): ref count is "
Chris@762 222 << m_refCountMap[m_url] << std::endl;
Chris@327 223 #endif
Chris@316 224 if (m_refCountMap[m_url] > 0) {
Chris@316 225 m_refCountMap[m_url]++;
Chris@327 226 #ifdef DEBUG_FILE_SOURCE
Chris@316 227 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 228 #endif
Chris@316 229 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 230 m_refCounted = true;
Chris@316 231 } else {
Chris@316 232 m_ok = false;
Chris@316 233 m_lastStatus = 404;
Chris@316 234 }
Chris@316 235 }
Chris@316 236
Chris@316 237 m_done = true;
Chris@497 238
Chris@504 239 #ifdef DEBUG_FILE_SOURCE
Chris@762 240 std::cerr << "FileSource::FileSource(" << m_url.toString() << ") [copy ctor]: note: local filename is \"" << m_localFilename << "\"" << std::endl;
Chris@527 241 #endif
Chris@527 242
Chris@527 243 #ifdef DEBUG_FILE_SOURCE
Chris@762 244 std::cerr << "FileSource::FileSource(copy ctor) exiting" << std::endl;
Chris@504 245 #endif
Chris@316 246 }
Chris@316 247
Chris@317 248 FileSource::~FileSource()
Chris@316 249 {
Chris@327 250 #ifdef DEBUG_FILE_SOURCE
Chris@686 251 std::cerr << "FileSource(" << m_url.toString() << ")::~FileSource" << std::endl;
Chris@529 252 decCount(m_url.toString());
Chris@327 253 #endif
Chris@316 254
Chris@316 255 cleanup();
Chris@316 256
Chris@469 257 if (isRemote() && !m_leaveLocalFile) deleteCacheFile();
Chris@316 258 }
Chris@316 259
Chris@316 260 void
Chris@357 261 FileSource::init()
Chris@316 262 {
Chris@706 263 if (isResource()) {
Chris@355 264 #ifdef DEBUG_FILE_SOURCE
Chris@706 265 std::cerr << "FileSource::init: Is a resource" << std::endl;
Chris@706 266 #endif
Chris@706 267 QString resourceFile = m_url.toString();
Chris@706 268 resourceFile.replace(QRegExp("^qrc:"), ":");
Chris@706 269
Chris@706 270 if (!QFileInfo(resourceFile).exists()) {
Chris@706 271 #ifdef DEBUG_FILE_SOURCE
Chris@706 272 std::cerr << "FileSource::init: Resource file of this name does not exist, switching to non-resource URL" << std::endl;
Chris@706 273 #endif
Chris@706 274 m_url = resourceFile;
Chris@706 275 m_resource = false;
Chris@706 276 }
Chris@706 277 }
Chris@706 278
Chris@706 279 if (!isRemote() && !isResource()) {
Chris@706 280 #ifdef DEBUG_FILE_SOURCE
Chris@706 281 std::cerr << "FileSource::init: Not a remote URL" << std::endl;
Chris@355 282 #endif
Chris@355 283 bool literal = false;
Chris@316 284 m_localFilename = m_url.toLocalFile();
Chris@342 285 if (m_localFilename == "") {
Chris@342 286 // QUrl may have mishandled the scheme (e.g. in a DOS path)
Chris@342 287 m_localFilename = m_url.toString();
Chris@355 288 literal = true;
Chris@342 289 }
Chris@439 290 m_localFilename = QFileInfo(m_localFilename).absoluteFilePath();
Chris@439 291
Chris@355 292 #ifdef DEBUG_FILE_SOURCE
Chris@706 293 std::cerr << "FileSource::init: URL translates to local filename \""
Chris@706 294 << m_localFilename << "\" (with literal=" << literal << ")"
Chris@706 295 << std::endl;
Chris@355 296 #endif
Chris@316 297 m_ok = true;
Chris@355 298 m_lastStatus = 200;
Chris@355 299
Chris@316 300 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 301 if (literal) {
Chris@355 302 m_lastStatus = 404;
Chris@355 303 } else {
Chris@614 304 #ifdef DEBUG_FILE_SOURCE
Chris@706 305 std::cerr << "FileSource::init: Local file of this name does not exist, trying URL as a literal filename" << std::endl;
Chris@614 306 #endif
Chris@355 307 // Again, QUrl may have been mistreating us --
Chris@355 308 // e.g. dropping a part that looks like query data
Chris@355 309 m_localFilename = m_url.toString();
Chris@355 310 literal = true;
Chris@355 311 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 312 m_lastStatus = 404;
Chris@355 313 }
Chris@355 314 }
Chris@316 315 }
Chris@355 316
Chris@316 317 m_done = true;
Chris@316 318 return;
Chris@316 319 }
Chris@316 320
Chris@316 321 if (createCacheFile()) {
Chris@327 322 #ifdef DEBUG_FILE_SOURCE
Chris@706 323 std::cerr << "FileSource::init: Already have this one" << std::endl;
Chris@327 324 #endif
Chris@316 325 m_ok = true;
Chris@316 326 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 327 m_lastStatus = 404;
Chris@316 328 } else {
Chris@316 329 m_lastStatus = 200;
Chris@316 330 }
Chris@316 331 m_done = true;
Chris@316 332 return;
Chris@316 333 }
Chris@316 334
Chris@208 335 if (m_localFilename == "") return;
Chris@706 336
Chris@208 337 m_localFile = new QFile(m_localFilename);
Chris@208 338 m_localFile->open(QFile::WriteOnly);
Chris@208 339
Chris@706 340 if (isResource()) {
Chris@706 341
Chris@706 342 // Absent resource file case was dealt with at the top -- this
Chris@706 343 // is the successful case
Chris@706 344
Chris@706 345 QString resourceFileName = m_url.toString();
Chris@706 346 resourceFileName.replace(QRegExp("^qrc:"), ":");
Chris@706 347 QFile resourceFile(resourceFileName);
Chris@706 348 resourceFile.open(QFile::ReadOnly);
Chris@706 349 QByteArray ba(resourceFile.readAll());
Chris@706 350
Chris@706 351 #ifdef DEBUG_FILE_SOURCE
Chris@706 352 std::cerr << "Copying " << ba.size() << " bytes from resource file to cache file" << std::endl;
Chris@706 353 #endif
Chris@706 354
Chris@706 355 qint64 written = m_localFile->write(ba);
Chris@706 356 m_localFile->close();
Chris@706 357 delete m_localFile;
Chris@706 358 m_localFile = 0;
Chris@706 359
Chris@706 360 if (written != ba.size()) {
Chris@706 361 #ifdef DEBUG_FILE_SOURCE
Chris@706 362 std::cerr << "Copy failed (wrote " << written << " bytes)" << std::endl;
Chris@706 363 #endif
Chris@706 364 m_ok = false;
Chris@706 365 return;
Chris@706 366 } else {
Chris@706 367 m_ok = true;
Chris@706 368 m_lastStatus = 200;
Chris@706 369 m_done = true;
Chris@706 370 }
Chris@706 371
Chris@706 372 } else {
Chris@706 373
Chris@706 374 QString scheme = m_url.scheme().toLower();
Chris@316 375
Chris@327 376 #ifdef DEBUG_FILE_SOURCE
Chris@706 377 std::cerr << "FileSource::init: Don't have local copy of \""
Chris@706 378 << m_url.toString() << "\", retrieving" << std::endl;
Chris@327 379 #endif
Chris@208 380
Chris@762 381 if (scheme == "http" || scheme == "https" || scheme == "ftp") {
Chris@762 382 initRemote();
Chris@520 383 #ifdef DEBUG_FILE_SOURCE
Chris@762 384 std::cerr << "FileSource: initRemote returned" << std::endl;
Chris@520 385 #endif
Chris@706 386 } else {
Chris@706 387 m_remote = false;
Chris@706 388 m_ok = false;
Chris@706 389 }
Chris@208 390 }
Chris@208 391
Chris@208 392 if (m_ok) {
Chris@316 393
Chris@316 394 QMutexLocker locker(&m_mapMutex);
Chris@316 395
Chris@316 396 if (m_refCountMap[m_url] > 0) {
Chris@316 397 // someone else has been doing the same thing at the same time,
Chris@316 398 // but has got there first
Chris@316 399 cleanup();
Chris@316 400 m_refCountMap[m_url]++;
Chris@327 401 #ifdef DEBUG_FILE_SOURCE
Chris@706 402 std::cerr << "FileSource::init: Another FileSource has got there first, abandoning our download and using theirs" << std::endl;
Chris@327 403 #endif
Chris@316 404 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 405 m_refCounted = true;
Chris@316 406 m_ok = true;
Chris@316 407 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 408 m_lastStatus = 404;
Chris@316 409 }
Chris@316 410 m_done = true;
Chris@316 411 return;
Chris@316 412 }
Chris@304 413
Chris@304 414 m_remoteLocalMap[m_url] = m_localFilename;
Chris@304 415 m_refCountMap[m_url]++;
Chris@316 416 m_refCounted = true;
Chris@304 417
Chris@706 418 if (m_reporter && !m_done) {
Chris@392 419 m_reporter->setMessage
Chris@392 420 (tr("Downloading %1...").arg(m_url.toString()));
Chris@392 421 connect(m_reporter, SIGNAL(cancelled()), this, SLOT(cancelled()));
Chris@357 422 connect(this, SIGNAL(progress(int)),
Chris@392 423 m_reporter, SLOT(setProgress(int)));
Chris@316 424 }
Chris@208 425 }
Chris@208 426 }
Chris@208 427
Chris@316 428 void
Chris@762 429 FileSource::initRemote()
Chris@208 430 {
Chris@316 431 m_ok = true;
Chris@316 432
Chris@762 433 QNetworkRequest req;
Chris@762 434 req.setUrl(m_url);
Chris@762 435
Chris@762 436 if (m_preferredContentType != "") {
Chris@520 437 #ifdef DEBUG_FILE_SOURCE
Chris@520 438 std::cerr << "FileSource: indicating preferred content type of \""
Chris@686 439 << m_preferredContentType << "\"" << std::endl;
Chris@520 440 #endif
Chris@762 441 req.setRawHeader
Chris@762 442 ("Accept",
Chris@762 443 QString("%1, */*").arg(m_preferredContentType).toLatin1());
Chris@520 444 }
Chris@316 445
Chris@762 446 m_reply = nm.get(req);
Chris@762 447
Chris@762 448 connect(m_reply, SIGNAL(readyRead()),
Chris@762 449 this, SLOT(readyRead()));
Chris@762 450 connect(m_reply, SIGNAL(error(QNetworkReply::NetworkError)),
Chris@762 451 this, SLOT(replyFailed(QNetworkReply::NetworkError)));
Chris@762 452 connect(m_reply, SIGNAL(finished()),
Chris@762 453 this, SLOT(replyFinished()));
Chris@764 454 connect(m_reply, SIGNAL(metaDataChanged()),
Chris@764 455 this, SLOT(metaDataChanged()));
Chris@762 456 connect(m_reply, SIGNAL(downloadProgress(qint64, qint64)),
Chris@762 457 this, SLOT(downloadProgress(qint64, qint64)));
Chris@211 458 }
Chris@211 459
Chris@211 460 void
Chris@317 461 FileSource::cleanup()
Chris@211 462 {
Chris@470 463 if (m_done) {
Chris@470 464 delete m_localFile; // does not actually delete the file
Chris@470 465 m_localFile = 0;
Chris@470 466 }
Chris@211 467 m_done = true;
Chris@762 468 if (m_reply) {
Chris@762 469 QNetworkReply *r = m_reply;
Chris@762 470 m_reply = 0;
Chris@762 471 r->abort();
Chris@762 472 r->deleteLater();
Chris@214 473 }
Chris@470 474 if (m_localFile) {
Chris@470 475 delete m_localFile; // does not actually delete the file
Chris@470 476 m_localFile = 0;
Chris@470 477 }
Chris@208 478 }
Chris@208 479
Chris@208 480 bool
Chris@317 481 FileSource::isRemote(QString fileOrUrl)
Chris@304 482 {
Chris@342 483 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@316 484 QString scheme = QUrl(fileOrUrl).scheme().toLower();
Chris@342 485 if (scheme == "" || scheme == "file" || scheme.length() == 1) return false;
Chris@342 486 return true;
Chris@304 487 }
Chris@304 488
Chris@304 489 bool
Chris@317 490 FileSource::canHandleScheme(QUrl url)
Chris@208 491 {
Chris@342 492 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@208 493 QString scheme = url.scheme().toLower();
Chris@762 494 return (scheme == "http" || scheme == "https" ||
Chris@762 495 scheme == "ftp" || scheme == "file" || scheme == "qrc" ||
Chris@706 496 scheme == "" || scheme.length() == 1);
Chris@208 497 }
Chris@208 498
Chris@210 499 bool
Chris@317 500 FileSource::isAvailable()
Chris@210 501 {
Chris@316 502 waitForStatus();
Chris@211 503 bool available = true;
Chris@211 504 if (!m_ok) available = false;
Chris@211 505 else available = (m_lastStatus / 100 == 2);
Chris@327 506 #ifdef DEBUG_FILE_SOURCE
Chris@762 507 std::cerr << "FileSource::isAvailable: " << (available ? "yes" : "no")
Chris@762 508 << std::endl;
Chris@327 509 #endif
Chris@211 510 return available;
Chris@210 511 }
Chris@210 512
Chris@208 513 void
Chris@317 514 FileSource::waitForStatus()
Chris@316 515 {
Chris@316 516 while (m_ok && (!m_done && m_lastStatus == 0)) {
Chris@316 517 // std::cerr << "waitForStatus: processing (last status " << m_lastStatus << ")" << std::endl;
Chris@392 518 QCoreApplication::processEvents();
Chris@316 519 }
Chris@316 520 }
Chris@316 521
Chris@316 522 void
Chris@317 523 FileSource::waitForData()
Chris@208 524 {
Chris@211 525 while (m_ok && !m_done) {
Chris@762 526 // std::cerr << "FileSource::waitForData: calling QApplication::processEvents" << std::endl;
Chris@392 527 QCoreApplication::processEvents();
Chris@497 528 usleep(10000);
Chris@208 529 }
Chris@208 530 }
Chris@208 531
Chris@316 532 void
Chris@317 533 FileSource::setLeaveLocalFile(bool leave)
Chris@316 534 {
Chris@316 535 m_leaveLocalFile = leave;
Chris@316 536 }
Chris@316 537
Chris@208 538 bool
Chris@317 539 FileSource::isOK() const
Chris@208 540 {
Chris@208 541 return m_ok;
Chris@208 542 }
Chris@208 543
Chris@208 544 bool
Chris@317 545 FileSource::isDone() const
Chris@208 546 {
Chris@208 547 return m_done;
Chris@208 548 }
Chris@208 549
Chris@316 550 bool
Chris@706 551 FileSource::isResource() const
Chris@706 552 {
Chris@706 553 return m_resource;
Chris@706 554 }
Chris@706 555
Chris@706 556 bool
Chris@317 557 FileSource::isRemote() const
Chris@316 558 {
Chris@316 559 return m_remote;
Chris@316 560 }
Chris@316 561
Chris@316 562 QString
Chris@317 563 FileSource::getLocation() const
Chris@316 564 {
Chris@316 565 return m_url.toString();
Chris@316 566 }
Chris@316 567
Chris@208 568 QString
Chris@317 569 FileSource::getLocalFilename() const
Chris@208 570 {
Chris@208 571 return m_localFilename;
Chris@208 572 }
Chris@208 573
Chris@208 574 QString
Chris@678 575 FileSource::getBasename() const
Chris@678 576 {
Chris@678 577 return QFileInfo(m_localFilename).fileName();
Chris@678 578 }
Chris@678 579
Chris@678 580 QString
Chris@317 581 FileSource::getContentType() const
Chris@316 582 {
Chris@316 583 return m_contentType;
Chris@316 584 }
Chris@316 585
Chris@316 586 QString
Chris@317 587 FileSource::getExtension() const
Chris@316 588 {
Chris@316 589 if (m_localFilename != "") {
Chris@316 590 return QFileInfo(m_localFilename).suffix().toLower();
Chris@316 591 } else {
Chris@316 592 return QFileInfo(m_url.toLocalFile()).suffix().toLower();
Chris@316 593 }
Chris@316 594 }
Chris@316 595
Chris@316 596 QString
Chris@317 597 FileSource::getErrorString() const
Chris@208 598 {
Chris@208 599 return m_errorString;
Chris@208 600 }
Chris@208 601
Chris@208 602 void
Chris@762 603 FileSource::readyRead()
Chris@208 604 {
Chris@762 605 m_localFile->write(m_reply->readAll());
Chris@208 606 }
Chris@208 607
Chris@208 608 void
Chris@764 609 FileSource::metaDataChanged()
Chris@210 610 {
Chris@520 611 #ifdef DEBUG_FILE_SOURCE
Chris@764 612 std::cerr << "FileSource::metaDataChanged" << std::endl;
Chris@520 613 #endif
Chris@497 614
Chris@762 615 if (!m_reply) {
Chris@764 616 std::cerr << "WARNING: FileSource::metaDataChanged() called without a reply object being known to us" << std::endl;
Chris@762 617 return;
Chris@762 618 }
Chris@762 619
Chris@762 620 int status =
Chris@762 621 m_reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
Chris@762 622
Chris@762 623 if (status / 100 == 3) {
Chris@762 624 QString location = m_reply->header
Chris@762 625 (QNetworkRequest::LocationHeader).toString();
Chris@496 626 #ifdef DEBUG_FILE_SOURCE
Chris@764 627 std::cerr << "FileSource::metaDataChanged: redirect to \""
Chris@762 628 << location << "\" received" << std::endl;
Chris@496 629 #endif
Chris@496 630 if (location != "") {
Chris@496 631 QUrl newUrl(location);
Chris@496 632 if (newUrl != m_url) {
Chris@497 633 cleanup();
Chris@497 634 deleteCacheFile();
Chris@529 635 #ifdef DEBUG_FILE_SOURCE
Chris@529 636 decCount(m_url.toString());
Chris@529 637 incCount(newUrl.toString());
Chris@529 638 #endif
Chris@496 639 m_url = newUrl;
Chris@497 640 m_localFile = 0;
Chris@496 641 m_lastStatus = 0;
Chris@497 642 m_done = false;
Chris@497 643 m_refCounted = false;
Chris@496 644 init();
Chris@496 645 return;
Chris@496 646 }
Chris@496 647 }
Chris@496 648 }
Chris@497 649
Chris@762 650 m_lastStatus = status;
Chris@210 651 if (m_lastStatus / 100 >= 4) {
Chris@210 652 m_errorString = QString("%1 %2")
Chris@762 653 .arg(status)
Chris@762 654 .arg(m_reply->attribute
Chris@762 655 (QNetworkRequest::HttpReasonPhraseAttribute).toString());
Chris@327 656 #ifdef DEBUG_FILE_SOURCE
Chris@764 657 std::cerr << "FileSource::metaDataChanged: "
Chris@762 658 << m_errorString << std::endl;
Chris@327 659 #endif
Chris@211 660 } else {
Chris@327 661 #ifdef DEBUG_FILE_SOURCE
Chris@764 662 std::cerr << "FileSource::metaDataChanged: "
Chris@762 663 << m_lastStatus << std::endl;
Chris@327 664 #endif
Chris@762 665 m_contentType =
Chris@762 666 m_reply->header(QNetworkRequest::ContentTypeHeader).toString();
Chris@325 667 }
Chris@325 668 emit statusAvailable();
Chris@210 669 }
Chris@210 670
Chris@210 671 void
Chris@762 672 FileSource::downloadProgress(qint64 done, qint64 total)
Chris@208 673 {
Chris@208 674 int percent = int((double(done) / double(total)) * 100.0 - 0.1);
Chris@208 675 emit progress(percent);
Chris@210 676 }
Chris@210 677
Chris@210 678 void
Chris@317 679 FileSource::cancelled()
Chris@210 680 {
Chris@210 681 m_done = true;
Chris@316 682 cleanup();
Chris@316 683
Chris@210 684 m_ok = false;
Chris@210 685 m_errorString = tr("Download cancelled");
Chris@208 686 }
Chris@208 687
Chris@208 688 void
Chris@762 689 FileSource::replyFinished()
Chris@208 690 {
Chris@327 691 emit progress(100);
Chris@327 692
Chris@327 693 #ifdef DEBUG_FILE_SOURCE
Chris@762 694 std::cerr << "FileSource::replyFinished()" << std::endl;
Chris@327 695 #endif
Chris@211 696
Chris@211 697 if (m_done) return;
Chris@211 698
Chris@762 699 bool error = (m_lastStatus / 100 >= 4);
Chris@210 700
Chris@211 701 cleanup();
Chris@208 702
Chris@211 703 if (!error) {
Chris@208 704 QFileInfo fi(m_localFilename);
Chris@208 705 if (!fi.exists()) {
Chris@208 706 m_errorString = tr("Failed to create local file %1").arg(m_localFilename);
Chris@211 707 error = true;
Chris@208 708 } else if (fi.size() == 0) {
Chris@208 709 m_errorString = tr("File contains no data!");
Chris@211 710 error = true;
Chris@208 711 }
Chris@208 712 }
Chris@211 713
Chris@211 714 if (error) {
Chris@327 715 #ifdef DEBUG_FILE_SOURCE
Chris@469 716 std::cerr << "FileSource::done: error is " << error << ", deleting cache file" << std::endl;
Chris@327 717 #endif
Chris@316 718 deleteCacheFile();
Chris@211 719 }
Chris@211 720
Chris@211 721 m_ok = !error;
Chris@520 722 if (m_localFile) m_localFile->flush();
Chris@211 723 m_done = true;
Chris@304 724 emit ready();
Chris@211 725 }
Chris@211 726
Chris@211 727 void
Chris@763 728 FileSource::replyFailed(QNetworkReply::NetworkError)
Chris@763 729 {
Chris@763 730 emit progress(100);
Chris@763 731 m_errorString = m_reply->errorString();
Chris@763 732 m_ok = false;
Chris@763 733 m_done = true;
Chris@763 734 cleanup();
Chris@763 735 emit ready();
Chris@763 736 }
Chris@763 737
Chris@763 738 void
Chris@317 739 FileSource::deleteCacheFile()
Chris@211 740 {
Chris@327 741 #ifdef DEBUG_FILE_SOURCE
Chris@762 742 std::cerr << "FileSource::deleteCacheFile(\"" << m_localFilename << "\")" << std::endl;
Chris@327 743 #endif
Chris@211 744
Chris@211 745 cleanup();
Chris@211 746
Chris@316 747 if (m_localFilename == "") {
Chris@316 748 return;
Chris@316 749 }
Chris@211 750
Chris@316 751 if (!isRemote()) {
Chris@327 752 #ifdef DEBUG_FILE_SOURCE
Chris@316 753 std::cerr << "not a cache file" << std::endl;
Chris@327 754 #endif
Chris@316 755 return;
Chris@316 756 }
Chris@316 757
Chris@316 758 if (m_refCounted) {
Chris@304 759
Chris@304 760 QMutexLocker locker(&m_mapMutex);
Chris@316 761 m_refCounted = false;
Chris@304 762
Chris@304 763 if (m_refCountMap[m_url] > 0) {
Chris@304 764 m_refCountMap[m_url]--;
Chris@327 765 #ifdef DEBUG_FILE_SOURCE
Chris@316 766 std::cerr << "reduced ref count to " << m_refCountMap[m_url] << std::endl;
Chris@327 767 #endif
Chris@304 768 if (m_refCountMap[m_url] > 0) {
Chris@304 769 m_done = true;
Chris@304 770 return;
Chris@304 771 }
Chris@304 772 }
Chris@304 773 }
Chris@304 774
Chris@211 775 m_fileCreationMutex.lock();
Chris@211 776
Chris@211 777 if (!QFile(m_localFilename).remove()) {
Chris@469 778 #ifdef DEBUG_FILE_SOURCE
Chris@686 779 std::cerr << "FileSource::deleteCacheFile: ERROR: Failed to delete file \"" << m_localFilename << "\"" << std::endl;
Chris@469 780 #endif
Chris@211 781 } else {
Chris@327 782 #ifdef DEBUG_FILE_SOURCE
Chris@762 783 std::cerr << "FileSource::deleteCacheFile: Deleted cache file \"" << m_localFilename << "\"" << std::endl;
Chris@327 784 #endif
Chris@211 785 m_localFilename = "";
Chris@211 786 }
Chris@211 787
Chris@211 788 m_fileCreationMutex.unlock();
Chris@211 789
Chris@208 790 m_done = true;
Chris@208 791 }
Chris@208 792
Chris@316 793 bool
Chris@317 794 FileSource::createCacheFile()
Chris@208 795 {
Chris@316 796 {
Chris@316 797 QMutexLocker locker(&m_mapMutex);
Chris@316 798
Chris@327 799 #ifdef DEBUG_FILE_SOURCE
Chris@762 800 std::cerr << "FileSource::createCacheFile: refcount is " << m_refCountMap[m_url] << std::endl;
Chris@327 801 #endif
Chris@316 802
Chris@316 803 if (m_refCountMap[m_url] > 0) {
Chris@316 804 m_refCountMap[m_url]++;
Chris@316 805 m_localFilename = m_remoteLocalMap[m_url];
Chris@327 806 #ifdef DEBUG_FILE_SOURCE
Chris@316 807 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 808 #endif
Chris@316 809 m_refCounted = true;
Chris@316 810 return true;
Chris@316 811 }
Chris@316 812 }
Chris@316 813
Chris@208 814 QDir dir;
Chris@208 815 try {
Chris@208 816 dir = TempDirectory::getInstance()->getSubDirectoryPath("download");
Chris@208 817 } catch (DirectoryCreationFailed f) {
Chris@327 818 #ifdef DEBUG_FILE_SOURCE
Chris@317 819 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create temporary directory: " << f.what() << std::endl;
Chris@327 820 #endif
Chris@208 821 return "";
Chris@208 822 }
Chris@208 823
Chris@316 824 QString filepart = m_url.path().section('/', -1, -1,
Chris@316 825 QString::SectionSkipEmpty);
Chris@208 826
Chris@457 827 QString extension = "";
Chris@457 828 if (filepart.contains('.')) extension = filepart.section('.', -1);
Chris@457 829
Chris@208 830 QString base = filepart;
Chris@208 831 if (extension != "") {
Chris@208 832 base = base.left(base.length() - extension.length() - 1);
Chris@208 833 }
Chris@208 834 if (base == "") base = "remote";
Chris@208 835
Chris@208 836 QString filename;
Chris@208 837
Chris@208 838 if (extension == "") {
Chris@208 839 filename = base;
Chris@208 840 } else {
Chris@208 841 filename = QString("%1.%2").arg(base).arg(extension);
Chris@208 842 }
Chris@208 843
Chris@208 844 QString filepath(dir.filePath(filename));
Chris@208 845
Chris@327 846 #ifdef DEBUG_FILE_SOURCE
Chris@706 847 std::cerr << "FileSource::createCacheFile: URL is \"" << m_url.toString() << "\", dir is \"" << dir.path() << "\", base \"" << base << "\", extension \"" << extension << "\", filebase \"" << filename << "\", filename \"" << filepath << "\"" << std::endl;
Chris@327 848 #endif
Chris@208 849
Chris@316 850 QMutexLocker fcLocker(&m_fileCreationMutex);
Chris@316 851
Chris@208 852 ++m_count;
Chris@208 853
Chris@208 854 if (QFileInfo(filepath).exists() ||
Chris@208 855 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 856
Chris@327 857 #ifdef DEBUG_FILE_SOURCE
Chris@317 858 std::cerr << "FileSource::createCacheFile: Failed to create local file \""
Chris@686 859 << filepath << "\" for URL \""
Chris@686 860 << m_url.toString() << "\" (or file already exists): appending suffix instead" << std::endl;
Chris@327 861 #endif
Chris@208 862
Chris@208 863 if (extension == "") {
Chris@208 864 filename = QString("%1_%2").arg(base).arg(m_count);
Chris@208 865 } else {
Chris@208 866 filename = QString("%1_%2.%3").arg(base).arg(m_count).arg(extension);
Chris@208 867 }
Chris@208 868 filepath = dir.filePath(filename);
Chris@208 869
Chris@208 870 if (QFileInfo(filepath).exists() ||
Chris@208 871 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 872
Chris@327 873 #ifdef DEBUG_FILE_SOURCE
Chris@317 874 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create local file \""
Chris@686 875 << filepath << "\" for URL \""
Chris@686 876 << m_url.toString() << "\" (or file already exists)" << std::endl;
Chris@327 877 #endif
Chris@208 878
Chris@208 879 return "";
Chris@208 880 }
Chris@208 881 }
Chris@208 882
Chris@327 883 #ifdef DEBUG_FILE_SOURCE
Chris@706 884 std::cerr << "FileSource::createCacheFile: url "
Chris@686 885 << m_url.toString() << " -> local filename "
Chris@706 886 << filepath << std::endl;
Chris@327 887 #endif
Chris@316 888
Chris@316 889 m_localFilename = filepath;
Chris@208 890
Chris@316 891 return false;
Chris@208 892 }
Chris@327 893