annotate data/fileio/FileSource.cpp @ 743:964a20b5a747 coreaudio_tests

Overhaul CoreAudioFileReader -- it now at least superficially appears to work
author Chris Cannam
date Mon, 02 Jul 2012 09:39:55 +0100
parents 579b2da21e7a
children 3b2409646cc0
rev   line source
Chris@208 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@208 2
Chris@208 3 /*
Chris@208 4 Sonic Visualiser
Chris@208 5 An audio file viewer and annotation editor.
Chris@208 6 Centre for Digital Music, Queen Mary, University of London.
Chris@208 7 This file copyright 2007 QMUL.
Chris@208 8
Chris@208 9 This program is free software; you can redistribute it and/or
Chris@208 10 modify it under the terms of the GNU General Public License as
Chris@208 11 published by the Free Software Foundation; either version 2 of the
Chris@208 12 License, or (at your option) any later version. See the file
Chris@208 13 COPYING included with this distribution for more information.
Chris@208 14 */
Chris@208 15
Chris@317 16 #include "FileSource.h"
Chris@357 17
Chris@208 18 #include "base/TempDirectory.h"
Chris@208 19 #include "base/Exceptions.h"
Chris@392 20 #include "base/ProgressReporter.h"
Chris@502 21 #include "system/System.h"
Chris@208 22
Chris@208 23 #include <QHttp>
Chris@208 24 #include <QFtp>
Chris@208 25 #include <QFileInfo>
Chris@208 26 #include <QDir>
Chris@392 27 #include <QCoreApplication>
Chris@210 28 #include <QHttpResponseHeader>
Chris@208 29
Chris@208 30 #include <iostream>
Chris@405 31 #include <cstdlib>
Chris@208 32
Chris@608 33 #include <unistd.h>
Chris@608 34
Chris@528 35 //#define DEBUG_FILE_SOURCE 1
Chris@327 36
Chris@208 37 int
Chris@317 38 FileSource::m_count = 0;
Chris@208 39
Chris@208 40 QMutex
Chris@317 41 FileSource::m_fileCreationMutex;
Chris@208 42
Chris@317 43 FileSource::RemoteRefCountMap
Chris@317 44 FileSource::m_refCountMap;
Chris@304 45
Chris@317 46 FileSource::RemoteLocalMap
Chris@317 47 FileSource::m_remoteLocalMap;
Chris@304 48
Chris@304 49 QMutex
Chris@317 50 FileSource::m_mapMutex;
Chris@304 51
Chris@529 52 #ifdef DEBUG_FILE_SOURCE
Chris@529 53 static int extantCount = 0;
Chris@529 54 static std::map<QString, int> urlExtantCountMap;
Chris@529 55 static void incCount(QString url) {
Chris@529 56 ++extantCount;
Chris@529 57 if (urlExtantCountMap.find(url) == urlExtantCountMap.end()) {
Chris@529 58 urlExtantCountMap[url] = 1;
Chris@529 59 } else {
Chris@529 60 ++urlExtantCountMap[url];
Chris@529 61 }
Chris@529 62 std::cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << std::endl;
Chris@529 63 }
Chris@529 64 static void decCount(QString url) {
Chris@529 65 --extantCount;
Chris@529 66 --urlExtantCountMap[url];
Chris@529 67 std::cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << std::endl;
Chris@529 68 }
Chris@529 69 #endif
Chris@529 70
Chris@520 71 FileSource::FileSource(QString fileOrUrl, ProgressReporter *reporter,
Chris@520 72 QString preferredContentType) :
Chris@614 73 m_url(fileOrUrl, QUrl::StrictMode),
Chris@316 74 m_ftp(0),
Chris@316 75 m_http(0),
Chris@316 76 m_localFile(0),
Chris@520 77 m_preferredContentType(preferredContentType),
Chris@316 78 m_ok(false),
Chris@316 79 m_lastStatus(0),
Chris@706 80 m_resource(fileOrUrl.startsWith(':')),
Chris@316 81 m_remote(isRemote(fileOrUrl)),
Chris@316 82 m_done(false),
Chris@316 83 m_leaveLocalFile(false),
Chris@392 84 m_reporter(reporter),
Chris@316 85 m_refCounted(false)
Chris@316 86 {
Chris@706 87 if (m_resource) { // qrc file
Chris@706 88 m_url = QUrl("qrc" + fileOrUrl);
Chris@706 89 }
Chris@706 90
Chris@661 91 if (m_url.toString() == "") {
Chris@661 92 m_url = QUrl(fileOrUrl, QUrl::TolerantMode);
Chris@661 93 }
Chris@661 94
Chris@327 95 #ifdef DEBUG_FILE_SOURCE
Chris@706 96 std::cerr << "FileSource::FileSource(" << fileOrUrl << "): url <" << m_url.toString() << ">" << std::endl;
Chris@529 97 incCount(m_url.toString());
Chris@327 98 #endif
Chris@316 99
Chris@316 100 if (!canHandleScheme(m_url)) {
Chris@690 101 SVDEBUG << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316 102 m_errorString = tr("Unsupported scheme in URL");
Chris@316 103 return;
Chris@316 104 }
Chris@316 105
Chris@357 106 init();
Chris@316 107
Chris@614 108 if (!isRemote() &&
Chris@614 109 !isAvailable()) {
Chris@614 110 #ifdef DEBUG_FILE_SOURCE
Chris@706 111 std::cerr << "FileSource::FileSource: Failed to open local file with URL \"" << m_url.toString() << "\"; trying again assuming filename was encoded" << std::endl;
Chris@614 112 #endif
Chris@616 113 m_url = QUrl::fromEncoded(fileOrUrl.toAscii());
Chris@614 114 init();
Chris@614 115 }
Chris@614 116
Chris@316 117 if (isRemote() &&
Chris@316 118 (fileOrUrl.contains('%') ||
Chris@316 119 fileOrUrl.contains("--"))) { // for IDNA
Chris@316 120
Chris@316 121 waitForStatus();
Chris@316 122
Chris@316 123 if (!isAvailable()) {
Chris@336 124
Chris@316 125 // The URL was created on the assumption that the string
Chris@316 126 // was human-readable. Let's try again, this time
Chris@316 127 // assuming it was already encoded.
Chris@317 128 std::cerr << "FileSource::FileSource: Failed to retrieve URL \""
Chris@316 129 << fileOrUrl.toStdString()
Chris@316 130 << "\" as human-readable URL; "
Chris@316 131 << "trying again treating it as encoded URL"
Chris@316 132 << std::endl;
Chris@336 133
Chris@336 134 // even though our cache file doesn't exist (because the
Chris@336 135 // resource was 404), we still need to ensure we're no
Chris@336 136 // longer associating a filename with this url in the
Chris@336 137 // refcount map -- or createCacheFile will think we've
Chris@336 138 // already done all the work and no request will be sent
Chris@336 139 deleteCacheFile();
Chris@336 140
Chris@316 141 m_url.setEncodedUrl(fileOrUrl.toAscii());
Chris@336 142
Chris@336 143 m_ok = false;
Chris@336 144 m_done = false;
Chris@336 145 m_lastStatus = 0;
Chris@357 146 init();
Chris@316 147 }
Chris@316 148 }
Chris@325 149
Chris@325 150 if (!isRemote()) {
Chris@325 151 emit statusAvailable();
Chris@325 152 emit ready();
Chris@325 153 }
Chris@497 154
Chris@504 155 #ifdef DEBUG_FILE_SOURCE
Chris@690 156 SVDEBUG << "FileSource::FileSource(string) exiting" << endl;
Chris@504 157 #endif
Chris@316 158 }
Chris@316 159
Chris@469 160 FileSource::FileSource(QUrl url, ProgressReporter *reporter) :
Chris@304 161 m_url(url),
Chris@208 162 m_ftp(0),
Chris@208 163 m_http(0),
Chris@208 164 m_localFile(0),
Chris@208 165 m_ok(false),
Chris@210 166 m_lastStatus(0),
Chris@706 167 m_resource(false),
Chris@316 168 m_remote(isRemote(url.toString())),
Chris@208 169 m_done(false),
Chris@316 170 m_leaveLocalFile(false),
Chris@392 171 m_reporter(reporter),
Chris@316 172 m_refCounted(false)
Chris@208 173 {
Chris@327 174 #ifdef DEBUG_FILE_SOURCE
Chris@690 175 SVDEBUG << "FileSource::FileSource(" << url.toString() << ") [as url]" << endl;
Chris@529 176 incCount(m_url.toString());
Chris@327 177 #endif
Chris@316 178
Chris@316 179 if (!canHandleScheme(m_url)) {
Chris@690 180 SVDEBUG << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316 181 m_errorString = tr("Unsupported scheme in URL");
Chris@208 182 return;
Chris@208 183 }
Chris@208 184
Chris@357 185 init();
Chris@497 186
Chris@504 187 #ifdef DEBUG_FILE_SOURCE
Chris@690 188 SVDEBUG << "FileSource::FileSource(url) exiting" << endl;
Chris@504 189 #endif
Chris@316 190 }
Chris@304 191
Chris@317 192 FileSource::FileSource(const FileSource &rf) :
Chris@316 193 QObject(),
Chris@316 194 m_url(rf.m_url),
Chris@316 195 m_ftp(0),
Chris@316 196 m_http(0),
Chris@316 197 m_localFile(0),
Chris@316 198 m_ok(rf.m_ok),
Chris@316 199 m_lastStatus(rf.m_lastStatus),
Chris@706 200 m_resource(rf.m_resource),
Chris@316 201 m_remote(rf.m_remote),
Chris@316 202 m_done(false),
Chris@316 203 m_leaveLocalFile(false),
Chris@392 204 m_reporter(rf.m_reporter),
Chris@316 205 m_refCounted(false)
Chris@316 206 {
Chris@327 207 #ifdef DEBUG_FILE_SOURCE
Chris@690 208 SVDEBUG << "FileSource::FileSource(" << m_url.toString() << ") [copy ctor]" << endl;
Chris@529 209 incCount(m_url.toString());
Chris@327 210 #endif
Chris@304 211
Chris@316 212 if (!canHandleScheme(m_url)) {
Chris@690 213 SVDEBUG << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316 214 m_errorString = tr("Unsupported scheme in URL");
Chris@304 215 return;
Chris@304 216 }
Chris@304 217
Chris@469 218 if (!isRemote()) {
Chris@316 219 m_localFilename = rf.m_localFilename;
Chris@316 220 } else {
Chris@316 221 QMutexLocker locker(&m_mapMutex);
Chris@327 222 #ifdef DEBUG_FILE_SOURCE
Chris@690 223 SVDEBUG << "FileSource::FileSource(copy ctor): ref count is "
Chris@687 224 << m_refCountMap[m_url] << endl;
Chris@327 225 #endif
Chris@316 226 if (m_refCountMap[m_url] > 0) {
Chris@316 227 m_refCountMap[m_url]++;
Chris@327 228 #ifdef DEBUG_FILE_SOURCE
Chris@316 229 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 230 #endif
Chris@316 231 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 232 m_refCounted = true;
Chris@316 233 } else {
Chris@316 234 m_ok = false;
Chris@316 235 m_lastStatus = 404;
Chris@316 236 }
Chris@316 237 }
Chris@316 238
Chris@316 239 m_done = true;
Chris@497 240
Chris@504 241 #ifdef DEBUG_FILE_SOURCE
Chris@690 242 SVDEBUG << "FileSource::FileSource(" << m_url.toString() << ") [copy ctor]: note: local filename is \"" << m_localFilename << "\"" << endl;
Chris@527 243 #endif
Chris@527 244
Chris@527 245 #ifdef DEBUG_FILE_SOURCE
Chris@690 246 SVDEBUG << "FileSource::FileSource(copy ctor) exiting" << endl;
Chris@504 247 #endif
Chris@316 248 }
Chris@316 249
Chris@317 250 FileSource::~FileSource()
Chris@316 251 {
Chris@327 252 #ifdef DEBUG_FILE_SOURCE
Chris@686 253 std::cerr << "FileSource(" << m_url.toString() << ")::~FileSource" << std::endl;
Chris@529 254 decCount(m_url.toString());
Chris@327 255 #endif
Chris@316 256
Chris@316 257 cleanup();
Chris@316 258
Chris@469 259 if (isRemote() && !m_leaveLocalFile) deleteCacheFile();
Chris@316 260 }
Chris@316 261
Chris@316 262 void
Chris@357 263 FileSource::init()
Chris@316 264 {
Chris@706 265 if (isResource()) {
Chris@355 266 #ifdef DEBUG_FILE_SOURCE
Chris@706 267 std::cerr << "FileSource::init: Is a resource" << std::endl;
Chris@706 268 #endif
Chris@706 269 QString resourceFile = m_url.toString();
Chris@706 270 resourceFile.replace(QRegExp("^qrc:"), ":");
Chris@706 271
Chris@706 272 if (!QFileInfo(resourceFile).exists()) {
Chris@706 273 #ifdef DEBUG_FILE_SOURCE
Chris@706 274 std::cerr << "FileSource::init: Resource file of this name does not exist, switching to non-resource URL" << std::endl;
Chris@706 275 #endif
Chris@706 276 m_url = resourceFile;
Chris@706 277 m_resource = false;
Chris@706 278 }
Chris@706 279 }
Chris@706 280
Chris@706 281 if (!isRemote() && !isResource()) {
Chris@706 282 #ifdef DEBUG_FILE_SOURCE
Chris@706 283 std::cerr << "FileSource::init: Not a remote URL" << std::endl;
Chris@355 284 #endif
Chris@355 285 bool literal = false;
Chris@316 286 m_localFilename = m_url.toLocalFile();
Chris@342 287 if (m_localFilename == "") {
Chris@342 288 // QUrl may have mishandled the scheme (e.g. in a DOS path)
Chris@342 289 m_localFilename = m_url.toString();
Chris@355 290 literal = true;
Chris@342 291 }
Chris@439 292 m_localFilename = QFileInfo(m_localFilename).absoluteFilePath();
Chris@439 293
Chris@355 294 #ifdef DEBUG_FILE_SOURCE
Chris@706 295 std::cerr << "FileSource::init: URL translates to local filename \""
Chris@706 296 << m_localFilename << "\" (with literal=" << literal << ")"
Chris@706 297 << std::endl;
Chris@355 298 #endif
Chris@316 299 m_ok = true;
Chris@355 300 m_lastStatus = 200;
Chris@355 301
Chris@316 302 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 303 if (literal) {
Chris@355 304 m_lastStatus = 404;
Chris@355 305 } else {
Chris@614 306 #ifdef DEBUG_FILE_SOURCE
Chris@706 307 std::cerr << "FileSource::init: Local file of this name does not exist, trying URL as a literal filename" << std::endl;
Chris@614 308 #endif
Chris@355 309 // Again, QUrl may have been mistreating us --
Chris@355 310 // e.g. dropping a part that looks like query data
Chris@355 311 m_localFilename = m_url.toString();
Chris@355 312 literal = true;
Chris@355 313 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 314 m_lastStatus = 404;
Chris@355 315 }
Chris@355 316 }
Chris@316 317 }
Chris@355 318
Chris@316 319 m_done = true;
Chris@316 320 return;
Chris@316 321 }
Chris@316 322
Chris@316 323 if (createCacheFile()) {
Chris@327 324 #ifdef DEBUG_FILE_SOURCE
Chris@706 325 std::cerr << "FileSource::init: Already have this one" << std::endl;
Chris@327 326 #endif
Chris@316 327 m_ok = true;
Chris@316 328 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 329 m_lastStatus = 404;
Chris@316 330 } else {
Chris@316 331 m_lastStatus = 200;
Chris@316 332 }
Chris@316 333 m_done = true;
Chris@316 334 return;
Chris@316 335 }
Chris@316 336
Chris@208 337 if (m_localFilename == "") return;
Chris@706 338
Chris@208 339 m_localFile = new QFile(m_localFilename);
Chris@208 340 m_localFile->open(QFile::WriteOnly);
Chris@208 341
Chris@706 342 if (isResource()) {
Chris@706 343
Chris@706 344 // Absent resource file case was dealt with at the top -- this
Chris@706 345 // is the successful case
Chris@706 346
Chris@706 347 QString resourceFileName = m_url.toString();
Chris@706 348 resourceFileName.replace(QRegExp("^qrc:"), ":");
Chris@706 349 QFile resourceFile(resourceFileName);
Chris@706 350 resourceFile.open(QFile::ReadOnly);
Chris@706 351 QByteArray ba(resourceFile.readAll());
Chris@706 352
Chris@706 353 #ifdef DEBUG_FILE_SOURCE
Chris@706 354 std::cerr << "Copying " << ba.size() << " bytes from resource file to cache file" << std::endl;
Chris@706 355 #endif
Chris@706 356
Chris@706 357 qint64 written = m_localFile->write(ba);
Chris@706 358 m_localFile->close();
Chris@706 359 delete m_localFile;
Chris@706 360 m_localFile = 0;
Chris@706 361
Chris@706 362 if (written != ba.size()) {
Chris@706 363 #ifdef DEBUG_FILE_SOURCE
Chris@706 364 std::cerr << "Copy failed (wrote " << written << " bytes)" << std::endl;
Chris@706 365 #endif
Chris@706 366 m_ok = false;
Chris@706 367 return;
Chris@706 368 } else {
Chris@706 369 m_ok = true;
Chris@706 370 m_lastStatus = 200;
Chris@706 371 m_done = true;
Chris@706 372 }
Chris@706 373
Chris@706 374 } else {
Chris@706 375
Chris@706 376 QString scheme = m_url.scheme().toLower();
Chris@316 377
Chris@327 378 #ifdef DEBUG_FILE_SOURCE
Chris@706 379 std::cerr << "FileSource::init: Don't have local copy of \""
Chris@706 380 << m_url.toString() << "\", retrieving" << std::endl;
Chris@327 381 #endif
Chris@208 382
Chris@706 383 if (scheme == "http") {
Chris@706 384 initHttp();
Chris@520 385 #ifdef DEBUG_FILE_SOURCE
Chris@706 386 std::cerr << "FileSource: initHttp succeeded" << std::endl;
Chris@520 387 #endif
Chris@706 388 } else if (scheme == "ftp") {
Chris@706 389 initFtp();
Chris@706 390 } else {
Chris@706 391 m_remote = false;
Chris@706 392 m_ok = false;
Chris@706 393 }
Chris@208 394 }
Chris@208 395
Chris@208 396 if (m_ok) {
Chris@316 397
Chris@316 398 QMutexLocker locker(&m_mapMutex);
Chris@316 399
Chris@316 400 if (m_refCountMap[m_url] > 0) {
Chris@316 401 // someone else has been doing the same thing at the same time,
Chris@316 402 // but has got there first
Chris@316 403 cleanup();
Chris@316 404 m_refCountMap[m_url]++;
Chris@327 405 #ifdef DEBUG_FILE_SOURCE
Chris@706 406 std::cerr << "FileSource::init: Another FileSource has got there first, abandoning our download and using theirs" << std::endl;
Chris@327 407 #endif
Chris@316 408 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 409 m_refCounted = true;
Chris@316 410 m_ok = true;
Chris@316 411 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 412 m_lastStatus = 404;
Chris@316 413 }
Chris@316 414 m_done = true;
Chris@316 415 return;
Chris@316 416 }
Chris@304 417
Chris@304 418 m_remoteLocalMap[m_url] = m_localFilename;
Chris@304 419 m_refCountMap[m_url]++;
Chris@316 420 m_refCounted = true;
Chris@304 421
Chris@706 422 if (m_reporter && !m_done) {
Chris@392 423 m_reporter->setMessage
Chris@392 424 (tr("Downloading %1...").arg(m_url.toString()));
Chris@392 425 connect(m_reporter, SIGNAL(cancelled()), this, SLOT(cancelled()));
Chris@357 426 connect(this, SIGNAL(progress(int)),
Chris@392 427 m_reporter, SLOT(setProgress(int)));
Chris@316 428 }
Chris@208 429 }
Chris@208 430 }
Chris@208 431
Chris@316 432 void
Chris@317 433 FileSource::initHttp()
Chris@208 434 {
Chris@316 435 m_ok = true;
Chris@497 436 int port = m_url.port();
Chris@497 437 m_http = new QHttp(m_url.host(), port < 0 ? 80 : port);
Chris@316 438 connect(m_http, SIGNAL(done(bool)), this, SLOT(done(bool)));
Chris@316 439 connect(m_http, SIGNAL(dataReadProgress(int, int)),
Chris@316 440 this, SLOT(dataReadProgress(int, int)));
Chris@316 441 connect(m_http, SIGNAL(responseHeaderReceived(const QHttpResponseHeader &)),
Chris@316 442 this, SLOT(httpResponseHeaderReceived(const QHttpResponseHeader &)));
Chris@316 443
Chris@316 444 // I don't quite understand this. url.path() returns a path
Chris@316 445 // without percent encoding; for example, spaces appear as
Chris@316 446 // literal spaces. This generally won't work if sent to the
Chris@316 447 // server directly. You can retrieve a correctly encoded URL
Chris@316 448 // from QUrl using url.toEncoded(), but that gives you the
Chris@316 449 // whole URL; there doesn't seem to be any way to retrieve
Chris@316 450 // only an encoded path. Furthermore there doesn't seem to be
Chris@316 451 // any way to convert a retrieved path into an encoded path
Chris@316 452 // without explicitly specifying that you don't want the path
Chris@316 453 // separators ("/") to be encoded. (Besides being painful to
Chris@316 454 // manage, I don't see how this can work correctly in any case
Chris@316 455 // where a percent-encoded "/" is supposed to appear within a
Chris@316 456 // path element?) There also seems to be no way to retrieve
Chris@316 457 // the path plus query string, i.e. everything that I need to
Chris@316 458 // send to the HTTP server. And no way for QHttp to take a
Chris@316 459 // QUrl argument. I'm obviously missing something.
Chris@316 460
Chris@316 461 // So, two ways to do this: query the bits from the URL,
Chris@316 462 // encode them individually, and glue them back together
Chris@316 463 // again...
Chris@316 464 /*
Chris@316 465 QString path = QUrl::toPercentEncoding(m_url.path(), "/");
Chris@316 466 QList<QPair<QString, QString> > query = m_url.queryItems();
Chris@316 467 if (!query.empty()) {
Chris@316 468 QStringList q2;
Chris@316 469 for (QList<QPair<QString, QString> >::iterator i = query.begin();
Chris@316 470 i != query.end(); ++i) {
Chris@316 471 q2.push_back(QString("%1=%3")
Chris@316 472 .arg(QString(QUrl::toPercentEncoding(i->first)))
Chris@316 473 .arg(QString(QUrl::toPercentEncoding(i->second))));
Chris@316 474 }
Chris@316 475 path = QString("%1%2%3")
Chris@316 476 .arg(path).arg("?")
Chris@316 477 .arg(q2.join("&"));
Chris@316 478 }
Chris@316 479 */
Chris@316 480
Chris@316 481 // ...or, much simpler but relying on knowledge about the
Chris@316 482 // scheme://host/path/path/query etc format of the URL, we can
Chris@316 483 // get the whole URL ready-encoded and then split it on "/" as
Chris@316 484 // appropriate...
Chris@316 485
Chris@316 486 QString path = "/" + QString(m_url.toEncoded()).section('/', 3);
Chris@316 487
Chris@327 488 #ifdef DEBUG_FILE_SOURCE
Chris@690 489 SVDEBUG << "FileSource: path is \""
Chris@687 490 << path << "\"" << endl;
Chris@327 491 #endif
Chris@316 492
Chris@520 493 if (m_preferredContentType == "") {
Chris@520 494 m_http->get(path, m_localFile);
Chris@520 495 } else {
Chris@520 496 #ifdef DEBUG_FILE_SOURCE
Chris@520 497 std::cerr << "FileSource: indicating preferred content type of \""
Chris@686 498 << m_preferredContentType << "\"" << std::endl;
Chris@520 499 #endif
Chris@520 500 QHttpRequestHeader header("GET", path);
Chris@520 501 header.setValue("Host", m_url.host());
Chris@520 502 header.setValue("Accept", QString("%1, */*").arg(m_preferredContentType));
Chris@520 503 m_http->request(header, 0, m_localFile);
Chris@520 504 }
Chris@316 505 }
Chris@316 506
Chris@316 507 void
Chris@317 508 FileSource::initFtp()
Chris@316 509 {
Chris@316 510 m_ok = true;
Chris@316 511 m_ftp = new QFtp;
Chris@316 512 connect(m_ftp, SIGNAL(done(bool)), this, SLOT(done(bool)));
Chris@316 513 connect(m_ftp, SIGNAL(commandFinished(int, bool)),
Chris@316 514 this, SLOT(ftpCommandFinished(int, bool)));
Chris@316 515 connect(m_ftp, SIGNAL(dataTransferProgress(qint64, qint64)),
Chris@316 516 this, SLOT(dataTransferProgress(qint64, qint64)));
Chris@316 517 m_ftp->connectToHost(m_url.host(), m_url.port(21));
Chris@316 518
Chris@316 519 QString username = m_url.userName();
Chris@316 520 if (username == "") {
Chris@316 521 username = "anonymous";
Chris@316 522 }
Chris@316 523
Chris@316 524 QString password = m_url.password();
Chris@316 525 if (password == "") {
Chris@316 526 password = QString("%1@%2").arg(getenv("USER")).arg(getenv("HOST"));
Chris@316 527 }
Chris@316 528
Chris@316 529 m_ftp->login(username, password);
Chris@316 530
Chris@316 531 QString dirpath = m_url.path().section('/', 0, -2);
Chris@316 532 QString filename = m_url.path().section('/', -1);
Chris@316 533
Chris@316 534 if (dirpath == "") dirpath = "/";
Chris@316 535 m_ftp->cd(dirpath);
Chris@316 536 m_ftp->get(filename, m_localFile);
Chris@211 537 }
Chris@211 538
Chris@211 539 void
Chris@317 540 FileSource::cleanup()
Chris@211 541 {
Chris@470 542 if (m_done) {
Chris@470 543 delete m_localFile; // does not actually delete the file
Chris@470 544 m_localFile = 0;
Chris@470 545 }
Chris@211 546 m_done = true;
Chris@214 547 if (m_http) {
Chris@287 548 QHttp *h = m_http;
Chris@214 549 m_http = 0;
Chris@287 550 h->abort();
Chris@287 551 h->deleteLater();
Chris@214 552 }
Chris@214 553 if (m_ftp) {
Chris@287 554 QFtp *f = m_ftp;
Chris@214 555 m_ftp = 0;
Chris@287 556 f->abort();
Chris@287 557 f->deleteLater();
Chris@214 558 }
Chris@470 559 if (m_localFile) {
Chris@470 560 delete m_localFile; // does not actually delete the file
Chris@470 561 m_localFile = 0;
Chris@470 562 }
Chris@208 563 }
Chris@208 564
Chris@208 565 bool
Chris@317 566 FileSource::isRemote(QString fileOrUrl)
Chris@304 567 {
Chris@342 568 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@316 569 QString scheme = QUrl(fileOrUrl).scheme().toLower();
Chris@342 570 if (scheme == "" || scheme == "file" || scheme.length() == 1) return false;
Chris@342 571 return true;
Chris@304 572 }
Chris@304 573
Chris@304 574 bool
Chris@317 575 FileSource::canHandleScheme(QUrl url)
Chris@208 576 {
Chris@342 577 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@208 578 QString scheme = url.scheme().toLower();
Chris@316 579 return (scheme == "http" || scheme == "ftp" ||
Chris@706 580 scheme == "file" || scheme == "qrc" ||
Chris@706 581 scheme == "" || scheme.length() == 1);
Chris@208 582 }
Chris@208 583
Chris@210 584 bool
Chris@317 585 FileSource::isAvailable()
Chris@210 586 {
Chris@316 587 waitForStatus();
Chris@211 588 bool available = true;
Chris@211 589 if (!m_ok) available = false;
Chris@211 590 else available = (m_lastStatus / 100 == 2);
Chris@327 591 #ifdef DEBUG_FILE_SOURCE
Chris@690 592 SVDEBUG << "FileSource::isAvailable: " << (available ? "yes" : "no")
Chris@687 593 << endl;
Chris@327 594 #endif
Chris@211 595 return available;
Chris@210 596 }
Chris@210 597
Chris@208 598 void
Chris@317 599 FileSource::waitForStatus()
Chris@316 600 {
Chris@316 601 while (m_ok && (!m_done && m_lastStatus == 0)) {
Chris@316 602 // std::cerr << "waitForStatus: processing (last status " << m_lastStatus << ")" << std::endl;
Chris@392 603 QCoreApplication::processEvents();
Chris@316 604 }
Chris@316 605 }
Chris@316 606
Chris@316 607 void
Chris@317 608 FileSource::waitForData()
Chris@208 609 {
Chris@211 610 while (m_ok && !m_done) {
Chris@690 611 // SVDEBUG << "FileSource::waitForData: calling QApplication::processEvents" << endl;
Chris@392 612 QCoreApplication::processEvents();
Chris@497 613 usleep(10000);
Chris@208 614 }
Chris@208 615 }
Chris@208 616
Chris@316 617 void
Chris@317 618 FileSource::setLeaveLocalFile(bool leave)
Chris@316 619 {
Chris@316 620 m_leaveLocalFile = leave;
Chris@316 621 }
Chris@316 622
Chris@208 623 bool
Chris@317 624 FileSource::isOK() const
Chris@208 625 {
Chris@208 626 return m_ok;
Chris@208 627 }
Chris@208 628
Chris@208 629 bool
Chris@317 630 FileSource::isDone() const
Chris@208 631 {
Chris@208 632 return m_done;
Chris@208 633 }
Chris@208 634
Chris@316 635 bool
Chris@706 636 FileSource::isResource() const
Chris@706 637 {
Chris@706 638 return m_resource;
Chris@706 639 }
Chris@706 640
Chris@706 641 bool
Chris@317 642 FileSource::isRemote() const
Chris@316 643 {
Chris@316 644 return m_remote;
Chris@316 645 }
Chris@316 646
Chris@316 647 QString
Chris@317 648 FileSource::getLocation() const
Chris@316 649 {
Chris@316 650 return m_url.toString();
Chris@316 651 }
Chris@316 652
Chris@208 653 QString
Chris@317 654 FileSource::getLocalFilename() const
Chris@208 655 {
Chris@208 656 return m_localFilename;
Chris@208 657 }
Chris@208 658
Chris@208 659 QString
Chris@678 660 FileSource::getBasename() const
Chris@678 661 {
Chris@678 662 return QFileInfo(m_localFilename).fileName();
Chris@678 663 }
Chris@678 664
Chris@678 665 QString
Chris@317 666 FileSource::getContentType() const
Chris@316 667 {
Chris@316 668 return m_contentType;
Chris@316 669 }
Chris@316 670
Chris@316 671 QString
Chris@317 672 FileSource::getExtension() const
Chris@316 673 {
Chris@316 674 if (m_localFilename != "") {
Chris@316 675 return QFileInfo(m_localFilename).suffix().toLower();
Chris@316 676 } else {
Chris@316 677 return QFileInfo(m_url.toLocalFile()).suffix().toLower();
Chris@316 678 }
Chris@316 679 }
Chris@316 680
Chris@316 681 QString
Chris@317 682 FileSource::getErrorString() const
Chris@208 683 {
Chris@208 684 return m_errorString;
Chris@208 685 }
Chris@208 686
Chris@208 687 void
Chris@317 688 FileSource::dataReadProgress(int done, int total)
Chris@208 689 {
Chris@208 690 dataTransferProgress(done, total);
Chris@208 691 }
Chris@208 692
Chris@208 693 void
Chris@317 694 FileSource::httpResponseHeaderReceived(const QHttpResponseHeader &resp)
Chris@210 695 {
Chris@520 696 #ifdef DEBUG_FILE_SOURCE
Chris@690 697 SVDEBUG << "FileSource::httpResponseHeaderReceived" << endl;
Chris@520 698 #endif
Chris@497 699
Chris@497 700 if (resp.statusCode() / 100 == 3) {
Chris@496 701 QString location = resp.value("Location");
Chris@496 702 #ifdef DEBUG_FILE_SOURCE
Chris@690 703 SVDEBUG << "FileSource::responseHeaderReceived: redirect to \""
Chris@687 704 << location << "\" received" << endl;
Chris@496 705 #endif
Chris@496 706 if (location != "") {
Chris@496 707 QUrl newUrl(location);
Chris@496 708 if (newUrl != m_url) {
Chris@497 709 cleanup();
Chris@497 710 deleteCacheFile();
Chris@529 711 #ifdef DEBUG_FILE_SOURCE
Chris@529 712 decCount(m_url.toString());
Chris@529 713 incCount(newUrl.toString());
Chris@529 714 #endif
Chris@496 715 m_url = newUrl;
Chris@497 716 m_localFile = 0;
Chris@496 717 m_lastStatus = 0;
Chris@497 718 m_done = false;
Chris@497 719 m_refCounted = false;
Chris@496 720 init();
Chris@496 721 return;
Chris@496 722 }
Chris@496 723 }
Chris@496 724 }
Chris@497 725
Chris@497 726 m_lastStatus = resp.statusCode();
Chris@210 727 if (m_lastStatus / 100 >= 4) {
Chris@210 728 m_errorString = QString("%1 %2")
Chris@210 729 .arg(resp.statusCode()).arg(resp.reasonPhrase());
Chris@327 730 #ifdef DEBUG_FILE_SOURCE
Chris@690 731 SVDEBUG << "FileSource::responseHeaderReceived: "
Chris@687 732 << m_errorString << endl;
Chris@327 733 #endif
Chris@211 734 } else {
Chris@327 735 #ifdef DEBUG_FILE_SOURCE
Chris@690 736 SVDEBUG << "FileSource::responseHeaderReceived: "
Chris@687 737 << m_lastStatus << endl;
Chris@327 738 #endif
Chris@315 739 if (resp.hasContentType()) m_contentType = resp.contentType();
Chris@325 740 }
Chris@325 741 emit statusAvailable();
Chris@210 742 }
Chris@210 743
Chris@210 744 void
Chris@317 745 FileSource::ftpCommandFinished(int id, bool error)
Chris@214 746 {
Chris@327 747 #ifdef DEBUG_FILE_SOURCE
Chris@317 748 std::cerr << "FileSource::ftpCommandFinished(" << id << ", " << error << ")" << std::endl;
Chris@327 749 #endif
Chris@214 750
Chris@214 751 if (!m_ftp) return;
Chris@214 752
Chris@214 753 QFtp::Command command = m_ftp->currentCommand();
Chris@214 754
Chris@214 755 if (!error) {
Chris@327 756 #ifdef DEBUG_FILE_SOURCE
Chris@690 757 SVDEBUG << "FileSource::ftpCommandFinished: success for command "
Chris@687 758 << command << endl;
Chris@327 759 #endif
Chris@214 760 return;
Chris@214 761 }
Chris@214 762
Chris@214 763 if (command == QFtp::ConnectToHost) {
Chris@214 764 m_errorString = tr("Failed to connect to FTP server");
Chris@214 765 } else if (command == QFtp::Login) {
Chris@214 766 m_errorString = tr("Login failed");
Chris@214 767 } else if (command == QFtp::Cd) {
Chris@214 768 m_errorString = tr("Failed to change to correct directory");
Chris@214 769 } else if (command == QFtp::Get) {
Chris@214 770 m_errorString = tr("FTP download aborted");
Chris@214 771 }
Chris@214 772
Chris@214 773 m_lastStatus = 400; // for done()
Chris@214 774 }
Chris@214 775
Chris@214 776 void
Chris@317 777 FileSource::dataTransferProgress(qint64 done, qint64 total)
Chris@208 778 {
Chris@208 779 int percent = int((double(done) / double(total)) * 100.0 - 0.1);
Chris@208 780 emit progress(percent);
Chris@210 781 }
Chris@210 782
Chris@210 783 void
Chris@317 784 FileSource::cancelled()
Chris@210 785 {
Chris@210 786 m_done = true;
Chris@316 787 cleanup();
Chris@316 788
Chris@210 789 m_ok = false;
Chris@210 790 m_errorString = tr("Download cancelled");
Chris@208 791 }
Chris@208 792
Chris@208 793 void
Chris@317 794 FileSource::done(bool error)
Chris@208 795 {
Chris@327 796 emit progress(100);
Chris@327 797
Chris@327 798 #ifdef DEBUG_FILE_SOURCE
Chris@317 799 std::cerr << "FileSource::done(" << error << ")" << std::endl;
Chris@327 800 #endif
Chris@211 801
Chris@211 802 if (m_done) return;
Chris@211 803
Chris@208 804 if (error) {
Chris@208 805 if (m_http) {
Chris@208 806 m_errorString = m_http->errorString();
Chris@208 807 } else if (m_ftp) {
Chris@208 808 m_errorString = m_ftp->errorString();
Chris@208 809 }
Chris@208 810 }
Chris@208 811
Chris@210 812 if (m_lastStatus / 100 >= 4) {
Chris@211 813 error = true;
Chris@210 814 }
Chris@210 815
Chris@211 816 cleanup();
Chris@208 817
Chris@211 818 if (!error) {
Chris@208 819 QFileInfo fi(m_localFilename);
Chris@208 820 if (!fi.exists()) {
Chris@208 821 m_errorString = tr("Failed to create local file %1").arg(m_localFilename);
Chris@211 822 error = true;
Chris@208 823 } else if (fi.size() == 0) {
Chris@208 824 m_errorString = tr("File contains no data!");
Chris@211 825 error = true;
Chris@208 826 }
Chris@208 827 }
Chris@211 828
Chris@211 829 if (error) {
Chris@327 830 #ifdef DEBUG_FILE_SOURCE
Chris@469 831 std::cerr << "FileSource::done: error is " << error << ", deleting cache file" << std::endl;
Chris@327 832 #endif
Chris@316 833 deleteCacheFile();
Chris@211 834 }
Chris@211 835
Chris@211 836 m_ok = !error;
Chris@520 837 if (m_localFile) m_localFile->flush();
Chris@211 838 m_done = true;
Chris@304 839 emit ready();
Chris@211 840 }
Chris@211 841
Chris@211 842 void
Chris@317 843 FileSource::deleteCacheFile()
Chris@211 844 {
Chris@327 845 #ifdef DEBUG_FILE_SOURCE
Chris@690 846 SVDEBUG << "FileSource::deleteCacheFile(\"" << m_localFilename << "\")" << endl;
Chris@327 847 #endif
Chris@211 848
Chris@211 849 cleanup();
Chris@211 850
Chris@316 851 if (m_localFilename == "") {
Chris@316 852 return;
Chris@316 853 }
Chris@211 854
Chris@316 855 if (!isRemote()) {
Chris@327 856 #ifdef DEBUG_FILE_SOURCE
Chris@316 857 std::cerr << "not a cache file" << std::endl;
Chris@327 858 #endif
Chris@316 859 return;
Chris@316 860 }
Chris@316 861
Chris@316 862 if (m_refCounted) {
Chris@304 863
Chris@304 864 QMutexLocker locker(&m_mapMutex);
Chris@316 865 m_refCounted = false;
Chris@304 866
Chris@304 867 if (m_refCountMap[m_url] > 0) {
Chris@304 868 m_refCountMap[m_url]--;
Chris@327 869 #ifdef DEBUG_FILE_SOURCE
Chris@316 870 std::cerr << "reduced ref count to " << m_refCountMap[m_url] << std::endl;
Chris@327 871 #endif
Chris@304 872 if (m_refCountMap[m_url] > 0) {
Chris@304 873 m_done = true;
Chris@304 874 return;
Chris@304 875 }
Chris@304 876 }
Chris@304 877 }
Chris@304 878
Chris@211 879 m_fileCreationMutex.lock();
Chris@211 880
Chris@211 881 if (!QFile(m_localFilename).remove()) {
Chris@469 882 #ifdef DEBUG_FILE_SOURCE
Chris@686 883 std::cerr << "FileSource::deleteCacheFile: ERROR: Failed to delete file \"" << m_localFilename << "\"" << std::endl;
Chris@469 884 #endif
Chris@211 885 } else {
Chris@327 886 #ifdef DEBUG_FILE_SOURCE
Chris@690 887 SVDEBUG << "FileSource::deleteCacheFile: Deleted cache file \"" << m_localFilename << "\"" << endl;
Chris@327 888 #endif
Chris@211 889 m_localFilename = "";
Chris@211 890 }
Chris@211 891
Chris@211 892 m_fileCreationMutex.unlock();
Chris@211 893
Chris@208 894 m_done = true;
Chris@208 895 }
Chris@208 896
Chris@316 897 bool
Chris@317 898 FileSource::createCacheFile()
Chris@208 899 {
Chris@316 900 {
Chris@316 901 QMutexLocker locker(&m_mapMutex);
Chris@316 902
Chris@327 903 #ifdef DEBUG_FILE_SOURCE
Chris@690 904 SVDEBUG << "FileSource::createCacheFile: refcount is " << m_refCountMap[m_url] << endl;
Chris@327 905 #endif
Chris@316 906
Chris@316 907 if (m_refCountMap[m_url] > 0) {
Chris@316 908 m_refCountMap[m_url]++;
Chris@316 909 m_localFilename = m_remoteLocalMap[m_url];
Chris@327 910 #ifdef DEBUG_FILE_SOURCE
Chris@316 911 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 912 #endif
Chris@316 913 m_refCounted = true;
Chris@316 914 return true;
Chris@316 915 }
Chris@316 916 }
Chris@316 917
Chris@208 918 QDir dir;
Chris@208 919 try {
Chris@208 920 dir = TempDirectory::getInstance()->getSubDirectoryPath("download");
Chris@208 921 } catch (DirectoryCreationFailed f) {
Chris@327 922 #ifdef DEBUG_FILE_SOURCE
Chris@317 923 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create temporary directory: " << f.what() << std::endl;
Chris@327 924 #endif
Chris@208 925 return "";
Chris@208 926 }
Chris@208 927
Chris@316 928 QString filepart = m_url.path().section('/', -1, -1,
Chris@316 929 QString::SectionSkipEmpty);
Chris@208 930
Chris@457 931 QString extension = "";
Chris@457 932 if (filepart.contains('.')) extension = filepart.section('.', -1);
Chris@457 933
Chris@208 934 QString base = filepart;
Chris@208 935 if (extension != "") {
Chris@208 936 base = base.left(base.length() - extension.length() - 1);
Chris@208 937 }
Chris@208 938 if (base == "") base = "remote";
Chris@208 939
Chris@208 940 QString filename;
Chris@208 941
Chris@208 942 if (extension == "") {
Chris@208 943 filename = base;
Chris@208 944 } else {
Chris@208 945 filename = QString("%1.%2").arg(base).arg(extension);
Chris@208 946 }
Chris@208 947
Chris@208 948 QString filepath(dir.filePath(filename));
Chris@208 949
Chris@327 950 #ifdef DEBUG_FILE_SOURCE
Chris@706 951 std::cerr << "FileSource::createCacheFile: URL is \"" << m_url.toString() << "\", dir is \"" << dir.path() << "\", base \"" << base << "\", extension \"" << extension << "\", filebase \"" << filename << "\", filename \"" << filepath << "\"" << std::endl;
Chris@327 952 #endif
Chris@208 953
Chris@316 954 QMutexLocker fcLocker(&m_fileCreationMutex);
Chris@316 955
Chris@208 956 ++m_count;
Chris@208 957
Chris@208 958 if (QFileInfo(filepath).exists() ||
Chris@208 959 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 960
Chris@327 961 #ifdef DEBUG_FILE_SOURCE
Chris@317 962 std::cerr << "FileSource::createCacheFile: Failed to create local file \""
Chris@686 963 << filepath << "\" for URL \""
Chris@686 964 << m_url.toString() << "\" (or file already exists): appending suffix instead" << std::endl;
Chris@327 965 #endif
Chris@208 966
Chris@208 967 if (extension == "") {
Chris@208 968 filename = QString("%1_%2").arg(base).arg(m_count);
Chris@208 969 } else {
Chris@208 970 filename = QString("%1_%2.%3").arg(base).arg(m_count).arg(extension);
Chris@208 971 }
Chris@208 972 filepath = dir.filePath(filename);
Chris@208 973
Chris@208 974 if (QFileInfo(filepath).exists() ||
Chris@208 975 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 976
Chris@327 977 #ifdef DEBUG_FILE_SOURCE
Chris@317 978 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create local file \""
Chris@686 979 << filepath << "\" for URL \""
Chris@686 980 << m_url.toString() << "\" (or file already exists)" << std::endl;
Chris@327 981 #endif
Chris@208 982
Chris@208 983 return "";
Chris@208 984 }
Chris@208 985 }
Chris@208 986
Chris@327 987 #ifdef DEBUG_FILE_SOURCE
Chris@706 988 std::cerr << "FileSource::createCacheFile: url "
Chris@686 989 << m_url.toString() << " -> local filename "
Chris@706 990 << filepath << std::endl;
Chris@327 991 #endif
Chris@316 992
Chris@316 993 m_localFilename = filepath;
Chris@208 994
Chris@316 995 return false;
Chris@208 996 }
Chris@327 997