annotate data/fileio/FileSource.cpp @ 692:cb5eb032c28b sonification

Add queueExampleNote to AudioPlaySource interface
author Chris Cannam
date Fri, 24 Jun 2011 16:42:13 +0100 (2011-06-24)
parents 1424aa29ae95
children 579b2da21e7a
rev   line source
Chris@208 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@208 2
Chris@208 3 /*
Chris@208 4 Sonic Visualiser
Chris@208 5 An audio file viewer and annotation editor.
Chris@208 6 Centre for Digital Music, Queen Mary, University of London.
Chris@208 7 This file copyright 2007 QMUL.
Chris@208 8
Chris@208 9 This program is free software; you can redistribute it and/or
Chris@208 10 modify it under the terms of the GNU General Public License as
Chris@208 11 published by the Free Software Foundation; either version 2 of the
Chris@208 12 License, or (at your option) any later version. See the file
Chris@208 13 COPYING included with this distribution for more information.
Chris@208 14 */
Chris@208 15
Chris@317 16 #include "FileSource.h"
Chris@357 17
Chris@208 18 #include "base/TempDirectory.h"
Chris@208 19 #include "base/Exceptions.h"
Chris@392 20 #include "base/ProgressReporter.h"
Chris@502 21 #include "system/System.h"
Chris@208 22
Chris@208 23 #include <QHttp>
Chris@208 24 #include <QFtp>
Chris@208 25 #include <QFileInfo>
Chris@208 26 #include <QDir>
Chris@392 27 #include <QCoreApplication>
Chris@210 28 #include <QHttpResponseHeader>
Chris@208 29
Chris@208 30 #include <iostream>
Chris@405 31 #include <cstdlib>
Chris@208 32
Chris@608 33 #include <unistd.h>
Chris@608 34
Chris@528 35 //#define DEBUG_FILE_SOURCE 1
Chris@327 36
Chris@208 37 int
Chris@317 38 FileSource::m_count = 0;
Chris@208 39
Chris@208 40 QMutex
Chris@317 41 FileSource::m_fileCreationMutex;
Chris@208 42
Chris@317 43 FileSource::RemoteRefCountMap
Chris@317 44 FileSource::m_refCountMap;
Chris@304 45
Chris@317 46 FileSource::RemoteLocalMap
Chris@317 47 FileSource::m_remoteLocalMap;
Chris@304 48
Chris@304 49 QMutex
Chris@317 50 FileSource::m_mapMutex;
Chris@304 51
Chris@529 52 #ifdef DEBUG_FILE_SOURCE
Chris@529 53 static int extantCount = 0;
Chris@529 54 static std::map<QString, int> urlExtantCountMap;
Chris@529 55 static void incCount(QString url) {
Chris@529 56 ++extantCount;
Chris@529 57 if (urlExtantCountMap.find(url) == urlExtantCountMap.end()) {
Chris@529 58 urlExtantCountMap[url] = 1;
Chris@529 59 } else {
Chris@529 60 ++urlExtantCountMap[url];
Chris@529 61 }
Chris@529 62 std::cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << std::endl;
Chris@529 63 }
Chris@529 64 static void decCount(QString url) {
Chris@529 65 --extantCount;
Chris@529 66 --urlExtantCountMap[url];
Chris@529 67 std::cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << std::endl;
Chris@529 68 }
Chris@529 69 #endif
Chris@529 70
Chris@520 71 FileSource::FileSource(QString fileOrUrl, ProgressReporter *reporter,
Chris@520 72 QString preferredContentType) :
Chris@614 73 m_url(fileOrUrl, QUrl::StrictMode),
Chris@316 74 m_ftp(0),
Chris@316 75 m_http(0),
Chris@316 76 m_localFile(0),
Chris@520 77 m_preferredContentType(preferredContentType),
Chris@316 78 m_ok(false),
Chris@316 79 m_lastStatus(0),
Chris@316 80 m_remote(isRemote(fileOrUrl)),
Chris@316 81 m_done(false),
Chris@316 82 m_leaveLocalFile(false),
Chris@392 83 m_reporter(reporter),
Chris@316 84 m_refCounted(false)
Chris@316 85 {
Chris@661 86 if (m_url.toString() == "") {
Chris@661 87 m_url = QUrl(fileOrUrl, QUrl::TolerantMode);
Chris@661 88 }
Chris@661 89
Chris@327 90 #ifdef DEBUG_FILE_SOURCE
Chris@690 91 SVDEBUG << "FileSource::FileSource(" << fileOrUrl << "): url <" << m_url.toString() << ">" << endl;
Chris@529 92 incCount(m_url.toString());
Chris@327 93 #endif
Chris@316 94
Chris@316 95 if (!canHandleScheme(m_url)) {
Chris@690 96 SVDEBUG << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316 97 m_errorString = tr("Unsupported scheme in URL");
Chris@316 98 return;
Chris@316 99 }
Chris@316 100
Chris@357 101 init();
Chris@316 102
Chris@614 103 if (!isRemote() &&
Chris@614 104 !isAvailable()) {
Chris@614 105 #ifdef DEBUG_FILE_SOURCE
Chris@686 106 std::cerr << "FileSource::FileSource: Failed to open local file with URL \"" << m_url.toString() << "; trying again assuming filename was encoded" << std::endl;
Chris@614 107 #endif
Chris@616 108 m_url = QUrl::fromEncoded(fileOrUrl.toAscii());
Chris@614 109 init();
Chris@614 110 }
Chris@614 111
Chris@316 112 if (isRemote() &&
Chris@316 113 (fileOrUrl.contains('%') ||
Chris@316 114 fileOrUrl.contains("--"))) { // for IDNA
Chris@316 115
Chris@316 116 waitForStatus();
Chris@316 117
Chris@316 118 if (!isAvailable()) {
Chris@336 119
Chris@316 120 // The URL was created on the assumption that the string
Chris@316 121 // was human-readable. Let's try again, this time
Chris@316 122 // assuming it was already encoded.
Chris@317 123 std::cerr << "FileSource::FileSource: Failed to retrieve URL \""
Chris@316 124 << fileOrUrl.toStdString()
Chris@316 125 << "\" as human-readable URL; "
Chris@316 126 << "trying again treating it as encoded URL"
Chris@316 127 << std::endl;
Chris@336 128
Chris@336 129 // even though our cache file doesn't exist (because the
Chris@336 130 // resource was 404), we still need to ensure we're no
Chris@336 131 // longer associating a filename with this url in the
Chris@336 132 // refcount map -- or createCacheFile will think we've
Chris@336 133 // already done all the work and no request will be sent
Chris@336 134 deleteCacheFile();
Chris@336 135
Chris@316 136 m_url.setEncodedUrl(fileOrUrl.toAscii());
Chris@336 137
Chris@336 138 m_ok = false;
Chris@336 139 m_done = false;
Chris@336 140 m_lastStatus = 0;
Chris@357 141 init();
Chris@316 142 }
Chris@316 143 }
Chris@325 144
Chris@325 145 if (!isRemote()) {
Chris@325 146 emit statusAvailable();
Chris@325 147 emit ready();
Chris@325 148 }
Chris@497 149
Chris@504 150 #ifdef DEBUG_FILE_SOURCE
Chris@690 151 SVDEBUG << "FileSource::FileSource(string) exiting" << endl;
Chris@504 152 #endif
Chris@316 153 }
Chris@316 154
Chris@469 155 FileSource::FileSource(QUrl url, ProgressReporter *reporter) :
Chris@304 156 m_url(url),
Chris@208 157 m_ftp(0),
Chris@208 158 m_http(0),
Chris@208 159 m_localFile(0),
Chris@208 160 m_ok(false),
Chris@210 161 m_lastStatus(0),
Chris@316 162 m_remote(isRemote(url.toString())),
Chris@208 163 m_done(false),
Chris@316 164 m_leaveLocalFile(false),
Chris@392 165 m_reporter(reporter),
Chris@316 166 m_refCounted(false)
Chris@208 167 {
Chris@327 168 #ifdef DEBUG_FILE_SOURCE
Chris@690 169 SVDEBUG << "FileSource::FileSource(" << url.toString() << ") [as url]" << endl;
Chris@529 170 incCount(m_url.toString());
Chris@327 171 #endif
Chris@316 172
Chris@316 173 if (!canHandleScheme(m_url)) {
Chris@690 174 SVDEBUG << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316 175 m_errorString = tr("Unsupported scheme in URL");
Chris@208 176 return;
Chris@208 177 }
Chris@208 178
Chris@357 179 init();
Chris@497 180
Chris@504 181 #ifdef DEBUG_FILE_SOURCE
Chris@690 182 SVDEBUG << "FileSource::FileSource(url) exiting" << endl;
Chris@504 183 #endif
Chris@316 184 }
Chris@304 185
Chris@317 186 FileSource::FileSource(const FileSource &rf) :
Chris@316 187 QObject(),
Chris@316 188 m_url(rf.m_url),
Chris@316 189 m_ftp(0),
Chris@316 190 m_http(0),
Chris@316 191 m_localFile(0),
Chris@316 192 m_ok(rf.m_ok),
Chris@316 193 m_lastStatus(rf.m_lastStatus),
Chris@316 194 m_remote(rf.m_remote),
Chris@316 195 m_done(false),
Chris@316 196 m_leaveLocalFile(false),
Chris@392 197 m_reporter(rf.m_reporter),
Chris@316 198 m_refCounted(false)
Chris@316 199 {
Chris@327 200 #ifdef DEBUG_FILE_SOURCE
Chris@690 201 SVDEBUG << "FileSource::FileSource(" << m_url.toString() << ") [copy ctor]" << endl;
Chris@529 202 incCount(m_url.toString());
Chris@327 203 #endif
Chris@304 204
Chris@316 205 if (!canHandleScheme(m_url)) {
Chris@690 206 SVDEBUG << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString() << "\"" << endl;
Chris@316 207 m_errorString = tr("Unsupported scheme in URL");
Chris@304 208 return;
Chris@304 209 }
Chris@304 210
Chris@469 211 if (!isRemote()) {
Chris@316 212 m_localFilename = rf.m_localFilename;
Chris@316 213 } else {
Chris@316 214 QMutexLocker locker(&m_mapMutex);
Chris@327 215 #ifdef DEBUG_FILE_SOURCE
Chris@690 216 SVDEBUG << "FileSource::FileSource(copy ctor): ref count is "
Chris@687 217 << m_refCountMap[m_url] << endl;
Chris@327 218 #endif
Chris@316 219 if (m_refCountMap[m_url] > 0) {
Chris@316 220 m_refCountMap[m_url]++;
Chris@327 221 #ifdef DEBUG_FILE_SOURCE
Chris@316 222 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 223 #endif
Chris@316 224 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 225 m_refCounted = true;
Chris@316 226 } else {
Chris@316 227 m_ok = false;
Chris@316 228 m_lastStatus = 404;
Chris@316 229 }
Chris@316 230 }
Chris@316 231
Chris@316 232 m_done = true;
Chris@497 233
Chris@504 234 #ifdef DEBUG_FILE_SOURCE
Chris@690 235 SVDEBUG << "FileSource::FileSource(" << m_url.toString() << ") [copy ctor]: note: local filename is \"" << m_localFilename << "\"" << endl;
Chris@527 236 #endif
Chris@527 237
Chris@527 238 #ifdef DEBUG_FILE_SOURCE
Chris@690 239 SVDEBUG << "FileSource::FileSource(copy ctor) exiting" << endl;
Chris@504 240 #endif
Chris@316 241 }
Chris@316 242
Chris@317 243 FileSource::~FileSource()
Chris@316 244 {
Chris@327 245 #ifdef DEBUG_FILE_SOURCE
Chris@686 246 std::cerr << "FileSource(" << m_url.toString() << ")::~FileSource" << std::endl;
Chris@529 247 decCount(m_url.toString());
Chris@327 248 #endif
Chris@316 249
Chris@316 250 cleanup();
Chris@316 251
Chris@469 252 if (isRemote() && !m_leaveLocalFile) deleteCacheFile();
Chris@316 253 }
Chris@316 254
Chris@316 255 void
Chris@357 256 FileSource::init()
Chris@316 257 {
Chris@316 258 if (!isRemote()) {
Chris@355 259 #ifdef DEBUG_FILE_SOURCE
Chris@690 260 SVDEBUG << "FileSource::init: Not a remote URL" << endl;
Chris@355 261 #endif
Chris@355 262 bool literal = false;
Chris@316 263 m_localFilename = m_url.toLocalFile();
Chris@342 264 if (m_localFilename == "") {
Chris@342 265 // QUrl may have mishandled the scheme (e.g. in a DOS path)
Chris@342 266 m_localFilename = m_url.toString();
Chris@355 267 literal = true;
Chris@342 268 }
Chris@439 269 m_localFilename = QFileInfo(m_localFilename).absoluteFilePath();
Chris@439 270
Chris@355 271 #ifdef DEBUG_FILE_SOURCE
Chris@690 272 SVDEBUG << "FileSource::init: URL translates to local filename \""
Chris@687 273 << m_localFilename << "\" (with literal=" << literal << ")" << endl;
Chris@355 274 #endif
Chris@316 275 m_ok = true;
Chris@355 276 m_lastStatus = 200;
Chris@355 277
Chris@316 278 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 279 if (literal) {
Chris@355 280 m_lastStatus = 404;
Chris@355 281 } else {
Chris@614 282 #ifdef DEBUG_FILE_SOURCE
Chris@690 283 SVDEBUG << "FileSource::init: Local file of this name does not exist, trying URL as a literal filename" << endl;
Chris@614 284 #endif
Chris@355 285 // Again, QUrl may have been mistreating us --
Chris@355 286 // e.g. dropping a part that looks like query data
Chris@355 287 m_localFilename = m_url.toString();
Chris@355 288 literal = true;
Chris@355 289 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 290 m_lastStatus = 404;
Chris@355 291 }
Chris@355 292 }
Chris@316 293 }
Chris@355 294
Chris@316 295 m_done = true;
Chris@316 296 return;
Chris@316 297 }
Chris@316 298
Chris@316 299 if (createCacheFile()) {
Chris@327 300 #ifdef DEBUG_FILE_SOURCE
Chris@690 301 SVDEBUG << "FileSource::init: Already have this one" << endl;
Chris@327 302 #endif
Chris@316 303 m_ok = true;
Chris@316 304 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 305 m_lastStatus = 404;
Chris@316 306 } else {
Chris@316 307 m_lastStatus = 200;
Chris@316 308 }
Chris@316 309 m_done = true;
Chris@316 310 return;
Chris@316 311 }
Chris@316 312
Chris@208 313 if (m_localFilename == "") return;
Chris@208 314 m_localFile = new QFile(m_localFilename);
Chris@208 315 m_localFile->open(QFile::WriteOnly);
Chris@208 316
Chris@316 317 QString scheme = m_url.scheme().toLower();
Chris@316 318
Chris@327 319 #ifdef DEBUG_FILE_SOURCE
Chris@690 320 SVDEBUG << "FileSource::init: Don't have local copy of \""
Chris@687 321 << m_url.toString() << "\", retrieving" << endl;
Chris@327 322 #endif
Chris@208 323
Chris@208 324 if (scheme == "http") {
Chris@316 325 initHttp();
Chris@520 326 #ifdef DEBUG_FILE_SOURCE
Chris@357 327 std::cerr << "FileSource: initHttp succeeded" << std::endl;
Chris@520 328 #endif
Chris@208 329 } else if (scheme == "ftp") {
Chris@316 330 initFtp();
Chris@316 331 } else {
Chris@316 332 m_remote = false;
Chris@316 333 m_ok = false;
Chris@208 334 }
Chris@208 335
Chris@208 336 if (m_ok) {
Chris@316 337
Chris@316 338 QMutexLocker locker(&m_mapMutex);
Chris@316 339
Chris@316 340 if (m_refCountMap[m_url] > 0) {
Chris@316 341 // someone else has been doing the same thing at the same time,
Chris@316 342 // but has got there first
Chris@316 343 cleanup();
Chris@316 344 m_refCountMap[m_url]++;
Chris@327 345 #ifdef DEBUG_FILE_SOURCE
Chris@690 346 SVDEBUG << "FileSource::init: Another FileSource has got there first, abandoning our download and using theirs" << endl;
Chris@327 347 #endif
Chris@316 348 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 349 m_refCounted = true;
Chris@316 350 m_ok = true;
Chris@316 351 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 352 m_lastStatus = 404;
Chris@316 353 }
Chris@316 354 m_done = true;
Chris@316 355 return;
Chris@316 356 }
Chris@304 357
Chris@304 358 m_remoteLocalMap[m_url] = m_localFilename;
Chris@304 359 m_refCountMap[m_url]++;
Chris@316 360 m_refCounted = true;
Chris@304 361
Chris@392 362 if (m_reporter) {
Chris@392 363 m_reporter->setMessage
Chris@392 364 (tr("Downloading %1...").arg(m_url.toString()));
Chris@392 365 connect(m_reporter, SIGNAL(cancelled()), this, SLOT(cancelled()));
Chris@357 366 connect(this, SIGNAL(progress(int)),
Chris@392 367 m_reporter, SLOT(setProgress(int)));
Chris@316 368 }
Chris@208 369 }
Chris@208 370 }
Chris@208 371
Chris@316 372 void
Chris@317 373 FileSource::initHttp()
Chris@208 374 {
Chris@316 375 m_ok = true;
Chris@497 376 int port = m_url.port();
Chris@497 377 m_http = new QHttp(m_url.host(), port < 0 ? 80 : port);
Chris@316 378 connect(m_http, SIGNAL(done(bool)), this, SLOT(done(bool)));
Chris@316 379 connect(m_http, SIGNAL(dataReadProgress(int, int)),
Chris@316 380 this, SLOT(dataReadProgress(int, int)));
Chris@316 381 connect(m_http, SIGNAL(responseHeaderReceived(const QHttpResponseHeader &)),
Chris@316 382 this, SLOT(httpResponseHeaderReceived(const QHttpResponseHeader &)));
Chris@316 383
Chris@316 384 // I don't quite understand this. url.path() returns a path
Chris@316 385 // without percent encoding; for example, spaces appear as
Chris@316 386 // literal spaces. This generally won't work if sent to the
Chris@316 387 // server directly. You can retrieve a correctly encoded URL
Chris@316 388 // from QUrl using url.toEncoded(), but that gives you the
Chris@316 389 // whole URL; there doesn't seem to be any way to retrieve
Chris@316 390 // only an encoded path. Furthermore there doesn't seem to be
Chris@316 391 // any way to convert a retrieved path into an encoded path
Chris@316 392 // without explicitly specifying that you don't want the path
Chris@316 393 // separators ("/") to be encoded. (Besides being painful to
Chris@316 394 // manage, I don't see how this can work correctly in any case
Chris@316 395 // where a percent-encoded "/" is supposed to appear within a
Chris@316 396 // path element?) There also seems to be no way to retrieve
Chris@316 397 // the path plus query string, i.e. everything that I need to
Chris@316 398 // send to the HTTP server. And no way for QHttp to take a
Chris@316 399 // QUrl argument. I'm obviously missing something.
Chris@316 400
Chris@316 401 // So, two ways to do this: query the bits from the URL,
Chris@316 402 // encode them individually, and glue them back together
Chris@316 403 // again...
Chris@316 404 /*
Chris@316 405 QString path = QUrl::toPercentEncoding(m_url.path(), "/");
Chris@316 406 QList<QPair<QString, QString> > query = m_url.queryItems();
Chris@316 407 if (!query.empty()) {
Chris@316 408 QStringList q2;
Chris@316 409 for (QList<QPair<QString, QString> >::iterator i = query.begin();
Chris@316 410 i != query.end(); ++i) {
Chris@316 411 q2.push_back(QString("%1=%3")
Chris@316 412 .arg(QString(QUrl::toPercentEncoding(i->first)))
Chris@316 413 .arg(QString(QUrl::toPercentEncoding(i->second))));
Chris@316 414 }
Chris@316 415 path = QString("%1%2%3")
Chris@316 416 .arg(path).arg("?")
Chris@316 417 .arg(q2.join("&"));
Chris@316 418 }
Chris@316 419 */
Chris@316 420
Chris@316 421 // ...or, much simpler but relying on knowledge about the
Chris@316 422 // scheme://host/path/path/query etc format of the URL, we can
Chris@316 423 // get the whole URL ready-encoded and then split it on "/" as
Chris@316 424 // appropriate...
Chris@316 425
Chris@316 426 QString path = "/" + QString(m_url.toEncoded()).section('/', 3);
Chris@316 427
Chris@327 428 #ifdef DEBUG_FILE_SOURCE
Chris@690 429 SVDEBUG << "FileSource: path is \""
Chris@687 430 << path << "\"" << endl;
Chris@327 431 #endif
Chris@316 432
Chris@520 433 if (m_preferredContentType == "") {
Chris@520 434 m_http->get(path, m_localFile);
Chris@520 435 } else {
Chris@520 436 #ifdef DEBUG_FILE_SOURCE
Chris@520 437 std::cerr << "FileSource: indicating preferred content type of \""
Chris@686 438 << m_preferredContentType << "\"" << std::endl;
Chris@520 439 #endif
Chris@520 440 QHttpRequestHeader header("GET", path);
Chris@520 441 header.setValue("Host", m_url.host());
Chris@520 442 header.setValue("Accept", QString("%1, */*").arg(m_preferredContentType));
Chris@520 443 m_http->request(header, 0, m_localFile);
Chris@520 444 }
Chris@316 445 }
Chris@316 446
Chris@316 447 void
Chris@317 448 FileSource::initFtp()
Chris@316 449 {
Chris@316 450 m_ok = true;
Chris@316 451 m_ftp = new QFtp;
Chris@316 452 connect(m_ftp, SIGNAL(done(bool)), this, SLOT(done(bool)));
Chris@316 453 connect(m_ftp, SIGNAL(commandFinished(int, bool)),
Chris@316 454 this, SLOT(ftpCommandFinished(int, bool)));
Chris@316 455 connect(m_ftp, SIGNAL(dataTransferProgress(qint64, qint64)),
Chris@316 456 this, SLOT(dataTransferProgress(qint64, qint64)));
Chris@316 457 m_ftp->connectToHost(m_url.host(), m_url.port(21));
Chris@316 458
Chris@316 459 QString username = m_url.userName();
Chris@316 460 if (username == "") {
Chris@316 461 username = "anonymous";
Chris@316 462 }
Chris@316 463
Chris@316 464 QString password = m_url.password();
Chris@316 465 if (password == "") {
Chris@316 466 password = QString("%1@%2").arg(getenv("USER")).arg(getenv("HOST"));
Chris@316 467 }
Chris@316 468
Chris@316 469 m_ftp->login(username, password);
Chris@316 470
Chris@316 471 QString dirpath = m_url.path().section('/', 0, -2);
Chris@316 472 QString filename = m_url.path().section('/', -1);
Chris@316 473
Chris@316 474 if (dirpath == "") dirpath = "/";
Chris@316 475 m_ftp->cd(dirpath);
Chris@316 476 m_ftp->get(filename, m_localFile);
Chris@211 477 }
Chris@211 478
Chris@211 479 void
Chris@317 480 FileSource::cleanup()
Chris@211 481 {
Chris@470 482 if (m_done) {
Chris@470 483 delete m_localFile; // does not actually delete the file
Chris@470 484 m_localFile = 0;
Chris@470 485 }
Chris@211 486 m_done = true;
Chris@214 487 if (m_http) {
Chris@287 488 QHttp *h = m_http;
Chris@214 489 m_http = 0;
Chris@287 490 h->abort();
Chris@287 491 h->deleteLater();
Chris@214 492 }
Chris@214 493 if (m_ftp) {
Chris@287 494 QFtp *f = m_ftp;
Chris@214 495 m_ftp = 0;
Chris@287 496 f->abort();
Chris@287 497 f->deleteLater();
Chris@214 498 }
Chris@470 499 if (m_localFile) {
Chris@470 500 delete m_localFile; // does not actually delete the file
Chris@470 501 m_localFile = 0;
Chris@470 502 }
Chris@208 503 }
Chris@208 504
Chris@208 505 bool
Chris@317 506 FileSource::isRemote(QString fileOrUrl)
Chris@304 507 {
Chris@342 508 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@316 509 QString scheme = QUrl(fileOrUrl).scheme().toLower();
Chris@342 510 if (scheme == "" || scheme == "file" || scheme.length() == 1) return false;
Chris@342 511 return true;
Chris@304 512 }
Chris@304 513
Chris@304 514 bool
Chris@317 515 FileSource::canHandleScheme(QUrl url)
Chris@208 516 {
Chris@342 517 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@208 518 QString scheme = url.scheme().toLower();
Chris@316 519 return (scheme == "http" || scheme == "ftp" ||
Chris@342 520 scheme == "file" || scheme == "" || scheme.length() == 1);
Chris@208 521 }
Chris@208 522
Chris@210 523 bool
Chris@317 524 FileSource::isAvailable()
Chris@210 525 {
Chris@316 526 waitForStatus();
Chris@211 527 bool available = true;
Chris@211 528 if (!m_ok) available = false;
Chris@211 529 else available = (m_lastStatus / 100 == 2);
Chris@327 530 #ifdef DEBUG_FILE_SOURCE
Chris@690 531 SVDEBUG << "FileSource::isAvailable: " << (available ? "yes" : "no")
Chris@687 532 << endl;
Chris@327 533 #endif
Chris@211 534 return available;
Chris@210 535 }
Chris@210 536
Chris@208 537 void
Chris@317 538 FileSource::waitForStatus()
Chris@316 539 {
Chris@316 540 while (m_ok && (!m_done && m_lastStatus == 0)) {
Chris@316 541 // std::cerr << "waitForStatus: processing (last status " << m_lastStatus << ")" << std::endl;
Chris@392 542 QCoreApplication::processEvents();
Chris@316 543 }
Chris@316 544 }
Chris@316 545
Chris@316 546 void
Chris@317 547 FileSource::waitForData()
Chris@208 548 {
Chris@211 549 while (m_ok && !m_done) {
Chris@690 550 // SVDEBUG << "FileSource::waitForData: calling QApplication::processEvents" << endl;
Chris@392 551 QCoreApplication::processEvents();
Chris@497 552 usleep(10000);
Chris@208 553 }
Chris@208 554 }
Chris@208 555
Chris@316 556 void
Chris@317 557 FileSource::setLeaveLocalFile(bool leave)
Chris@316 558 {
Chris@316 559 m_leaveLocalFile = leave;
Chris@316 560 }
Chris@316 561
Chris@208 562 bool
Chris@317 563 FileSource::isOK() const
Chris@208 564 {
Chris@208 565 return m_ok;
Chris@208 566 }
Chris@208 567
Chris@208 568 bool
Chris@317 569 FileSource::isDone() const
Chris@208 570 {
Chris@208 571 return m_done;
Chris@208 572 }
Chris@208 573
Chris@316 574 bool
Chris@317 575 FileSource::isRemote() const
Chris@316 576 {
Chris@316 577 return m_remote;
Chris@316 578 }
Chris@316 579
Chris@316 580 QString
Chris@317 581 FileSource::getLocation() const
Chris@316 582 {
Chris@316 583 return m_url.toString();
Chris@316 584 }
Chris@316 585
Chris@208 586 QString
Chris@317 587 FileSource::getLocalFilename() const
Chris@208 588 {
Chris@208 589 return m_localFilename;
Chris@208 590 }
Chris@208 591
Chris@208 592 QString
Chris@678 593 FileSource::getBasename() const
Chris@678 594 {
Chris@678 595 return QFileInfo(m_localFilename).fileName();
Chris@678 596 }
Chris@678 597
Chris@678 598 QString
Chris@317 599 FileSource::getContentType() const
Chris@316 600 {
Chris@316 601 return m_contentType;
Chris@316 602 }
Chris@316 603
Chris@316 604 QString
Chris@317 605 FileSource::getExtension() const
Chris@316 606 {
Chris@316 607 if (m_localFilename != "") {
Chris@316 608 return QFileInfo(m_localFilename).suffix().toLower();
Chris@316 609 } else {
Chris@316 610 return QFileInfo(m_url.toLocalFile()).suffix().toLower();
Chris@316 611 }
Chris@316 612 }
Chris@316 613
Chris@316 614 QString
Chris@317 615 FileSource::getErrorString() const
Chris@208 616 {
Chris@208 617 return m_errorString;
Chris@208 618 }
Chris@208 619
Chris@208 620 void
Chris@317 621 FileSource::dataReadProgress(int done, int total)
Chris@208 622 {
Chris@208 623 dataTransferProgress(done, total);
Chris@208 624 }
Chris@208 625
Chris@208 626 void
Chris@317 627 FileSource::httpResponseHeaderReceived(const QHttpResponseHeader &resp)
Chris@210 628 {
Chris@520 629 #ifdef DEBUG_FILE_SOURCE
Chris@690 630 SVDEBUG << "FileSource::httpResponseHeaderReceived" << endl;
Chris@520 631 #endif
Chris@497 632
Chris@497 633 if (resp.statusCode() / 100 == 3) {
Chris@496 634 QString location = resp.value("Location");
Chris@496 635 #ifdef DEBUG_FILE_SOURCE
Chris@690 636 SVDEBUG << "FileSource::responseHeaderReceived: redirect to \""
Chris@687 637 << location << "\" received" << endl;
Chris@496 638 #endif
Chris@496 639 if (location != "") {
Chris@496 640 QUrl newUrl(location);
Chris@496 641 if (newUrl != m_url) {
Chris@497 642 cleanup();
Chris@497 643 deleteCacheFile();
Chris@529 644 #ifdef DEBUG_FILE_SOURCE
Chris@529 645 decCount(m_url.toString());
Chris@529 646 incCount(newUrl.toString());
Chris@529 647 #endif
Chris@496 648 m_url = newUrl;
Chris@497 649 m_localFile = 0;
Chris@496 650 m_lastStatus = 0;
Chris@497 651 m_done = false;
Chris@497 652 m_refCounted = false;
Chris@496 653 init();
Chris@496 654 return;
Chris@496 655 }
Chris@496 656 }
Chris@496 657 }
Chris@497 658
Chris@497 659 m_lastStatus = resp.statusCode();
Chris@210 660 if (m_lastStatus / 100 >= 4) {
Chris@210 661 m_errorString = QString("%1 %2")
Chris@210 662 .arg(resp.statusCode()).arg(resp.reasonPhrase());
Chris@327 663 #ifdef DEBUG_FILE_SOURCE
Chris@690 664 SVDEBUG << "FileSource::responseHeaderReceived: "
Chris@687 665 << m_errorString << endl;
Chris@327 666 #endif
Chris@211 667 } else {
Chris@327 668 #ifdef DEBUG_FILE_SOURCE
Chris@690 669 SVDEBUG << "FileSource::responseHeaderReceived: "
Chris@687 670 << m_lastStatus << endl;
Chris@327 671 #endif
Chris@315 672 if (resp.hasContentType()) m_contentType = resp.contentType();
Chris@325 673 }
Chris@325 674 emit statusAvailable();
Chris@210 675 }
Chris@210 676
Chris@210 677 void
Chris@317 678 FileSource::ftpCommandFinished(int id, bool error)
Chris@214 679 {
Chris@327 680 #ifdef DEBUG_FILE_SOURCE
Chris@317 681 std::cerr << "FileSource::ftpCommandFinished(" << id << ", " << error << ")" << std::endl;
Chris@327 682 #endif
Chris@214 683
Chris@214 684 if (!m_ftp) return;
Chris@214 685
Chris@214 686 QFtp::Command command = m_ftp->currentCommand();
Chris@214 687
Chris@214 688 if (!error) {
Chris@327 689 #ifdef DEBUG_FILE_SOURCE
Chris@690 690 SVDEBUG << "FileSource::ftpCommandFinished: success for command "
Chris@687 691 << command << endl;
Chris@327 692 #endif
Chris@214 693 return;
Chris@214 694 }
Chris@214 695
Chris@214 696 if (command == QFtp::ConnectToHost) {
Chris@214 697 m_errorString = tr("Failed to connect to FTP server");
Chris@214 698 } else if (command == QFtp::Login) {
Chris@214 699 m_errorString = tr("Login failed");
Chris@214 700 } else if (command == QFtp::Cd) {
Chris@214 701 m_errorString = tr("Failed to change to correct directory");
Chris@214 702 } else if (command == QFtp::Get) {
Chris@214 703 m_errorString = tr("FTP download aborted");
Chris@214 704 }
Chris@214 705
Chris@214 706 m_lastStatus = 400; // for done()
Chris@214 707 }
Chris@214 708
Chris@214 709 void
Chris@317 710 FileSource::dataTransferProgress(qint64 done, qint64 total)
Chris@208 711 {
Chris@208 712 int percent = int((double(done) / double(total)) * 100.0 - 0.1);
Chris@208 713 emit progress(percent);
Chris@210 714 }
Chris@210 715
Chris@210 716 void
Chris@317 717 FileSource::cancelled()
Chris@210 718 {
Chris@210 719 m_done = true;
Chris@316 720 cleanup();
Chris@316 721
Chris@210 722 m_ok = false;
Chris@210 723 m_errorString = tr("Download cancelled");
Chris@208 724 }
Chris@208 725
Chris@208 726 void
Chris@317 727 FileSource::done(bool error)
Chris@208 728 {
Chris@327 729 emit progress(100);
Chris@327 730
Chris@327 731 #ifdef DEBUG_FILE_SOURCE
Chris@317 732 std::cerr << "FileSource::done(" << error << ")" << std::endl;
Chris@327 733 #endif
Chris@211 734
Chris@211 735 if (m_done) return;
Chris@211 736
Chris@208 737 if (error) {
Chris@208 738 if (m_http) {
Chris@208 739 m_errorString = m_http->errorString();
Chris@208 740 } else if (m_ftp) {
Chris@208 741 m_errorString = m_ftp->errorString();
Chris@208 742 }
Chris@208 743 }
Chris@208 744
Chris@210 745 if (m_lastStatus / 100 >= 4) {
Chris@211 746 error = true;
Chris@210 747 }
Chris@210 748
Chris@211 749 cleanup();
Chris@208 750
Chris@211 751 if (!error) {
Chris@208 752 QFileInfo fi(m_localFilename);
Chris@208 753 if (!fi.exists()) {
Chris@208 754 m_errorString = tr("Failed to create local file %1").arg(m_localFilename);
Chris@211 755 error = true;
Chris@208 756 } else if (fi.size() == 0) {
Chris@208 757 m_errorString = tr("File contains no data!");
Chris@211 758 error = true;
Chris@208 759 }
Chris@208 760 }
Chris@211 761
Chris@211 762 if (error) {
Chris@327 763 #ifdef DEBUG_FILE_SOURCE
Chris@469 764 std::cerr << "FileSource::done: error is " << error << ", deleting cache file" << std::endl;
Chris@327 765 #endif
Chris@316 766 deleteCacheFile();
Chris@211 767 }
Chris@211 768
Chris@211 769 m_ok = !error;
Chris@520 770 if (m_localFile) m_localFile->flush();
Chris@211 771 m_done = true;
Chris@304 772 emit ready();
Chris@211 773 }
Chris@211 774
Chris@211 775 void
Chris@317 776 FileSource::deleteCacheFile()
Chris@211 777 {
Chris@327 778 #ifdef DEBUG_FILE_SOURCE
Chris@690 779 SVDEBUG << "FileSource::deleteCacheFile(\"" << m_localFilename << "\")" << endl;
Chris@327 780 #endif
Chris@211 781
Chris@211 782 cleanup();
Chris@211 783
Chris@316 784 if (m_localFilename == "") {
Chris@316 785 return;
Chris@316 786 }
Chris@211 787
Chris@316 788 if (!isRemote()) {
Chris@327 789 #ifdef DEBUG_FILE_SOURCE
Chris@316 790 std::cerr << "not a cache file" << std::endl;
Chris@327 791 #endif
Chris@316 792 return;
Chris@316 793 }
Chris@316 794
Chris@316 795 if (m_refCounted) {
Chris@304 796
Chris@304 797 QMutexLocker locker(&m_mapMutex);
Chris@316 798 m_refCounted = false;
Chris@304 799
Chris@304 800 if (m_refCountMap[m_url] > 0) {
Chris@304 801 m_refCountMap[m_url]--;
Chris@327 802 #ifdef DEBUG_FILE_SOURCE
Chris@316 803 std::cerr << "reduced ref count to " << m_refCountMap[m_url] << std::endl;
Chris@327 804 #endif
Chris@304 805 if (m_refCountMap[m_url] > 0) {
Chris@304 806 m_done = true;
Chris@304 807 return;
Chris@304 808 }
Chris@304 809 }
Chris@304 810 }
Chris@304 811
Chris@211 812 m_fileCreationMutex.lock();
Chris@211 813
Chris@211 814 if (!QFile(m_localFilename).remove()) {
Chris@469 815 #ifdef DEBUG_FILE_SOURCE
Chris@686 816 std::cerr << "FileSource::deleteCacheFile: ERROR: Failed to delete file \"" << m_localFilename << "\"" << std::endl;
Chris@469 817 #endif
Chris@211 818 } else {
Chris@327 819 #ifdef DEBUG_FILE_SOURCE
Chris@690 820 SVDEBUG << "FileSource::deleteCacheFile: Deleted cache file \"" << m_localFilename << "\"" << endl;
Chris@327 821 #endif
Chris@211 822 m_localFilename = "";
Chris@211 823 }
Chris@211 824
Chris@211 825 m_fileCreationMutex.unlock();
Chris@211 826
Chris@208 827 m_done = true;
Chris@208 828 }
Chris@208 829
Chris@316 830 bool
Chris@317 831 FileSource::createCacheFile()
Chris@208 832 {
Chris@316 833 {
Chris@316 834 QMutexLocker locker(&m_mapMutex);
Chris@316 835
Chris@327 836 #ifdef DEBUG_FILE_SOURCE
Chris@690 837 SVDEBUG << "FileSource::createCacheFile: refcount is " << m_refCountMap[m_url] << endl;
Chris@327 838 #endif
Chris@316 839
Chris@316 840 if (m_refCountMap[m_url] > 0) {
Chris@316 841 m_refCountMap[m_url]++;
Chris@316 842 m_localFilename = m_remoteLocalMap[m_url];
Chris@327 843 #ifdef DEBUG_FILE_SOURCE
Chris@316 844 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 845 #endif
Chris@316 846 m_refCounted = true;
Chris@316 847 return true;
Chris@316 848 }
Chris@316 849 }
Chris@316 850
Chris@208 851 QDir dir;
Chris@208 852 try {
Chris@208 853 dir = TempDirectory::getInstance()->getSubDirectoryPath("download");
Chris@208 854 } catch (DirectoryCreationFailed f) {
Chris@327 855 #ifdef DEBUG_FILE_SOURCE
Chris@317 856 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create temporary directory: " << f.what() << std::endl;
Chris@327 857 #endif
Chris@208 858 return "";
Chris@208 859 }
Chris@208 860
Chris@316 861 QString filepart = m_url.path().section('/', -1, -1,
Chris@316 862 QString::SectionSkipEmpty);
Chris@208 863
Chris@457 864 QString extension = "";
Chris@457 865 if (filepart.contains('.')) extension = filepart.section('.', -1);
Chris@457 866
Chris@208 867 QString base = filepart;
Chris@208 868 if (extension != "") {
Chris@208 869 base = base.left(base.length() - extension.length() - 1);
Chris@208 870 }
Chris@208 871 if (base == "") base = "remote";
Chris@208 872
Chris@208 873 QString filename;
Chris@208 874
Chris@208 875 if (extension == "") {
Chris@208 876 filename = base;
Chris@208 877 } else {
Chris@208 878 filename = QString("%1.%2").arg(base).arg(extension);
Chris@208 879 }
Chris@208 880
Chris@208 881 QString filepath(dir.filePath(filename));
Chris@208 882
Chris@327 883 #ifdef DEBUG_FILE_SOURCE
Chris@690 884 SVDEBUG << "FileSource::createCacheFile: URL is \"" << m_url.toString() << "\", dir is \"" << dir.path() << "\", base \"" << base << "\", extension \"" << extension << "\", filebase \"" << filename << "\", filename \"" << filepath << "\"" << endl;
Chris@327 885 #endif
Chris@208 886
Chris@316 887 QMutexLocker fcLocker(&m_fileCreationMutex);
Chris@316 888
Chris@208 889 ++m_count;
Chris@208 890
Chris@208 891 if (QFileInfo(filepath).exists() ||
Chris@208 892 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 893
Chris@327 894 #ifdef DEBUG_FILE_SOURCE
Chris@317 895 std::cerr << "FileSource::createCacheFile: Failed to create local file \""
Chris@686 896 << filepath << "\" for URL \""
Chris@686 897 << m_url.toString() << "\" (or file already exists): appending suffix instead" << std::endl;
Chris@327 898 #endif
Chris@208 899
Chris@208 900 if (extension == "") {
Chris@208 901 filename = QString("%1_%2").arg(base).arg(m_count);
Chris@208 902 } else {
Chris@208 903 filename = QString("%1_%2.%3").arg(base).arg(m_count).arg(extension);
Chris@208 904 }
Chris@208 905 filepath = dir.filePath(filename);
Chris@208 906
Chris@208 907 if (QFileInfo(filepath).exists() ||
Chris@208 908 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 909
Chris@327 910 #ifdef DEBUG_FILE_SOURCE
Chris@317 911 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create local file \""
Chris@686 912 << filepath << "\" for URL \""
Chris@686 913 << m_url.toString() << "\" (or file already exists)" << std::endl;
Chris@327 914 #endif
Chris@208 915
Chris@208 916 return "";
Chris@208 917 }
Chris@208 918 }
Chris@208 919
Chris@327 920 #ifdef DEBUG_FILE_SOURCE
Chris@690 921 SVDEBUG << "FileSource::createCacheFile: url "
Chris@686 922 << m_url.toString() << " -> local filename "
Chris@687 923 << filepath << endl;
Chris@327 924 #endif
Chris@316 925
Chris@316 926 m_localFilename = filepath;
Chris@208 927
Chris@316 928 return false;
Chris@208 929 }
Chris@327 930