annotate data/fileio/FileSource.cpp @ 642:6f6aa00a404c

* Print error if qmake fails
author Chris Cannam
date Fri, 17 Sep 2010 13:17:08 +0100
parents 3a61a4df6d53
children a4faa1840384
rev   line source
Chris@208 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@208 2
Chris@208 3 /*
Chris@208 4 Sonic Visualiser
Chris@208 5 An audio file viewer and annotation editor.
Chris@208 6 Centre for Digital Music, Queen Mary, University of London.
Chris@208 7 This file copyright 2007 QMUL.
Chris@208 8
Chris@208 9 This program is free software; you can redistribute it and/or
Chris@208 10 modify it under the terms of the GNU General Public License as
Chris@208 11 published by the Free Software Foundation; either version 2 of the
Chris@208 12 License, or (at your option) any later version. See the file
Chris@208 13 COPYING included with this distribution for more information.
Chris@208 14 */
Chris@208 15
Chris@317 16 #include "FileSource.h"
Chris@357 17
Chris@208 18 #include "base/TempDirectory.h"
Chris@208 19 #include "base/Exceptions.h"
Chris@392 20 #include "base/ProgressReporter.h"
Chris@502 21 #include "system/System.h"
Chris@208 22
Chris@208 23 #include <QHttp>
Chris@208 24 #include <QFtp>
Chris@208 25 #include <QFileInfo>
Chris@208 26 #include <QDir>
Chris@392 27 #include <QCoreApplication>
Chris@210 28 #include <QHttpResponseHeader>
Chris@208 29
Chris@208 30 #include <iostream>
Chris@405 31 #include <cstdlib>
Chris@208 32
Chris@608 33 #include <unistd.h>
Chris@608 34
Chris@528 35 //#define DEBUG_FILE_SOURCE 1
Chris@327 36
Chris@208 37 int
Chris@317 38 FileSource::m_count = 0;
Chris@208 39
Chris@208 40 QMutex
Chris@317 41 FileSource::m_fileCreationMutex;
Chris@208 42
Chris@317 43 FileSource::RemoteRefCountMap
Chris@317 44 FileSource::m_refCountMap;
Chris@304 45
Chris@317 46 FileSource::RemoteLocalMap
Chris@317 47 FileSource::m_remoteLocalMap;
Chris@304 48
Chris@304 49 QMutex
Chris@317 50 FileSource::m_mapMutex;
Chris@304 51
Chris@529 52 #ifdef DEBUG_FILE_SOURCE
Chris@529 53 static int extantCount = 0;
Chris@529 54 static std::map<QString, int> urlExtantCountMap;
Chris@529 55 static void incCount(QString url) {
Chris@529 56 ++extantCount;
Chris@529 57 if (urlExtantCountMap.find(url) == urlExtantCountMap.end()) {
Chris@529 58 urlExtantCountMap[url] = 1;
Chris@529 59 } else {
Chris@529 60 ++urlExtantCountMap[url];
Chris@529 61 }
Chris@529 62 std::cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << std::endl;
Chris@529 63 }
Chris@529 64 static void decCount(QString url) {
Chris@529 65 --extantCount;
Chris@529 66 --urlExtantCountMap[url];
Chris@529 67 std::cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << std::endl;
Chris@529 68 }
Chris@529 69 #endif
Chris@529 70
Chris@520 71 FileSource::FileSource(QString fileOrUrl, ProgressReporter *reporter,
Chris@520 72 QString preferredContentType) :
Chris@614 73 m_url(fileOrUrl, QUrl::StrictMode),
Chris@316 74 m_ftp(0),
Chris@316 75 m_http(0),
Chris@316 76 m_localFile(0),
Chris@520 77 m_preferredContentType(preferredContentType),
Chris@316 78 m_ok(false),
Chris@316 79 m_lastStatus(0),
Chris@316 80 m_remote(isRemote(fileOrUrl)),
Chris@316 81 m_done(false),
Chris@316 82 m_leaveLocalFile(false),
Chris@392 83 m_reporter(reporter),
Chris@316 84 m_refCounted(false)
Chris@316 85 {
Chris@327 86 #ifdef DEBUG_FILE_SOURCE
Chris@469 87 std::cerr << "FileSource::FileSource(" << fileOrUrl.toStdString() << ")" << std::endl;
Chris@529 88 incCount(m_url.toString());
Chris@327 89 #endif
Chris@316 90
Chris@316 91 if (!canHandleScheme(m_url)) {
Chris@317 92 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString().toStdString() << "\"" << std::endl;
Chris@316 93 m_errorString = tr("Unsupported scheme in URL");
Chris@316 94 return;
Chris@316 95 }
Chris@316 96
Chris@357 97 init();
Chris@316 98
Chris@614 99 if (!isRemote() &&
Chris@614 100 !isAvailable()) {
Chris@614 101 #ifdef DEBUG_FILE_SOURCE
Chris@616 102 std::cerr << "FileSource::FileSource: Failed to open local file with URL \"" << m_url.toString().toStdString() << "; trying again assuming filename was encoded" << std::endl;
Chris@614 103 #endif
Chris@616 104 m_url = QUrl::fromEncoded(fileOrUrl.toAscii());
Chris@614 105 init();
Chris@614 106 }
Chris@614 107
Chris@316 108 if (isRemote() &&
Chris@316 109 (fileOrUrl.contains('%') ||
Chris@316 110 fileOrUrl.contains("--"))) { // for IDNA
Chris@316 111
Chris@316 112 waitForStatus();
Chris@316 113
Chris@316 114 if (!isAvailable()) {
Chris@336 115
Chris@316 116 // The URL was created on the assumption that the string
Chris@316 117 // was human-readable. Let's try again, this time
Chris@316 118 // assuming it was already encoded.
Chris@317 119 std::cerr << "FileSource::FileSource: Failed to retrieve URL \""
Chris@316 120 << fileOrUrl.toStdString()
Chris@316 121 << "\" as human-readable URL; "
Chris@316 122 << "trying again treating it as encoded URL"
Chris@316 123 << std::endl;
Chris@336 124
Chris@336 125 // even though our cache file doesn't exist (because the
Chris@336 126 // resource was 404), we still need to ensure we're no
Chris@336 127 // longer associating a filename with this url in the
Chris@336 128 // refcount map -- or createCacheFile will think we've
Chris@336 129 // already done all the work and no request will be sent
Chris@336 130 deleteCacheFile();
Chris@336 131
Chris@316 132 m_url.setEncodedUrl(fileOrUrl.toAscii());
Chris@336 133
Chris@336 134 m_ok = false;
Chris@336 135 m_done = false;
Chris@336 136 m_lastStatus = 0;
Chris@357 137 init();
Chris@316 138 }
Chris@316 139 }
Chris@325 140
Chris@325 141 if (!isRemote()) {
Chris@325 142 emit statusAvailable();
Chris@325 143 emit ready();
Chris@325 144 }
Chris@497 145
Chris@504 146 #ifdef DEBUG_FILE_SOURCE
Chris@497 147 std::cerr << "FileSource::FileSource(string) exiting" << std::endl;
Chris@504 148 #endif
Chris@316 149 }
Chris@316 150
Chris@469 151 FileSource::FileSource(QUrl url, ProgressReporter *reporter) :
Chris@304 152 m_url(url),
Chris@208 153 m_ftp(0),
Chris@208 154 m_http(0),
Chris@208 155 m_localFile(0),
Chris@208 156 m_ok(false),
Chris@210 157 m_lastStatus(0),
Chris@316 158 m_remote(isRemote(url.toString())),
Chris@208 159 m_done(false),
Chris@316 160 m_leaveLocalFile(false),
Chris@392 161 m_reporter(reporter),
Chris@316 162 m_refCounted(false)
Chris@208 163 {
Chris@327 164 #ifdef DEBUG_FILE_SOURCE
Chris@317 165 std::cerr << "FileSource::FileSource(" << url.toString().toStdString() << ") [as url]" << std::endl;
Chris@529 166 incCount(m_url.toString());
Chris@327 167 #endif
Chris@316 168
Chris@316 169 if (!canHandleScheme(m_url)) {
Chris@317 170 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString().toStdString() << "\"" << std::endl;
Chris@316 171 m_errorString = tr("Unsupported scheme in URL");
Chris@208 172 return;
Chris@208 173 }
Chris@208 174
Chris@357 175 init();
Chris@497 176
Chris@504 177 #ifdef DEBUG_FILE_SOURCE
Chris@497 178 std::cerr << "FileSource::FileSource(url) exiting" << std::endl;
Chris@504 179 #endif
Chris@316 180 }
Chris@304 181
Chris@317 182 FileSource::FileSource(const FileSource &rf) :
Chris@316 183 QObject(),
Chris@316 184 m_url(rf.m_url),
Chris@316 185 m_ftp(0),
Chris@316 186 m_http(0),
Chris@316 187 m_localFile(0),
Chris@316 188 m_ok(rf.m_ok),
Chris@316 189 m_lastStatus(rf.m_lastStatus),
Chris@316 190 m_remote(rf.m_remote),
Chris@316 191 m_done(false),
Chris@316 192 m_leaveLocalFile(false),
Chris@392 193 m_reporter(rf.m_reporter),
Chris@316 194 m_refCounted(false)
Chris@316 195 {
Chris@327 196 #ifdef DEBUG_FILE_SOURCE
Chris@317 197 std::cerr << "FileSource::FileSource(" << m_url.toString().toStdString() << ") [copy ctor]" << std::endl;
Chris@529 198 incCount(m_url.toString());
Chris@327 199 #endif
Chris@304 200
Chris@316 201 if (!canHandleScheme(m_url)) {
Chris@317 202 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString().toStdString() << "\"" << std::endl;
Chris@316 203 m_errorString = tr("Unsupported scheme in URL");
Chris@304 204 return;
Chris@304 205 }
Chris@304 206
Chris@469 207 if (!isRemote()) {
Chris@316 208 m_localFilename = rf.m_localFilename;
Chris@316 209 } else {
Chris@316 210 QMutexLocker locker(&m_mapMutex);
Chris@327 211 #ifdef DEBUG_FILE_SOURCE
Chris@317 212 std::cerr << "FileSource::FileSource(copy ctor): ref count is "
Chris@316 213 << m_refCountMap[m_url] << std::endl;
Chris@327 214 #endif
Chris@316 215 if (m_refCountMap[m_url] > 0) {
Chris@316 216 m_refCountMap[m_url]++;
Chris@327 217 #ifdef DEBUG_FILE_SOURCE
Chris@316 218 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 219 #endif
Chris@316 220 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 221 m_refCounted = true;
Chris@316 222 } else {
Chris@316 223 m_ok = false;
Chris@316 224 m_lastStatus = 404;
Chris@316 225 }
Chris@316 226 }
Chris@316 227
Chris@316 228 m_done = true;
Chris@497 229
Chris@504 230 #ifdef DEBUG_FILE_SOURCE
Chris@527 231 std::cerr << "FileSource::FileSource(" << m_url.toString().toStdString() << ") [copy ctor]: note: local filename is \"" << m_localFilename.toStdString() << "\"" << std::endl;
Chris@527 232 #endif
Chris@527 233
Chris@527 234 #ifdef DEBUG_FILE_SOURCE
Chris@497 235 std::cerr << "FileSource::FileSource(copy ctor) exiting" << std::endl;
Chris@504 236 #endif
Chris@316 237 }
Chris@316 238
Chris@317 239 FileSource::~FileSource()
Chris@316 240 {
Chris@327 241 #ifdef DEBUG_FILE_SOURCE
Chris@317 242 std::cerr << "FileSource(" << m_url.toString().toStdString() << ")::~FileSource" << std::endl;
Chris@529 243 decCount(m_url.toString());
Chris@327 244 #endif
Chris@316 245
Chris@316 246 cleanup();
Chris@316 247
Chris@469 248 if (isRemote() && !m_leaveLocalFile) deleteCacheFile();
Chris@316 249 }
Chris@316 250
Chris@316 251 void
Chris@357 252 FileSource::init()
Chris@316 253 {
Chris@316 254 if (!isRemote()) {
Chris@355 255 #ifdef DEBUG_FILE_SOURCE
Chris@355 256 std::cerr << "FileSource::init: Not a remote URL" << std::endl;
Chris@355 257 #endif
Chris@355 258 bool literal = false;
Chris@316 259 m_localFilename = m_url.toLocalFile();
Chris@342 260 if (m_localFilename == "") {
Chris@342 261 // QUrl may have mishandled the scheme (e.g. in a DOS path)
Chris@342 262 m_localFilename = m_url.toString();
Chris@355 263 literal = true;
Chris@342 264 }
Chris@439 265 m_localFilename = QFileInfo(m_localFilename).absoluteFilePath();
Chris@439 266
Chris@355 267 #ifdef DEBUG_FILE_SOURCE
Chris@355 268 std::cerr << "FileSource::init: URL translates to local filename \""
Chris@355 269 << m_localFilename.toStdString() << "\"" << std::endl;
Chris@355 270 #endif
Chris@316 271 m_ok = true;
Chris@355 272 m_lastStatus = 200;
Chris@355 273
Chris@316 274 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 275 if (literal) {
Chris@355 276 m_lastStatus = 404;
Chris@355 277 } else {
Chris@614 278 #ifdef DEBUG_FILE_SOURCE
Chris@614 279 std::cerr << "FileSource::init: Local file of this name does not exist, trying URL as a literal filename" << std::endl;
Chris@614 280 #endif
Chris@355 281 // Again, QUrl may have been mistreating us --
Chris@355 282 // e.g. dropping a part that looks like query data
Chris@355 283 m_localFilename = m_url.toString();
Chris@355 284 literal = true;
Chris@355 285 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 286 m_lastStatus = 404;
Chris@355 287 }
Chris@355 288 }
Chris@316 289 }
Chris@355 290
Chris@316 291 m_done = true;
Chris@316 292 return;
Chris@316 293 }
Chris@316 294
Chris@316 295 if (createCacheFile()) {
Chris@327 296 #ifdef DEBUG_FILE_SOURCE
Chris@469 297 std::cerr << "FileSource::init: Already have this one" << std::endl;
Chris@327 298 #endif
Chris@316 299 m_ok = true;
Chris@316 300 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 301 m_lastStatus = 404;
Chris@316 302 } else {
Chris@316 303 m_lastStatus = 200;
Chris@316 304 }
Chris@316 305 m_done = true;
Chris@316 306 return;
Chris@316 307 }
Chris@316 308
Chris@208 309 if (m_localFilename == "") return;
Chris@208 310 m_localFile = new QFile(m_localFilename);
Chris@208 311 m_localFile->open(QFile::WriteOnly);
Chris@208 312
Chris@316 313 QString scheme = m_url.scheme().toLower();
Chris@316 314
Chris@327 315 #ifdef DEBUG_FILE_SOURCE
Chris@317 316 std::cerr << "FileSource::init: Don't have local copy of \""
Chris@469 317 << m_url.toString().toStdString() << "\", retrieving" << std::endl;
Chris@327 318 #endif
Chris@208 319
Chris@208 320 if (scheme == "http") {
Chris@316 321 initHttp();
Chris@520 322 #ifdef DEBUG_FILE_SOURCE
Chris@357 323 std::cerr << "FileSource: initHttp succeeded" << std::endl;
Chris@520 324 #endif
Chris@208 325 } else if (scheme == "ftp") {
Chris@316 326 initFtp();
Chris@316 327 } else {
Chris@316 328 m_remote = false;
Chris@316 329 m_ok = false;
Chris@208 330 }
Chris@208 331
Chris@208 332 if (m_ok) {
Chris@316 333
Chris@316 334 QMutexLocker locker(&m_mapMutex);
Chris@316 335
Chris@316 336 if (m_refCountMap[m_url] > 0) {
Chris@316 337 // someone else has been doing the same thing at the same time,
Chris@316 338 // but has got there first
Chris@316 339 cleanup();
Chris@316 340 m_refCountMap[m_url]++;
Chris@327 341 #ifdef DEBUG_FILE_SOURCE
Chris@317 342 std::cerr << "FileSource::init: Another FileSource has got there first, abandoning our download and using theirs" << std::endl;
Chris@327 343 #endif
Chris@316 344 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 345 m_refCounted = true;
Chris@316 346 m_ok = true;
Chris@316 347 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 348 m_lastStatus = 404;
Chris@316 349 }
Chris@316 350 m_done = true;
Chris@316 351 return;
Chris@316 352 }
Chris@304 353
Chris@304 354 m_remoteLocalMap[m_url] = m_localFilename;
Chris@304 355 m_refCountMap[m_url]++;
Chris@316 356 m_refCounted = true;
Chris@304 357
Chris@392 358 if (m_reporter) {
Chris@392 359 m_reporter->setMessage
Chris@392 360 (tr("Downloading %1...").arg(m_url.toString()));
Chris@392 361 connect(m_reporter, SIGNAL(cancelled()), this, SLOT(cancelled()));
Chris@357 362 connect(this, SIGNAL(progress(int)),
Chris@392 363 m_reporter, SLOT(setProgress(int)));
Chris@316 364 }
Chris@208 365 }
Chris@208 366 }
Chris@208 367
Chris@316 368 void
Chris@317 369 FileSource::initHttp()
Chris@208 370 {
Chris@316 371 m_ok = true;
Chris@497 372 int port = m_url.port();
Chris@497 373 m_http = new QHttp(m_url.host(), port < 0 ? 80 : port);
Chris@316 374 connect(m_http, SIGNAL(done(bool)), this, SLOT(done(bool)));
Chris@316 375 connect(m_http, SIGNAL(dataReadProgress(int, int)),
Chris@316 376 this, SLOT(dataReadProgress(int, int)));
Chris@316 377 connect(m_http, SIGNAL(responseHeaderReceived(const QHttpResponseHeader &)),
Chris@316 378 this, SLOT(httpResponseHeaderReceived(const QHttpResponseHeader &)));
Chris@316 379
Chris@316 380 // I don't quite understand this. url.path() returns a path
Chris@316 381 // without percent encoding; for example, spaces appear as
Chris@316 382 // literal spaces. This generally won't work if sent to the
Chris@316 383 // server directly. You can retrieve a correctly encoded URL
Chris@316 384 // from QUrl using url.toEncoded(), but that gives you the
Chris@316 385 // whole URL; there doesn't seem to be any way to retrieve
Chris@316 386 // only an encoded path. Furthermore there doesn't seem to be
Chris@316 387 // any way to convert a retrieved path into an encoded path
Chris@316 388 // without explicitly specifying that you don't want the path
Chris@316 389 // separators ("/") to be encoded. (Besides being painful to
Chris@316 390 // manage, I don't see how this can work correctly in any case
Chris@316 391 // where a percent-encoded "/" is supposed to appear within a
Chris@316 392 // path element?) There also seems to be no way to retrieve
Chris@316 393 // the path plus query string, i.e. everything that I need to
Chris@316 394 // send to the HTTP server. And no way for QHttp to take a
Chris@316 395 // QUrl argument. I'm obviously missing something.
Chris@316 396
Chris@316 397 // So, two ways to do this: query the bits from the URL,
Chris@316 398 // encode them individually, and glue them back together
Chris@316 399 // again...
Chris@316 400 /*
Chris@316 401 QString path = QUrl::toPercentEncoding(m_url.path(), "/");
Chris@316 402 QList<QPair<QString, QString> > query = m_url.queryItems();
Chris@316 403 if (!query.empty()) {
Chris@316 404 QStringList q2;
Chris@316 405 for (QList<QPair<QString, QString> >::iterator i = query.begin();
Chris@316 406 i != query.end(); ++i) {
Chris@316 407 q2.push_back(QString("%1=%3")
Chris@316 408 .arg(QString(QUrl::toPercentEncoding(i->first)))
Chris@316 409 .arg(QString(QUrl::toPercentEncoding(i->second))));
Chris@316 410 }
Chris@316 411 path = QString("%1%2%3")
Chris@316 412 .arg(path).arg("?")
Chris@316 413 .arg(q2.join("&"));
Chris@316 414 }
Chris@316 415 */
Chris@316 416
Chris@316 417 // ...or, much simpler but relying on knowledge about the
Chris@316 418 // scheme://host/path/path/query etc format of the URL, we can
Chris@316 419 // get the whole URL ready-encoded and then split it on "/" as
Chris@316 420 // appropriate...
Chris@316 421
Chris@316 422 QString path = "/" + QString(m_url.toEncoded()).section('/', 3);
Chris@316 423
Chris@327 424 #ifdef DEBUG_FILE_SOURCE
Chris@317 425 std::cerr << "FileSource: path is \""
Chris@316 426 << path.toStdString() << "\"" << std::endl;
Chris@327 427 #endif
Chris@316 428
Chris@520 429 if (m_preferredContentType == "") {
Chris@520 430 m_http->get(path, m_localFile);
Chris@520 431 } else {
Chris@520 432 #ifdef DEBUG_FILE_SOURCE
Chris@520 433 std::cerr << "FileSource: indicating preferred content type of \""
Chris@520 434 << m_preferredContentType.toStdString() << "\"" << std::endl;
Chris@520 435 #endif
Chris@520 436 QHttpRequestHeader header("GET", path);
Chris@520 437 header.setValue("Host", m_url.host());
Chris@520 438 header.setValue("Accept", QString("%1, */*").arg(m_preferredContentType));
Chris@520 439 m_http->request(header, 0, m_localFile);
Chris@520 440 }
Chris@316 441 }
Chris@316 442
Chris@316 443 void
Chris@317 444 FileSource::initFtp()
Chris@316 445 {
Chris@316 446 m_ok = true;
Chris@316 447 m_ftp = new QFtp;
Chris@316 448 connect(m_ftp, SIGNAL(done(bool)), this, SLOT(done(bool)));
Chris@316 449 connect(m_ftp, SIGNAL(commandFinished(int, bool)),
Chris@316 450 this, SLOT(ftpCommandFinished(int, bool)));
Chris@316 451 connect(m_ftp, SIGNAL(dataTransferProgress(qint64, qint64)),
Chris@316 452 this, SLOT(dataTransferProgress(qint64, qint64)));
Chris@316 453 m_ftp->connectToHost(m_url.host(), m_url.port(21));
Chris@316 454
Chris@316 455 QString username = m_url.userName();
Chris@316 456 if (username == "") {
Chris@316 457 username = "anonymous";
Chris@316 458 }
Chris@316 459
Chris@316 460 QString password = m_url.password();
Chris@316 461 if (password == "") {
Chris@316 462 password = QString("%1@%2").arg(getenv("USER")).arg(getenv("HOST"));
Chris@316 463 }
Chris@316 464
Chris@316 465 m_ftp->login(username, password);
Chris@316 466
Chris@316 467 QString dirpath = m_url.path().section('/', 0, -2);
Chris@316 468 QString filename = m_url.path().section('/', -1);
Chris@316 469
Chris@316 470 if (dirpath == "") dirpath = "/";
Chris@316 471 m_ftp->cd(dirpath);
Chris@316 472 m_ftp->get(filename, m_localFile);
Chris@211 473 }
Chris@211 474
Chris@211 475 void
Chris@317 476 FileSource::cleanup()
Chris@211 477 {
Chris@470 478 if (m_done) {
Chris@470 479 delete m_localFile; // does not actually delete the file
Chris@470 480 m_localFile = 0;
Chris@470 481 }
Chris@211 482 m_done = true;
Chris@214 483 if (m_http) {
Chris@287 484 QHttp *h = m_http;
Chris@214 485 m_http = 0;
Chris@287 486 h->abort();
Chris@287 487 h->deleteLater();
Chris@214 488 }
Chris@214 489 if (m_ftp) {
Chris@287 490 QFtp *f = m_ftp;
Chris@214 491 m_ftp = 0;
Chris@287 492 f->abort();
Chris@287 493 f->deleteLater();
Chris@214 494 }
Chris@470 495 if (m_localFile) {
Chris@470 496 delete m_localFile; // does not actually delete the file
Chris@470 497 m_localFile = 0;
Chris@470 498 }
Chris@208 499 }
Chris@208 500
Chris@208 501 bool
Chris@317 502 FileSource::isRemote(QString fileOrUrl)
Chris@304 503 {
Chris@342 504 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@316 505 QString scheme = QUrl(fileOrUrl).scheme().toLower();
Chris@342 506 if (scheme == "" || scheme == "file" || scheme.length() == 1) return false;
Chris@342 507 return true;
Chris@304 508 }
Chris@304 509
Chris@304 510 bool
Chris@317 511 FileSource::canHandleScheme(QUrl url)
Chris@208 512 {
Chris@342 513 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@208 514 QString scheme = url.scheme().toLower();
Chris@316 515 return (scheme == "http" || scheme == "ftp" ||
Chris@342 516 scheme == "file" || scheme == "" || scheme.length() == 1);
Chris@208 517 }
Chris@208 518
Chris@210 519 bool
Chris@317 520 FileSource::isAvailable()
Chris@210 521 {
Chris@316 522 waitForStatus();
Chris@211 523 bool available = true;
Chris@211 524 if (!m_ok) available = false;
Chris@211 525 else available = (m_lastStatus / 100 == 2);
Chris@327 526 #ifdef DEBUG_FILE_SOURCE
Chris@317 527 std::cerr << "FileSource::isAvailable: " << (available ? "yes" : "no")
Chris@211 528 << std::endl;
Chris@327 529 #endif
Chris@211 530 return available;
Chris@210 531 }
Chris@210 532
Chris@208 533 void
Chris@317 534 FileSource::waitForStatus()
Chris@316 535 {
Chris@316 536 while (m_ok && (!m_done && m_lastStatus == 0)) {
Chris@316 537 // std::cerr << "waitForStatus: processing (last status " << m_lastStatus << ")" << std::endl;
Chris@392 538 QCoreApplication::processEvents();
Chris@316 539 }
Chris@316 540 }
Chris@316 541
Chris@316 542 void
Chris@317 543 FileSource::waitForData()
Chris@208 544 {
Chris@211 545 while (m_ok && !m_done) {
Chris@357 546 // std::cerr << "FileSource::waitForData: calling QApplication::processEvents" << std::endl;
Chris@392 547 QCoreApplication::processEvents();
Chris@497 548 usleep(10000);
Chris@208 549 }
Chris@208 550 }
Chris@208 551
Chris@316 552 void
Chris@317 553 FileSource::setLeaveLocalFile(bool leave)
Chris@316 554 {
Chris@316 555 m_leaveLocalFile = leave;
Chris@316 556 }
Chris@316 557
Chris@208 558 bool
Chris@317 559 FileSource::isOK() const
Chris@208 560 {
Chris@208 561 return m_ok;
Chris@208 562 }
Chris@208 563
Chris@208 564 bool
Chris@317 565 FileSource::isDone() const
Chris@208 566 {
Chris@208 567 return m_done;
Chris@208 568 }
Chris@208 569
Chris@316 570 bool
Chris@317 571 FileSource::isRemote() const
Chris@316 572 {
Chris@316 573 return m_remote;
Chris@316 574 }
Chris@316 575
Chris@316 576 QString
Chris@317 577 FileSource::getLocation() const
Chris@316 578 {
Chris@316 579 return m_url.toString();
Chris@316 580 }
Chris@316 581
Chris@208 582 QString
Chris@317 583 FileSource::getLocalFilename() const
Chris@208 584 {
Chris@208 585 return m_localFilename;
Chris@208 586 }
Chris@208 587
Chris@208 588 QString
Chris@317 589 FileSource::getContentType() const
Chris@316 590 {
Chris@316 591 return m_contentType;
Chris@316 592 }
Chris@316 593
Chris@316 594 QString
Chris@317 595 FileSource::getExtension() const
Chris@316 596 {
Chris@316 597 if (m_localFilename != "") {
Chris@316 598 return QFileInfo(m_localFilename).suffix().toLower();
Chris@316 599 } else {
Chris@316 600 return QFileInfo(m_url.toLocalFile()).suffix().toLower();
Chris@316 601 }
Chris@316 602 }
Chris@316 603
Chris@316 604 QString
Chris@317 605 FileSource::getErrorString() const
Chris@208 606 {
Chris@208 607 return m_errorString;
Chris@208 608 }
Chris@208 609
Chris@208 610 void
Chris@317 611 FileSource::dataReadProgress(int done, int total)
Chris@208 612 {
Chris@208 613 dataTransferProgress(done, total);
Chris@208 614 }
Chris@208 615
Chris@208 616 void
Chris@317 617 FileSource::httpResponseHeaderReceived(const QHttpResponseHeader &resp)
Chris@210 618 {
Chris@520 619 #ifdef DEBUG_FILE_SOURCE
Chris@497 620 std::cerr << "FileSource::httpResponseHeaderReceived" << std::endl;
Chris@520 621 #endif
Chris@497 622
Chris@497 623 if (resp.statusCode() / 100 == 3) {
Chris@496 624 QString location = resp.value("Location");
Chris@496 625 #ifdef DEBUG_FILE_SOURCE
Chris@496 626 std::cerr << "FileSource::responseHeaderReceived: redirect to \""
Chris@496 627 << location.toStdString() << "\" received" << std::endl;
Chris@496 628 #endif
Chris@496 629 if (location != "") {
Chris@496 630 QUrl newUrl(location);
Chris@496 631 if (newUrl != m_url) {
Chris@497 632 cleanup();
Chris@497 633 deleteCacheFile();
Chris@529 634 #ifdef DEBUG_FILE_SOURCE
Chris@529 635 decCount(m_url.toString());
Chris@529 636 incCount(newUrl.toString());
Chris@529 637 #endif
Chris@496 638 m_url = newUrl;
Chris@497 639 m_localFile = 0;
Chris@496 640 m_lastStatus = 0;
Chris@497 641 m_done = false;
Chris@497 642 m_refCounted = false;
Chris@496 643 init();
Chris@496 644 return;
Chris@496 645 }
Chris@496 646 }
Chris@496 647 }
Chris@497 648
Chris@497 649 m_lastStatus = resp.statusCode();
Chris@210 650 if (m_lastStatus / 100 >= 4) {
Chris@210 651 m_errorString = QString("%1 %2")
Chris@210 652 .arg(resp.statusCode()).arg(resp.reasonPhrase());
Chris@327 653 #ifdef DEBUG_FILE_SOURCE
Chris@317 654 std::cerr << "FileSource::responseHeaderReceived: "
Chris@211 655 << m_errorString.toStdString() << std::endl;
Chris@327 656 #endif
Chris@211 657 } else {
Chris@327 658 #ifdef DEBUG_FILE_SOURCE
Chris@317 659 std::cerr << "FileSource::responseHeaderReceived: "
Chris@211 660 << m_lastStatus << std::endl;
Chris@327 661 #endif
Chris@315 662 if (resp.hasContentType()) m_contentType = resp.contentType();
Chris@325 663 }
Chris@325 664 emit statusAvailable();
Chris@210 665 }
Chris@210 666
Chris@210 667 void
Chris@317 668 FileSource::ftpCommandFinished(int id, bool error)
Chris@214 669 {
Chris@327 670 #ifdef DEBUG_FILE_SOURCE
Chris@317 671 std::cerr << "FileSource::ftpCommandFinished(" << id << ", " << error << ")" << std::endl;
Chris@327 672 #endif
Chris@214 673
Chris@214 674 if (!m_ftp) return;
Chris@214 675
Chris@214 676 QFtp::Command command = m_ftp->currentCommand();
Chris@214 677
Chris@214 678 if (!error) {
Chris@327 679 #ifdef DEBUG_FILE_SOURCE
Chris@317 680 std::cerr << "FileSource::ftpCommandFinished: success for command "
Chris@214 681 << command << std::endl;
Chris@327 682 #endif
Chris@214 683 return;
Chris@214 684 }
Chris@214 685
Chris@214 686 if (command == QFtp::ConnectToHost) {
Chris@214 687 m_errorString = tr("Failed to connect to FTP server");
Chris@214 688 } else if (command == QFtp::Login) {
Chris@214 689 m_errorString = tr("Login failed");
Chris@214 690 } else if (command == QFtp::Cd) {
Chris@214 691 m_errorString = tr("Failed to change to correct directory");
Chris@214 692 } else if (command == QFtp::Get) {
Chris@214 693 m_errorString = tr("FTP download aborted");
Chris@214 694 }
Chris@214 695
Chris@214 696 m_lastStatus = 400; // for done()
Chris@214 697 }
Chris@214 698
Chris@214 699 void
Chris@317 700 FileSource::dataTransferProgress(qint64 done, qint64 total)
Chris@208 701 {
Chris@208 702 int percent = int((double(done) / double(total)) * 100.0 - 0.1);
Chris@208 703 emit progress(percent);
Chris@210 704 }
Chris@210 705
Chris@210 706 void
Chris@317 707 FileSource::cancelled()
Chris@210 708 {
Chris@210 709 m_done = true;
Chris@316 710 cleanup();
Chris@316 711
Chris@210 712 m_ok = false;
Chris@210 713 m_errorString = tr("Download cancelled");
Chris@208 714 }
Chris@208 715
Chris@208 716 void
Chris@317 717 FileSource::done(bool error)
Chris@208 718 {
Chris@327 719 emit progress(100);
Chris@327 720
Chris@327 721 #ifdef DEBUG_FILE_SOURCE
Chris@317 722 std::cerr << "FileSource::done(" << error << ")" << std::endl;
Chris@327 723 #endif
Chris@211 724
Chris@211 725 if (m_done) return;
Chris@211 726
Chris@208 727 if (error) {
Chris@208 728 if (m_http) {
Chris@208 729 m_errorString = m_http->errorString();
Chris@208 730 } else if (m_ftp) {
Chris@208 731 m_errorString = m_ftp->errorString();
Chris@208 732 }
Chris@208 733 }
Chris@208 734
Chris@210 735 if (m_lastStatus / 100 >= 4) {
Chris@211 736 error = true;
Chris@210 737 }
Chris@210 738
Chris@211 739 cleanup();
Chris@208 740
Chris@211 741 if (!error) {
Chris@208 742 QFileInfo fi(m_localFilename);
Chris@208 743 if (!fi.exists()) {
Chris@208 744 m_errorString = tr("Failed to create local file %1").arg(m_localFilename);
Chris@211 745 error = true;
Chris@208 746 } else if (fi.size() == 0) {
Chris@208 747 m_errorString = tr("File contains no data!");
Chris@211 748 error = true;
Chris@208 749 }
Chris@208 750 }
Chris@211 751
Chris@211 752 if (error) {
Chris@327 753 #ifdef DEBUG_FILE_SOURCE
Chris@469 754 std::cerr << "FileSource::done: error is " << error << ", deleting cache file" << std::endl;
Chris@327 755 #endif
Chris@316 756 deleteCacheFile();
Chris@211 757 }
Chris@211 758
Chris@211 759 m_ok = !error;
Chris@520 760 if (m_localFile) m_localFile->flush();
Chris@211 761 m_done = true;
Chris@304 762 emit ready();
Chris@211 763 }
Chris@211 764
Chris@211 765 void
Chris@317 766 FileSource::deleteCacheFile()
Chris@211 767 {
Chris@327 768 #ifdef DEBUG_FILE_SOURCE
Chris@317 769 std::cerr << "FileSource::deleteCacheFile(\"" << m_localFilename.toStdString() << "\")" << std::endl;
Chris@327 770 #endif
Chris@211 771
Chris@211 772 cleanup();
Chris@211 773
Chris@316 774 if (m_localFilename == "") {
Chris@316 775 return;
Chris@316 776 }
Chris@211 777
Chris@316 778 if (!isRemote()) {
Chris@327 779 #ifdef DEBUG_FILE_SOURCE
Chris@316 780 std::cerr << "not a cache file" << std::endl;
Chris@327 781 #endif
Chris@316 782 return;
Chris@316 783 }
Chris@316 784
Chris@316 785 if (m_refCounted) {
Chris@304 786
Chris@304 787 QMutexLocker locker(&m_mapMutex);
Chris@316 788 m_refCounted = false;
Chris@304 789
Chris@304 790 if (m_refCountMap[m_url] > 0) {
Chris@304 791 m_refCountMap[m_url]--;
Chris@327 792 #ifdef DEBUG_FILE_SOURCE
Chris@316 793 std::cerr << "reduced ref count to " << m_refCountMap[m_url] << std::endl;
Chris@327 794 #endif
Chris@304 795 if (m_refCountMap[m_url] > 0) {
Chris@304 796 m_done = true;
Chris@304 797 return;
Chris@304 798 }
Chris@304 799 }
Chris@304 800 }
Chris@304 801
Chris@211 802 m_fileCreationMutex.lock();
Chris@211 803
Chris@211 804 if (!QFile(m_localFilename).remove()) {
Chris@469 805 #ifdef DEBUG_FILE_SOURCE
Chris@317 806 std::cerr << "FileSource::deleteCacheFile: ERROR: Failed to delete file \"" << m_localFilename.toStdString() << "\"" << std::endl;
Chris@469 807 #endif
Chris@211 808 } else {
Chris@327 809 #ifdef DEBUG_FILE_SOURCE
Chris@317 810 std::cerr << "FileSource::deleteCacheFile: Deleted cache file \"" << m_localFilename.toStdString() << "\"" << std::endl;
Chris@327 811 #endif
Chris@211 812 m_localFilename = "";
Chris@211 813 }
Chris@211 814
Chris@211 815 m_fileCreationMutex.unlock();
Chris@211 816
Chris@208 817 m_done = true;
Chris@208 818 }
Chris@208 819
Chris@316 820 bool
Chris@317 821 FileSource::createCacheFile()
Chris@208 822 {
Chris@316 823 {
Chris@316 824 QMutexLocker locker(&m_mapMutex);
Chris@316 825
Chris@327 826 #ifdef DEBUG_FILE_SOURCE
Chris@317 827 std::cerr << "FileSource::createCacheFile: refcount is " << m_refCountMap[m_url] << std::endl;
Chris@327 828 #endif
Chris@316 829
Chris@316 830 if (m_refCountMap[m_url] > 0) {
Chris@316 831 m_refCountMap[m_url]++;
Chris@316 832 m_localFilename = m_remoteLocalMap[m_url];
Chris@327 833 #ifdef DEBUG_FILE_SOURCE
Chris@316 834 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 835 #endif
Chris@316 836 m_refCounted = true;
Chris@316 837 return true;
Chris@316 838 }
Chris@316 839 }
Chris@316 840
Chris@208 841 QDir dir;
Chris@208 842 try {
Chris@208 843 dir = TempDirectory::getInstance()->getSubDirectoryPath("download");
Chris@208 844 } catch (DirectoryCreationFailed f) {
Chris@327 845 #ifdef DEBUG_FILE_SOURCE
Chris@317 846 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create temporary directory: " << f.what() << std::endl;
Chris@327 847 #endif
Chris@208 848 return "";
Chris@208 849 }
Chris@208 850
Chris@316 851 QString filepart = m_url.path().section('/', -1, -1,
Chris@316 852 QString::SectionSkipEmpty);
Chris@208 853
Chris@457 854 QString extension = "";
Chris@457 855 if (filepart.contains('.')) extension = filepart.section('.', -1);
Chris@457 856
Chris@208 857 QString base = filepart;
Chris@208 858 if (extension != "") {
Chris@208 859 base = base.left(base.length() - extension.length() - 1);
Chris@208 860 }
Chris@208 861 if (base == "") base = "remote";
Chris@208 862
Chris@208 863 QString filename;
Chris@208 864
Chris@208 865 if (extension == "") {
Chris@208 866 filename = base;
Chris@208 867 } else {
Chris@208 868 filename = QString("%1.%2").arg(base).arg(extension);
Chris@208 869 }
Chris@208 870
Chris@208 871 QString filepath(dir.filePath(filename));
Chris@208 872
Chris@327 873 #ifdef DEBUG_FILE_SOURCE
Chris@317 874 std::cerr << "FileSource::createCacheFile: URL is \"" << m_url.toString().toStdString() << "\", dir is \"" << dir.path().toStdString() << "\", base \"" << base.toStdString() << "\", extension \"" << extension.toStdString() << "\", filebase \"" << filename.toStdString() << "\", filename \"" << filepath.toStdString() << "\"" << std::endl;
Chris@327 875 #endif
Chris@208 876
Chris@316 877 QMutexLocker fcLocker(&m_fileCreationMutex);
Chris@316 878
Chris@208 879 ++m_count;
Chris@208 880
Chris@208 881 if (QFileInfo(filepath).exists() ||
Chris@208 882 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 883
Chris@327 884 #ifdef DEBUG_FILE_SOURCE
Chris@317 885 std::cerr << "FileSource::createCacheFile: Failed to create local file \""
Chris@208 886 << filepath.toStdString() << "\" for URL \""
Chris@316 887 << m_url.toString().toStdString() << "\" (or file already exists): appending suffix instead" << std::endl;
Chris@327 888 #endif
Chris@208 889
Chris@208 890 if (extension == "") {
Chris@208 891 filename = QString("%1_%2").arg(base).arg(m_count);
Chris@208 892 } else {
Chris@208 893 filename = QString("%1_%2.%3").arg(base).arg(m_count).arg(extension);
Chris@208 894 }
Chris@208 895 filepath = dir.filePath(filename);
Chris@208 896
Chris@208 897 if (QFileInfo(filepath).exists() ||
Chris@208 898 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 899
Chris@327 900 #ifdef DEBUG_FILE_SOURCE
Chris@317 901 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create local file \""
Chris@208 902 << filepath.toStdString() << "\" for URL \""
Chris@316 903 << m_url.toString().toStdString() << "\" (or file already exists)" << std::endl;
Chris@327 904 #endif
Chris@208 905
Chris@208 906 return "";
Chris@208 907 }
Chris@208 908 }
Chris@208 909
Chris@327 910 #ifdef DEBUG_FILE_SOURCE
Chris@317 911 std::cerr << "FileSource::createCacheFile: url "
Chris@316 912 << m_url.toString().toStdString() << " -> local filename "
Chris@316 913 << filepath.toStdString() << std::endl;
Chris@327 914 #endif
Chris@316 915
Chris@316 916 m_localFilename = filepath;
Chris@208 917
Chris@316 918 return false;
Chris@208 919 }
Chris@327 920