annotate data/fileio/FileSource.cpp @ 608:d7f3dfe6f9a4

* solaris build fixes
author Chris Cannam
date Thu, 10 Sep 2009 18:44:45 +0000
parents 993e0a4f1343
children 87b81f959706
rev   line source
Chris@208 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@208 2
Chris@208 3 /*
Chris@208 4 Sonic Visualiser
Chris@208 5 An audio file viewer and annotation editor.
Chris@208 6 Centre for Digital Music, Queen Mary, University of London.
Chris@208 7 This file copyright 2007 QMUL.
Chris@208 8
Chris@208 9 This program is free software; you can redistribute it and/or
Chris@208 10 modify it under the terms of the GNU General Public License as
Chris@208 11 published by the Free Software Foundation; either version 2 of the
Chris@208 12 License, or (at your option) any later version. See the file
Chris@208 13 COPYING included with this distribution for more information.
Chris@208 14 */
Chris@208 15
Chris@317 16 #include "FileSource.h"
Chris@357 17
Chris@208 18 #include "base/TempDirectory.h"
Chris@208 19 #include "base/Exceptions.h"
Chris@392 20 #include "base/ProgressReporter.h"
Chris@502 21 #include "system/System.h"
Chris@208 22
Chris@208 23 #include <QHttp>
Chris@208 24 #include <QFtp>
Chris@208 25 #include <QFileInfo>
Chris@208 26 #include <QDir>
Chris@392 27 #include <QCoreApplication>
Chris@210 28 #include <QHttpResponseHeader>
Chris@208 29
Chris@208 30 #include <iostream>
Chris@405 31 #include <cstdlib>
Chris@208 32
Chris@608 33 #include <unistd.h>
Chris@608 34
Chris@528 35 //#define DEBUG_FILE_SOURCE 1
Chris@327 36
Chris@208 37 int
Chris@317 38 FileSource::m_count = 0;
Chris@208 39
Chris@208 40 QMutex
Chris@317 41 FileSource::m_fileCreationMutex;
Chris@208 42
Chris@317 43 FileSource::RemoteRefCountMap
Chris@317 44 FileSource::m_refCountMap;
Chris@304 45
Chris@317 46 FileSource::RemoteLocalMap
Chris@317 47 FileSource::m_remoteLocalMap;
Chris@304 48
Chris@304 49 QMutex
Chris@317 50 FileSource::m_mapMutex;
Chris@304 51
Chris@529 52 #ifdef DEBUG_FILE_SOURCE
Chris@529 53 static int extantCount = 0;
Chris@529 54 static std::map<QString, int> urlExtantCountMap;
Chris@529 55 static void incCount(QString url) {
Chris@529 56 ++extantCount;
Chris@529 57 if (urlExtantCountMap.find(url) == urlExtantCountMap.end()) {
Chris@529 58 urlExtantCountMap[url] = 1;
Chris@529 59 } else {
Chris@529 60 ++urlExtantCountMap[url];
Chris@529 61 }
Chris@529 62 std::cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << std::endl;
Chris@529 63 }
Chris@529 64 static void decCount(QString url) {
Chris@529 65 --extantCount;
Chris@529 66 --urlExtantCountMap[url];
Chris@529 67 std::cerr << "FileSource: Now " << urlExtantCountMap[url] << " for this url, " << extantCount << " total" << std::endl;
Chris@529 68 }
Chris@529 69 #endif
Chris@529 70
Chris@520 71 FileSource::FileSource(QString fileOrUrl, ProgressReporter *reporter,
Chris@520 72 QString preferredContentType) :
Chris@316 73 m_url(fileOrUrl),
Chris@316 74 m_ftp(0),
Chris@316 75 m_http(0),
Chris@316 76 m_localFile(0),
Chris@520 77 m_preferredContentType(preferredContentType),
Chris@316 78 m_ok(false),
Chris@316 79 m_lastStatus(0),
Chris@316 80 m_remote(isRemote(fileOrUrl)),
Chris@316 81 m_done(false),
Chris@316 82 m_leaveLocalFile(false),
Chris@392 83 m_reporter(reporter),
Chris@316 84 m_refCounted(false)
Chris@316 85 {
Chris@327 86 #ifdef DEBUG_FILE_SOURCE
Chris@469 87 std::cerr << "FileSource::FileSource(" << fileOrUrl.toStdString() << ")" << std::endl;
Chris@529 88 incCount(m_url.toString());
Chris@327 89 #endif
Chris@316 90
Chris@316 91 if (!canHandleScheme(m_url)) {
Chris@317 92 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString().toStdString() << "\"" << std::endl;
Chris@316 93 m_errorString = tr("Unsupported scheme in URL");
Chris@316 94 return;
Chris@316 95 }
Chris@316 96
Chris@357 97 init();
Chris@316 98
Chris@316 99 if (isRemote() &&
Chris@316 100 (fileOrUrl.contains('%') ||
Chris@316 101 fileOrUrl.contains("--"))) { // for IDNA
Chris@316 102
Chris@316 103 waitForStatus();
Chris@316 104
Chris@316 105 if (!isAvailable()) {
Chris@336 106
Chris@316 107 // The URL was created on the assumption that the string
Chris@316 108 // was human-readable. Let's try again, this time
Chris@316 109 // assuming it was already encoded.
Chris@317 110 std::cerr << "FileSource::FileSource: Failed to retrieve URL \""
Chris@316 111 << fileOrUrl.toStdString()
Chris@316 112 << "\" as human-readable URL; "
Chris@316 113 << "trying again treating it as encoded URL"
Chris@316 114 << std::endl;
Chris@336 115
Chris@336 116 // even though our cache file doesn't exist (because the
Chris@336 117 // resource was 404), we still need to ensure we're no
Chris@336 118 // longer associating a filename with this url in the
Chris@336 119 // refcount map -- or createCacheFile will think we've
Chris@336 120 // already done all the work and no request will be sent
Chris@336 121 deleteCacheFile();
Chris@336 122
Chris@316 123 m_url.setEncodedUrl(fileOrUrl.toAscii());
Chris@336 124
Chris@336 125 m_ok = false;
Chris@336 126 m_done = false;
Chris@336 127 m_lastStatus = 0;
Chris@357 128 init();
Chris@316 129 }
Chris@316 130 }
Chris@325 131
Chris@325 132 if (!isRemote()) {
Chris@325 133 emit statusAvailable();
Chris@325 134 emit ready();
Chris@325 135 }
Chris@497 136
Chris@504 137 #ifdef DEBUG_FILE_SOURCE
Chris@497 138 std::cerr << "FileSource::FileSource(string) exiting" << std::endl;
Chris@504 139 #endif
Chris@316 140 }
Chris@316 141
Chris@469 142 FileSource::FileSource(QUrl url, ProgressReporter *reporter) :
Chris@304 143 m_url(url),
Chris@208 144 m_ftp(0),
Chris@208 145 m_http(0),
Chris@208 146 m_localFile(0),
Chris@208 147 m_ok(false),
Chris@210 148 m_lastStatus(0),
Chris@316 149 m_remote(isRemote(url.toString())),
Chris@208 150 m_done(false),
Chris@316 151 m_leaveLocalFile(false),
Chris@392 152 m_reporter(reporter),
Chris@316 153 m_refCounted(false)
Chris@208 154 {
Chris@327 155 #ifdef DEBUG_FILE_SOURCE
Chris@317 156 std::cerr << "FileSource::FileSource(" << url.toString().toStdString() << ") [as url]" << std::endl;
Chris@529 157 incCount(m_url.toString());
Chris@327 158 #endif
Chris@316 159
Chris@316 160 if (!canHandleScheme(m_url)) {
Chris@317 161 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString().toStdString() << "\"" << std::endl;
Chris@316 162 m_errorString = tr("Unsupported scheme in URL");
Chris@208 163 return;
Chris@208 164 }
Chris@208 165
Chris@357 166 init();
Chris@497 167
Chris@504 168 #ifdef DEBUG_FILE_SOURCE
Chris@497 169 std::cerr << "FileSource::FileSource(url) exiting" << std::endl;
Chris@504 170 #endif
Chris@316 171 }
Chris@304 172
Chris@317 173 FileSource::FileSource(const FileSource &rf) :
Chris@316 174 QObject(),
Chris@316 175 m_url(rf.m_url),
Chris@316 176 m_ftp(0),
Chris@316 177 m_http(0),
Chris@316 178 m_localFile(0),
Chris@316 179 m_ok(rf.m_ok),
Chris@316 180 m_lastStatus(rf.m_lastStatus),
Chris@316 181 m_remote(rf.m_remote),
Chris@316 182 m_done(false),
Chris@316 183 m_leaveLocalFile(false),
Chris@392 184 m_reporter(rf.m_reporter),
Chris@316 185 m_refCounted(false)
Chris@316 186 {
Chris@327 187 #ifdef DEBUG_FILE_SOURCE
Chris@317 188 std::cerr << "FileSource::FileSource(" << m_url.toString().toStdString() << ") [copy ctor]" << std::endl;
Chris@529 189 incCount(m_url.toString());
Chris@327 190 #endif
Chris@304 191
Chris@316 192 if (!canHandleScheme(m_url)) {
Chris@317 193 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString().toStdString() << "\"" << std::endl;
Chris@316 194 m_errorString = tr("Unsupported scheme in URL");
Chris@304 195 return;
Chris@304 196 }
Chris@304 197
Chris@469 198 if (!isRemote()) {
Chris@316 199 m_localFilename = rf.m_localFilename;
Chris@316 200 } else {
Chris@316 201 QMutexLocker locker(&m_mapMutex);
Chris@327 202 #ifdef DEBUG_FILE_SOURCE
Chris@317 203 std::cerr << "FileSource::FileSource(copy ctor): ref count is "
Chris@316 204 << m_refCountMap[m_url] << std::endl;
Chris@327 205 #endif
Chris@316 206 if (m_refCountMap[m_url] > 0) {
Chris@316 207 m_refCountMap[m_url]++;
Chris@327 208 #ifdef DEBUG_FILE_SOURCE
Chris@316 209 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 210 #endif
Chris@316 211 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 212 m_refCounted = true;
Chris@316 213 } else {
Chris@316 214 m_ok = false;
Chris@316 215 m_lastStatus = 404;
Chris@316 216 }
Chris@316 217 }
Chris@316 218
Chris@316 219 m_done = true;
Chris@497 220
Chris@504 221 #ifdef DEBUG_FILE_SOURCE
Chris@527 222 std::cerr << "FileSource::FileSource(" << m_url.toString().toStdString() << ") [copy ctor]: note: local filename is \"" << m_localFilename.toStdString() << "\"" << std::endl;
Chris@527 223 #endif
Chris@527 224
Chris@527 225 #ifdef DEBUG_FILE_SOURCE
Chris@497 226 std::cerr << "FileSource::FileSource(copy ctor) exiting" << std::endl;
Chris@504 227 #endif
Chris@316 228 }
Chris@316 229
Chris@317 230 FileSource::~FileSource()
Chris@316 231 {
Chris@327 232 #ifdef DEBUG_FILE_SOURCE
Chris@317 233 std::cerr << "FileSource(" << m_url.toString().toStdString() << ")::~FileSource" << std::endl;
Chris@529 234 decCount(m_url.toString());
Chris@327 235 #endif
Chris@316 236
Chris@316 237 cleanup();
Chris@316 238
Chris@469 239 if (isRemote() && !m_leaveLocalFile) deleteCacheFile();
Chris@316 240 }
Chris@316 241
Chris@316 242 void
Chris@357 243 FileSource::init()
Chris@316 244 {
Chris@316 245 if (!isRemote()) {
Chris@355 246 #ifdef DEBUG_FILE_SOURCE
Chris@355 247 std::cerr << "FileSource::init: Not a remote URL" << std::endl;
Chris@355 248 #endif
Chris@355 249 bool literal = false;
Chris@316 250 m_localFilename = m_url.toLocalFile();
Chris@342 251 if (m_localFilename == "") {
Chris@342 252 // QUrl may have mishandled the scheme (e.g. in a DOS path)
Chris@342 253 m_localFilename = m_url.toString();
Chris@355 254 literal = true;
Chris@342 255 }
Chris@439 256 m_localFilename = QFileInfo(m_localFilename).absoluteFilePath();
Chris@439 257
Chris@355 258 #ifdef DEBUG_FILE_SOURCE
Chris@355 259 std::cerr << "FileSource::init: URL translates to local filename \""
Chris@355 260 << m_localFilename.toStdString() << "\"" << std::endl;
Chris@355 261 #endif
Chris@316 262 m_ok = true;
Chris@355 263 m_lastStatus = 200;
Chris@355 264
Chris@316 265 if (!QFileInfo(m_localFilename).exists()) {
Chris@527 266 #ifdef DEBUG_FILE_SOURCE
Chris@527 267 std::cerr << "FileSource::init: Local file of this name does not exist, trying URL as a literal filename" << std::endl;
Chris@527 268 #endif
Chris@355 269 if (literal) {
Chris@355 270 m_lastStatus = 404;
Chris@355 271 } else {
Chris@355 272 // Again, QUrl may have been mistreating us --
Chris@355 273 // e.g. dropping a part that looks like query data
Chris@355 274 m_localFilename = m_url.toString();
Chris@355 275 literal = true;
Chris@355 276 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 277 m_lastStatus = 404;
Chris@355 278 }
Chris@355 279 }
Chris@316 280 }
Chris@355 281
Chris@316 282 m_done = true;
Chris@316 283 return;
Chris@316 284 }
Chris@316 285
Chris@316 286 if (createCacheFile()) {
Chris@327 287 #ifdef DEBUG_FILE_SOURCE
Chris@469 288 std::cerr << "FileSource::init: Already have this one" << std::endl;
Chris@327 289 #endif
Chris@316 290 m_ok = true;
Chris@316 291 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 292 m_lastStatus = 404;
Chris@316 293 } else {
Chris@316 294 m_lastStatus = 200;
Chris@316 295 }
Chris@316 296 m_done = true;
Chris@316 297 return;
Chris@316 298 }
Chris@316 299
Chris@208 300 if (m_localFilename == "") return;
Chris@208 301 m_localFile = new QFile(m_localFilename);
Chris@208 302 m_localFile->open(QFile::WriteOnly);
Chris@208 303
Chris@316 304 QString scheme = m_url.scheme().toLower();
Chris@316 305
Chris@327 306 #ifdef DEBUG_FILE_SOURCE
Chris@317 307 std::cerr << "FileSource::init: Don't have local copy of \""
Chris@469 308 << m_url.toString().toStdString() << "\", retrieving" << std::endl;
Chris@327 309 #endif
Chris@208 310
Chris@208 311 if (scheme == "http") {
Chris@316 312 initHttp();
Chris@520 313 #ifdef DEBUG_FILE_SOURCE
Chris@357 314 std::cerr << "FileSource: initHttp succeeded" << std::endl;
Chris@520 315 #endif
Chris@208 316 } else if (scheme == "ftp") {
Chris@316 317 initFtp();
Chris@316 318 } else {
Chris@316 319 m_remote = false;
Chris@316 320 m_ok = false;
Chris@208 321 }
Chris@208 322
Chris@208 323 if (m_ok) {
Chris@316 324
Chris@316 325 QMutexLocker locker(&m_mapMutex);
Chris@316 326
Chris@316 327 if (m_refCountMap[m_url] > 0) {
Chris@316 328 // someone else has been doing the same thing at the same time,
Chris@316 329 // but has got there first
Chris@316 330 cleanup();
Chris@316 331 m_refCountMap[m_url]++;
Chris@327 332 #ifdef DEBUG_FILE_SOURCE
Chris@317 333 std::cerr << "FileSource::init: Another FileSource has got there first, abandoning our download and using theirs" << std::endl;
Chris@327 334 #endif
Chris@316 335 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 336 m_refCounted = true;
Chris@316 337 m_ok = true;
Chris@316 338 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 339 m_lastStatus = 404;
Chris@316 340 }
Chris@316 341 m_done = true;
Chris@316 342 return;
Chris@316 343 }
Chris@304 344
Chris@304 345 m_remoteLocalMap[m_url] = m_localFilename;
Chris@304 346 m_refCountMap[m_url]++;
Chris@316 347 m_refCounted = true;
Chris@304 348
Chris@392 349 if (m_reporter) {
Chris@392 350 m_reporter->setMessage
Chris@392 351 (tr("Downloading %1...").arg(m_url.toString()));
Chris@392 352 connect(m_reporter, SIGNAL(cancelled()), this, SLOT(cancelled()));
Chris@357 353 connect(this, SIGNAL(progress(int)),
Chris@392 354 m_reporter, SLOT(setProgress(int)));
Chris@316 355 }
Chris@208 356 }
Chris@208 357 }
Chris@208 358
Chris@316 359 void
Chris@317 360 FileSource::initHttp()
Chris@208 361 {
Chris@316 362 m_ok = true;
Chris@497 363 int port = m_url.port();
Chris@497 364 m_http = new QHttp(m_url.host(), port < 0 ? 80 : port);
Chris@316 365 connect(m_http, SIGNAL(done(bool)), this, SLOT(done(bool)));
Chris@316 366 connect(m_http, SIGNAL(dataReadProgress(int, int)),
Chris@316 367 this, SLOT(dataReadProgress(int, int)));
Chris@316 368 connect(m_http, SIGNAL(responseHeaderReceived(const QHttpResponseHeader &)),
Chris@316 369 this, SLOT(httpResponseHeaderReceived(const QHttpResponseHeader &)));
Chris@316 370
Chris@316 371 // I don't quite understand this. url.path() returns a path
Chris@316 372 // without percent encoding; for example, spaces appear as
Chris@316 373 // literal spaces. This generally won't work if sent to the
Chris@316 374 // server directly. You can retrieve a correctly encoded URL
Chris@316 375 // from QUrl using url.toEncoded(), but that gives you the
Chris@316 376 // whole URL; there doesn't seem to be any way to retrieve
Chris@316 377 // only an encoded path. Furthermore there doesn't seem to be
Chris@316 378 // any way to convert a retrieved path into an encoded path
Chris@316 379 // without explicitly specifying that you don't want the path
Chris@316 380 // separators ("/") to be encoded. (Besides being painful to
Chris@316 381 // manage, I don't see how this can work correctly in any case
Chris@316 382 // where a percent-encoded "/" is supposed to appear within a
Chris@316 383 // path element?) There also seems to be no way to retrieve
Chris@316 384 // the path plus query string, i.e. everything that I need to
Chris@316 385 // send to the HTTP server. And no way for QHttp to take a
Chris@316 386 // QUrl argument. I'm obviously missing something.
Chris@316 387
Chris@316 388 // So, two ways to do this: query the bits from the URL,
Chris@316 389 // encode them individually, and glue them back together
Chris@316 390 // again...
Chris@316 391 /*
Chris@316 392 QString path = QUrl::toPercentEncoding(m_url.path(), "/");
Chris@316 393 QList<QPair<QString, QString> > query = m_url.queryItems();
Chris@316 394 if (!query.empty()) {
Chris@316 395 QStringList q2;
Chris@316 396 for (QList<QPair<QString, QString> >::iterator i = query.begin();
Chris@316 397 i != query.end(); ++i) {
Chris@316 398 q2.push_back(QString("%1=%3")
Chris@316 399 .arg(QString(QUrl::toPercentEncoding(i->first)))
Chris@316 400 .arg(QString(QUrl::toPercentEncoding(i->second))));
Chris@316 401 }
Chris@316 402 path = QString("%1%2%3")
Chris@316 403 .arg(path).arg("?")
Chris@316 404 .arg(q2.join("&"));
Chris@316 405 }
Chris@316 406 */
Chris@316 407
Chris@316 408 // ...or, much simpler but relying on knowledge about the
Chris@316 409 // scheme://host/path/path/query etc format of the URL, we can
Chris@316 410 // get the whole URL ready-encoded and then split it on "/" as
Chris@316 411 // appropriate...
Chris@316 412
Chris@316 413 QString path = "/" + QString(m_url.toEncoded()).section('/', 3);
Chris@316 414
Chris@327 415 #ifdef DEBUG_FILE_SOURCE
Chris@317 416 std::cerr << "FileSource: path is \""
Chris@316 417 << path.toStdString() << "\"" << std::endl;
Chris@327 418 #endif
Chris@316 419
Chris@520 420 if (m_preferredContentType == "") {
Chris@520 421 m_http->get(path, m_localFile);
Chris@520 422 } else {
Chris@520 423 #ifdef DEBUG_FILE_SOURCE
Chris@520 424 std::cerr << "FileSource: indicating preferred content type of \""
Chris@520 425 << m_preferredContentType.toStdString() << "\"" << std::endl;
Chris@520 426 #endif
Chris@520 427 QHttpRequestHeader header("GET", path);
Chris@520 428 header.setValue("Host", m_url.host());
Chris@520 429 header.setValue("Accept", QString("%1, */*").arg(m_preferredContentType));
Chris@520 430 m_http->request(header, 0, m_localFile);
Chris@520 431 }
Chris@316 432 }
Chris@316 433
Chris@316 434 void
Chris@317 435 FileSource::initFtp()
Chris@316 436 {
Chris@316 437 m_ok = true;
Chris@316 438 m_ftp = new QFtp;
Chris@316 439 connect(m_ftp, SIGNAL(done(bool)), this, SLOT(done(bool)));
Chris@316 440 connect(m_ftp, SIGNAL(commandFinished(int, bool)),
Chris@316 441 this, SLOT(ftpCommandFinished(int, bool)));
Chris@316 442 connect(m_ftp, SIGNAL(dataTransferProgress(qint64, qint64)),
Chris@316 443 this, SLOT(dataTransferProgress(qint64, qint64)));
Chris@316 444 m_ftp->connectToHost(m_url.host(), m_url.port(21));
Chris@316 445
Chris@316 446 QString username = m_url.userName();
Chris@316 447 if (username == "") {
Chris@316 448 username = "anonymous";
Chris@316 449 }
Chris@316 450
Chris@316 451 QString password = m_url.password();
Chris@316 452 if (password == "") {
Chris@316 453 password = QString("%1@%2").arg(getenv("USER")).arg(getenv("HOST"));
Chris@316 454 }
Chris@316 455
Chris@316 456 m_ftp->login(username, password);
Chris@316 457
Chris@316 458 QString dirpath = m_url.path().section('/', 0, -2);
Chris@316 459 QString filename = m_url.path().section('/', -1);
Chris@316 460
Chris@316 461 if (dirpath == "") dirpath = "/";
Chris@316 462 m_ftp->cd(dirpath);
Chris@316 463 m_ftp->get(filename, m_localFile);
Chris@211 464 }
Chris@211 465
Chris@211 466 void
Chris@317 467 FileSource::cleanup()
Chris@211 468 {
Chris@470 469 if (m_done) {
Chris@470 470 delete m_localFile; // does not actually delete the file
Chris@470 471 m_localFile = 0;
Chris@470 472 }
Chris@211 473 m_done = true;
Chris@214 474 if (m_http) {
Chris@287 475 QHttp *h = m_http;
Chris@214 476 m_http = 0;
Chris@287 477 h->abort();
Chris@287 478 h->deleteLater();
Chris@214 479 }
Chris@214 480 if (m_ftp) {
Chris@287 481 QFtp *f = m_ftp;
Chris@214 482 m_ftp = 0;
Chris@287 483 f->abort();
Chris@287 484 f->deleteLater();
Chris@214 485 }
Chris@470 486 if (m_localFile) {
Chris@470 487 delete m_localFile; // does not actually delete the file
Chris@470 488 m_localFile = 0;
Chris@470 489 }
Chris@208 490 }
Chris@208 491
Chris@208 492 bool
Chris@317 493 FileSource::isRemote(QString fileOrUrl)
Chris@304 494 {
Chris@342 495 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@316 496 QString scheme = QUrl(fileOrUrl).scheme().toLower();
Chris@342 497 if (scheme == "" || scheme == "file" || scheme.length() == 1) return false;
Chris@342 498 return true;
Chris@304 499 }
Chris@304 500
Chris@304 501 bool
Chris@317 502 FileSource::canHandleScheme(QUrl url)
Chris@208 503 {
Chris@342 504 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@208 505 QString scheme = url.scheme().toLower();
Chris@316 506 return (scheme == "http" || scheme == "ftp" ||
Chris@342 507 scheme == "file" || scheme == "" || scheme.length() == 1);
Chris@208 508 }
Chris@208 509
Chris@210 510 bool
Chris@317 511 FileSource::isAvailable()
Chris@210 512 {
Chris@316 513 waitForStatus();
Chris@211 514 bool available = true;
Chris@211 515 if (!m_ok) available = false;
Chris@211 516 else available = (m_lastStatus / 100 == 2);
Chris@327 517 #ifdef DEBUG_FILE_SOURCE
Chris@317 518 std::cerr << "FileSource::isAvailable: " << (available ? "yes" : "no")
Chris@211 519 << std::endl;
Chris@327 520 #endif
Chris@211 521 return available;
Chris@210 522 }
Chris@210 523
Chris@208 524 void
Chris@317 525 FileSource::waitForStatus()
Chris@316 526 {
Chris@316 527 while (m_ok && (!m_done && m_lastStatus == 0)) {
Chris@316 528 // std::cerr << "waitForStatus: processing (last status " << m_lastStatus << ")" << std::endl;
Chris@392 529 QCoreApplication::processEvents();
Chris@316 530 }
Chris@316 531 }
Chris@316 532
Chris@316 533 void
Chris@317 534 FileSource::waitForData()
Chris@208 535 {
Chris@211 536 while (m_ok && !m_done) {
Chris@357 537 // std::cerr << "FileSource::waitForData: calling QApplication::processEvents" << std::endl;
Chris@392 538 QCoreApplication::processEvents();
Chris@497 539 usleep(10000);
Chris@208 540 }
Chris@208 541 }
Chris@208 542
Chris@316 543 void
Chris@317 544 FileSource::setLeaveLocalFile(bool leave)
Chris@316 545 {
Chris@316 546 m_leaveLocalFile = leave;
Chris@316 547 }
Chris@316 548
Chris@208 549 bool
Chris@317 550 FileSource::isOK() const
Chris@208 551 {
Chris@208 552 return m_ok;
Chris@208 553 }
Chris@208 554
Chris@208 555 bool
Chris@317 556 FileSource::isDone() const
Chris@208 557 {
Chris@208 558 return m_done;
Chris@208 559 }
Chris@208 560
Chris@316 561 bool
Chris@317 562 FileSource::isRemote() const
Chris@316 563 {
Chris@316 564 return m_remote;
Chris@316 565 }
Chris@316 566
Chris@316 567 QString
Chris@317 568 FileSource::getLocation() const
Chris@316 569 {
Chris@316 570 return m_url.toString();
Chris@316 571 }
Chris@316 572
Chris@208 573 QString
Chris@317 574 FileSource::getLocalFilename() const
Chris@208 575 {
Chris@208 576 return m_localFilename;
Chris@208 577 }
Chris@208 578
Chris@208 579 QString
Chris@317 580 FileSource::getContentType() const
Chris@316 581 {
Chris@316 582 return m_contentType;
Chris@316 583 }
Chris@316 584
Chris@316 585 QString
Chris@317 586 FileSource::getExtension() const
Chris@316 587 {
Chris@316 588 if (m_localFilename != "") {
Chris@316 589 return QFileInfo(m_localFilename).suffix().toLower();
Chris@316 590 } else {
Chris@316 591 return QFileInfo(m_url.toLocalFile()).suffix().toLower();
Chris@316 592 }
Chris@316 593 }
Chris@316 594
Chris@316 595 QString
Chris@317 596 FileSource::getErrorString() const
Chris@208 597 {
Chris@208 598 return m_errorString;
Chris@208 599 }
Chris@208 600
Chris@208 601 void
Chris@317 602 FileSource::dataReadProgress(int done, int total)
Chris@208 603 {
Chris@208 604 dataTransferProgress(done, total);
Chris@208 605 }
Chris@208 606
Chris@208 607 void
Chris@317 608 FileSource::httpResponseHeaderReceived(const QHttpResponseHeader &resp)
Chris@210 609 {
Chris@520 610 #ifdef DEBUG_FILE_SOURCE
Chris@497 611 std::cerr << "FileSource::httpResponseHeaderReceived" << std::endl;
Chris@520 612 #endif
Chris@497 613
Chris@497 614 if (resp.statusCode() / 100 == 3) {
Chris@496 615 QString location = resp.value("Location");
Chris@496 616 #ifdef DEBUG_FILE_SOURCE
Chris@496 617 std::cerr << "FileSource::responseHeaderReceived: redirect to \""
Chris@496 618 << location.toStdString() << "\" received" << std::endl;
Chris@496 619 #endif
Chris@496 620 if (location != "") {
Chris@496 621 QUrl newUrl(location);
Chris@496 622 if (newUrl != m_url) {
Chris@497 623 cleanup();
Chris@497 624 deleteCacheFile();
Chris@529 625 #ifdef DEBUG_FILE_SOURCE
Chris@529 626 decCount(m_url.toString());
Chris@529 627 incCount(newUrl.toString());
Chris@529 628 #endif
Chris@496 629 m_url = newUrl;
Chris@497 630 m_localFile = 0;
Chris@496 631 m_lastStatus = 0;
Chris@497 632 m_done = false;
Chris@497 633 m_refCounted = false;
Chris@496 634 init();
Chris@496 635 return;
Chris@496 636 }
Chris@496 637 }
Chris@496 638 }
Chris@497 639
Chris@497 640 m_lastStatus = resp.statusCode();
Chris@210 641 if (m_lastStatus / 100 >= 4) {
Chris@210 642 m_errorString = QString("%1 %2")
Chris@210 643 .arg(resp.statusCode()).arg(resp.reasonPhrase());
Chris@327 644 #ifdef DEBUG_FILE_SOURCE
Chris@317 645 std::cerr << "FileSource::responseHeaderReceived: "
Chris@211 646 << m_errorString.toStdString() << std::endl;
Chris@327 647 #endif
Chris@211 648 } else {
Chris@327 649 #ifdef DEBUG_FILE_SOURCE
Chris@317 650 std::cerr << "FileSource::responseHeaderReceived: "
Chris@211 651 << m_lastStatus << std::endl;
Chris@327 652 #endif
Chris@315 653 if (resp.hasContentType()) m_contentType = resp.contentType();
Chris@325 654 }
Chris@325 655 emit statusAvailable();
Chris@210 656 }
Chris@210 657
Chris@210 658 void
Chris@317 659 FileSource::ftpCommandFinished(int id, bool error)
Chris@214 660 {
Chris@327 661 #ifdef DEBUG_FILE_SOURCE
Chris@317 662 std::cerr << "FileSource::ftpCommandFinished(" << id << ", " << error << ")" << std::endl;
Chris@327 663 #endif
Chris@214 664
Chris@214 665 if (!m_ftp) return;
Chris@214 666
Chris@214 667 QFtp::Command command = m_ftp->currentCommand();
Chris@214 668
Chris@214 669 if (!error) {
Chris@327 670 #ifdef DEBUG_FILE_SOURCE
Chris@317 671 std::cerr << "FileSource::ftpCommandFinished: success for command "
Chris@214 672 << command << std::endl;
Chris@327 673 #endif
Chris@214 674 return;
Chris@214 675 }
Chris@214 676
Chris@214 677 if (command == QFtp::ConnectToHost) {
Chris@214 678 m_errorString = tr("Failed to connect to FTP server");
Chris@214 679 } else if (command == QFtp::Login) {
Chris@214 680 m_errorString = tr("Login failed");
Chris@214 681 } else if (command == QFtp::Cd) {
Chris@214 682 m_errorString = tr("Failed to change to correct directory");
Chris@214 683 } else if (command == QFtp::Get) {
Chris@214 684 m_errorString = tr("FTP download aborted");
Chris@214 685 }
Chris@214 686
Chris@214 687 m_lastStatus = 400; // for done()
Chris@214 688 }
Chris@214 689
Chris@214 690 void
Chris@317 691 FileSource::dataTransferProgress(qint64 done, qint64 total)
Chris@208 692 {
Chris@208 693 int percent = int((double(done) / double(total)) * 100.0 - 0.1);
Chris@208 694 emit progress(percent);
Chris@210 695 }
Chris@210 696
Chris@210 697 void
Chris@317 698 FileSource::cancelled()
Chris@210 699 {
Chris@210 700 m_done = true;
Chris@316 701 cleanup();
Chris@316 702
Chris@210 703 m_ok = false;
Chris@210 704 m_errorString = tr("Download cancelled");
Chris@208 705 }
Chris@208 706
Chris@208 707 void
Chris@317 708 FileSource::done(bool error)
Chris@208 709 {
Chris@327 710 emit progress(100);
Chris@327 711
Chris@327 712 #ifdef DEBUG_FILE_SOURCE
Chris@317 713 std::cerr << "FileSource::done(" << error << ")" << std::endl;
Chris@327 714 #endif
Chris@211 715
Chris@211 716 if (m_done) return;
Chris@211 717
Chris@208 718 if (error) {
Chris@208 719 if (m_http) {
Chris@208 720 m_errorString = m_http->errorString();
Chris@208 721 } else if (m_ftp) {
Chris@208 722 m_errorString = m_ftp->errorString();
Chris@208 723 }
Chris@208 724 }
Chris@208 725
Chris@210 726 if (m_lastStatus / 100 >= 4) {
Chris@211 727 error = true;
Chris@210 728 }
Chris@210 729
Chris@211 730 cleanup();
Chris@208 731
Chris@211 732 if (!error) {
Chris@208 733 QFileInfo fi(m_localFilename);
Chris@208 734 if (!fi.exists()) {
Chris@208 735 m_errorString = tr("Failed to create local file %1").arg(m_localFilename);
Chris@211 736 error = true;
Chris@208 737 } else if (fi.size() == 0) {
Chris@208 738 m_errorString = tr("File contains no data!");
Chris@211 739 error = true;
Chris@208 740 }
Chris@208 741 }
Chris@211 742
Chris@211 743 if (error) {
Chris@327 744 #ifdef DEBUG_FILE_SOURCE
Chris@469 745 std::cerr << "FileSource::done: error is " << error << ", deleting cache file" << std::endl;
Chris@327 746 #endif
Chris@316 747 deleteCacheFile();
Chris@211 748 }
Chris@211 749
Chris@211 750 m_ok = !error;
Chris@520 751 if (m_localFile) m_localFile->flush();
Chris@211 752 m_done = true;
Chris@304 753 emit ready();
Chris@211 754 }
Chris@211 755
Chris@211 756 void
Chris@317 757 FileSource::deleteCacheFile()
Chris@211 758 {
Chris@327 759 #ifdef DEBUG_FILE_SOURCE
Chris@317 760 std::cerr << "FileSource::deleteCacheFile(\"" << m_localFilename.toStdString() << "\")" << std::endl;
Chris@327 761 #endif
Chris@211 762
Chris@211 763 cleanup();
Chris@211 764
Chris@316 765 if (m_localFilename == "") {
Chris@316 766 return;
Chris@316 767 }
Chris@211 768
Chris@316 769 if (!isRemote()) {
Chris@327 770 #ifdef DEBUG_FILE_SOURCE
Chris@316 771 std::cerr << "not a cache file" << std::endl;
Chris@327 772 #endif
Chris@316 773 return;
Chris@316 774 }
Chris@316 775
Chris@316 776 if (m_refCounted) {
Chris@304 777
Chris@304 778 QMutexLocker locker(&m_mapMutex);
Chris@316 779 m_refCounted = false;
Chris@304 780
Chris@304 781 if (m_refCountMap[m_url] > 0) {
Chris@304 782 m_refCountMap[m_url]--;
Chris@327 783 #ifdef DEBUG_FILE_SOURCE
Chris@316 784 std::cerr << "reduced ref count to " << m_refCountMap[m_url] << std::endl;
Chris@327 785 #endif
Chris@304 786 if (m_refCountMap[m_url] > 0) {
Chris@304 787 m_done = true;
Chris@304 788 return;
Chris@304 789 }
Chris@304 790 }
Chris@304 791 }
Chris@304 792
Chris@211 793 m_fileCreationMutex.lock();
Chris@211 794
Chris@211 795 if (!QFile(m_localFilename).remove()) {
Chris@469 796 #ifdef DEBUG_FILE_SOURCE
Chris@317 797 std::cerr << "FileSource::deleteCacheFile: ERROR: Failed to delete file \"" << m_localFilename.toStdString() << "\"" << std::endl;
Chris@469 798 #endif
Chris@211 799 } else {
Chris@327 800 #ifdef DEBUG_FILE_SOURCE
Chris@317 801 std::cerr << "FileSource::deleteCacheFile: Deleted cache file \"" << m_localFilename.toStdString() << "\"" << std::endl;
Chris@327 802 #endif
Chris@211 803 m_localFilename = "";
Chris@211 804 }
Chris@211 805
Chris@211 806 m_fileCreationMutex.unlock();
Chris@211 807
Chris@208 808 m_done = true;
Chris@208 809 }
Chris@208 810
Chris@316 811 bool
Chris@317 812 FileSource::createCacheFile()
Chris@208 813 {
Chris@316 814 {
Chris@316 815 QMutexLocker locker(&m_mapMutex);
Chris@316 816
Chris@327 817 #ifdef DEBUG_FILE_SOURCE
Chris@317 818 std::cerr << "FileSource::createCacheFile: refcount is " << m_refCountMap[m_url] << std::endl;
Chris@327 819 #endif
Chris@316 820
Chris@316 821 if (m_refCountMap[m_url] > 0) {
Chris@316 822 m_refCountMap[m_url]++;
Chris@316 823 m_localFilename = m_remoteLocalMap[m_url];
Chris@327 824 #ifdef DEBUG_FILE_SOURCE
Chris@316 825 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 826 #endif
Chris@316 827 m_refCounted = true;
Chris@316 828 return true;
Chris@316 829 }
Chris@316 830 }
Chris@316 831
Chris@208 832 QDir dir;
Chris@208 833 try {
Chris@208 834 dir = TempDirectory::getInstance()->getSubDirectoryPath("download");
Chris@208 835 } catch (DirectoryCreationFailed f) {
Chris@327 836 #ifdef DEBUG_FILE_SOURCE
Chris@317 837 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create temporary directory: " << f.what() << std::endl;
Chris@327 838 #endif
Chris@208 839 return "";
Chris@208 840 }
Chris@208 841
Chris@316 842 QString filepart = m_url.path().section('/', -1, -1,
Chris@316 843 QString::SectionSkipEmpty);
Chris@208 844
Chris@457 845 QString extension = "";
Chris@457 846 if (filepart.contains('.')) extension = filepart.section('.', -1);
Chris@457 847
Chris@208 848 QString base = filepart;
Chris@208 849 if (extension != "") {
Chris@208 850 base = base.left(base.length() - extension.length() - 1);
Chris@208 851 }
Chris@208 852 if (base == "") base = "remote";
Chris@208 853
Chris@208 854 QString filename;
Chris@208 855
Chris@208 856 if (extension == "") {
Chris@208 857 filename = base;
Chris@208 858 } else {
Chris@208 859 filename = QString("%1.%2").arg(base).arg(extension);
Chris@208 860 }
Chris@208 861
Chris@208 862 QString filepath(dir.filePath(filename));
Chris@208 863
Chris@327 864 #ifdef DEBUG_FILE_SOURCE
Chris@317 865 std::cerr << "FileSource::createCacheFile: URL is \"" << m_url.toString().toStdString() << "\", dir is \"" << dir.path().toStdString() << "\", base \"" << base.toStdString() << "\", extension \"" << extension.toStdString() << "\", filebase \"" << filename.toStdString() << "\", filename \"" << filepath.toStdString() << "\"" << std::endl;
Chris@327 866 #endif
Chris@208 867
Chris@316 868 QMutexLocker fcLocker(&m_fileCreationMutex);
Chris@316 869
Chris@208 870 ++m_count;
Chris@208 871
Chris@208 872 if (QFileInfo(filepath).exists() ||
Chris@208 873 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 874
Chris@327 875 #ifdef DEBUG_FILE_SOURCE
Chris@317 876 std::cerr << "FileSource::createCacheFile: Failed to create local file \""
Chris@208 877 << filepath.toStdString() << "\" for URL \""
Chris@316 878 << m_url.toString().toStdString() << "\" (or file already exists): appending suffix instead" << std::endl;
Chris@327 879 #endif
Chris@208 880
Chris@208 881 if (extension == "") {
Chris@208 882 filename = QString("%1_%2").arg(base).arg(m_count);
Chris@208 883 } else {
Chris@208 884 filename = QString("%1_%2.%3").arg(base).arg(m_count).arg(extension);
Chris@208 885 }
Chris@208 886 filepath = dir.filePath(filename);
Chris@208 887
Chris@208 888 if (QFileInfo(filepath).exists() ||
Chris@208 889 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 890
Chris@327 891 #ifdef DEBUG_FILE_SOURCE
Chris@317 892 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create local file \""
Chris@208 893 << filepath.toStdString() << "\" for URL \""
Chris@316 894 << m_url.toString().toStdString() << "\" (or file already exists)" << std::endl;
Chris@327 895 #endif
Chris@208 896
Chris@208 897 return "";
Chris@208 898 }
Chris@208 899 }
Chris@208 900
Chris@327 901 #ifdef DEBUG_FILE_SOURCE
Chris@317 902 std::cerr << "FileSource::createCacheFile: url "
Chris@316 903 << m_url.toString().toStdString() << " -> local filename "
Chris@316 904 << filepath.toStdString() << std::endl;
Chris@327 905 #endif
Chris@316 906
Chris@316 907 m_localFilename = filepath;
Chris@208 908
Chris@316 909 return false;
Chris@208 910 }
Chris@327 911