annotate data/fileio/FileSource.cpp @ 520:e340b2fb9471

* Make FileSource able to indicate its preferred content type (e.g. application/rdf+xml in preference to text/html for rdf content) * Temp directory on Windows -- make $HOME expand to homedrive+homepath instead of user profile location (which may be networked)
author Chris Cannam
date Thu, 11 Dec 2008 15:18:10 +0000
parents 3376dc26dece
children 3f68eab92857
rev   line source
Chris@208 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@208 2
Chris@208 3 /*
Chris@208 4 Sonic Visualiser
Chris@208 5 An audio file viewer and annotation editor.
Chris@208 6 Centre for Digital Music, Queen Mary, University of London.
Chris@208 7 This file copyright 2007 QMUL.
Chris@208 8
Chris@208 9 This program is free software; you can redistribute it and/or
Chris@208 10 modify it under the terms of the GNU General Public License as
Chris@208 11 published by the Free Software Foundation; either version 2 of the
Chris@208 12 License, or (at your option) any later version. See the file
Chris@208 13 COPYING included with this distribution for more information.
Chris@208 14 */
Chris@208 15
Chris@317 16 #include "FileSource.h"
Chris@357 17
Chris@208 18 #include "base/TempDirectory.h"
Chris@208 19 #include "base/Exceptions.h"
Chris@392 20 #include "base/ProgressReporter.h"
Chris@502 21 #include "system/System.h"
Chris@208 22
Chris@208 23 #include <QHttp>
Chris@208 24 #include <QFtp>
Chris@208 25 #include <QFileInfo>
Chris@208 26 #include <QDir>
Chris@392 27 #include <QCoreApplication>
Chris@210 28 #include <QHttpResponseHeader>
Chris@208 29
Chris@208 30 #include <iostream>
Chris@405 31 #include <cstdlib>
Chris@208 32
Chris@520 33 #define DEBUG_FILE_SOURCE 1
Chris@327 34
Chris@208 35 int
Chris@317 36 FileSource::m_count = 0;
Chris@208 37
Chris@208 38 QMutex
Chris@317 39 FileSource::m_fileCreationMutex;
Chris@208 40
Chris@317 41 FileSource::RemoteRefCountMap
Chris@317 42 FileSource::m_refCountMap;
Chris@304 43
Chris@317 44 FileSource::RemoteLocalMap
Chris@317 45 FileSource::m_remoteLocalMap;
Chris@304 46
Chris@304 47 QMutex
Chris@317 48 FileSource::m_mapMutex;
Chris@304 49
Chris@520 50 FileSource::FileSource(QString fileOrUrl, ProgressReporter *reporter,
Chris@520 51 QString preferredContentType) :
Chris@316 52 m_url(fileOrUrl),
Chris@316 53 m_ftp(0),
Chris@316 54 m_http(0),
Chris@316 55 m_localFile(0),
Chris@520 56 m_preferredContentType(preferredContentType),
Chris@316 57 m_ok(false),
Chris@316 58 m_lastStatus(0),
Chris@316 59 m_remote(isRemote(fileOrUrl)),
Chris@316 60 m_done(false),
Chris@316 61 m_leaveLocalFile(false),
Chris@392 62 m_reporter(reporter),
Chris@316 63 m_refCounted(false)
Chris@316 64 {
Chris@327 65 #ifdef DEBUG_FILE_SOURCE
Chris@469 66 std::cerr << "FileSource::FileSource(" << fileOrUrl.toStdString() << ")" << std::endl;
Chris@327 67 #endif
Chris@316 68
Chris@316 69 if (!canHandleScheme(m_url)) {
Chris@317 70 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString().toStdString() << "\"" << std::endl;
Chris@316 71 m_errorString = tr("Unsupported scheme in URL");
Chris@316 72 return;
Chris@316 73 }
Chris@316 74
Chris@357 75 init();
Chris@316 76
Chris@316 77 if (isRemote() &&
Chris@316 78 (fileOrUrl.contains('%') ||
Chris@316 79 fileOrUrl.contains("--"))) { // for IDNA
Chris@316 80
Chris@316 81 waitForStatus();
Chris@316 82
Chris@316 83 if (!isAvailable()) {
Chris@336 84
Chris@316 85 // The URL was created on the assumption that the string
Chris@316 86 // was human-readable. Let's try again, this time
Chris@316 87 // assuming it was already encoded.
Chris@317 88 std::cerr << "FileSource::FileSource: Failed to retrieve URL \""
Chris@316 89 << fileOrUrl.toStdString()
Chris@316 90 << "\" as human-readable URL; "
Chris@316 91 << "trying again treating it as encoded URL"
Chris@316 92 << std::endl;
Chris@336 93
Chris@336 94 // even though our cache file doesn't exist (because the
Chris@336 95 // resource was 404), we still need to ensure we're no
Chris@336 96 // longer associating a filename with this url in the
Chris@336 97 // refcount map -- or createCacheFile will think we've
Chris@336 98 // already done all the work and no request will be sent
Chris@336 99 deleteCacheFile();
Chris@336 100
Chris@316 101 m_url.setEncodedUrl(fileOrUrl.toAscii());
Chris@336 102
Chris@336 103 m_ok = false;
Chris@336 104 m_done = false;
Chris@336 105 m_lastStatus = 0;
Chris@357 106 init();
Chris@316 107 }
Chris@316 108 }
Chris@325 109
Chris@325 110 if (!isRemote()) {
Chris@325 111 emit statusAvailable();
Chris@325 112 emit ready();
Chris@325 113 }
Chris@497 114
Chris@504 115 #ifdef DEBUG_FILE_SOURCE
Chris@497 116 std::cerr << "FileSource::FileSource(string) exiting" << std::endl;
Chris@504 117 #endif
Chris@316 118 }
Chris@316 119
Chris@469 120 FileSource::FileSource(QUrl url, ProgressReporter *reporter) :
Chris@304 121 m_url(url),
Chris@208 122 m_ftp(0),
Chris@208 123 m_http(0),
Chris@208 124 m_localFile(0),
Chris@208 125 m_ok(false),
Chris@210 126 m_lastStatus(0),
Chris@316 127 m_remote(isRemote(url.toString())),
Chris@208 128 m_done(false),
Chris@316 129 m_leaveLocalFile(false),
Chris@392 130 m_reporter(reporter),
Chris@316 131 m_refCounted(false)
Chris@208 132 {
Chris@327 133 #ifdef DEBUG_FILE_SOURCE
Chris@317 134 std::cerr << "FileSource::FileSource(" << url.toString().toStdString() << ") [as url]" << std::endl;
Chris@327 135 #endif
Chris@316 136
Chris@316 137 if (!canHandleScheme(m_url)) {
Chris@317 138 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString().toStdString() << "\"" << std::endl;
Chris@316 139 m_errorString = tr("Unsupported scheme in URL");
Chris@208 140 return;
Chris@208 141 }
Chris@208 142
Chris@357 143 init();
Chris@497 144
Chris@504 145 #ifdef DEBUG_FILE_SOURCE
Chris@497 146 std::cerr << "FileSource::FileSource(url) exiting" << std::endl;
Chris@504 147 #endif
Chris@316 148 }
Chris@304 149
Chris@317 150 FileSource::FileSource(const FileSource &rf) :
Chris@316 151 QObject(),
Chris@316 152 m_url(rf.m_url),
Chris@316 153 m_ftp(0),
Chris@316 154 m_http(0),
Chris@316 155 m_localFile(0),
Chris@316 156 m_ok(rf.m_ok),
Chris@316 157 m_lastStatus(rf.m_lastStatus),
Chris@316 158 m_remote(rf.m_remote),
Chris@316 159 m_done(false),
Chris@316 160 m_leaveLocalFile(false),
Chris@392 161 m_reporter(rf.m_reporter),
Chris@316 162 m_refCounted(false)
Chris@316 163 {
Chris@327 164 #ifdef DEBUG_FILE_SOURCE
Chris@317 165 std::cerr << "FileSource::FileSource(" << m_url.toString().toStdString() << ") [copy ctor]" << std::endl;
Chris@327 166 #endif
Chris@304 167
Chris@316 168 if (!canHandleScheme(m_url)) {
Chris@317 169 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString().toStdString() << "\"" << std::endl;
Chris@316 170 m_errorString = tr("Unsupported scheme in URL");
Chris@304 171 return;
Chris@304 172 }
Chris@304 173
Chris@469 174 if (!isRemote()) {
Chris@316 175 m_localFilename = rf.m_localFilename;
Chris@316 176 } else {
Chris@316 177 QMutexLocker locker(&m_mapMutex);
Chris@327 178 #ifdef DEBUG_FILE_SOURCE
Chris@317 179 std::cerr << "FileSource::FileSource(copy ctor): ref count is "
Chris@316 180 << m_refCountMap[m_url] << std::endl;
Chris@327 181 #endif
Chris@316 182 if (m_refCountMap[m_url] > 0) {
Chris@316 183 m_refCountMap[m_url]++;
Chris@327 184 #ifdef DEBUG_FILE_SOURCE
Chris@316 185 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 186 #endif
Chris@316 187 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 188 m_refCounted = true;
Chris@316 189 } else {
Chris@316 190 m_ok = false;
Chris@316 191 m_lastStatus = 404;
Chris@316 192 }
Chris@316 193 }
Chris@316 194
Chris@316 195 m_done = true;
Chris@497 196
Chris@504 197 #ifdef DEBUG_FILE_SOURCE
Chris@497 198 std::cerr << "FileSource::FileSource(copy ctor) exiting" << std::endl;
Chris@504 199 #endif
Chris@316 200 }
Chris@316 201
Chris@317 202 FileSource::~FileSource()
Chris@316 203 {
Chris@327 204 #ifdef DEBUG_FILE_SOURCE
Chris@317 205 std::cerr << "FileSource(" << m_url.toString().toStdString() << ")::~FileSource" << std::endl;
Chris@327 206 #endif
Chris@316 207
Chris@316 208 cleanup();
Chris@316 209
Chris@469 210 if (isRemote() && !m_leaveLocalFile) deleteCacheFile();
Chris@316 211 }
Chris@316 212
Chris@316 213 void
Chris@357 214 FileSource::init()
Chris@316 215 {
Chris@316 216 if (!isRemote()) {
Chris@355 217 #ifdef DEBUG_FILE_SOURCE
Chris@355 218 std::cerr << "FileSource::init: Not a remote URL" << std::endl;
Chris@355 219 #endif
Chris@355 220 bool literal = false;
Chris@316 221 m_localFilename = m_url.toLocalFile();
Chris@342 222 if (m_localFilename == "") {
Chris@342 223 // QUrl may have mishandled the scheme (e.g. in a DOS path)
Chris@342 224 m_localFilename = m_url.toString();
Chris@355 225 literal = true;
Chris@342 226 }
Chris@439 227 m_localFilename = QFileInfo(m_localFilename).absoluteFilePath();
Chris@439 228
Chris@355 229 #ifdef DEBUG_FILE_SOURCE
Chris@355 230 std::cerr << "FileSource::init: URL translates to local filename \""
Chris@355 231 << m_localFilename.toStdString() << "\"" << std::endl;
Chris@355 232 #endif
Chris@316 233 m_ok = true;
Chris@355 234 m_lastStatus = 200;
Chris@355 235
Chris@316 236 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 237 if (literal) {
Chris@355 238 m_lastStatus = 404;
Chris@355 239 } else {
Chris@355 240 // Again, QUrl may have been mistreating us --
Chris@355 241 // e.g. dropping a part that looks like query data
Chris@355 242 m_localFilename = m_url.toString();
Chris@355 243 literal = true;
Chris@355 244 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 245 m_lastStatus = 404;
Chris@355 246 }
Chris@355 247 }
Chris@316 248 }
Chris@355 249
Chris@316 250 m_done = true;
Chris@316 251 return;
Chris@316 252 }
Chris@316 253
Chris@316 254 if (createCacheFile()) {
Chris@327 255 #ifdef DEBUG_FILE_SOURCE
Chris@469 256 std::cerr << "FileSource::init: Already have this one" << std::endl;
Chris@327 257 #endif
Chris@316 258 m_ok = true;
Chris@316 259 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 260 m_lastStatus = 404;
Chris@316 261 } else {
Chris@316 262 m_lastStatus = 200;
Chris@316 263 }
Chris@316 264 m_done = true;
Chris@316 265 return;
Chris@316 266 }
Chris@316 267
Chris@208 268 if (m_localFilename == "") return;
Chris@208 269 m_localFile = new QFile(m_localFilename);
Chris@208 270 m_localFile->open(QFile::WriteOnly);
Chris@208 271
Chris@316 272 QString scheme = m_url.scheme().toLower();
Chris@316 273
Chris@327 274 #ifdef DEBUG_FILE_SOURCE
Chris@317 275 std::cerr << "FileSource::init: Don't have local copy of \""
Chris@469 276 << m_url.toString().toStdString() << "\", retrieving" << std::endl;
Chris@327 277 #endif
Chris@208 278
Chris@208 279 if (scheme == "http") {
Chris@316 280 initHttp();
Chris@520 281 #ifdef DEBUG_FILE_SOURCE
Chris@357 282 std::cerr << "FileSource: initHttp succeeded" << std::endl;
Chris@520 283 #endif
Chris@208 284 } else if (scheme == "ftp") {
Chris@316 285 initFtp();
Chris@316 286 } else {
Chris@316 287 m_remote = false;
Chris@316 288 m_ok = false;
Chris@208 289 }
Chris@208 290
Chris@208 291 if (m_ok) {
Chris@316 292
Chris@316 293 QMutexLocker locker(&m_mapMutex);
Chris@316 294
Chris@316 295 if (m_refCountMap[m_url] > 0) {
Chris@316 296 // someone else has been doing the same thing at the same time,
Chris@316 297 // but has got there first
Chris@316 298 cleanup();
Chris@316 299 m_refCountMap[m_url]++;
Chris@327 300 #ifdef DEBUG_FILE_SOURCE
Chris@317 301 std::cerr << "FileSource::init: Another FileSource has got there first, abandoning our download and using theirs" << std::endl;
Chris@327 302 #endif
Chris@316 303 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 304 m_refCounted = true;
Chris@316 305 m_ok = true;
Chris@316 306 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 307 m_lastStatus = 404;
Chris@316 308 }
Chris@316 309 m_done = true;
Chris@316 310 return;
Chris@316 311 }
Chris@304 312
Chris@304 313 m_remoteLocalMap[m_url] = m_localFilename;
Chris@304 314 m_refCountMap[m_url]++;
Chris@316 315 m_refCounted = true;
Chris@304 316
Chris@392 317 if (m_reporter) {
Chris@392 318 m_reporter->setMessage
Chris@392 319 (tr("Downloading %1...").arg(m_url.toString()));
Chris@392 320 connect(m_reporter, SIGNAL(cancelled()), this, SLOT(cancelled()));
Chris@357 321 connect(this, SIGNAL(progress(int)),
Chris@392 322 m_reporter, SLOT(setProgress(int)));
Chris@316 323 }
Chris@208 324 }
Chris@208 325 }
Chris@208 326
Chris@316 327 void
Chris@317 328 FileSource::initHttp()
Chris@208 329 {
Chris@316 330 m_ok = true;
Chris@497 331 int port = m_url.port();
Chris@497 332 m_http = new QHttp(m_url.host(), port < 0 ? 80 : port);
Chris@316 333 connect(m_http, SIGNAL(done(bool)), this, SLOT(done(bool)));
Chris@316 334 connect(m_http, SIGNAL(dataReadProgress(int, int)),
Chris@316 335 this, SLOT(dataReadProgress(int, int)));
Chris@316 336 connect(m_http, SIGNAL(responseHeaderReceived(const QHttpResponseHeader &)),
Chris@316 337 this, SLOT(httpResponseHeaderReceived(const QHttpResponseHeader &)));
Chris@316 338
Chris@316 339 // I don't quite understand this. url.path() returns a path
Chris@316 340 // without percent encoding; for example, spaces appear as
Chris@316 341 // literal spaces. This generally won't work if sent to the
Chris@316 342 // server directly. You can retrieve a correctly encoded URL
Chris@316 343 // from QUrl using url.toEncoded(), but that gives you the
Chris@316 344 // whole URL; there doesn't seem to be any way to retrieve
Chris@316 345 // only an encoded path. Furthermore there doesn't seem to be
Chris@316 346 // any way to convert a retrieved path into an encoded path
Chris@316 347 // without explicitly specifying that you don't want the path
Chris@316 348 // separators ("/") to be encoded. (Besides being painful to
Chris@316 349 // manage, I don't see how this can work correctly in any case
Chris@316 350 // where a percent-encoded "/" is supposed to appear within a
Chris@316 351 // path element?) There also seems to be no way to retrieve
Chris@316 352 // the path plus query string, i.e. everything that I need to
Chris@316 353 // send to the HTTP server. And no way for QHttp to take a
Chris@316 354 // QUrl argument. I'm obviously missing something.
Chris@316 355
Chris@316 356 // So, two ways to do this: query the bits from the URL,
Chris@316 357 // encode them individually, and glue them back together
Chris@316 358 // again...
Chris@316 359 /*
Chris@316 360 QString path = QUrl::toPercentEncoding(m_url.path(), "/");
Chris@316 361 QList<QPair<QString, QString> > query = m_url.queryItems();
Chris@316 362 if (!query.empty()) {
Chris@316 363 QStringList q2;
Chris@316 364 for (QList<QPair<QString, QString> >::iterator i = query.begin();
Chris@316 365 i != query.end(); ++i) {
Chris@316 366 q2.push_back(QString("%1=%3")
Chris@316 367 .arg(QString(QUrl::toPercentEncoding(i->first)))
Chris@316 368 .arg(QString(QUrl::toPercentEncoding(i->second))));
Chris@316 369 }
Chris@316 370 path = QString("%1%2%3")
Chris@316 371 .arg(path).arg("?")
Chris@316 372 .arg(q2.join("&"));
Chris@316 373 }
Chris@316 374 */
Chris@316 375
Chris@316 376 // ...or, much simpler but relying on knowledge about the
Chris@316 377 // scheme://host/path/path/query etc format of the URL, we can
Chris@316 378 // get the whole URL ready-encoded and then split it on "/" as
Chris@316 379 // appropriate...
Chris@316 380
Chris@316 381 QString path = "/" + QString(m_url.toEncoded()).section('/', 3);
Chris@316 382
Chris@327 383 #ifdef DEBUG_FILE_SOURCE
Chris@317 384 std::cerr << "FileSource: path is \""
Chris@316 385 << path.toStdString() << "\"" << std::endl;
Chris@327 386 #endif
Chris@316 387
Chris@520 388 if (m_preferredContentType == "") {
Chris@520 389 m_http->get(path, m_localFile);
Chris@520 390 } else {
Chris@520 391 #ifdef DEBUG_FILE_SOURCE
Chris@520 392 std::cerr << "FileSource: indicating preferred content type of \""
Chris@520 393 << m_preferredContentType.toStdString() << "\"" << std::endl;
Chris@520 394 #endif
Chris@520 395 QHttpRequestHeader header("GET", path);
Chris@520 396 header.setValue("Host", m_url.host());
Chris@520 397 header.setValue("Accept", QString("%1, */*").arg(m_preferredContentType));
Chris@520 398 m_http->request(header, 0, m_localFile);
Chris@520 399 }
Chris@316 400 }
Chris@316 401
Chris@316 402 void
Chris@317 403 FileSource::initFtp()
Chris@316 404 {
Chris@316 405 m_ok = true;
Chris@316 406 m_ftp = new QFtp;
Chris@316 407 connect(m_ftp, SIGNAL(done(bool)), this, SLOT(done(bool)));
Chris@316 408 connect(m_ftp, SIGNAL(commandFinished(int, bool)),
Chris@316 409 this, SLOT(ftpCommandFinished(int, bool)));
Chris@316 410 connect(m_ftp, SIGNAL(dataTransferProgress(qint64, qint64)),
Chris@316 411 this, SLOT(dataTransferProgress(qint64, qint64)));
Chris@316 412 m_ftp->connectToHost(m_url.host(), m_url.port(21));
Chris@316 413
Chris@316 414 QString username = m_url.userName();
Chris@316 415 if (username == "") {
Chris@316 416 username = "anonymous";
Chris@316 417 }
Chris@316 418
Chris@316 419 QString password = m_url.password();
Chris@316 420 if (password == "") {
Chris@316 421 password = QString("%1@%2").arg(getenv("USER")).arg(getenv("HOST"));
Chris@316 422 }
Chris@316 423
Chris@316 424 m_ftp->login(username, password);
Chris@316 425
Chris@316 426 QString dirpath = m_url.path().section('/', 0, -2);
Chris@316 427 QString filename = m_url.path().section('/', -1);
Chris@316 428
Chris@316 429 if (dirpath == "") dirpath = "/";
Chris@316 430 m_ftp->cd(dirpath);
Chris@316 431 m_ftp->get(filename, m_localFile);
Chris@211 432 }
Chris@211 433
Chris@211 434 void
Chris@317 435 FileSource::cleanup()
Chris@211 436 {
Chris@470 437 if (m_done) {
Chris@470 438 delete m_localFile; // does not actually delete the file
Chris@470 439 m_localFile = 0;
Chris@470 440 }
Chris@211 441 m_done = true;
Chris@214 442 if (m_http) {
Chris@287 443 QHttp *h = m_http;
Chris@214 444 m_http = 0;
Chris@287 445 h->abort();
Chris@287 446 h->deleteLater();
Chris@214 447 }
Chris@214 448 if (m_ftp) {
Chris@287 449 QFtp *f = m_ftp;
Chris@214 450 m_ftp = 0;
Chris@287 451 f->abort();
Chris@287 452 f->deleteLater();
Chris@214 453 }
Chris@470 454 if (m_localFile) {
Chris@470 455 delete m_localFile; // does not actually delete the file
Chris@470 456 m_localFile = 0;
Chris@470 457 }
Chris@208 458 }
Chris@208 459
Chris@208 460 bool
Chris@317 461 FileSource::isRemote(QString fileOrUrl)
Chris@304 462 {
Chris@342 463 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@316 464 QString scheme = QUrl(fileOrUrl).scheme().toLower();
Chris@342 465 if (scheme == "" || scheme == "file" || scheme.length() == 1) return false;
Chris@342 466 return true;
Chris@304 467 }
Chris@304 468
Chris@304 469 bool
Chris@317 470 FileSource::canHandleScheme(QUrl url)
Chris@208 471 {
Chris@342 472 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@208 473 QString scheme = url.scheme().toLower();
Chris@316 474 return (scheme == "http" || scheme == "ftp" ||
Chris@342 475 scheme == "file" || scheme == "" || scheme.length() == 1);
Chris@208 476 }
Chris@208 477
Chris@210 478 bool
Chris@317 479 FileSource::isAvailable()
Chris@210 480 {
Chris@316 481 waitForStatus();
Chris@211 482 bool available = true;
Chris@211 483 if (!m_ok) available = false;
Chris@211 484 else available = (m_lastStatus / 100 == 2);
Chris@327 485 #ifdef DEBUG_FILE_SOURCE
Chris@317 486 std::cerr << "FileSource::isAvailable: " << (available ? "yes" : "no")
Chris@211 487 << std::endl;
Chris@327 488 #endif
Chris@211 489 return available;
Chris@210 490 }
Chris@210 491
Chris@208 492 void
Chris@317 493 FileSource::waitForStatus()
Chris@316 494 {
Chris@316 495 while (m_ok && (!m_done && m_lastStatus == 0)) {
Chris@316 496 // std::cerr << "waitForStatus: processing (last status " << m_lastStatus << ")" << std::endl;
Chris@392 497 QCoreApplication::processEvents();
Chris@316 498 }
Chris@316 499 }
Chris@316 500
Chris@316 501 void
Chris@317 502 FileSource::waitForData()
Chris@208 503 {
Chris@211 504 while (m_ok && !m_done) {
Chris@357 505 // std::cerr << "FileSource::waitForData: calling QApplication::processEvents" << std::endl;
Chris@392 506 QCoreApplication::processEvents();
Chris@497 507 usleep(10000);
Chris@208 508 }
Chris@208 509 }
Chris@208 510
Chris@316 511 void
Chris@317 512 FileSource::setLeaveLocalFile(bool leave)
Chris@316 513 {
Chris@316 514 m_leaveLocalFile = leave;
Chris@316 515 }
Chris@316 516
Chris@208 517 bool
Chris@317 518 FileSource::isOK() const
Chris@208 519 {
Chris@208 520 return m_ok;
Chris@208 521 }
Chris@208 522
Chris@208 523 bool
Chris@317 524 FileSource::isDone() const
Chris@208 525 {
Chris@208 526 return m_done;
Chris@208 527 }
Chris@208 528
Chris@316 529 bool
Chris@317 530 FileSource::isRemote() const
Chris@316 531 {
Chris@316 532 return m_remote;
Chris@316 533 }
Chris@316 534
Chris@316 535 QString
Chris@317 536 FileSource::getLocation() const
Chris@316 537 {
Chris@316 538 return m_url.toString();
Chris@316 539 }
Chris@316 540
Chris@208 541 QString
Chris@317 542 FileSource::getLocalFilename() const
Chris@208 543 {
Chris@208 544 return m_localFilename;
Chris@208 545 }
Chris@208 546
Chris@208 547 QString
Chris@317 548 FileSource::getContentType() const
Chris@316 549 {
Chris@316 550 return m_contentType;
Chris@316 551 }
Chris@316 552
Chris@316 553 QString
Chris@317 554 FileSource::getExtension() const
Chris@316 555 {
Chris@316 556 if (m_localFilename != "") {
Chris@316 557 return QFileInfo(m_localFilename).suffix().toLower();
Chris@316 558 } else {
Chris@316 559 return QFileInfo(m_url.toLocalFile()).suffix().toLower();
Chris@316 560 }
Chris@316 561 }
Chris@316 562
Chris@316 563 QString
Chris@317 564 FileSource::getErrorString() const
Chris@208 565 {
Chris@208 566 return m_errorString;
Chris@208 567 }
Chris@208 568
Chris@208 569 void
Chris@317 570 FileSource::dataReadProgress(int done, int total)
Chris@208 571 {
Chris@208 572 dataTransferProgress(done, total);
Chris@208 573 }
Chris@208 574
Chris@208 575 void
Chris@317 576 FileSource::httpResponseHeaderReceived(const QHttpResponseHeader &resp)
Chris@210 577 {
Chris@520 578 #ifdef DEBUG_FILE_SOURCE
Chris@497 579 std::cerr << "FileSource::httpResponseHeaderReceived" << std::endl;
Chris@520 580 #endif
Chris@497 581
Chris@497 582 if (resp.statusCode() / 100 == 3) {
Chris@496 583 QString location = resp.value("Location");
Chris@496 584 #ifdef DEBUG_FILE_SOURCE
Chris@496 585 std::cerr << "FileSource::responseHeaderReceived: redirect to \""
Chris@496 586 << location.toStdString() << "\" received" << std::endl;
Chris@496 587 #endif
Chris@496 588 if (location != "") {
Chris@496 589 QUrl newUrl(location);
Chris@496 590 if (newUrl != m_url) {
Chris@497 591 cleanup();
Chris@497 592 deleteCacheFile();
Chris@496 593 m_url = newUrl;
Chris@497 594 m_localFile = 0;
Chris@496 595 m_lastStatus = 0;
Chris@497 596 m_done = false;
Chris@497 597 m_refCounted = false;
Chris@496 598 init();
Chris@496 599 return;
Chris@496 600 }
Chris@496 601 }
Chris@496 602 }
Chris@497 603
Chris@497 604 m_lastStatus = resp.statusCode();
Chris@210 605 if (m_lastStatus / 100 >= 4) {
Chris@210 606 m_errorString = QString("%1 %2")
Chris@210 607 .arg(resp.statusCode()).arg(resp.reasonPhrase());
Chris@327 608 #ifdef DEBUG_FILE_SOURCE
Chris@317 609 std::cerr << "FileSource::responseHeaderReceived: "
Chris@211 610 << m_errorString.toStdString() << std::endl;
Chris@327 611 #endif
Chris@211 612 } else {
Chris@327 613 #ifdef DEBUG_FILE_SOURCE
Chris@317 614 std::cerr << "FileSource::responseHeaderReceived: "
Chris@211 615 << m_lastStatus << std::endl;
Chris@327 616 #endif
Chris@315 617 if (resp.hasContentType()) m_contentType = resp.contentType();
Chris@325 618 }
Chris@325 619 emit statusAvailable();
Chris@210 620 }
Chris@210 621
Chris@210 622 void
Chris@317 623 FileSource::ftpCommandFinished(int id, bool error)
Chris@214 624 {
Chris@327 625 #ifdef DEBUG_FILE_SOURCE
Chris@317 626 std::cerr << "FileSource::ftpCommandFinished(" << id << ", " << error << ")" << std::endl;
Chris@327 627 #endif
Chris@214 628
Chris@214 629 if (!m_ftp) return;
Chris@214 630
Chris@214 631 QFtp::Command command = m_ftp->currentCommand();
Chris@214 632
Chris@214 633 if (!error) {
Chris@327 634 #ifdef DEBUG_FILE_SOURCE
Chris@317 635 std::cerr << "FileSource::ftpCommandFinished: success for command "
Chris@214 636 << command << std::endl;
Chris@327 637 #endif
Chris@214 638 return;
Chris@214 639 }
Chris@214 640
Chris@214 641 if (command == QFtp::ConnectToHost) {
Chris@214 642 m_errorString = tr("Failed to connect to FTP server");
Chris@214 643 } else if (command == QFtp::Login) {
Chris@214 644 m_errorString = tr("Login failed");
Chris@214 645 } else if (command == QFtp::Cd) {
Chris@214 646 m_errorString = tr("Failed to change to correct directory");
Chris@214 647 } else if (command == QFtp::Get) {
Chris@214 648 m_errorString = tr("FTP download aborted");
Chris@214 649 }
Chris@214 650
Chris@214 651 m_lastStatus = 400; // for done()
Chris@214 652 }
Chris@214 653
Chris@214 654 void
Chris@317 655 FileSource::dataTransferProgress(qint64 done, qint64 total)
Chris@208 656 {
Chris@208 657 int percent = int((double(done) / double(total)) * 100.0 - 0.1);
Chris@208 658 emit progress(percent);
Chris@210 659 }
Chris@210 660
Chris@210 661 void
Chris@317 662 FileSource::cancelled()
Chris@210 663 {
Chris@210 664 m_done = true;
Chris@316 665 cleanup();
Chris@316 666
Chris@210 667 m_ok = false;
Chris@210 668 m_errorString = tr("Download cancelled");
Chris@208 669 }
Chris@208 670
Chris@208 671 void
Chris@317 672 FileSource::done(bool error)
Chris@208 673 {
Chris@327 674 emit progress(100);
Chris@327 675
Chris@327 676 #ifdef DEBUG_FILE_SOURCE
Chris@317 677 std::cerr << "FileSource::done(" << error << ")" << std::endl;
Chris@327 678 #endif
Chris@211 679
Chris@211 680 if (m_done) return;
Chris@211 681
Chris@208 682 if (error) {
Chris@208 683 if (m_http) {
Chris@208 684 m_errorString = m_http->errorString();
Chris@208 685 } else if (m_ftp) {
Chris@208 686 m_errorString = m_ftp->errorString();
Chris@208 687 }
Chris@208 688 }
Chris@208 689
Chris@210 690 if (m_lastStatus / 100 >= 4) {
Chris@211 691 error = true;
Chris@210 692 }
Chris@210 693
Chris@211 694 cleanup();
Chris@208 695
Chris@211 696 if (!error) {
Chris@208 697 QFileInfo fi(m_localFilename);
Chris@208 698 if (!fi.exists()) {
Chris@208 699 m_errorString = tr("Failed to create local file %1").arg(m_localFilename);
Chris@211 700 error = true;
Chris@208 701 } else if (fi.size() == 0) {
Chris@208 702 m_errorString = tr("File contains no data!");
Chris@211 703 error = true;
Chris@208 704 }
Chris@208 705 }
Chris@211 706
Chris@211 707 if (error) {
Chris@327 708 #ifdef DEBUG_FILE_SOURCE
Chris@469 709 std::cerr << "FileSource::done: error is " << error << ", deleting cache file" << std::endl;
Chris@327 710 #endif
Chris@316 711 deleteCacheFile();
Chris@211 712 }
Chris@211 713
Chris@211 714 m_ok = !error;
Chris@520 715 if (m_localFile) m_localFile->flush();
Chris@211 716 m_done = true;
Chris@304 717 emit ready();
Chris@211 718 }
Chris@211 719
Chris@211 720 void
Chris@317 721 FileSource::deleteCacheFile()
Chris@211 722 {
Chris@327 723 #ifdef DEBUG_FILE_SOURCE
Chris@317 724 std::cerr << "FileSource::deleteCacheFile(\"" << m_localFilename.toStdString() << "\")" << std::endl;
Chris@327 725 #endif
Chris@211 726
Chris@211 727 cleanup();
Chris@211 728
Chris@316 729 if (m_localFilename == "") {
Chris@316 730 return;
Chris@316 731 }
Chris@211 732
Chris@316 733 if (!isRemote()) {
Chris@327 734 #ifdef DEBUG_FILE_SOURCE
Chris@316 735 std::cerr << "not a cache file" << std::endl;
Chris@327 736 #endif
Chris@316 737 return;
Chris@316 738 }
Chris@316 739
Chris@316 740 if (m_refCounted) {
Chris@304 741
Chris@304 742 QMutexLocker locker(&m_mapMutex);
Chris@316 743 m_refCounted = false;
Chris@304 744
Chris@304 745 if (m_refCountMap[m_url] > 0) {
Chris@304 746 m_refCountMap[m_url]--;
Chris@327 747 #ifdef DEBUG_FILE_SOURCE
Chris@316 748 std::cerr << "reduced ref count to " << m_refCountMap[m_url] << std::endl;
Chris@327 749 #endif
Chris@304 750 if (m_refCountMap[m_url] > 0) {
Chris@304 751 m_done = true;
Chris@304 752 return;
Chris@304 753 }
Chris@304 754 }
Chris@304 755 }
Chris@304 756
Chris@211 757 m_fileCreationMutex.lock();
Chris@211 758
Chris@211 759 if (!QFile(m_localFilename).remove()) {
Chris@469 760 #ifdef DEBUG_FILE_SOURCE
Chris@317 761 std::cerr << "FileSource::deleteCacheFile: ERROR: Failed to delete file \"" << m_localFilename.toStdString() << "\"" << std::endl;
Chris@469 762 #endif
Chris@211 763 } else {
Chris@327 764 #ifdef DEBUG_FILE_SOURCE
Chris@317 765 std::cerr << "FileSource::deleteCacheFile: Deleted cache file \"" << m_localFilename.toStdString() << "\"" << std::endl;
Chris@327 766 #endif
Chris@211 767 m_localFilename = "";
Chris@211 768 }
Chris@211 769
Chris@211 770 m_fileCreationMutex.unlock();
Chris@211 771
Chris@208 772 m_done = true;
Chris@208 773 }
Chris@208 774
Chris@316 775 bool
Chris@317 776 FileSource::createCacheFile()
Chris@208 777 {
Chris@316 778 {
Chris@316 779 QMutexLocker locker(&m_mapMutex);
Chris@316 780
Chris@327 781 #ifdef DEBUG_FILE_SOURCE
Chris@317 782 std::cerr << "FileSource::createCacheFile: refcount is " << m_refCountMap[m_url] << std::endl;
Chris@327 783 #endif
Chris@316 784
Chris@316 785 if (m_refCountMap[m_url] > 0) {
Chris@316 786 m_refCountMap[m_url]++;
Chris@316 787 m_localFilename = m_remoteLocalMap[m_url];
Chris@327 788 #ifdef DEBUG_FILE_SOURCE
Chris@316 789 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 790 #endif
Chris@316 791 m_refCounted = true;
Chris@316 792 return true;
Chris@316 793 }
Chris@316 794 }
Chris@316 795
Chris@208 796 QDir dir;
Chris@208 797 try {
Chris@208 798 dir = TempDirectory::getInstance()->getSubDirectoryPath("download");
Chris@208 799 } catch (DirectoryCreationFailed f) {
Chris@327 800 #ifdef DEBUG_FILE_SOURCE
Chris@317 801 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create temporary directory: " << f.what() << std::endl;
Chris@327 802 #endif
Chris@208 803 return "";
Chris@208 804 }
Chris@208 805
Chris@316 806 QString filepart = m_url.path().section('/', -1, -1,
Chris@316 807 QString::SectionSkipEmpty);
Chris@208 808
Chris@457 809 QString extension = "";
Chris@457 810 if (filepart.contains('.')) extension = filepart.section('.', -1);
Chris@457 811
Chris@208 812 QString base = filepart;
Chris@208 813 if (extension != "") {
Chris@208 814 base = base.left(base.length() - extension.length() - 1);
Chris@208 815 }
Chris@208 816 if (base == "") base = "remote";
Chris@208 817
Chris@208 818 QString filename;
Chris@208 819
Chris@208 820 if (extension == "") {
Chris@208 821 filename = base;
Chris@208 822 } else {
Chris@208 823 filename = QString("%1.%2").arg(base).arg(extension);
Chris@208 824 }
Chris@208 825
Chris@208 826 QString filepath(dir.filePath(filename));
Chris@208 827
Chris@327 828 #ifdef DEBUG_FILE_SOURCE
Chris@317 829 std::cerr << "FileSource::createCacheFile: URL is \"" << m_url.toString().toStdString() << "\", dir is \"" << dir.path().toStdString() << "\", base \"" << base.toStdString() << "\", extension \"" << extension.toStdString() << "\", filebase \"" << filename.toStdString() << "\", filename \"" << filepath.toStdString() << "\"" << std::endl;
Chris@327 830 #endif
Chris@208 831
Chris@316 832 QMutexLocker fcLocker(&m_fileCreationMutex);
Chris@316 833
Chris@208 834 ++m_count;
Chris@208 835
Chris@208 836 if (QFileInfo(filepath).exists() ||
Chris@208 837 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 838
Chris@327 839 #ifdef DEBUG_FILE_SOURCE
Chris@317 840 std::cerr << "FileSource::createCacheFile: Failed to create local file \""
Chris@208 841 << filepath.toStdString() << "\" for URL \""
Chris@316 842 << m_url.toString().toStdString() << "\" (or file already exists): appending suffix instead" << std::endl;
Chris@327 843 #endif
Chris@208 844
Chris@208 845 if (extension == "") {
Chris@208 846 filename = QString("%1_%2").arg(base).arg(m_count);
Chris@208 847 } else {
Chris@208 848 filename = QString("%1_%2.%3").arg(base).arg(m_count).arg(extension);
Chris@208 849 }
Chris@208 850 filepath = dir.filePath(filename);
Chris@208 851
Chris@208 852 if (QFileInfo(filepath).exists() ||
Chris@208 853 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 854
Chris@327 855 #ifdef DEBUG_FILE_SOURCE
Chris@317 856 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create local file \""
Chris@208 857 << filepath.toStdString() << "\" for URL \""
Chris@316 858 << m_url.toString().toStdString() << "\" (or file already exists)" << std::endl;
Chris@327 859 #endif
Chris@208 860
Chris@208 861 return "";
Chris@208 862 }
Chris@208 863 }
Chris@208 864
Chris@327 865 #ifdef DEBUG_FILE_SOURCE
Chris@317 866 std::cerr << "FileSource::createCacheFile: url "
Chris@316 867 << m_url.toString().toStdString() << " -> local filename "
Chris@316 868 << filepath.toStdString() << std::endl;
Chris@327 869 #endif
Chris@316 870
Chris@316 871 m_localFilename = filepath;
Chris@208 872
Chris@316 873 return false;
Chris@208 874 }
Chris@327 875