annotate data/fileio/FileSource.cpp @ 504:3376dc26dece

* Add FeatureWriter method to set basic track metadata; write it out from RDFFeatureWriter if present
author Chris Cannam
date Fri, 05 Dec 2008 11:09:44 +0000
parents bd7c46636bd0
children e340b2fb9471
rev   line source
Chris@208 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@208 2
Chris@208 3 /*
Chris@208 4 Sonic Visualiser
Chris@208 5 An audio file viewer and annotation editor.
Chris@208 6 Centre for Digital Music, Queen Mary, University of London.
Chris@208 7 This file copyright 2007 QMUL.
Chris@208 8
Chris@208 9 This program is free software; you can redistribute it and/or
Chris@208 10 modify it under the terms of the GNU General Public License as
Chris@208 11 published by the Free Software Foundation; either version 2 of the
Chris@208 12 License, or (at your option) any later version. See the file
Chris@208 13 COPYING included with this distribution for more information.
Chris@208 14 */
Chris@208 15
Chris@317 16 #include "FileSource.h"
Chris@357 17
Chris@208 18 #include "base/TempDirectory.h"
Chris@208 19 #include "base/Exceptions.h"
Chris@392 20 #include "base/ProgressReporter.h"
Chris@502 21 #include "system/System.h"
Chris@208 22
Chris@208 23 #include <QHttp>
Chris@208 24 #include <QFtp>
Chris@208 25 #include <QFileInfo>
Chris@208 26 #include <QDir>
Chris@392 27 #include <QCoreApplication>
Chris@210 28 #include <QHttpResponseHeader>
Chris@208 29
Chris@208 30 #include <iostream>
Chris@405 31 #include <cstdlib>
Chris@208 32
Chris@497 33 //#define DEBUG_FILE_SOURCE 1
Chris@327 34
Chris@208 35 int
Chris@317 36 FileSource::m_count = 0;
Chris@208 37
Chris@208 38 QMutex
Chris@317 39 FileSource::m_fileCreationMutex;
Chris@208 40
Chris@317 41 FileSource::RemoteRefCountMap
Chris@317 42 FileSource::m_refCountMap;
Chris@304 43
Chris@317 44 FileSource::RemoteLocalMap
Chris@317 45 FileSource::m_remoteLocalMap;
Chris@304 46
Chris@304 47 QMutex
Chris@317 48 FileSource::m_mapMutex;
Chris@304 49
Chris@469 50 FileSource::FileSource(QString fileOrUrl, ProgressReporter *reporter) :
Chris@316 51 m_url(fileOrUrl),
Chris@316 52 m_ftp(0),
Chris@316 53 m_http(0),
Chris@316 54 m_localFile(0),
Chris@316 55 m_ok(false),
Chris@316 56 m_lastStatus(0),
Chris@316 57 m_remote(isRemote(fileOrUrl)),
Chris@316 58 m_done(false),
Chris@316 59 m_leaveLocalFile(false),
Chris@392 60 m_reporter(reporter),
Chris@316 61 m_refCounted(false)
Chris@316 62 {
Chris@327 63 #ifdef DEBUG_FILE_SOURCE
Chris@469 64 std::cerr << "FileSource::FileSource(" << fileOrUrl.toStdString() << ")" << std::endl;
Chris@327 65 #endif
Chris@316 66
Chris@316 67 if (!canHandleScheme(m_url)) {
Chris@317 68 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString().toStdString() << "\"" << std::endl;
Chris@316 69 m_errorString = tr("Unsupported scheme in URL");
Chris@316 70 return;
Chris@316 71 }
Chris@316 72
Chris@357 73 init();
Chris@316 74
Chris@316 75 if (isRemote() &&
Chris@316 76 (fileOrUrl.contains('%') ||
Chris@316 77 fileOrUrl.contains("--"))) { // for IDNA
Chris@316 78
Chris@316 79 waitForStatus();
Chris@316 80
Chris@316 81 if (!isAvailable()) {
Chris@336 82
Chris@316 83 // The URL was created on the assumption that the string
Chris@316 84 // was human-readable. Let's try again, this time
Chris@316 85 // assuming it was already encoded.
Chris@317 86 std::cerr << "FileSource::FileSource: Failed to retrieve URL \""
Chris@316 87 << fileOrUrl.toStdString()
Chris@316 88 << "\" as human-readable URL; "
Chris@316 89 << "trying again treating it as encoded URL"
Chris@316 90 << std::endl;
Chris@336 91
Chris@336 92 // even though our cache file doesn't exist (because the
Chris@336 93 // resource was 404), we still need to ensure we're no
Chris@336 94 // longer associating a filename with this url in the
Chris@336 95 // refcount map -- or createCacheFile will think we've
Chris@336 96 // already done all the work and no request will be sent
Chris@336 97 deleteCacheFile();
Chris@336 98
Chris@316 99 m_url.setEncodedUrl(fileOrUrl.toAscii());
Chris@336 100
Chris@336 101 m_ok = false;
Chris@336 102 m_done = false;
Chris@336 103 m_lastStatus = 0;
Chris@357 104 init();
Chris@316 105 }
Chris@316 106 }
Chris@325 107
Chris@325 108 if (!isRemote()) {
Chris@325 109 emit statusAvailable();
Chris@325 110 emit ready();
Chris@325 111 }
Chris@497 112
Chris@504 113 #ifdef DEBUG_FILE_SOURCE
Chris@497 114 std::cerr << "FileSource::FileSource(string) exiting" << std::endl;
Chris@504 115 #endif
Chris@316 116 }
Chris@316 117
Chris@469 118 FileSource::FileSource(QUrl url, ProgressReporter *reporter) :
Chris@304 119 m_url(url),
Chris@208 120 m_ftp(0),
Chris@208 121 m_http(0),
Chris@208 122 m_localFile(0),
Chris@208 123 m_ok(false),
Chris@210 124 m_lastStatus(0),
Chris@316 125 m_remote(isRemote(url.toString())),
Chris@208 126 m_done(false),
Chris@316 127 m_leaveLocalFile(false),
Chris@392 128 m_reporter(reporter),
Chris@316 129 m_refCounted(false)
Chris@208 130 {
Chris@327 131 #ifdef DEBUG_FILE_SOURCE
Chris@317 132 std::cerr << "FileSource::FileSource(" << url.toString().toStdString() << ") [as url]" << std::endl;
Chris@327 133 #endif
Chris@316 134
Chris@316 135 if (!canHandleScheme(m_url)) {
Chris@317 136 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString().toStdString() << "\"" << std::endl;
Chris@316 137 m_errorString = tr("Unsupported scheme in URL");
Chris@208 138 return;
Chris@208 139 }
Chris@208 140
Chris@357 141 init();
Chris@497 142
Chris@504 143 #ifdef DEBUG_FILE_SOURCE
Chris@497 144 std::cerr << "FileSource::FileSource(url) exiting" << std::endl;
Chris@504 145 #endif
Chris@316 146 }
Chris@304 147
Chris@317 148 FileSource::FileSource(const FileSource &rf) :
Chris@316 149 QObject(),
Chris@316 150 m_url(rf.m_url),
Chris@316 151 m_ftp(0),
Chris@316 152 m_http(0),
Chris@316 153 m_localFile(0),
Chris@316 154 m_ok(rf.m_ok),
Chris@316 155 m_lastStatus(rf.m_lastStatus),
Chris@316 156 m_remote(rf.m_remote),
Chris@316 157 m_done(false),
Chris@316 158 m_leaveLocalFile(false),
Chris@392 159 m_reporter(rf.m_reporter),
Chris@316 160 m_refCounted(false)
Chris@316 161 {
Chris@327 162 #ifdef DEBUG_FILE_SOURCE
Chris@317 163 std::cerr << "FileSource::FileSource(" << m_url.toString().toStdString() << ") [copy ctor]" << std::endl;
Chris@327 164 #endif
Chris@304 165
Chris@316 166 if (!canHandleScheme(m_url)) {
Chris@317 167 std::cerr << "FileSource::FileSource: ERROR: Unsupported scheme in URL \"" << m_url.toString().toStdString() << "\"" << std::endl;
Chris@316 168 m_errorString = tr("Unsupported scheme in URL");
Chris@304 169 return;
Chris@304 170 }
Chris@304 171
Chris@469 172 if (!isRemote()) {
Chris@316 173 m_localFilename = rf.m_localFilename;
Chris@316 174 } else {
Chris@316 175 QMutexLocker locker(&m_mapMutex);
Chris@327 176 #ifdef DEBUG_FILE_SOURCE
Chris@317 177 std::cerr << "FileSource::FileSource(copy ctor): ref count is "
Chris@316 178 << m_refCountMap[m_url] << std::endl;
Chris@327 179 #endif
Chris@316 180 if (m_refCountMap[m_url] > 0) {
Chris@316 181 m_refCountMap[m_url]++;
Chris@327 182 #ifdef DEBUG_FILE_SOURCE
Chris@316 183 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 184 #endif
Chris@316 185 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 186 m_refCounted = true;
Chris@316 187 } else {
Chris@316 188 m_ok = false;
Chris@316 189 m_lastStatus = 404;
Chris@316 190 }
Chris@316 191 }
Chris@316 192
Chris@316 193 m_done = true;
Chris@497 194
Chris@504 195 #ifdef DEBUG_FILE_SOURCE
Chris@497 196 std::cerr << "FileSource::FileSource(copy ctor) exiting" << std::endl;
Chris@504 197 #endif
Chris@316 198 }
Chris@316 199
Chris@317 200 FileSource::~FileSource()
Chris@316 201 {
Chris@327 202 #ifdef DEBUG_FILE_SOURCE
Chris@317 203 std::cerr << "FileSource(" << m_url.toString().toStdString() << ")::~FileSource" << std::endl;
Chris@327 204 #endif
Chris@316 205
Chris@316 206 cleanup();
Chris@316 207
Chris@469 208 if (isRemote() && !m_leaveLocalFile) deleteCacheFile();
Chris@316 209 }
Chris@316 210
Chris@316 211 void
Chris@357 212 FileSource::init()
Chris@316 213 {
Chris@316 214 if (!isRemote()) {
Chris@355 215 #ifdef DEBUG_FILE_SOURCE
Chris@355 216 std::cerr << "FileSource::init: Not a remote URL" << std::endl;
Chris@355 217 #endif
Chris@355 218 bool literal = false;
Chris@316 219 m_localFilename = m_url.toLocalFile();
Chris@342 220 if (m_localFilename == "") {
Chris@342 221 // QUrl may have mishandled the scheme (e.g. in a DOS path)
Chris@342 222 m_localFilename = m_url.toString();
Chris@355 223 literal = true;
Chris@342 224 }
Chris@439 225 m_localFilename = QFileInfo(m_localFilename).absoluteFilePath();
Chris@439 226
Chris@355 227 #ifdef DEBUG_FILE_SOURCE
Chris@355 228 std::cerr << "FileSource::init: URL translates to local filename \""
Chris@355 229 << m_localFilename.toStdString() << "\"" << std::endl;
Chris@355 230 #endif
Chris@316 231 m_ok = true;
Chris@355 232 m_lastStatus = 200;
Chris@355 233
Chris@316 234 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 235 if (literal) {
Chris@355 236 m_lastStatus = 404;
Chris@355 237 } else {
Chris@355 238 // Again, QUrl may have been mistreating us --
Chris@355 239 // e.g. dropping a part that looks like query data
Chris@355 240 m_localFilename = m_url.toString();
Chris@355 241 literal = true;
Chris@355 242 if (!QFileInfo(m_localFilename).exists()) {
Chris@355 243 m_lastStatus = 404;
Chris@355 244 }
Chris@355 245 }
Chris@316 246 }
Chris@355 247
Chris@316 248 m_done = true;
Chris@316 249 return;
Chris@316 250 }
Chris@316 251
Chris@316 252 if (createCacheFile()) {
Chris@327 253 #ifdef DEBUG_FILE_SOURCE
Chris@469 254 std::cerr << "FileSource::init: Already have this one" << std::endl;
Chris@327 255 #endif
Chris@316 256 m_ok = true;
Chris@316 257 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 258 m_lastStatus = 404;
Chris@316 259 } else {
Chris@316 260 m_lastStatus = 200;
Chris@316 261 }
Chris@316 262 m_done = true;
Chris@316 263 return;
Chris@316 264 }
Chris@316 265
Chris@208 266 if (m_localFilename == "") return;
Chris@208 267 m_localFile = new QFile(m_localFilename);
Chris@208 268 m_localFile->open(QFile::WriteOnly);
Chris@208 269
Chris@316 270 QString scheme = m_url.scheme().toLower();
Chris@316 271
Chris@327 272 #ifdef DEBUG_FILE_SOURCE
Chris@317 273 std::cerr << "FileSource::init: Don't have local copy of \""
Chris@469 274 << m_url.toString().toStdString() << "\", retrieving" << std::endl;
Chris@327 275 #endif
Chris@208 276
Chris@208 277 if (scheme == "http") {
Chris@316 278 initHttp();
Chris@357 279 std::cerr << "FileSource: initHttp succeeded" << std::endl;
Chris@208 280 } else if (scheme == "ftp") {
Chris@316 281 initFtp();
Chris@316 282 } else {
Chris@316 283 m_remote = false;
Chris@316 284 m_ok = false;
Chris@208 285 }
Chris@208 286
Chris@208 287 if (m_ok) {
Chris@316 288
Chris@316 289 QMutexLocker locker(&m_mapMutex);
Chris@316 290
Chris@316 291 if (m_refCountMap[m_url] > 0) {
Chris@316 292 // someone else has been doing the same thing at the same time,
Chris@316 293 // but has got there first
Chris@316 294 cleanup();
Chris@316 295 m_refCountMap[m_url]++;
Chris@327 296 #ifdef DEBUG_FILE_SOURCE
Chris@317 297 std::cerr << "FileSource::init: Another FileSource has got there first, abandoning our download and using theirs" << std::endl;
Chris@327 298 #endif
Chris@316 299 m_localFilename = m_remoteLocalMap[m_url];
Chris@316 300 m_refCounted = true;
Chris@316 301 m_ok = true;
Chris@316 302 if (!QFileInfo(m_localFilename).exists()) {
Chris@316 303 m_lastStatus = 404;
Chris@316 304 }
Chris@316 305 m_done = true;
Chris@316 306 return;
Chris@316 307 }
Chris@304 308
Chris@304 309 m_remoteLocalMap[m_url] = m_localFilename;
Chris@304 310 m_refCountMap[m_url]++;
Chris@316 311 m_refCounted = true;
Chris@304 312
Chris@392 313 if (m_reporter) {
Chris@392 314 m_reporter->setMessage
Chris@392 315 (tr("Downloading %1...").arg(m_url.toString()));
Chris@392 316 connect(m_reporter, SIGNAL(cancelled()), this, SLOT(cancelled()));
Chris@357 317 connect(this, SIGNAL(progress(int)),
Chris@392 318 m_reporter, SLOT(setProgress(int)));
Chris@316 319 }
Chris@208 320 }
Chris@208 321 }
Chris@208 322
Chris@316 323 void
Chris@317 324 FileSource::initHttp()
Chris@208 325 {
Chris@316 326 m_ok = true;
Chris@497 327 int port = m_url.port();
Chris@497 328 m_http = new QHttp(m_url.host(), port < 0 ? 80 : port);
Chris@316 329 connect(m_http, SIGNAL(done(bool)), this, SLOT(done(bool)));
Chris@316 330 connect(m_http, SIGNAL(dataReadProgress(int, int)),
Chris@316 331 this, SLOT(dataReadProgress(int, int)));
Chris@316 332 connect(m_http, SIGNAL(responseHeaderReceived(const QHttpResponseHeader &)),
Chris@316 333 this, SLOT(httpResponseHeaderReceived(const QHttpResponseHeader &)));
Chris@316 334
Chris@316 335 // I don't quite understand this. url.path() returns a path
Chris@316 336 // without percent encoding; for example, spaces appear as
Chris@316 337 // literal spaces. This generally won't work if sent to the
Chris@316 338 // server directly. You can retrieve a correctly encoded URL
Chris@316 339 // from QUrl using url.toEncoded(), but that gives you the
Chris@316 340 // whole URL; there doesn't seem to be any way to retrieve
Chris@316 341 // only an encoded path. Furthermore there doesn't seem to be
Chris@316 342 // any way to convert a retrieved path into an encoded path
Chris@316 343 // without explicitly specifying that you don't want the path
Chris@316 344 // separators ("/") to be encoded. (Besides being painful to
Chris@316 345 // manage, I don't see how this can work correctly in any case
Chris@316 346 // where a percent-encoded "/" is supposed to appear within a
Chris@316 347 // path element?) There also seems to be no way to retrieve
Chris@316 348 // the path plus query string, i.e. everything that I need to
Chris@316 349 // send to the HTTP server. And no way for QHttp to take a
Chris@316 350 // QUrl argument. I'm obviously missing something.
Chris@316 351
Chris@316 352 // So, two ways to do this: query the bits from the URL,
Chris@316 353 // encode them individually, and glue them back together
Chris@316 354 // again...
Chris@316 355 /*
Chris@316 356 QString path = QUrl::toPercentEncoding(m_url.path(), "/");
Chris@316 357 QList<QPair<QString, QString> > query = m_url.queryItems();
Chris@316 358 if (!query.empty()) {
Chris@316 359 QStringList q2;
Chris@316 360 for (QList<QPair<QString, QString> >::iterator i = query.begin();
Chris@316 361 i != query.end(); ++i) {
Chris@316 362 q2.push_back(QString("%1=%3")
Chris@316 363 .arg(QString(QUrl::toPercentEncoding(i->first)))
Chris@316 364 .arg(QString(QUrl::toPercentEncoding(i->second))));
Chris@316 365 }
Chris@316 366 path = QString("%1%2%3")
Chris@316 367 .arg(path).arg("?")
Chris@316 368 .arg(q2.join("&"));
Chris@316 369 }
Chris@316 370 */
Chris@316 371
Chris@316 372 // ...or, much simpler but relying on knowledge about the
Chris@316 373 // scheme://host/path/path/query etc format of the URL, we can
Chris@316 374 // get the whole URL ready-encoded and then split it on "/" as
Chris@316 375 // appropriate...
Chris@316 376
Chris@316 377 QString path = "/" + QString(m_url.toEncoded()).section('/', 3);
Chris@316 378
Chris@327 379 #ifdef DEBUG_FILE_SOURCE
Chris@317 380 std::cerr << "FileSource: path is \""
Chris@316 381 << path.toStdString() << "\"" << std::endl;
Chris@327 382 #endif
Chris@316 383
Chris@316 384 m_http->get(path, m_localFile);
Chris@316 385 }
Chris@316 386
Chris@316 387 void
Chris@317 388 FileSource::initFtp()
Chris@316 389 {
Chris@316 390 m_ok = true;
Chris@316 391 m_ftp = new QFtp;
Chris@316 392 connect(m_ftp, SIGNAL(done(bool)), this, SLOT(done(bool)));
Chris@316 393 connect(m_ftp, SIGNAL(commandFinished(int, bool)),
Chris@316 394 this, SLOT(ftpCommandFinished(int, bool)));
Chris@316 395 connect(m_ftp, SIGNAL(dataTransferProgress(qint64, qint64)),
Chris@316 396 this, SLOT(dataTransferProgress(qint64, qint64)));
Chris@316 397 m_ftp->connectToHost(m_url.host(), m_url.port(21));
Chris@316 398
Chris@316 399 QString username = m_url.userName();
Chris@316 400 if (username == "") {
Chris@316 401 username = "anonymous";
Chris@316 402 }
Chris@316 403
Chris@316 404 QString password = m_url.password();
Chris@316 405 if (password == "") {
Chris@316 406 password = QString("%1@%2").arg(getenv("USER")).arg(getenv("HOST"));
Chris@316 407 }
Chris@316 408
Chris@316 409 m_ftp->login(username, password);
Chris@316 410
Chris@316 411 QString dirpath = m_url.path().section('/', 0, -2);
Chris@316 412 QString filename = m_url.path().section('/', -1);
Chris@316 413
Chris@316 414 if (dirpath == "") dirpath = "/";
Chris@316 415 m_ftp->cd(dirpath);
Chris@316 416 m_ftp->get(filename, m_localFile);
Chris@211 417 }
Chris@211 418
Chris@211 419 void
Chris@317 420 FileSource::cleanup()
Chris@211 421 {
Chris@470 422 if (m_done) {
Chris@470 423 delete m_localFile; // does not actually delete the file
Chris@470 424 m_localFile = 0;
Chris@470 425 }
Chris@211 426 m_done = true;
Chris@214 427 if (m_http) {
Chris@287 428 QHttp *h = m_http;
Chris@214 429 m_http = 0;
Chris@287 430 h->abort();
Chris@287 431 h->deleteLater();
Chris@214 432 }
Chris@214 433 if (m_ftp) {
Chris@287 434 QFtp *f = m_ftp;
Chris@214 435 m_ftp = 0;
Chris@287 436 f->abort();
Chris@287 437 f->deleteLater();
Chris@214 438 }
Chris@470 439 if (m_localFile) {
Chris@470 440 delete m_localFile; // does not actually delete the file
Chris@470 441 m_localFile = 0;
Chris@470 442 }
Chris@208 443 }
Chris@208 444
Chris@208 445 bool
Chris@317 446 FileSource::isRemote(QString fileOrUrl)
Chris@304 447 {
Chris@342 448 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@316 449 QString scheme = QUrl(fileOrUrl).scheme().toLower();
Chris@342 450 if (scheme == "" || scheme == "file" || scheme.length() == 1) return false;
Chris@342 451 return true;
Chris@304 452 }
Chris@304 453
Chris@304 454 bool
Chris@317 455 FileSource::canHandleScheme(QUrl url)
Chris@208 456 {
Chris@342 457 // Note that a "scheme" with length 1 is probably a DOS drive letter
Chris@208 458 QString scheme = url.scheme().toLower();
Chris@316 459 return (scheme == "http" || scheme == "ftp" ||
Chris@342 460 scheme == "file" || scheme == "" || scheme.length() == 1);
Chris@208 461 }
Chris@208 462
Chris@210 463 bool
Chris@317 464 FileSource::isAvailable()
Chris@210 465 {
Chris@316 466 waitForStatus();
Chris@211 467 bool available = true;
Chris@211 468 if (!m_ok) available = false;
Chris@211 469 else available = (m_lastStatus / 100 == 2);
Chris@327 470 #ifdef DEBUG_FILE_SOURCE
Chris@317 471 std::cerr << "FileSource::isAvailable: " << (available ? "yes" : "no")
Chris@211 472 << std::endl;
Chris@327 473 #endif
Chris@211 474 return available;
Chris@210 475 }
Chris@210 476
Chris@208 477 void
Chris@317 478 FileSource::waitForStatus()
Chris@316 479 {
Chris@316 480 while (m_ok && (!m_done && m_lastStatus == 0)) {
Chris@316 481 // std::cerr << "waitForStatus: processing (last status " << m_lastStatus << ")" << std::endl;
Chris@392 482 QCoreApplication::processEvents();
Chris@316 483 }
Chris@316 484 }
Chris@316 485
Chris@316 486 void
Chris@317 487 FileSource::waitForData()
Chris@208 488 {
Chris@211 489 while (m_ok && !m_done) {
Chris@357 490 // std::cerr << "FileSource::waitForData: calling QApplication::processEvents" << std::endl;
Chris@392 491 QCoreApplication::processEvents();
Chris@497 492 usleep(10000);
Chris@208 493 }
Chris@208 494 }
Chris@208 495
Chris@316 496 void
Chris@317 497 FileSource::setLeaveLocalFile(bool leave)
Chris@316 498 {
Chris@316 499 m_leaveLocalFile = leave;
Chris@316 500 }
Chris@316 501
Chris@208 502 bool
Chris@317 503 FileSource::isOK() const
Chris@208 504 {
Chris@208 505 return m_ok;
Chris@208 506 }
Chris@208 507
Chris@208 508 bool
Chris@317 509 FileSource::isDone() const
Chris@208 510 {
Chris@208 511 return m_done;
Chris@208 512 }
Chris@208 513
Chris@316 514 bool
Chris@317 515 FileSource::isRemote() const
Chris@316 516 {
Chris@316 517 return m_remote;
Chris@316 518 }
Chris@316 519
Chris@316 520 QString
Chris@317 521 FileSource::getLocation() const
Chris@316 522 {
Chris@316 523 return m_url.toString();
Chris@316 524 }
Chris@316 525
Chris@208 526 QString
Chris@317 527 FileSource::getLocalFilename() const
Chris@208 528 {
Chris@208 529 return m_localFilename;
Chris@208 530 }
Chris@208 531
Chris@208 532 QString
Chris@317 533 FileSource::getContentType() const
Chris@316 534 {
Chris@316 535 return m_contentType;
Chris@316 536 }
Chris@316 537
Chris@316 538 QString
Chris@317 539 FileSource::getExtension() const
Chris@316 540 {
Chris@316 541 if (m_localFilename != "") {
Chris@316 542 return QFileInfo(m_localFilename).suffix().toLower();
Chris@316 543 } else {
Chris@316 544 return QFileInfo(m_url.toLocalFile()).suffix().toLower();
Chris@316 545 }
Chris@316 546 }
Chris@316 547
Chris@316 548 QString
Chris@317 549 FileSource::getErrorString() const
Chris@208 550 {
Chris@208 551 return m_errorString;
Chris@208 552 }
Chris@208 553
Chris@208 554 void
Chris@317 555 FileSource::dataReadProgress(int done, int total)
Chris@208 556 {
Chris@208 557 dataTransferProgress(done, total);
Chris@208 558 }
Chris@208 559
Chris@208 560 void
Chris@317 561 FileSource::httpResponseHeaderReceived(const QHttpResponseHeader &resp)
Chris@210 562 {
Chris@497 563 std::cerr << "FileSource::httpResponseHeaderReceived" << std::endl;
Chris@497 564
Chris@497 565 if (resp.statusCode() / 100 == 3) {
Chris@496 566 QString location = resp.value("Location");
Chris@496 567 #ifdef DEBUG_FILE_SOURCE
Chris@496 568 std::cerr << "FileSource::responseHeaderReceived: redirect to \""
Chris@496 569 << location.toStdString() << "\" received" << std::endl;
Chris@496 570 #endif
Chris@496 571 if (location != "") {
Chris@496 572 QUrl newUrl(location);
Chris@496 573 if (newUrl != m_url) {
Chris@497 574 cleanup();
Chris@497 575 deleteCacheFile();
Chris@496 576 m_url = newUrl;
Chris@497 577 m_localFile = 0;
Chris@496 578 m_lastStatus = 0;
Chris@497 579 m_done = false;
Chris@497 580 m_refCounted = false;
Chris@496 581 init();
Chris@496 582 return;
Chris@496 583 }
Chris@496 584 }
Chris@496 585 }
Chris@497 586
Chris@497 587 m_lastStatus = resp.statusCode();
Chris@210 588 if (m_lastStatus / 100 >= 4) {
Chris@210 589 m_errorString = QString("%1 %2")
Chris@210 590 .arg(resp.statusCode()).arg(resp.reasonPhrase());
Chris@327 591 #ifdef DEBUG_FILE_SOURCE
Chris@317 592 std::cerr << "FileSource::responseHeaderReceived: "
Chris@211 593 << m_errorString.toStdString() << std::endl;
Chris@327 594 #endif
Chris@211 595 } else {
Chris@327 596 #ifdef DEBUG_FILE_SOURCE
Chris@317 597 std::cerr << "FileSource::responseHeaderReceived: "
Chris@211 598 << m_lastStatus << std::endl;
Chris@327 599 #endif
Chris@315 600 if (resp.hasContentType()) m_contentType = resp.contentType();
Chris@325 601 }
Chris@325 602 emit statusAvailable();
Chris@210 603 }
Chris@210 604
Chris@210 605 void
Chris@317 606 FileSource::ftpCommandFinished(int id, bool error)
Chris@214 607 {
Chris@327 608 #ifdef DEBUG_FILE_SOURCE
Chris@317 609 std::cerr << "FileSource::ftpCommandFinished(" << id << ", " << error << ")" << std::endl;
Chris@327 610 #endif
Chris@214 611
Chris@214 612 if (!m_ftp) return;
Chris@214 613
Chris@214 614 QFtp::Command command = m_ftp->currentCommand();
Chris@214 615
Chris@214 616 if (!error) {
Chris@327 617 #ifdef DEBUG_FILE_SOURCE
Chris@317 618 std::cerr << "FileSource::ftpCommandFinished: success for command "
Chris@214 619 << command << std::endl;
Chris@327 620 #endif
Chris@214 621 return;
Chris@214 622 }
Chris@214 623
Chris@214 624 if (command == QFtp::ConnectToHost) {
Chris@214 625 m_errorString = tr("Failed to connect to FTP server");
Chris@214 626 } else if (command == QFtp::Login) {
Chris@214 627 m_errorString = tr("Login failed");
Chris@214 628 } else if (command == QFtp::Cd) {
Chris@214 629 m_errorString = tr("Failed to change to correct directory");
Chris@214 630 } else if (command == QFtp::Get) {
Chris@214 631 m_errorString = tr("FTP download aborted");
Chris@214 632 }
Chris@214 633
Chris@214 634 m_lastStatus = 400; // for done()
Chris@214 635 }
Chris@214 636
Chris@214 637 void
Chris@317 638 FileSource::dataTransferProgress(qint64 done, qint64 total)
Chris@208 639 {
Chris@208 640 int percent = int((double(done) / double(total)) * 100.0 - 0.1);
Chris@208 641 emit progress(percent);
Chris@210 642 }
Chris@210 643
Chris@210 644 void
Chris@317 645 FileSource::cancelled()
Chris@210 646 {
Chris@210 647 m_done = true;
Chris@316 648 cleanup();
Chris@316 649
Chris@210 650 m_ok = false;
Chris@210 651 m_errorString = tr("Download cancelled");
Chris@208 652 }
Chris@208 653
Chris@208 654 void
Chris@317 655 FileSource::done(bool error)
Chris@208 656 {
Chris@327 657 emit progress(100);
Chris@327 658
Chris@327 659 #ifdef DEBUG_FILE_SOURCE
Chris@317 660 std::cerr << "FileSource::done(" << error << ")" << std::endl;
Chris@327 661 #endif
Chris@211 662
Chris@211 663 if (m_done) return;
Chris@211 664
Chris@208 665 if (error) {
Chris@208 666 if (m_http) {
Chris@208 667 m_errorString = m_http->errorString();
Chris@208 668 } else if (m_ftp) {
Chris@208 669 m_errorString = m_ftp->errorString();
Chris@208 670 }
Chris@208 671 }
Chris@208 672
Chris@210 673 if (m_lastStatus / 100 >= 4) {
Chris@211 674 error = true;
Chris@210 675 }
Chris@210 676
Chris@211 677 cleanup();
Chris@208 678
Chris@211 679 if (!error) {
Chris@208 680 QFileInfo fi(m_localFilename);
Chris@208 681 if (!fi.exists()) {
Chris@208 682 m_errorString = tr("Failed to create local file %1").arg(m_localFilename);
Chris@211 683 error = true;
Chris@208 684 } else if (fi.size() == 0) {
Chris@208 685 m_errorString = tr("File contains no data!");
Chris@211 686 error = true;
Chris@208 687 }
Chris@208 688 }
Chris@211 689
Chris@211 690 if (error) {
Chris@327 691 #ifdef DEBUG_FILE_SOURCE
Chris@469 692 std::cerr << "FileSource::done: error is " << error << ", deleting cache file" << std::endl;
Chris@327 693 #endif
Chris@316 694 deleteCacheFile();
Chris@211 695 }
Chris@211 696
Chris@211 697 m_ok = !error;
Chris@211 698 m_done = true;
Chris@304 699 emit ready();
Chris@211 700 }
Chris@211 701
Chris@211 702 void
Chris@317 703 FileSource::deleteCacheFile()
Chris@211 704 {
Chris@327 705 #ifdef DEBUG_FILE_SOURCE
Chris@317 706 std::cerr << "FileSource::deleteCacheFile(\"" << m_localFilename.toStdString() << "\")" << std::endl;
Chris@327 707 #endif
Chris@211 708
Chris@211 709 cleanup();
Chris@211 710
Chris@316 711 if (m_localFilename == "") {
Chris@316 712 return;
Chris@316 713 }
Chris@211 714
Chris@316 715 if (!isRemote()) {
Chris@327 716 #ifdef DEBUG_FILE_SOURCE
Chris@316 717 std::cerr << "not a cache file" << std::endl;
Chris@327 718 #endif
Chris@316 719 return;
Chris@316 720 }
Chris@316 721
Chris@316 722 if (m_refCounted) {
Chris@304 723
Chris@304 724 QMutexLocker locker(&m_mapMutex);
Chris@316 725 m_refCounted = false;
Chris@304 726
Chris@304 727 if (m_refCountMap[m_url] > 0) {
Chris@304 728 m_refCountMap[m_url]--;
Chris@327 729 #ifdef DEBUG_FILE_SOURCE
Chris@316 730 std::cerr << "reduced ref count to " << m_refCountMap[m_url] << std::endl;
Chris@327 731 #endif
Chris@304 732 if (m_refCountMap[m_url] > 0) {
Chris@304 733 m_done = true;
Chris@304 734 return;
Chris@304 735 }
Chris@304 736 }
Chris@304 737 }
Chris@304 738
Chris@211 739 m_fileCreationMutex.lock();
Chris@211 740
Chris@211 741 if (!QFile(m_localFilename).remove()) {
Chris@469 742 #ifdef DEBUG_FILE_SOURCE
Chris@317 743 std::cerr << "FileSource::deleteCacheFile: ERROR: Failed to delete file \"" << m_localFilename.toStdString() << "\"" << std::endl;
Chris@469 744 #endif
Chris@211 745 } else {
Chris@327 746 #ifdef DEBUG_FILE_SOURCE
Chris@317 747 std::cerr << "FileSource::deleteCacheFile: Deleted cache file \"" << m_localFilename.toStdString() << "\"" << std::endl;
Chris@327 748 #endif
Chris@211 749 m_localFilename = "";
Chris@211 750 }
Chris@211 751
Chris@211 752 m_fileCreationMutex.unlock();
Chris@211 753
Chris@208 754 m_done = true;
Chris@208 755 }
Chris@208 756
Chris@316 757 bool
Chris@317 758 FileSource::createCacheFile()
Chris@208 759 {
Chris@316 760 {
Chris@316 761 QMutexLocker locker(&m_mapMutex);
Chris@316 762
Chris@327 763 #ifdef DEBUG_FILE_SOURCE
Chris@317 764 std::cerr << "FileSource::createCacheFile: refcount is " << m_refCountMap[m_url] << std::endl;
Chris@327 765 #endif
Chris@316 766
Chris@316 767 if (m_refCountMap[m_url] > 0) {
Chris@316 768 m_refCountMap[m_url]++;
Chris@316 769 m_localFilename = m_remoteLocalMap[m_url];
Chris@327 770 #ifdef DEBUG_FILE_SOURCE
Chris@316 771 std::cerr << "raised it to " << m_refCountMap[m_url] << std::endl;
Chris@327 772 #endif
Chris@316 773 m_refCounted = true;
Chris@316 774 return true;
Chris@316 775 }
Chris@316 776 }
Chris@316 777
Chris@208 778 QDir dir;
Chris@208 779 try {
Chris@208 780 dir = TempDirectory::getInstance()->getSubDirectoryPath("download");
Chris@208 781 } catch (DirectoryCreationFailed f) {
Chris@327 782 #ifdef DEBUG_FILE_SOURCE
Chris@317 783 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create temporary directory: " << f.what() << std::endl;
Chris@327 784 #endif
Chris@208 785 return "";
Chris@208 786 }
Chris@208 787
Chris@316 788 QString filepart = m_url.path().section('/', -1, -1,
Chris@316 789 QString::SectionSkipEmpty);
Chris@208 790
Chris@457 791 QString extension = "";
Chris@457 792 if (filepart.contains('.')) extension = filepart.section('.', -1);
Chris@457 793
Chris@208 794 QString base = filepart;
Chris@208 795 if (extension != "") {
Chris@208 796 base = base.left(base.length() - extension.length() - 1);
Chris@208 797 }
Chris@208 798 if (base == "") base = "remote";
Chris@208 799
Chris@208 800 QString filename;
Chris@208 801
Chris@208 802 if (extension == "") {
Chris@208 803 filename = base;
Chris@208 804 } else {
Chris@208 805 filename = QString("%1.%2").arg(base).arg(extension);
Chris@208 806 }
Chris@208 807
Chris@208 808 QString filepath(dir.filePath(filename));
Chris@208 809
Chris@327 810 #ifdef DEBUG_FILE_SOURCE
Chris@317 811 std::cerr << "FileSource::createCacheFile: URL is \"" << m_url.toString().toStdString() << "\", dir is \"" << dir.path().toStdString() << "\", base \"" << base.toStdString() << "\", extension \"" << extension.toStdString() << "\", filebase \"" << filename.toStdString() << "\", filename \"" << filepath.toStdString() << "\"" << std::endl;
Chris@327 812 #endif
Chris@208 813
Chris@316 814 QMutexLocker fcLocker(&m_fileCreationMutex);
Chris@316 815
Chris@208 816 ++m_count;
Chris@208 817
Chris@208 818 if (QFileInfo(filepath).exists() ||
Chris@208 819 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 820
Chris@327 821 #ifdef DEBUG_FILE_SOURCE
Chris@317 822 std::cerr << "FileSource::createCacheFile: Failed to create local file \""
Chris@208 823 << filepath.toStdString() << "\" for URL \""
Chris@316 824 << m_url.toString().toStdString() << "\" (or file already exists): appending suffix instead" << std::endl;
Chris@327 825 #endif
Chris@208 826
Chris@208 827 if (extension == "") {
Chris@208 828 filename = QString("%1_%2").arg(base).arg(m_count);
Chris@208 829 } else {
Chris@208 830 filename = QString("%1_%2.%3").arg(base).arg(m_count).arg(extension);
Chris@208 831 }
Chris@208 832 filepath = dir.filePath(filename);
Chris@208 833
Chris@208 834 if (QFileInfo(filepath).exists() ||
Chris@208 835 !QFile(filepath).open(QFile::WriteOnly)) {
Chris@208 836
Chris@327 837 #ifdef DEBUG_FILE_SOURCE
Chris@317 838 std::cerr << "FileSource::createCacheFile: ERROR: Failed to create local file \""
Chris@208 839 << filepath.toStdString() << "\" for URL \""
Chris@316 840 << m_url.toString().toStdString() << "\" (or file already exists)" << std::endl;
Chris@327 841 #endif
Chris@208 842
Chris@208 843 return "";
Chris@208 844 }
Chris@208 845 }
Chris@208 846
Chris@327 847 #ifdef DEBUG_FILE_SOURCE
Chris@317 848 std::cerr << "FileSource::createCacheFile: url "
Chris@316 849 << m_url.toString().toStdString() << " -> local filename "
Chris@316 850 << filepath.toStdString() << std::endl;
Chris@327 851 #endif
Chris@316 852
Chris@316 853 m_localFilename = filepath;
Chris@208 854
Chris@316 855 return false;
Chris@208 856 }
Chris@327 857