annotate data/fileio/CachedFile.cpp @ 1833:21c792334c2e sensible-delimited-data-strings

Rewrite all the DelimitedDataString stuff so as to return vectors of individual cell strings rather than having the classes add the delimiters themselves. Rename accordingly to names based on StringExport. Take advantage of this in the CSV writer code so as to properly quote cells that contain delimiter characters.
author Chris Cannam
date Fri, 03 Apr 2020 17:11:05 +0100
parents f2fcb3ed51fa
children
rev   line source
Chris@465 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@465 2
Chris@465 3 /*
Chris@465 4 Sonic Visualiser
Chris@465 5 An audio file viewer and annotation editor.
Chris@465 6 Centre for Digital Music, Queen Mary, University of London.
Chris@465 7 This file copyright 2008 QMUL.
Chris@465 8
Chris@465 9 This program is free software; you can redistribute it and/or
Chris@465 10 modify it under the terms of the GNU General Public License as
Chris@465 11 published by the Free Software Foundation; either version 2 of the
Chris@465 12 License, or (at your option) any later version. See the file
Chris@465 13 COPYING included with this distribution for more information.
Chris@465 14 */
Chris@465 15
Chris@465 16 #include "CachedFile.h"
Chris@465 17
Chris@465 18 #include "base/TempDirectory.h"
Chris@465 19 #include "base/ProgressReporter.h"
Chris@465 20 #include "base/Exceptions.h"
Chris@465 21
Chris@466 22 #include "FileSource.h"
Chris@466 23
Chris@465 24 #include <QFileInfo>
Chris@465 25 #include <QSettings>
Chris@465 26 #include <QVariant>
Chris@465 27 #include <QMap>
Chris@465 28 #include <QDir>
Chris@465 29 #include <QCryptographicHash>
Chris@465 30
Chris@481 31 #include "base/Profiler.h"
Chris@481 32
Chris@466 33 #include <iostream>
Chris@466 34
Chris@481 35 CachedFile::OriginLocalFilenameMap
Chris@481 36 CachedFile::m_knownGoodCaches;
Chris@481 37
Chris@465 38 QString
Chris@465 39 CachedFile::getLocalFilenameFor(QUrl url)
Chris@465 40 {
Chris@481 41 Profiler p("CachedFile::getLocalFilenameFor");
Chris@481 42
Chris@465 43 QDir dir(getCacheDirectory());
Chris@465 44
Chris@465 45 QString filename =
Chris@465 46 QString::fromLocal8Bit
Chris@465 47 (QCryptographicHash::hash(url.toString().toLocal8Bit(),
Chris@465 48 QCryptographicHash::Sha1).toHex());
Chris@465 49
Chris@465 50 return dir.filePath(filename);
Chris@465 51 }
Chris@465 52
Chris@465 53 QString
Chris@465 54 CachedFile::getCacheDirectory()
Chris@465 55 {
Chris@465 56 QDir dir = TempDirectory::getInstance()->getContainingPath();
Chris@465 57
Chris@465 58 QString cacheDirName("cache");
Chris@465 59
Chris@465 60 QFileInfo fi(dir.filePath(cacheDirName));
Chris@465 61
Chris@465 62 if ((fi.exists() && !fi.isDir()) ||
Chris@465 63 (!fi.exists() && !dir.mkdir(cacheDirName))) {
Chris@465 64
Chris@465 65 throw DirectoryCreationFailed(fi.filePath());
Chris@465 66 }
Chris@465 67
Chris@465 68 return fi.filePath();
Chris@465 69 }
Chris@465 70
Chris@520 71 CachedFile::CachedFile(QString origin,
Chris@520 72 ProgressReporter *reporter,
Chris@520 73 QString preferredContentType) :
Chris@468 74 m_origin(origin),
Chris@520 75 m_preferredContentType(preferredContentType),
Chris@466 76 m_reporter(reporter),
Chris@465 77 m_ok(false)
Chris@465 78 {
Chris@481 79 Profiler p("CachedFile::CachedFile[1]");
Chris@481 80
Chris@690 81 SVDEBUG << "CachedFile::CachedFile: origin is \""
Chris@687 82 << origin << "\"" << endl;
Chris@691 83 checkFile();
Chris@467 84 }
Chris@467 85
Chris@520 86 CachedFile::CachedFile(QUrl url,
Chris@520 87 ProgressReporter *reporter,
Chris@520 88 QString preferredContentType) :
Chris@468 89 m_origin(url.toString()),
Chris@520 90 m_preferredContentType(preferredContentType),
Chris@467 91 m_reporter(reporter),
Chris@467 92 m_ok(false)
Chris@467 93 {
Chris@481 94 Profiler p("CachedFile::CachedFile[2]");
Chris@481 95
Chris@690 96 SVDEBUG << "CachedFile::CachedFile: url is \""
Chris@687 97 << url.toString() << "\"" << endl;
Chris@691 98 checkFile();
Chris@467 99 }
Chris@467 100
Chris@467 101 CachedFile::~CachedFile()
Chris@467 102 {
Chris@465 103 }
Chris@465 104
Chris@465 105 bool
Chris@465 106 CachedFile::isOK() const
Chris@465 107 {
Chris@465 108 return m_ok;
Chris@465 109 }
Chris@465 110
Chris@465 111 QString
Chris@465 112 CachedFile::getLocalFilename() const
Chris@465 113 {
Chris@465 114 return m_localFilename;
Chris@465 115 }
Chris@465 116
Chris@465 117 void
Chris@691 118 CachedFile::checkFile()
Chris@465 119 {
Chris@465 120 //!!! n.b. obvious race condition here if different CachedFile
Chris@465 121 // objects for same url used in more than one thread -- need to
Chris@465 122 // lock appropriately. also consider race condition between
Chris@481 123 // separate instances of the program!
Chris@481 124
Chris@481 125 OriginLocalFilenameMap::const_iterator i = m_knownGoodCaches.find(m_origin);
Chris@481 126 if (i != m_knownGoodCaches.end()) {
Chris@481 127 m_ok = true;
Chris@481 128 m_localFilename = i->second;
Chris@481 129 return;
Chris@481 130 }
Chris@481 131
Chris@481 132 m_localFilename = getLocalFilenameFor(m_origin);
Chris@465 133
Chris@465 134 if (!QFileInfo(m_localFilename).exists()) {
Chris@690 135 SVDEBUG << "CachedFile::check: Local file does not exist, making a note that it hasn't been retrieved" << endl;
Chris@465 136 updateLastRetrieval(false); // empirically!
Chris@465 137 }
Chris@465 138
Chris@465 139 QDateTime lastRetrieval = getLastRetrieval();
Chris@465 140
Chris@465 141 if (lastRetrieval.isValid()) {
Chris@690 142 SVDEBUG << "CachedFile::check: Valid last retrieval at "
Chris@687 143 << lastRetrieval.toString() << endl;
Chris@465 144 // this will not be the case if the file is missing, after
Chris@465 145 // updateLastRetrieval(false) was called above
Chris@465 146 m_ok = true;
Chris@465 147 if (lastRetrieval.addDays(2) < QDateTime::currentDateTime()) { //!!!
Chris@690 148 SVDEBUG << "CachedFile::check: Out of date; trying to retrieve again" << endl;
Chris@465 149 // doesn't matter if retrieval fails -- we just don't
Chris@465 150 // update the last retrieval time
Chris@465 151
Chris@465 152 //!!! but we do want an additional last-attempted
Chris@465 153 // timestamp so as to ensure we aren't retrying the
Chris@465 154 // retrieval every single time if it isn't working
Chris@465 155
Chris@465 156 if (retrieve()) {
Chris@690 157 SVDEBUG << "CachedFile::check: Retrieval succeeded" << endl;
Chris@465 158 updateLastRetrieval(true);
Chris@467 159 } else {
Chris@1486 160 SVCERR << "CachedFile::check: Retrieval failed, will try again later (using existing file for now)" << endl;
Chris@467 161 }
Chris@465 162 }
Chris@465 163 } else {
Chris@690 164 SVDEBUG << "CachedFile::check: No valid last retrieval" << endl;
Chris@465 165 // there is no acceptable file
Chris@465 166 if (retrieve()) {
Chris@690 167 SVDEBUG << "CachedFile::check: Retrieval succeeded" << endl;
Chris@465 168 m_ok = true;
Chris@465 169 updateLastRetrieval(true);
Chris@465 170 } else {
Chris@1486 171 SVCERR << "CachedFile::check: Retrieval failed, remaining in invalid state" << endl;
Chris@465 172 // again, we don't need to do anything here -- the last
Chris@465 173 // retrieval timestamp is already invalid
Chris@465 174 }
Chris@465 175 }
Chris@481 176
Chris@481 177 if (m_ok) {
Chris@481 178 m_knownGoodCaches[m_origin] = m_localFilename;
Chris@481 179 }
Chris@465 180 }
Chris@465 181
Chris@465 182 bool
Chris@465 183 CachedFile::retrieve()
Chris@465 184 {
Chris@465 185 //!!! need to work by retrieving the file to another name, and
Chris@465 186 //!!! then "atomically" moving it to its proper place (I'm not
Chris@465 187 //!!! sure we can do an atomic move to replace an existing file
Chris@465 188 //!!! using Qt classes, but a plain delete then copy is probably
Chris@465 189 //!!! good enough)
Chris@465 190
Chris@520 191 FileSource fs(m_origin, m_reporter, m_preferredContentType);
Chris@465 192
Chris@466 193 if (!fs.isOK() || !fs.isAvailable()) {
Chris@690 194 SVDEBUG << "CachedFile::retrieve: ERROR: FileSource reported unavailable or failure" << endl;
Chris@466 195 return false;
Chris@466 196 }
Chris@465 197
Chris@466 198 fs.waitForData();
Chris@465 199
Chris@466 200 if (!fs.isOK()) {
Chris@690 201 SVDEBUG << "CachedFile::retrieve: ERROR: FileSource reported failure during receive" << endl;
Chris@466 202 return false;
Chris@466 203 }
Chris@466 204
Chris@466 205 QString tempName = fs.getLocalFilename();
Chris@466 206 QFile tempFile(tempName);
Chris@466 207 if (!tempFile.exists()) {
Chris@690 208 SVDEBUG << "CachedFile::retrieve: ERROR: FileSource reported success, but local temporary file \"" << tempName << "\" does not exist" << endl;
Chris@466 209 return false;
Chris@466 210 }
Chris@466 211
Chris@466 212 QFile previous(m_localFilename);
Chris@466 213 if (previous.exists()) {
Chris@466 214 if (!previous.remove()) {
Chris@1486 215 SVCERR << "CachedFile::retrieve: ERROR: Failed to remove previous copy of cached file at \"" << m_localFilename << "\"" << endl;
Chris@466 216 return false;
Chris@466 217 }
Chris@466 218 }
Chris@466 219
Chris@466 220 //!!! This is not ideal, could leave us with nothing (old file
Chris@466 221 //!!! removed, new file not able to be copied in because e.g. no
Chris@466 222 //!!! disk space left)
Chris@466 223
Chris@466 224 if (!tempFile.copy(m_localFilename)) {
Chris@1486 225 SVCERR << "CachedFile::retrieve: ERROR: Failed to copy newly retrieved file from \"" << tempName << "\" to \"" << m_localFilename << "\"" << endl;
Chris@466 226 return false;
Chris@466 227 }
Chris@466 228
Chris@690 229 SVDEBUG << "CachedFile::retrieve: Successfully copied newly retrieved file \"" << tempName << "\" to its home at \"" << m_localFilename << "\"" << endl;
Chris@468 230
Chris@466 231 return true;
Chris@465 232 }
Chris@465 233
Chris@465 234 QDateTime
Chris@465 235 CachedFile::getLastRetrieval()
Chris@465 236 {
Chris@465 237 QSettings settings;
Chris@465 238 settings.beginGroup("FileCache");
Chris@465 239
Chris@465 240 QString key("last-retrieval-times");
Chris@465 241
Chris@465 242 QMap<QString, QVariant> timeMap = settings.value(key).toMap();
Chris@465 243 QDateTime lastTime = timeMap[m_localFilename].toDateTime();
Chris@465 244
Chris@465 245 settings.endGroup();
Chris@465 246 return lastTime;
Chris@465 247 }
Chris@465 248
Chris@465 249 void
Chris@465 250 CachedFile::updateLastRetrieval(bool successful)
Chris@465 251 {
Chris@465 252 //!!! note !successful does not mean "we failed to update the
Chris@465 253 //!!! file" (and so it remains the same as before); it means "the
Chris@465 254 //!!! file is not there at all"
Chris@465 255
Chris@465 256 QSettings settings;
Chris@465 257 settings.beginGroup("FileCache");
Chris@465 258
Chris@465 259 QString key("last-retrieval-times");
Chris@465 260
Chris@465 261 QMap<QString, QVariant> timeMap = settings.value(key).toMap();
Chris@465 262
Chris@465 263 QDateTime dt;
Chris@465 264 if (successful) dt = QDateTime::currentDateTime();
Chris@465 265
Chris@465 266 timeMap[m_localFilename] = dt;
Chris@465 267 settings.setValue(key, timeMap);
Chris@465 268
Chris@465 269 settings.endGroup();
Chris@465 270 }
Chris@465 271
Chris@465 272