annotate data/fileio/MatrixFile.cpp @ 458:f60360209e5c

* Fix race condition in FFTFileCache when reading from the same FFT model from multiple threads (e.g. when applying more than one plugin at once)
author Chris Cannam
date Wed, 15 Oct 2008 12:08:02 +0000
parents 3e0f1f7bec85
children 3cc4b7cd2aa5
rev   line source
Chris@148 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@148 2
Chris@148 3 /*
Chris@148 4 Sonic Visualiser
Chris@148 5 An audio file viewer and annotation editor.
Chris@148 6 Centre for Digital Music, Queen Mary, University of London.
Chris@148 7 This file copyright 2006 Chris Cannam.
Chris@148 8
Chris@148 9 This program is free software; you can redistribute it and/or
Chris@148 10 modify it under the terms of the GNU General Public License as
Chris@148 11 published by the Free Software Foundation; either version 2 of the
Chris@148 12 License, or (at your option) any later version. See the file
Chris@148 13 COPYING included with this distribution for more information.
Chris@148 14 */
Chris@148 15
Chris@148 16 #include "MatrixFile.h"
Chris@148 17 #include "base/TempDirectory.h"
Chris@150 18 #include "system/System.h"
Chris@148 19 #include "base/Profiler.h"
Chris@148 20 #include "base/Exceptions.h"
Chris@408 21 #include "base/Thread.h"
Chris@148 22
Chris@148 23 #include <sys/types.h>
Chris@148 24 #include <sys/stat.h>
Chris@148 25 #include <fcntl.h>
Chris@148 26 #include <unistd.h>
Chris@148 27
Chris@148 28 #include <iostream>
Chris@148 29
Chris@148 30 #include <cstdio>
Chris@148 31 #include <cassert>
Chris@148 32
Chris@405 33 #include <cstdlib>
Chris@405 34
Chris@148 35 #include <QFileInfo>
Chris@148 36 #include <QDir>
Chris@148 37
Chris@455 38 //#define DEBUG_MATRIX_FILE 1
Chris@403 39 //#define DEBUG_MATRIX_FILE_READ_SET 1
Chris@148 40
Chris@148 41 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@153 42 #ifndef DEBUG_MATRIX_FILE
Chris@148 43 #define DEBUG_MATRIX_FILE 1
Chris@148 44 #endif
Chris@153 45 #endif
Chris@148 46
Chris@148 47 std::map<QString, int> MatrixFile::m_refcount;
Chris@148 48 QMutex MatrixFile::m_refcountMutex;
Chris@148 49
Chris@148 50 MatrixFile::ResizeableBitsetMap MatrixFile::m_columnBitsets;
Chris@148 51 QMutex MatrixFile::m_columnBitsetWriteMutex;
Chris@148 52
Chris@148 53 FileReadThread *MatrixFile::m_readThread = 0;
Chris@148 54
Chris@148 55 static size_t totalStorage = 0;
Chris@148 56 static size_t totalMemory = 0;
Chris@148 57 static size_t totalCount = 0;
Chris@148 58
Chris@148 59 MatrixFile::MatrixFile(QString fileBase, Mode mode,
Chris@148 60 size_t cellSize, bool eagerCache) :
Chris@148 61 m_fd(-1),
Chris@148 62 m_mode(mode),
Chris@148 63 m_flags(0),
Chris@148 64 m_fmode(0),
Chris@148 65 m_cellSize(cellSize),
Chris@148 66 m_width(0),
Chris@148 67 m_height(0),
Chris@148 68 m_headerSize(2 * sizeof(size_t)),
Chris@148 69 m_defaultCacheWidth(1024),
Chris@148 70 m_prevX(0),
Chris@148 71 m_eagerCache(eagerCache),
Chris@148 72 m_requestToken(-1),
Chris@148 73 m_spareData(0),
Chris@148 74 m_columnBitset(0)
Chris@148 75 {
Chris@148 76 Profiler profiler("MatrixFile::MatrixFile", true);
Chris@148 77
Chris@148 78 if (!m_readThread) {
Chris@148 79 m_readThread = new FileReadThread;
Chris@148 80 m_readThread->start();
Chris@148 81 }
Chris@148 82
Chris@148 83 m_cache.data = 0;
Chris@148 84
Chris@148 85 QDir tempDir(TempDirectory::getInstance()->getPath());
Chris@148 86 QString fileName(tempDir.filePath(QString("%1.mfc").arg(fileBase)));
Chris@148 87 bool newFile = !QFileInfo(fileName).exists();
Chris@148 88
Chris@148 89 if (newFile && m_mode == ReadOnly) {
Chris@148 90 std::cerr << "ERROR: MatrixFile::MatrixFile: Read-only mode "
Chris@148 91 << "specified, but cache file does not exist" << std::endl;
Chris@148 92 throw FileNotFound(fileName);
Chris@148 93 }
Chris@148 94
Chris@148 95 if (!newFile && m_mode == ReadWrite) {
Chris@148 96 std::cerr << "Note: MatrixFile::MatrixFile: Read/write mode "
Chris@148 97 << "specified, but file already exists; falling back to "
Chris@148 98 << "read-only mode" << std::endl;
Chris@148 99 m_mode = ReadOnly;
Chris@148 100 }
Chris@148 101
Chris@148 102 if (!eagerCache && m_mode == ReadOnly) {
Chris@148 103 std::cerr << "WARNING: MatrixFile::MatrixFile: Eager cacheing not "
Chris@148 104 << "specified, but file is open in read-only mode -- cache "
Chris@148 105 << "will not be used" << std::endl;
Chris@148 106 }
Chris@148 107
Chris@148 108 m_flags = 0;
Chris@148 109 m_fmode = S_IRUSR | S_IWUSR;
Chris@148 110
Chris@148 111 if (m_mode == ReadWrite) {
Chris@148 112 m_flags = O_RDWR | O_CREAT;
Chris@148 113 } else {
Chris@148 114 m_flags = O_RDONLY;
Chris@148 115 }
Chris@148 116
Chris@233 117 #ifdef _WIN32
Chris@233 118 m_flags |= O_BINARY;
Chris@233 119 #endif
Chris@233 120
Chris@148 121 #ifdef DEBUG_MATRIX_FILE
Chris@148 122 std::cerr << "MatrixFile::MatrixFile: opening " << fileName.toStdString() << "..." << std::endl;
Chris@148 123 #endif
Chris@148 124
Chris@148 125 if ((m_fd = ::open(fileName.toLocal8Bit(), m_flags, m_fmode)) < 0) {
Chris@148 126 ::perror("Open failed");
Chris@148 127 std::cerr << "ERROR: MatrixFile::MatrixFile: "
Chris@148 128 << "Failed to open cache file \""
Chris@148 129 << fileName.toStdString() << "\"";
Chris@148 130 if (m_mode == ReadWrite) std::cerr << " for writing";
Chris@148 131 std::cerr << std::endl;
Chris@148 132 throw FailedToOpenFile(fileName);
Chris@148 133 }
Chris@148 134
Chris@148 135 if (newFile) {
Chris@148 136 resize(0, 0); // write header
Chris@148 137 } else {
Chris@148 138 size_t header[2];
Chris@148 139 if (::read(m_fd, header, 2 * sizeof(size_t)) < 0) {
Chris@236 140 ::perror("MatrixFile::MatrixFile: read failed");
Chris@148 141 std::cerr << "ERROR: MatrixFile::MatrixFile: "
Chris@148 142 << "Failed to read header (fd " << m_fd << ", file \""
Chris@148 143 << fileName.toStdString() << "\")" << std::endl;
Chris@148 144 throw FileReadFailed(fileName);
Chris@148 145 }
Chris@148 146 m_width = header[0];
Chris@148 147 m_height = header[1];
Chris@148 148 seekTo(0, 0);
Chris@148 149 }
Chris@148 150
Chris@148 151 m_fileName = fileName;
Chris@148 152
Chris@408 153 {
Chris@408 154 MutexLocker locker
Chris@408 155 (&m_columnBitsetWriteMutex,
Chris@408 156 "MatrixFile::MatrixFile::m_columnBitsetWriteMutex");
Chris@148 157
Chris@408 158 if (m_columnBitsets.find(m_fileName) == m_columnBitsets.end()) {
Chris@408 159 m_columnBitsets[m_fileName] = new ResizeableBitset;
Chris@408 160 }
Chris@408 161 m_columnBitset = m_columnBitsets[m_fileName];
Chris@148 162 }
Chris@148 163
Chris@408 164 MutexLocker locker(&m_refcountMutex,
Chris@408 165 "MatrixFile::MatrixFile::m_refcountMutex");
Chris@148 166 ++m_refcount[fileName];
Chris@148 167
Chris@259 168 // std::cerr << "MatrixFile(" << this << "): fd " << m_fd << ", file " << fileName.toStdString() << ", ref " << m_refcount[fileName] << std::endl;
Chris@148 169
Chris@148 170 // std::cerr << "MatrixFile::MatrixFile: Done, size is " << "(" << m_width << ", " << m_height << ")" << std::endl;
Chris@148 171
Chris@148 172 ++totalCount;
Chris@148 173
Chris@148 174 }
Chris@148 175
Chris@148 176 MatrixFile::~MatrixFile()
Chris@148 177 {
Chris@148 178 char *requestData = 0;
Chris@148 179
Chris@148 180 if (m_requestToken >= 0) {
Chris@148 181 FileReadThread::Request request;
Chris@148 182 if (m_readThread->getRequest(m_requestToken, request)) {
Chris@148 183 requestData = request.data;
Chris@148 184 }
Chris@148 185 m_readThread->cancel(m_requestToken);
Chris@148 186 }
Chris@148 187
Chris@148 188 if (requestData) free(requestData);
Chris@148 189 if (m_cache.data) free(m_cache.data);
Chris@148 190 if (m_spareData) free(m_spareData);
Chris@148 191
Chris@148 192 if (m_fd >= 0) {
Chris@148 193 if (::close(m_fd) < 0) {
Chris@148 194 ::perror("MatrixFile::~MatrixFile: close failed");
Chris@148 195 }
Chris@148 196 }
Chris@148 197
Chris@148 198 if (m_fileName != "") {
Chris@148 199
Chris@408 200 MutexLocker locker(&m_refcountMutex,
Chris@408 201 "MatrixFile::~MatrixFile::m_refcountMutex");
Chris@148 202
Chris@148 203 if (--m_refcount[m_fileName] == 0) {
Chris@148 204
Chris@148 205 if (::unlink(m_fileName.toLocal8Bit())) {
Chris@259 206 // ::perror("Unlink failed");
Chris@259 207 // std::cerr << "WARNING: MatrixFile::~MatrixFile: reference count reached 0, but failed to unlink file \"" << m_fileName.toStdString() << "\"" << std::endl;
Chris@148 208 } else {
Chris@259 209 // std::cerr << "deleted " << m_fileName.toStdString() << std::endl;
Chris@148 210 }
Chris@148 211
Chris@408 212 MutexLocker locker2
Chris@408 213 (&m_columnBitsetWriteMutex,
Chris@408 214 "MatrixFile::~MatrixFile::m_columnBitsetWriteMutex");
Chris@148 215 m_columnBitsets.erase(m_fileName);
Chris@148 216 delete m_columnBitset;
Chris@148 217 }
Chris@148 218 }
Chris@148 219
Chris@148 220 totalStorage -= (m_headerSize + (m_width * m_height * m_cellSize));
Chris@148 221 totalMemory -= (2 * m_defaultCacheWidth * m_height * m_cellSize);
Chris@148 222 totalCount --;
Chris@148 223
Chris@259 224 // std::cerr << "MatrixFile::~MatrixFile: " << std::endl;
Chris@259 225 // std::cerr << "Total storage now " << totalStorage/1024 << "K, theoretical max memory "
Chris@259 226 // << totalMemory/1024 << "K in " << totalCount << " instances" << std::endl;
Chris@148 227
Chris@148 228 }
Chris@148 229
Chris@148 230 void
Chris@148 231 MatrixFile::resize(size_t w, size_t h)
Chris@148 232 {
Chris@148 233 Profiler profiler("MatrixFile::resize", true);
Chris@148 234
Chris@148 235 assert(m_mode == ReadWrite);
Chris@148 236
Chris@408 237 MutexLocker locker(&m_fdMutex, "MatrixFile::resize::m_fdMutex");
Chris@148 238
Chris@148 239 totalStorage -= (m_headerSize + (m_width * m_height * m_cellSize));
Chris@148 240 totalMemory -= (2 * m_defaultCacheWidth * m_height * m_cellSize);
Chris@148 241
Chris@148 242 off_t off = m_headerSize + (w * h * m_cellSize);
Chris@148 243
Chris@148 244 #ifdef DEBUG_MATRIX_FILE
Chris@148 245 std::cerr << "MatrixFile::resize(" << w << ", " << h << "): resizing file" << std::endl;
Chris@148 246 #endif
Chris@148 247
Chris@148 248 if (w * h < m_width * m_height) {
Chris@148 249 if (::ftruncate(m_fd, off) < 0) {
Chris@148 250 ::perror("WARNING: MatrixFile::resize: ftruncate failed");
Chris@148 251 throw FileOperationFailed(m_fileName, "ftruncate");
Chris@148 252 }
Chris@148 253 }
Chris@148 254
Chris@148 255 m_width = 0;
Chris@148 256 m_height = 0;
Chris@148 257
Chris@148 258 if (::lseek(m_fd, 0, SEEK_SET) == (off_t)-1) {
Chris@148 259 ::perror("ERROR: MatrixFile::resize: Seek to write header failed");
Chris@148 260 throw FileOperationFailed(m_fileName, "lseek");
Chris@148 261 }
Chris@148 262
Chris@148 263 size_t header[2];
Chris@148 264 header[0] = w;
Chris@148 265 header[1] = h;
Chris@148 266 if (::write(m_fd, header, 2 * sizeof(size_t)) != 2 * sizeof(size_t)) {
Chris@148 267 ::perror("ERROR: MatrixFile::resize: Failed to write header");
Chris@148 268 throw FileOperationFailed(m_fileName, "write");
Chris@148 269 }
Chris@148 270
Chris@148 271 if (w > 0 && m_defaultCacheWidth > w) {
Chris@148 272 m_defaultCacheWidth = w;
Chris@148 273 }
Chris@148 274
Chris@411 275 //!!! static size_t maxCacheMB = 16;
Chris@411 276 static size_t maxCacheMB = 4;
Chris@148 277 if (2 * m_defaultCacheWidth * h * m_cellSize > maxCacheMB * 1024 * 1024) { //!!!
Chris@148 278 m_defaultCacheWidth = (maxCacheMB * 1024 * 1024) / (2 * h * m_cellSize);
Chris@148 279 if (m_defaultCacheWidth < 16) m_defaultCacheWidth = 16;
Chris@148 280 }
Chris@148 281
Chris@148 282 if (m_columnBitset) {
Chris@408 283 MutexLocker locker(&m_columnBitsetWriteMutex,
Chris@408 284 "MatrixFile::resize::m_columnBitsetWriteMutex");
Chris@148 285 m_columnBitset->resize(w);
Chris@148 286 }
Chris@148 287
Chris@148 288 if (m_cache.data) {
Chris@148 289 free(m_cache.data);
Chris@148 290 m_cache.data = 0;
Chris@148 291 }
Chris@148 292
Chris@148 293 if (m_spareData) {
Chris@148 294 free(m_spareData);
Chris@148 295 m_spareData = 0;
Chris@148 296 }
Chris@148 297
Chris@148 298 m_width = w;
Chris@148 299 m_height = h;
Chris@148 300
Chris@148 301 totalStorage += (m_headerSize + (m_width * m_height * m_cellSize));
Chris@148 302 totalMemory += (2 * m_defaultCacheWidth * m_height * m_cellSize);
Chris@148 303
Chris@148 304 #ifdef DEBUG_MATRIX_FILE
Chris@148 305 std::cerr << "MatrixFile::resize(" << w << ", " << h << "): cache width "
Chris@148 306 << m_defaultCacheWidth << ", storage "
Chris@148 307 << (m_headerSize + w * h * m_cellSize) << ", mem "
Chris@148 308 << (2 * h * m_defaultCacheWidth * m_cellSize) << std::endl;
Chris@148 309
Chris@148 310 std::cerr << "Total storage " << totalStorage/1024 << "K, theoretical max memory "
Chris@148 311 << totalMemory/1024 << "K in " << totalCount << " instances" << std::endl;
Chris@148 312 #endif
Chris@148 313
Chris@148 314 seekTo(0, 0);
Chris@148 315 }
Chris@148 316
Chris@148 317 void
Chris@148 318 MatrixFile::reset()
Chris@148 319 {
Chris@148 320 Profiler profiler("MatrixFile::reset", true);
Chris@148 321
Chris@148 322 assert (m_mode == ReadWrite);
Chris@148 323
Chris@148 324 if (m_eagerCache) {
Chris@148 325 void *emptyCol = calloc(m_height, m_cellSize);
Chris@148 326 for (size_t x = 0; x < m_width; ++x) setColumnAt(x, emptyCol);
Chris@148 327 free(emptyCol);
Chris@148 328 }
Chris@148 329
Chris@148 330 if (m_columnBitset) {
Chris@408 331 MutexLocker locker(&m_columnBitsetWriteMutex,
Chris@408 332 "MatrixFile::reset::m_columnBitsetWriteMutex");
Chris@148 333 m_columnBitset->resize(m_width);
Chris@148 334 }
Chris@148 335 }
Chris@148 336
Chris@148 337 void
Chris@148 338 MatrixFile::getColumnAt(size_t x, void *data)
Chris@148 339 {
Chris@408 340 Profiler profiler("MatrixFile::getColumnAt");
Chris@148 341
Chris@148 342 // assert(haveSetColumnAt(x));
Chris@148 343
Chris@148 344 if (getFromCache(x, 0, m_height, data)) return;
Chris@148 345
Chris@408 346 Profiler profiler2("MatrixFile::getColumnAt (uncached)");
Chris@148 347
Chris@148 348 ssize_t r = 0;
Chris@148 349
Chris@148 350 #ifdef DEBUG_MATRIX_FILE
Chris@148 351 std::cerr << "MatrixFile::getColumnAt(" << x << ")"
Chris@148 352 << ": reading the slow way";
Chris@148 353
Chris@148 354 if (m_requestToken >= 0 &&
Chris@148 355 x >= m_requestingX &&
Chris@148 356 x < m_requestingX + m_requestingWidth) {
Chris@148 357
Chris@148 358 std::cerr << " (awaiting " << m_requestingX << ", " << m_requestingWidth << " from disk)";
Chris@148 359 }
Chris@148 360
Chris@148 361 std::cerr << std::endl;
Chris@148 362 #endif
Chris@148 363
Chris@408 364 {
Chris@408 365 MutexLocker locker(&m_fdMutex, "MatrixFile::getColumnAt::m_fdMutex");
Chris@148 366
Chris@408 367 if (seekTo(x, 0)) {
Chris@408 368 r = ::read(m_fd, data, m_height * m_cellSize);
Chris@408 369 }
Chris@148 370 }
Chris@148 371
Chris@148 372 if (r < 0) {
Chris@148 373 ::perror("MatrixFile::getColumnAt: read failed");
Chris@236 374 std::cerr << "ERROR: MatrixFile::getColumnAt: "
Chris@236 375 << "Failed to read column " << x << " (height " << m_height << ", cell size " << m_cellSize << ", fd " << m_fd << ", file \""
Chris@236 376 << m_fileName.toStdString() << "\")" << std::endl;
Chris@148 377 throw FileReadFailed(m_fileName);
Chris@148 378 }
Chris@148 379
Chris@148 380 return;
Chris@148 381 }
Chris@148 382
Chris@148 383 bool
Chris@148 384 MatrixFile::getFromCache(size_t x, size_t ystart, size_t ycount, void *data)
Chris@148 385 {
Chris@408 386 bool fail = false;
Chris@408 387 bool primeLeft = false;
Chris@148 388
Chris@408 389 {
Chris@408 390 MutexLocker locker(&m_cacheMutex,
Chris@408 391 "MatrixFile::getFromCache::m_cacheMutex");
Chris@408 392
Chris@408 393 if (!m_cache.data || x < m_cache.x || x >= m_cache.x + m_cache.width) {
Chris@408 394 fail = true;
Chris@408 395 primeLeft = (m_cache.data && x < m_cache.x);
Chris@408 396 } else {
Chris@408 397 memcpy(data,
Chris@408 398 m_cache.data + m_cellSize * ((x - m_cache.x) * m_height + ystart),
Chris@408 399 ycount * m_cellSize);
Chris@408 400 }
Chris@408 401 }
Chris@408 402
Chris@408 403 if (fail) {
Chris@408 404 primeCache(x, primeLeft); // this doesn't take effect until a later callback
Chris@148 405 m_prevX = x;
Chris@148 406 return false;
Chris@148 407 }
Chris@148 408
Chris@148 409 if (m_cache.x > 0 && x < m_prevX && x < m_cache.x + m_cache.width/4) {
Chris@148 410 primeCache(x, true);
Chris@148 411 }
Chris@148 412
Chris@148 413 if (m_cache.x + m_cache.width < m_width &&
Chris@148 414 x > m_prevX &&
Chris@148 415 x > m_cache.x + (m_cache.width * 3) / 4) {
Chris@148 416 primeCache(x, false);
Chris@148 417 }
Chris@148 418
Chris@148 419 m_prevX = x;
Chris@148 420 return true;
Chris@148 421 }
Chris@148 422
Chris@148 423 void
Chris@148 424 MatrixFile::setColumnAt(size_t x, const void *data)
Chris@148 425 {
Chris@148 426 assert(m_mode == ReadWrite);
Chris@148 427
Chris@148 428 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@411 429 // std::cerr << "MatrixFile::setColumnAt(" << x << ")" << std::endl;
Chris@411 430 std::cerr << ".";
Chris@148 431 #endif
Chris@148 432
Chris@148 433 ssize_t w = 0;
Chris@148 434 bool seekFailed = false;
Chris@408 435
Chris@408 436 {
Chris@408 437 MutexLocker locker(&m_fdMutex, "MatrixFile::setColumnAt::m_fdMutex");
Chris@148 438
Chris@408 439 if (seekTo(x, 0)) {
Chris@408 440 w = ::write(m_fd, data, m_height * m_cellSize);
Chris@408 441 } else {
Chris@408 442 seekFailed = true;
Chris@408 443 }
Chris@148 444 }
Chris@148 445
Chris@148 446 if (!seekFailed && w != ssize_t(m_height * m_cellSize)) {
Chris@148 447 ::perror("WARNING: MatrixFile::setColumnAt: write failed");
Chris@148 448 throw FileOperationFailed(m_fileName, "write");
Chris@148 449 } else if (seekFailed) {
Chris@148 450 throw FileOperationFailed(m_fileName, "seek");
Chris@148 451 } else {
Chris@408 452 MutexLocker locker
Chris@408 453 (&m_columnBitsetWriteMutex,
Chris@408 454 "MatrixFile::setColumnAt::m_columnBitsetWriteMutex");
Chris@148 455 m_columnBitset->set(x);
Chris@148 456 }
Chris@148 457 }
Chris@148 458
Chris@148 459 void
Chris@148 460 MatrixFile::suspend()
Chris@148 461 {
Chris@408 462 MutexLocker locker(&m_fdMutex, "MatrixFile::suspend::m_fdMutex");
Chris@408 463 MutexLocker locker2(&m_cacheMutex, "MatrixFile::suspend::m_cacheMutex");
Chris@148 464
Chris@148 465 if (m_fd < 0) return; // already suspended
Chris@148 466
Chris@148 467 #ifdef DEBUG_MATRIX_FILE
Chris@148 468 std::cerr << "MatrixFile(" << this << ":" << m_fileName.toStdString() << ")::suspend(): fd was " << m_fd << std::endl;
Chris@148 469 #endif
Chris@148 470
Chris@148 471 if (m_requestToken >= 0) {
Chris@148 472 void *data = 0;
Chris@148 473 FileReadThread::Request request;
Chris@148 474 if (m_readThread->getRequest(m_requestToken, request)) {
Chris@148 475 data = request.data;
Chris@148 476 }
Chris@148 477 m_readThread->cancel(m_requestToken);
Chris@148 478 if (data) free(data);
Chris@148 479 m_requestToken = -1;
Chris@148 480 }
Chris@148 481
Chris@148 482 if (m_cache.data) {
Chris@148 483 free(m_cache.data);
Chris@148 484 m_cache.data = 0;
Chris@148 485 }
Chris@148 486
Chris@148 487 if (m_spareData) {
Chris@148 488 free(m_spareData);
Chris@148 489 m_spareData = 0;
Chris@148 490 }
Chris@148 491
Chris@148 492 if (::close(m_fd) < 0) {
Chris@148 493 ::perror("WARNING: MatrixFile::suspend: close failed");
Chris@148 494 throw FileOperationFailed(m_fileName, "close");
Chris@148 495 }
Chris@148 496
Chris@148 497 m_fd = -1;
Chris@148 498 }
Chris@148 499
Chris@148 500 void
Chris@148 501 MatrixFile::resume()
Chris@148 502 {
Chris@148 503 if (m_fd >= 0) return;
Chris@148 504
Chris@148 505 #ifdef DEBUG_MATRIX_FILE
Chris@148 506 std::cerr << "MatrixFile(" << this << ")::resume()" << std::endl;
Chris@148 507 #endif
Chris@148 508
Chris@148 509 if ((m_fd = ::open(m_fileName.toLocal8Bit(), m_flags, m_fmode)) < 0) {
Chris@148 510 ::perror("Open failed");
Chris@148 511 std::cerr << "ERROR: MatrixFile::resume: "
Chris@148 512 << "Failed to open cache file \""
Chris@148 513 << m_fileName.toStdString() << "\"";
Chris@148 514 if (m_mode == ReadWrite) std::cerr << " for writing";
Chris@148 515 std::cerr << std::endl;
Chris@148 516 throw FailedToOpenFile(m_fileName);
Chris@148 517 }
Chris@148 518
Chris@436 519 #ifdef DEBUG_MATRIX_FILE
Chris@148 520 std::cerr << "MatrixFile(" << this << ":" << m_fileName.toStdString() << ")::resume(): fd is " << m_fd << std::endl;
Chris@436 521 #endif
Chris@148 522 }
Chris@148 523
Chris@148 524 void
Chris@148 525 MatrixFile::primeCache(size_t x, bool goingLeft)
Chris@148 526 {
Chris@148 527 // Profiler profiler("MatrixFile::primeCache");
Chris@148 528
Chris@148 529 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@148 530 std::cerr << "MatrixFile::primeCache(" << x << ", " << goingLeft << ")" << std::endl;
Chris@148 531 #endif
Chris@148 532
Chris@148 533 size_t rx = x;
Chris@148 534 size_t rw = m_defaultCacheWidth;
Chris@148 535
Chris@148 536 size_t left = rw / 3;
Chris@148 537 if (goingLeft) left = (rw * 2) / 3;
Chris@148 538
Chris@148 539 if (rx > left) rx -= left;
Chris@148 540 else rx = 0;
Chris@148 541
Chris@148 542 if (rx + rw > m_width) rw = m_width - rx;
Chris@148 543
Chris@148 544 if (!m_eagerCache) {
Chris@148 545
Chris@148 546 size_t ti = 0;
Chris@148 547
Chris@148 548 for (ti = 0; ti < rw; ++ti) {
Chris@148 549 if (!m_columnBitset->get(rx + ti)) break;
Chris@148 550 }
Chris@148 551
Chris@148 552 #ifdef DEBUG_MATRIX_FILE
Chris@148 553 if (ti < rw) {
Chris@148 554 std::cerr << "eagerCache is false and there's a hole at "
Chris@148 555 << rx + ti << ", reducing rw from " << rw << " to "
Chris@148 556 << ti << std::endl;
Chris@148 557 }
Chris@148 558 #endif
Chris@148 559
Chris@148 560 rw = std::min(rw, ti);
Chris@148 561 if (rw < 10 || rx + rw <= x) return;
Chris@148 562 }
Chris@148 563
Chris@408 564 MutexLocker locker(&m_cacheMutex, "MatrixFile::primeCache::m_cacheMutex");
Chris@148 565
Chris@455 566 // Check for the existence of the request first; if it exists,
Chris@455 567 // check whether it's ready. Only when we know it's ready do we
Chris@455 568 // retrieve the actual request, because the reason we need the
Chris@455 569 // request is to check whether it was successful or not and
Chris@455 570 // extract data from it, and none of that can be relied upon if we
Chris@455 571 // retrieve the request before it's ready. (There used to be a
Chris@455 572 // race condition here, where we retrieved the request and only
Chris@455 573 // afterwards checked the ready status, pulling data from the
Chris@455 574 // request if it was found to be ready then.)
Chris@148 575
Chris@148 576 if (m_requestToken >= 0 &&
Chris@455 577 m_readThread->haveRequest(m_requestToken)) {
Chris@148 578
Chris@148 579 if (x >= m_requestingX &&
Chris@148 580 x < m_requestingX + m_requestingWidth) {
Chris@148 581
Chris@148 582 if (m_readThread->isReady(m_requestToken)) {
Chris@148 583
Chris@455 584 FileReadThread::Request request;
Chris@455 585 if (!m_readThread->getRequest(m_requestToken, request)) {
Chris@455 586 std::cerr << "ERROR: MatrixFile::primeCache: File read thread has lost our request!" << std::endl;
Chris@455 587 throw FileReadFailed(m_fileName);
Chris@455 588 }
Chris@455 589
Chris@148 590 if (!request.successful) {
Chris@148 591 std::cerr << "ERROR: MatrixFile::primeCache: Last request was unsuccessful" << std::endl;
Chris@148 592 throw FileReadFailed(m_fileName);
Chris@148 593 }
Chris@148 594
Chris@148 595 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@148 596 std::cerr << "last request is ready! (" << m_requestingX << ", "<< m_requestingWidth << ")" << std::endl;
Chris@148 597 #endif
Chris@148 598
Chris@148 599 m_cache.x = (request.start - m_headerSize) / (m_height * m_cellSize);
Chris@148 600 m_cache.width = request.size / (m_height * m_cellSize);
Chris@148 601
Chris@148 602 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@148 603 std::cerr << "received last request: actual size is: " << m_cache.x << ", " << m_cache.width << std::endl;
Chris@148 604 #endif
Chris@148 605
Chris@148 606 if (m_cache.data) {
Chris@411 607 if (m_spareData) {
Chris@412 608 // std::cerr << this << ": Freeing spare data" << std::endl;
Chris@411 609 free(m_spareData);
Chris@411 610 }
Chris@412 611 // std::cerr << this << ": Moving old cache data to spare" << std::endl;
Chris@148 612 m_spareData = m_cache.data;
Chris@148 613 }
Chris@412 614 // std::cerr << this << ": Moving request data to cache" << std::endl;
Chris@148 615 m_cache.data = request.data;
Chris@148 616
Chris@148 617 m_readThread->done(m_requestToken);
Chris@148 618 m_requestToken = -1;
Chris@148 619 }
Chris@148 620
Chris@148 621 // already requested something covering this area; wait for it
Chris@148 622 return;
Chris@148 623 }
Chris@148 624
Chris@455 625 FileReadThread::Request dud;
Chris@148 626
Chris@455 627 if (!m_readThread->getRequest(m_requestToken, dud)) {
Chris@455 628
Chris@455 629 std::cerr << "ERROR: MatrixFile::primeCache: Inconsistent replies from FileReadThread" << std::endl;
Chris@455 630
Chris@455 631 } else {
Chris@455 632
Chris@455 633 // current request is for the wrong area, so no longer of any use
Chris@455 634 m_readThread->cancel(m_requestToken);
Chris@455 635
Chris@455 636 // crude way to avoid leaking the data
Chris@455 637 while (!m_readThread->isCancelled(m_requestToken)) {
Chris@455 638 usleep(10000);
Chris@455 639 }
Chris@148 640
Chris@148 641 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@455 642 std::cerr << "cancelled " << m_requestToken << std::endl;
Chris@148 643 #endif
Chris@148 644
Chris@455 645 if (m_spareData) {
Chris@455 646 free(m_spareData);
Chris@455 647 }
Chris@455 648 m_spareData = dud.data;
Chris@455 649 m_readThread->done(m_requestToken);
Chris@411 650 }
Chris@148 651
Chris@148 652 m_requestToken = -1;
Chris@148 653 }
Chris@148 654
Chris@148 655 if (m_fd < 0) {
Chris@408 656 MutexLocker locker(&m_fdMutex, "MatrixFile::primeCache::m_fdMutex");
Chris@148 657 if (m_fd < 0) resume();
Chris@148 658 }
Chris@148 659
Chris@455 660 FileReadThread::Request request;
Chris@455 661
Chris@148 662 request.fd = m_fd;
Chris@148 663 request.mutex = &m_fdMutex;
Chris@148 664 request.start = m_headerSize + rx * m_height * m_cellSize;
Chris@148 665 request.size = rw * m_height * m_cellSize;
Chris@411 666
Chris@412 667 // std::cerr << this << ": Moving spare data to request, and resizing to " << rw * m_height * m_cellSize << std::endl;
Chris@411 668
Chris@148 669 request.data = (char *)realloc(m_spareData, rw * m_height * m_cellSize);
Chris@148 670 MUNLOCK(request.data, rw * m_height * m_cellSize);
Chris@148 671 m_spareData = 0;
Chris@148 672
Chris@148 673 m_requestingX = rx;
Chris@148 674 m_requestingWidth = rw;
Chris@148 675
Chris@148 676 int token = m_readThread->request(request);
Chris@148 677 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@148 678 std::cerr << "MatrixFile::primeCache: request token is "
Chris@148 679 << token << " (x = [" << rx << "], w = [" << rw << "], left = [" << goingLeft << "])" << std::endl;
Chris@148 680 #endif
Chris@148 681 m_requestToken = token;
Chris@148 682 }
Chris@148 683
Chris@148 684 bool
Chris@148 685 MatrixFile::seekTo(size_t x, size_t y)
Chris@148 686 {
Chris@148 687 if (m_fd < 0) resume();
Chris@148 688
Chris@148 689 off_t off = m_headerSize + (x * m_height + y) * m_cellSize;
Chris@148 690
Chris@148 691 if (::lseek(m_fd, off, SEEK_SET) == (off_t)-1) {
Chris@148 692 ::perror("Seek failed");
Chris@148 693 std::cerr << "ERROR: MatrixFile::seekTo(" << x << ", " << y
Chris@148 694 << ") failed" << std::endl;
Chris@148 695 return false;
Chris@148 696 }
Chris@148 697
Chris@148 698 return true;
Chris@148 699 }
Chris@148 700