annotate data/fileio/MatrixFile.cpp @ 453:1f15beefcd76

* debug
author Chris Cannam
date Thu, 09 Oct 2008 12:22:02 +0000
parents cff476cfce77
children 3e0f1f7bec85
rev   line source
Chris@148 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@148 2
Chris@148 3 /*
Chris@148 4 Sonic Visualiser
Chris@148 5 An audio file viewer and annotation editor.
Chris@148 6 Centre for Digital Music, Queen Mary, University of London.
Chris@148 7 This file copyright 2006 Chris Cannam.
Chris@148 8
Chris@148 9 This program is free software; you can redistribute it and/or
Chris@148 10 modify it under the terms of the GNU General Public License as
Chris@148 11 published by the Free Software Foundation; either version 2 of the
Chris@148 12 License, or (at your option) any later version. See the file
Chris@148 13 COPYING included with this distribution for more information.
Chris@148 14 */
Chris@148 15
Chris@148 16 #include "MatrixFile.h"
Chris@148 17 #include "base/TempDirectory.h"
Chris@150 18 #include "system/System.h"
Chris@148 19 #include "base/Profiler.h"
Chris@148 20 #include "base/Exceptions.h"
Chris@408 21 #include "base/Thread.h"
Chris@148 22
Chris@148 23 #include <sys/types.h>
Chris@148 24 #include <sys/stat.h>
Chris@148 25 #include <fcntl.h>
Chris@148 26 #include <unistd.h>
Chris@148 27
Chris@148 28 #include <iostream>
Chris@148 29
Chris@148 30 #include <cstdio>
Chris@148 31 #include <cassert>
Chris@148 32
Chris@405 33 #include <cstdlib>
Chris@405 34
Chris@148 35 #include <QFileInfo>
Chris@148 36 #include <QDir>
Chris@148 37
Chris@453 38 #define DEBUG_MATRIX_FILE 1
Chris@403 39 //#define DEBUG_MATRIX_FILE_READ_SET 1
Chris@148 40
Chris@148 41 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@153 42 #ifndef DEBUG_MATRIX_FILE
Chris@148 43 #define DEBUG_MATRIX_FILE 1
Chris@148 44 #endif
Chris@153 45 #endif
Chris@148 46
Chris@148 47 std::map<QString, int> MatrixFile::m_refcount;
Chris@148 48 QMutex MatrixFile::m_refcountMutex;
Chris@148 49
Chris@148 50 MatrixFile::ResizeableBitsetMap MatrixFile::m_columnBitsets;
Chris@148 51 QMutex MatrixFile::m_columnBitsetWriteMutex;
Chris@148 52
Chris@148 53 FileReadThread *MatrixFile::m_readThread = 0;
Chris@148 54
Chris@148 55 static size_t totalStorage = 0;
Chris@148 56 static size_t totalMemory = 0;
Chris@148 57 static size_t totalCount = 0;
Chris@148 58
Chris@148 59 MatrixFile::MatrixFile(QString fileBase, Mode mode,
Chris@148 60 size_t cellSize, bool eagerCache) :
Chris@148 61 m_fd(-1),
Chris@148 62 m_mode(mode),
Chris@148 63 m_flags(0),
Chris@148 64 m_fmode(0),
Chris@148 65 m_cellSize(cellSize),
Chris@148 66 m_width(0),
Chris@148 67 m_height(0),
Chris@148 68 m_headerSize(2 * sizeof(size_t)),
Chris@148 69 m_defaultCacheWidth(1024),
Chris@148 70 m_prevX(0),
Chris@148 71 m_eagerCache(eagerCache),
Chris@148 72 m_requestToken(-1),
Chris@148 73 m_spareData(0),
Chris@148 74 m_columnBitset(0)
Chris@148 75 {
Chris@148 76 Profiler profiler("MatrixFile::MatrixFile", true);
Chris@148 77
Chris@148 78 if (!m_readThread) {
Chris@148 79 m_readThread = new FileReadThread;
Chris@148 80 m_readThread->start();
Chris@148 81 }
Chris@148 82
Chris@148 83 m_cache.data = 0;
Chris@148 84
Chris@148 85 QDir tempDir(TempDirectory::getInstance()->getPath());
Chris@148 86 QString fileName(tempDir.filePath(QString("%1.mfc").arg(fileBase)));
Chris@148 87 bool newFile = !QFileInfo(fileName).exists();
Chris@148 88
Chris@148 89 if (newFile && m_mode == ReadOnly) {
Chris@148 90 std::cerr << "ERROR: MatrixFile::MatrixFile: Read-only mode "
Chris@148 91 << "specified, but cache file does not exist" << std::endl;
Chris@148 92 throw FileNotFound(fileName);
Chris@148 93 }
Chris@148 94
Chris@148 95 if (!newFile && m_mode == ReadWrite) {
Chris@148 96 std::cerr << "Note: MatrixFile::MatrixFile: Read/write mode "
Chris@148 97 << "specified, but file already exists; falling back to "
Chris@148 98 << "read-only mode" << std::endl;
Chris@148 99 m_mode = ReadOnly;
Chris@148 100 }
Chris@148 101
Chris@148 102 if (!eagerCache && m_mode == ReadOnly) {
Chris@148 103 std::cerr << "WARNING: MatrixFile::MatrixFile: Eager cacheing not "
Chris@148 104 << "specified, but file is open in read-only mode -- cache "
Chris@148 105 << "will not be used" << std::endl;
Chris@148 106 }
Chris@148 107
Chris@148 108 m_flags = 0;
Chris@148 109 m_fmode = S_IRUSR | S_IWUSR;
Chris@148 110
Chris@148 111 if (m_mode == ReadWrite) {
Chris@148 112 m_flags = O_RDWR | O_CREAT;
Chris@148 113 } else {
Chris@148 114 m_flags = O_RDONLY;
Chris@148 115 }
Chris@148 116
Chris@233 117 #ifdef _WIN32
Chris@233 118 m_flags |= O_BINARY;
Chris@233 119 #endif
Chris@233 120
Chris@148 121 #ifdef DEBUG_MATRIX_FILE
Chris@148 122 std::cerr << "MatrixFile::MatrixFile: opening " << fileName.toStdString() << "..." << std::endl;
Chris@148 123 #endif
Chris@148 124
Chris@148 125 if ((m_fd = ::open(fileName.toLocal8Bit(), m_flags, m_fmode)) < 0) {
Chris@148 126 ::perror("Open failed");
Chris@148 127 std::cerr << "ERROR: MatrixFile::MatrixFile: "
Chris@148 128 << "Failed to open cache file \""
Chris@148 129 << fileName.toStdString() << "\"";
Chris@148 130 if (m_mode == ReadWrite) std::cerr << " for writing";
Chris@148 131 std::cerr << std::endl;
Chris@148 132 throw FailedToOpenFile(fileName);
Chris@148 133 }
Chris@148 134
Chris@148 135 if (newFile) {
Chris@148 136 resize(0, 0); // write header
Chris@148 137 } else {
Chris@148 138 size_t header[2];
Chris@148 139 if (::read(m_fd, header, 2 * sizeof(size_t)) < 0) {
Chris@236 140 ::perror("MatrixFile::MatrixFile: read failed");
Chris@148 141 std::cerr << "ERROR: MatrixFile::MatrixFile: "
Chris@148 142 << "Failed to read header (fd " << m_fd << ", file \""
Chris@148 143 << fileName.toStdString() << "\")" << std::endl;
Chris@148 144 throw FileReadFailed(fileName);
Chris@148 145 }
Chris@148 146 m_width = header[0];
Chris@148 147 m_height = header[1];
Chris@148 148 seekTo(0, 0);
Chris@148 149 }
Chris@148 150
Chris@148 151 m_fileName = fileName;
Chris@148 152
Chris@408 153 {
Chris@408 154 MutexLocker locker
Chris@408 155 (&m_columnBitsetWriteMutex,
Chris@408 156 "MatrixFile::MatrixFile::m_columnBitsetWriteMutex");
Chris@148 157
Chris@408 158 if (m_columnBitsets.find(m_fileName) == m_columnBitsets.end()) {
Chris@408 159 m_columnBitsets[m_fileName] = new ResizeableBitset;
Chris@408 160 }
Chris@408 161 m_columnBitset = m_columnBitsets[m_fileName];
Chris@148 162 }
Chris@148 163
Chris@408 164 MutexLocker locker(&m_refcountMutex,
Chris@408 165 "MatrixFile::MatrixFile::m_refcountMutex");
Chris@148 166 ++m_refcount[fileName];
Chris@148 167
Chris@259 168 // std::cerr << "MatrixFile(" << this << "): fd " << m_fd << ", file " << fileName.toStdString() << ", ref " << m_refcount[fileName] << std::endl;
Chris@148 169
Chris@148 170 // std::cerr << "MatrixFile::MatrixFile: Done, size is " << "(" << m_width << ", " << m_height << ")" << std::endl;
Chris@148 171
Chris@148 172 ++totalCount;
Chris@148 173
Chris@148 174 }
Chris@148 175
Chris@148 176 MatrixFile::~MatrixFile()
Chris@148 177 {
Chris@148 178 char *requestData = 0;
Chris@148 179
Chris@148 180 if (m_requestToken >= 0) {
Chris@148 181 FileReadThread::Request request;
Chris@148 182 if (m_readThread->getRequest(m_requestToken, request)) {
Chris@148 183 requestData = request.data;
Chris@148 184 }
Chris@148 185 m_readThread->cancel(m_requestToken);
Chris@148 186 }
Chris@148 187
Chris@148 188 if (requestData) free(requestData);
Chris@148 189 if (m_cache.data) free(m_cache.data);
Chris@148 190 if (m_spareData) free(m_spareData);
Chris@148 191
Chris@148 192 if (m_fd >= 0) {
Chris@148 193 if (::close(m_fd) < 0) {
Chris@148 194 ::perror("MatrixFile::~MatrixFile: close failed");
Chris@148 195 }
Chris@148 196 }
Chris@148 197
Chris@148 198 if (m_fileName != "") {
Chris@148 199
Chris@408 200 MutexLocker locker(&m_refcountMutex,
Chris@408 201 "MatrixFile::~MatrixFile::m_refcountMutex");
Chris@148 202
Chris@148 203 if (--m_refcount[m_fileName] == 0) {
Chris@148 204
Chris@148 205 if (::unlink(m_fileName.toLocal8Bit())) {
Chris@259 206 // ::perror("Unlink failed");
Chris@259 207 // std::cerr << "WARNING: MatrixFile::~MatrixFile: reference count reached 0, but failed to unlink file \"" << m_fileName.toStdString() << "\"" << std::endl;
Chris@148 208 } else {
Chris@259 209 // std::cerr << "deleted " << m_fileName.toStdString() << std::endl;
Chris@148 210 }
Chris@148 211
Chris@408 212 MutexLocker locker2
Chris@408 213 (&m_columnBitsetWriteMutex,
Chris@408 214 "MatrixFile::~MatrixFile::m_columnBitsetWriteMutex");
Chris@148 215 m_columnBitsets.erase(m_fileName);
Chris@148 216 delete m_columnBitset;
Chris@148 217 }
Chris@148 218 }
Chris@148 219
Chris@148 220 totalStorage -= (m_headerSize + (m_width * m_height * m_cellSize));
Chris@148 221 totalMemory -= (2 * m_defaultCacheWidth * m_height * m_cellSize);
Chris@148 222 totalCount --;
Chris@148 223
Chris@259 224 // std::cerr << "MatrixFile::~MatrixFile: " << std::endl;
Chris@259 225 // std::cerr << "Total storage now " << totalStorage/1024 << "K, theoretical max memory "
Chris@259 226 // << totalMemory/1024 << "K in " << totalCount << " instances" << std::endl;
Chris@148 227
Chris@148 228 }
Chris@148 229
Chris@148 230 void
Chris@148 231 MatrixFile::resize(size_t w, size_t h)
Chris@148 232 {
Chris@148 233 Profiler profiler("MatrixFile::resize", true);
Chris@148 234
Chris@148 235 assert(m_mode == ReadWrite);
Chris@148 236
Chris@408 237 MutexLocker locker(&m_fdMutex, "MatrixFile::resize::m_fdMutex");
Chris@148 238
Chris@148 239 totalStorage -= (m_headerSize + (m_width * m_height * m_cellSize));
Chris@148 240 totalMemory -= (2 * m_defaultCacheWidth * m_height * m_cellSize);
Chris@148 241
Chris@148 242 off_t off = m_headerSize + (w * h * m_cellSize);
Chris@148 243
Chris@148 244 #ifdef DEBUG_MATRIX_FILE
Chris@148 245 std::cerr << "MatrixFile::resize(" << w << ", " << h << "): resizing file" << std::endl;
Chris@148 246 #endif
Chris@148 247
Chris@148 248 if (w * h < m_width * m_height) {
Chris@148 249 if (::ftruncate(m_fd, off) < 0) {
Chris@148 250 ::perror("WARNING: MatrixFile::resize: ftruncate failed");
Chris@148 251 throw FileOperationFailed(m_fileName, "ftruncate");
Chris@148 252 }
Chris@148 253 }
Chris@148 254
Chris@148 255 m_width = 0;
Chris@148 256 m_height = 0;
Chris@148 257
Chris@148 258 if (::lseek(m_fd, 0, SEEK_SET) == (off_t)-1) {
Chris@148 259 ::perror("ERROR: MatrixFile::resize: Seek to write header failed");
Chris@148 260 throw FileOperationFailed(m_fileName, "lseek");
Chris@148 261 }
Chris@148 262
Chris@148 263 size_t header[2];
Chris@148 264 header[0] = w;
Chris@148 265 header[1] = h;
Chris@148 266 if (::write(m_fd, header, 2 * sizeof(size_t)) != 2 * sizeof(size_t)) {
Chris@148 267 ::perror("ERROR: MatrixFile::resize: Failed to write header");
Chris@148 268 throw FileOperationFailed(m_fileName, "write");
Chris@148 269 }
Chris@148 270
Chris@148 271 if (w > 0 && m_defaultCacheWidth > w) {
Chris@148 272 m_defaultCacheWidth = w;
Chris@148 273 }
Chris@148 274
Chris@411 275 //!!! static size_t maxCacheMB = 16;
Chris@411 276 static size_t maxCacheMB = 4;
Chris@148 277 if (2 * m_defaultCacheWidth * h * m_cellSize > maxCacheMB * 1024 * 1024) { //!!!
Chris@148 278 m_defaultCacheWidth = (maxCacheMB * 1024 * 1024) / (2 * h * m_cellSize);
Chris@148 279 if (m_defaultCacheWidth < 16) m_defaultCacheWidth = 16;
Chris@148 280 }
Chris@148 281
Chris@148 282 if (m_columnBitset) {
Chris@408 283 MutexLocker locker(&m_columnBitsetWriteMutex,
Chris@408 284 "MatrixFile::resize::m_columnBitsetWriteMutex");
Chris@148 285 m_columnBitset->resize(w);
Chris@148 286 }
Chris@148 287
Chris@148 288 if (m_cache.data) {
Chris@148 289 free(m_cache.data);
Chris@148 290 m_cache.data = 0;
Chris@148 291 }
Chris@148 292
Chris@148 293 if (m_spareData) {
Chris@148 294 free(m_spareData);
Chris@148 295 m_spareData = 0;
Chris@148 296 }
Chris@148 297
Chris@148 298 m_width = w;
Chris@148 299 m_height = h;
Chris@148 300
Chris@148 301 totalStorage += (m_headerSize + (m_width * m_height * m_cellSize));
Chris@148 302 totalMemory += (2 * m_defaultCacheWidth * m_height * m_cellSize);
Chris@148 303
Chris@148 304 #ifdef DEBUG_MATRIX_FILE
Chris@148 305 std::cerr << "MatrixFile::resize(" << w << ", " << h << "): cache width "
Chris@148 306 << m_defaultCacheWidth << ", storage "
Chris@148 307 << (m_headerSize + w * h * m_cellSize) << ", mem "
Chris@148 308 << (2 * h * m_defaultCacheWidth * m_cellSize) << std::endl;
Chris@148 309
Chris@148 310 std::cerr << "Total storage " << totalStorage/1024 << "K, theoretical max memory "
Chris@148 311 << totalMemory/1024 << "K in " << totalCount << " instances" << std::endl;
Chris@148 312 #endif
Chris@148 313
Chris@148 314 seekTo(0, 0);
Chris@148 315 }
Chris@148 316
Chris@148 317 void
Chris@148 318 MatrixFile::reset()
Chris@148 319 {
Chris@148 320 Profiler profiler("MatrixFile::reset", true);
Chris@148 321
Chris@148 322 assert (m_mode == ReadWrite);
Chris@148 323
Chris@148 324 if (m_eagerCache) {
Chris@148 325 void *emptyCol = calloc(m_height, m_cellSize);
Chris@148 326 for (size_t x = 0; x < m_width; ++x) setColumnAt(x, emptyCol);
Chris@148 327 free(emptyCol);
Chris@148 328 }
Chris@148 329
Chris@148 330 if (m_columnBitset) {
Chris@408 331 MutexLocker locker(&m_columnBitsetWriteMutex,
Chris@408 332 "MatrixFile::reset::m_columnBitsetWriteMutex");
Chris@148 333 m_columnBitset->resize(m_width);
Chris@148 334 }
Chris@148 335 }
Chris@148 336
Chris@148 337 void
Chris@148 338 MatrixFile::getColumnAt(size_t x, void *data)
Chris@148 339 {
Chris@408 340 Profiler profiler("MatrixFile::getColumnAt");
Chris@148 341
Chris@148 342 // assert(haveSetColumnAt(x));
Chris@148 343
Chris@148 344 if (getFromCache(x, 0, m_height, data)) return;
Chris@148 345
Chris@408 346 Profiler profiler2("MatrixFile::getColumnAt (uncached)");
Chris@148 347
Chris@148 348 ssize_t r = 0;
Chris@148 349
Chris@148 350 #ifdef DEBUG_MATRIX_FILE
Chris@148 351 std::cerr << "MatrixFile::getColumnAt(" << x << ")"
Chris@148 352 << ": reading the slow way";
Chris@148 353
Chris@148 354 if (m_requestToken >= 0 &&
Chris@148 355 x >= m_requestingX &&
Chris@148 356 x < m_requestingX + m_requestingWidth) {
Chris@148 357
Chris@148 358 std::cerr << " (awaiting " << m_requestingX << ", " << m_requestingWidth << " from disk)";
Chris@148 359 }
Chris@148 360
Chris@148 361 std::cerr << std::endl;
Chris@148 362 #endif
Chris@148 363
Chris@408 364 {
Chris@408 365 MutexLocker locker(&m_fdMutex, "MatrixFile::getColumnAt::m_fdMutex");
Chris@148 366
Chris@408 367 if (seekTo(x, 0)) {
Chris@408 368 r = ::read(m_fd, data, m_height * m_cellSize);
Chris@408 369 }
Chris@148 370 }
Chris@148 371
Chris@148 372 if (r < 0) {
Chris@148 373 ::perror("MatrixFile::getColumnAt: read failed");
Chris@236 374 std::cerr << "ERROR: MatrixFile::getColumnAt: "
Chris@236 375 << "Failed to read column " << x << " (height " << m_height << ", cell size " << m_cellSize << ", fd " << m_fd << ", file \""
Chris@236 376 << m_fileName.toStdString() << "\")" << std::endl;
Chris@148 377 throw FileReadFailed(m_fileName);
Chris@148 378 }
Chris@148 379
Chris@148 380 return;
Chris@148 381 }
Chris@148 382
Chris@148 383 bool
Chris@148 384 MatrixFile::getFromCache(size_t x, size_t ystart, size_t ycount, void *data)
Chris@148 385 {
Chris@408 386 bool fail = false;
Chris@408 387 bool primeLeft = false;
Chris@148 388
Chris@408 389 {
Chris@408 390 MutexLocker locker(&m_cacheMutex,
Chris@408 391 "MatrixFile::getFromCache::m_cacheMutex");
Chris@408 392
Chris@408 393 if (!m_cache.data || x < m_cache.x || x >= m_cache.x + m_cache.width) {
Chris@408 394 fail = true;
Chris@408 395 primeLeft = (m_cache.data && x < m_cache.x);
Chris@408 396 } else {
Chris@408 397 memcpy(data,
Chris@408 398 m_cache.data + m_cellSize * ((x - m_cache.x) * m_height + ystart),
Chris@408 399 ycount * m_cellSize);
Chris@408 400 }
Chris@408 401 }
Chris@408 402
Chris@408 403 if (fail) {
Chris@408 404 primeCache(x, primeLeft); // this doesn't take effect until a later callback
Chris@148 405 m_prevX = x;
Chris@148 406 return false;
Chris@148 407 }
Chris@148 408
Chris@148 409 if (m_cache.x > 0 && x < m_prevX && x < m_cache.x + m_cache.width/4) {
Chris@148 410 primeCache(x, true);
Chris@148 411 }
Chris@148 412
Chris@148 413 if (m_cache.x + m_cache.width < m_width &&
Chris@148 414 x > m_prevX &&
Chris@148 415 x > m_cache.x + (m_cache.width * 3) / 4) {
Chris@148 416 primeCache(x, false);
Chris@148 417 }
Chris@148 418
Chris@148 419 m_prevX = x;
Chris@148 420 return true;
Chris@148 421 }
Chris@148 422
Chris@148 423 void
Chris@148 424 MatrixFile::setColumnAt(size_t x, const void *data)
Chris@148 425 {
Chris@148 426 assert(m_mode == ReadWrite);
Chris@148 427
Chris@148 428 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@411 429 // std::cerr << "MatrixFile::setColumnAt(" << x << ")" << std::endl;
Chris@411 430 std::cerr << ".";
Chris@148 431 #endif
Chris@148 432
Chris@148 433 ssize_t w = 0;
Chris@148 434 bool seekFailed = false;
Chris@408 435
Chris@408 436 {
Chris@408 437 MutexLocker locker(&m_fdMutex, "MatrixFile::setColumnAt::m_fdMutex");
Chris@148 438
Chris@408 439 if (seekTo(x, 0)) {
Chris@408 440 w = ::write(m_fd, data, m_height * m_cellSize);
Chris@408 441 } else {
Chris@408 442 seekFailed = true;
Chris@408 443 }
Chris@148 444 }
Chris@148 445
Chris@148 446 if (!seekFailed && w != ssize_t(m_height * m_cellSize)) {
Chris@148 447 ::perror("WARNING: MatrixFile::setColumnAt: write failed");
Chris@148 448 throw FileOperationFailed(m_fileName, "write");
Chris@148 449 } else if (seekFailed) {
Chris@148 450 throw FileOperationFailed(m_fileName, "seek");
Chris@148 451 } else {
Chris@408 452 MutexLocker locker
Chris@408 453 (&m_columnBitsetWriteMutex,
Chris@408 454 "MatrixFile::setColumnAt::m_columnBitsetWriteMutex");
Chris@148 455 m_columnBitset->set(x);
Chris@148 456 }
Chris@148 457 }
Chris@148 458
Chris@148 459 void
Chris@148 460 MatrixFile::suspend()
Chris@148 461 {
Chris@408 462 MutexLocker locker(&m_fdMutex, "MatrixFile::suspend::m_fdMutex");
Chris@408 463 MutexLocker locker2(&m_cacheMutex, "MatrixFile::suspend::m_cacheMutex");
Chris@148 464
Chris@148 465 if (m_fd < 0) return; // already suspended
Chris@148 466
Chris@148 467 #ifdef DEBUG_MATRIX_FILE
Chris@148 468 std::cerr << "MatrixFile(" << this << ":" << m_fileName.toStdString() << ")::suspend(): fd was " << m_fd << std::endl;
Chris@148 469 #endif
Chris@148 470
Chris@148 471 if (m_requestToken >= 0) {
Chris@148 472 void *data = 0;
Chris@148 473 FileReadThread::Request request;
Chris@148 474 if (m_readThread->getRequest(m_requestToken, request)) {
Chris@148 475 data = request.data;
Chris@148 476 }
Chris@148 477 m_readThread->cancel(m_requestToken);
Chris@148 478 if (data) free(data);
Chris@148 479 m_requestToken = -1;
Chris@148 480 }
Chris@148 481
Chris@148 482 if (m_cache.data) {
Chris@148 483 free(m_cache.data);
Chris@148 484 m_cache.data = 0;
Chris@148 485 }
Chris@148 486
Chris@148 487 if (m_spareData) {
Chris@148 488 free(m_spareData);
Chris@148 489 m_spareData = 0;
Chris@148 490 }
Chris@148 491
Chris@148 492 if (::close(m_fd) < 0) {
Chris@148 493 ::perror("WARNING: MatrixFile::suspend: close failed");
Chris@148 494 throw FileOperationFailed(m_fileName, "close");
Chris@148 495 }
Chris@148 496
Chris@148 497 m_fd = -1;
Chris@148 498 }
Chris@148 499
Chris@148 500 void
Chris@148 501 MatrixFile::resume()
Chris@148 502 {
Chris@148 503 if (m_fd >= 0) return;
Chris@148 504
Chris@148 505 #ifdef DEBUG_MATRIX_FILE
Chris@148 506 std::cerr << "MatrixFile(" << this << ")::resume()" << std::endl;
Chris@148 507 #endif
Chris@148 508
Chris@148 509 if ((m_fd = ::open(m_fileName.toLocal8Bit(), m_flags, m_fmode)) < 0) {
Chris@148 510 ::perror("Open failed");
Chris@148 511 std::cerr << "ERROR: MatrixFile::resume: "
Chris@148 512 << "Failed to open cache file \""
Chris@148 513 << m_fileName.toStdString() << "\"";
Chris@148 514 if (m_mode == ReadWrite) std::cerr << " for writing";
Chris@148 515 std::cerr << std::endl;
Chris@148 516 throw FailedToOpenFile(m_fileName);
Chris@148 517 }
Chris@148 518
Chris@436 519 #ifdef DEBUG_MATRIX_FILE
Chris@148 520 std::cerr << "MatrixFile(" << this << ":" << m_fileName.toStdString() << ")::resume(): fd is " << m_fd << std::endl;
Chris@436 521 #endif
Chris@148 522 }
Chris@148 523
Chris@411 524 static int alloc = 0;
Chris@411 525
Chris@148 526 void
Chris@148 527 MatrixFile::primeCache(size_t x, bool goingLeft)
Chris@148 528 {
Chris@148 529 // Profiler profiler("MatrixFile::primeCache");
Chris@148 530
Chris@148 531 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@148 532 std::cerr << "MatrixFile::primeCache(" << x << ", " << goingLeft << ")" << std::endl;
Chris@148 533 #endif
Chris@148 534
Chris@148 535 size_t rx = x;
Chris@148 536 size_t rw = m_defaultCacheWidth;
Chris@148 537
Chris@148 538 size_t left = rw / 3;
Chris@148 539 if (goingLeft) left = (rw * 2) / 3;
Chris@148 540
Chris@148 541 if (rx > left) rx -= left;
Chris@148 542 else rx = 0;
Chris@148 543
Chris@148 544 if (rx + rw > m_width) rw = m_width - rx;
Chris@148 545
Chris@148 546 if (!m_eagerCache) {
Chris@148 547
Chris@148 548 size_t ti = 0;
Chris@148 549
Chris@148 550 for (ti = 0; ti < rw; ++ti) {
Chris@148 551 if (!m_columnBitset->get(rx + ti)) break;
Chris@148 552 }
Chris@148 553
Chris@148 554 #ifdef DEBUG_MATRIX_FILE
Chris@148 555 if (ti < rw) {
Chris@148 556 std::cerr << "eagerCache is false and there's a hole at "
Chris@148 557 << rx + ti << ", reducing rw from " << rw << " to "
Chris@148 558 << ti << std::endl;
Chris@148 559 }
Chris@148 560 #endif
Chris@148 561
Chris@148 562 rw = std::min(rw, ti);
Chris@148 563 if (rw < 10 || rx + rw <= x) return;
Chris@148 564 }
Chris@148 565
Chris@408 566 MutexLocker locker(&m_cacheMutex, "MatrixFile::primeCache::m_cacheMutex");
Chris@148 567
Chris@148 568 FileReadThread::Request request;
Chris@148 569
Chris@148 570 if (m_requestToken >= 0 &&
Chris@148 571 m_readThread->getRequest(m_requestToken, request)) {
Chris@148 572
Chris@148 573 if (x >= m_requestingX &&
Chris@148 574 x < m_requestingX + m_requestingWidth) {
Chris@148 575
Chris@148 576 if (m_readThread->isReady(m_requestToken)) {
Chris@148 577
Chris@148 578 if (!request.successful) {
Chris@148 579 std::cerr << "ERROR: MatrixFile::primeCache: Last request was unsuccessful" << std::endl;
Chris@148 580 throw FileReadFailed(m_fileName);
Chris@148 581 }
Chris@148 582
Chris@148 583 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@148 584 std::cerr << "last request is ready! (" << m_requestingX << ", "<< m_requestingWidth << ")" << std::endl;
Chris@148 585 #endif
Chris@148 586
Chris@148 587 m_cache.x = (request.start - m_headerSize) / (m_height * m_cellSize);
Chris@148 588 m_cache.width = request.size / (m_height * m_cellSize);
Chris@148 589
Chris@148 590 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@148 591 std::cerr << "received last request: actual size is: " << m_cache.x << ", " << m_cache.width << std::endl;
Chris@148 592 #endif
Chris@148 593
Chris@148 594 if (m_cache.data) {
Chris@411 595 if (m_spareData) {
Chris@412 596 // std::cerr << this << ": Freeing spare data" << std::endl;
Chris@411 597 free(m_spareData);
Chris@411 598 }
Chris@412 599 // std::cerr << this << ": Moving old cache data to spare" << std::endl;
Chris@148 600 m_spareData = m_cache.data;
Chris@148 601 }
Chris@412 602 // std::cerr << this << ": Moving request data to cache" << std::endl;
Chris@148 603 m_cache.data = request.data;
Chris@148 604
Chris@148 605 m_readThread->done(m_requestToken);
Chris@148 606 m_requestToken = -1;
Chris@148 607 }
Chris@148 608
Chris@148 609 // already requested something covering this area; wait for it
Chris@148 610 return;
Chris@148 611 }
Chris@148 612
Chris@148 613 // the current request is no longer of any use
Chris@148 614 m_readThread->cancel(m_requestToken);
Chris@148 615
Chris@148 616 // crude way to avoid leaking the data
Chris@148 617 while (!m_readThread->isCancelled(m_requestToken)) {
Chris@148 618 usleep(10000);
Chris@148 619 }
Chris@148 620
Chris@148 621 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@148 622 std::cerr << "cancelled " << m_requestToken << std::endl;
Chris@148 623 #endif
Chris@148 624
Chris@411 625 if (m_spareData) {
Chris@412 626 // std::cerr << this << ": Freeing spare data" << std::endl;
Chris@411 627 free(m_spareData);
Chris@411 628 }
Chris@412 629 // std::cerr << this << ": Moving request data to spare" << std::endl;
Chris@148 630 m_spareData = request.data;
Chris@148 631 m_readThread->done(m_requestToken);
Chris@148 632
Chris@148 633 m_requestToken = -1;
Chris@148 634 }
Chris@148 635
Chris@148 636 if (m_fd < 0) {
Chris@408 637 MutexLocker locker(&m_fdMutex, "MatrixFile::primeCache::m_fdMutex");
Chris@148 638 if (m_fd < 0) resume();
Chris@148 639 }
Chris@148 640
Chris@148 641 request.fd = m_fd;
Chris@148 642 request.mutex = &m_fdMutex;
Chris@148 643 request.start = m_headerSize + rx * m_height * m_cellSize;
Chris@148 644 request.size = rw * m_height * m_cellSize;
Chris@411 645
Chris@412 646 // std::cerr << this << ": Moving spare data to request, and resizing to " << rw * m_height * m_cellSize << std::endl;
Chris@411 647
Chris@148 648 request.data = (char *)realloc(m_spareData, rw * m_height * m_cellSize);
Chris@148 649 MUNLOCK(request.data, rw * m_height * m_cellSize);
Chris@148 650 m_spareData = 0;
Chris@148 651
Chris@148 652 m_requestingX = rx;
Chris@148 653 m_requestingWidth = rw;
Chris@148 654
Chris@148 655 int token = m_readThread->request(request);
Chris@148 656 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@148 657 std::cerr << "MatrixFile::primeCache: request token is "
Chris@148 658 << token << " (x = [" << rx << "], w = [" << rw << "], left = [" << goingLeft << "])" << std::endl;
Chris@148 659 #endif
Chris@148 660 m_requestToken = token;
Chris@148 661 }
Chris@148 662
Chris@148 663 bool
Chris@148 664 MatrixFile::seekTo(size_t x, size_t y)
Chris@148 665 {
Chris@148 666 if (m_fd < 0) resume();
Chris@148 667
Chris@148 668 off_t off = m_headerSize + (x * m_height + y) * m_cellSize;
Chris@148 669
Chris@148 670 if (::lseek(m_fd, off, SEEK_SET) == (off_t)-1) {
Chris@148 671 ::perror("Seek failed");
Chris@148 672 std::cerr << "ERROR: MatrixFile::seekTo(" << x << ", " << y
Chris@148 673 << ") failed" << std::endl;
Chris@148 674 return false;
Chris@148 675 }
Chris@148 676
Chris@148 677 return true;
Chris@148 678 }
Chris@148 679