annotate data/fileio/MatrixFile.cpp @ 324:9cdc7a4efde2

* FileSource -> default to showProgress = false
author Chris Cannam
date Thu, 01 Nov 2007 10:45:28 +0000
parents dc46851837d6
children 7aa1de571880
rev   line source
Chris@148 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@148 2
Chris@148 3 /*
Chris@148 4 Sonic Visualiser
Chris@148 5 An audio file viewer and annotation editor.
Chris@148 6 Centre for Digital Music, Queen Mary, University of London.
Chris@148 7 This file copyright 2006 Chris Cannam.
Chris@148 8
Chris@148 9 This program is free software; you can redistribute it and/or
Chris@148 10 modify it under the terms of the GNU General Public License as
Chris@148 11 published by the Free Software Foundation; either version 2 of the
Chris@148 12 License, or (at your option) any later version. See the file
Chris@148 13 COPYING included with this distribution for more information.
Chris@148 14 */
Chris@148 15
Chris@148 16 #include "MatrixFile.h"
Chris@148 17 #include "base/TempDirectory.h"
Chris@150 18 #include "system/System.h"
Chris@148 19 #include "base/Profiler.h"
Chris@148 20 #include "base/Exceptions.h"
Chris@148 21
Chris@148 22 #include <sys/types.h>
Chris@148 23 #include <sys/stat.h>
Chris@148 24 #include <fcntl.h>
Chris@148 25 #include <unistd.h>
Chris@148 26
Chris@148 27 #include <iostream>
Chris@148 28
Chris@148 29 #include <cstdio>
Chris@148 30 #include <cassert>
Chris@148 31
Chris@148 32 #include <QFileInfo>
Chris@148 33 #include <QDir>
Chris@148 34
Chris@154 35 //#define DEBUG_MATRIX_FILE 1
Chris@154 36 //#define DEBUG_MATRIX_FILE_READ_SET 1
Chris@148 37
Chris@148 38 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@153 39 #ifndef DEBUG_MATRIX_FILE
Chris@148 40 #define DEBUG_MATRIX_FILE 1
Chris@148 41 #endif
Chris@153 42 #endif
Chris@148 43
Chris@148 44 std::map<QString, int> MatrixFile::m_refcount;
Chris@148 45 QMutex MatrixFile::m_refcountMutex;
Chris@148 46
Chris@148 47 MatrixFile::ResizeableBitsetMap MatrixFile::m_columnBitsets;
Chris@148 48 QMutex MatrixFile::m_columnBitsetWriteMutex;
Chris@148 49
Chris@148 50 FileReadThread *MatrixFile::m_readThread = 0;
Chris@148 51
Chris@148 52 static size_t totalStorage = 0;
Chris@148 53 static size_t totalMemory = 0;
Chris@148 54 static size_t totalCount = 0;
Chris@148 55
Chris@148 56 MatrixFile::MatrixFile(QString fileBase, Mode mode,
Chris@148 57 size_t cellSize, bool eagerCache) :
Chris@148 58 m_fd(-1),
Chris@148 59 m_mode(mode),
Chris@148 60 m_flags(0),
Chris@148 61 m_fmode(0),
Chris@148 62 m_cellSize(cellSize),
Chris@148 63 m_width(0),
Chris@148 64 m_height(0),
Chris@148 65 m_headerSize(2 * sizeof(size_t)),
Chris@148 66 m_defaultCacheWidth(1024),
Chris@148 67 m_prevX(0),
Chris@148 68 m_eagerCache(eagerCache),
Chris@148 69 m_requestToken(-1),
Chris@148 70 m_spareData(0),
Chris@148 71 m_columnBitset(0)
Chris@148 72 {
Chris@148 73 Profiler profiler("MatrixFile::MatrixFile", true);
Chris@148 74
Chris@148 75 if (!m_readThread) {
Chris@148 76 m_readThread = new FileReadThread;
Chris@148 77 m_readThread->start();
Chris@148 78 }
Chris@148 79
Chris@148 80 m_cache.data = 0;
Chris@148 81
Chris@148 82 QDir tempDir(TempDirectory::getInstance()->getPath());
Chris@148 83 QString fileName(tempDir.filePath(QString("%1.mfc").arg(fileBase)));
Chris@148 84 bool newFile = !QFileInfo(fileName).exists();
Chris@148 85
Chris@148 86 if (newFile && m_mode == ReadOnly) {
Chris@148 87 std::cerr << "ERROR: MatrixFile::MatrixFile: Read-only mode "
Chris@148 88 << "specified, but cache file does not exist" << std::endl;
Chris@148 89 throw FileNotFound(fileName);
Chris@148 90 }
Chris@148 91
Chris@148 92 if (!newFile && m_mode == ReadWrite) {
Chris@148 93 std::cerr << "Note: MatrixFile::MatrixFile: Read/write mode "
Chris@148 94 << "specified, but file already exists; falling back to "
Chris@148 95 << "read-only mode" << std::endl;
Chris@148 96 m_mode = ReadOnly;
Chris@148 97 }
Chris@148 98
Chris@148 99 if (!eagerCache && m_mode == ReadOnly) {
Chris@148 100 std::cerr << "WARNING: MatrixFile::MatrixFile: Eager cacheing not "
Chris@148 101 << "specified, but file is open in read-only mode -- cache "
Chris@148 102 << "will not be used" << std::endl;
Chris@148 103 }
Chris@148 104
Chris@148 105 m_flags = 0;
Chris@148 106 m_fmode = S_IRUSR | S_IWUSR;
Chris@148 107
Chris@148 108 if (m_mode == ReadWrite) {
Chris@148 109 m_flags = O_RDWR | O_CREAT;
Chris@148 110 } else {
Chris@148 111 m_flags = O_RDONLY;
Chris@148 112 }
Chris@148 113
Chris@233 114 #ifdef _WIN32
Chris@233 115 m_flags |= O_BINARY;
Chris@233 116 #endif
Chris@233 117
Chris@148 118 #ifdef DEBUG_MATRIX_FILE
Chris@148 119 std::cerr << "MatrixFile::MatrixFile: opening " << fileName.toStdString() << "..." << std::endl;
Chris@148 120 #endif
Chris@148 121
Chris@148 122 if ((m_fd = ::open(fileName.toLocal8Bit(), m_flags, m_fmode)) < 0) {
Chris@148 123 ::perror("Open failed");
Chris@148 124 std::cerr << "ERROR: MatrixFile::MatrixFile: "
Chris@148 125 << "Failed to open cache file \""
Chris@148 126 << fileName.toStdString() << "\"";
Chris@148 127 if (m_mode == ReadWrite) std::cerr << " for writing";
Chris@148 128 std::cerr << std::endl;
Chris@148 129 throw FailedToOpenFile(fileName);
Chris@148 130 }
Chris@148 131
Chris@148 132 if (newFile) {
Chris@148 133 resize(0, 0); // write header
Chris@148 134 } else {
Chris@148 135 size_t header[2];
Chris@148 136 if (::read(m_fd, header, 2 * sizeof(size_t)) < 0) {
Chris@236 137 ::perror("MatrixFile::MatrixFile: read failed");
Chris@148 138 std::cerr << "ERROR: MatrixFile::MatrixFile: "
Chris@148 139 << "Failed to read header (fd " << m_fd << ", file \""
Chris@148 140 << fileName.toStdString() << "\")" << std::endl;
Chris@148 141 throw FileReadFailed(fileName);
Chris@148 142 }
Chris@148 143 m_width = header[0];
Chris@148 144 m_height = header[1];
Chris@148 145 seekTo(0, 0);
Chris@148 146 }
Chris@148 147
Chris@148 148 m_fileName = fileName;
Chris@148 149
Chris@148 150 m_columnBitsetWriteMutex.lock();
Chris@148 151
Chris@148 152 if (m_columnBitsets.find(m_fileName) == m_columnBitsets.end()) {
Chris@148 153 m_columnBitsets[m_fileName] = new ResizeableBitset;
Chris@148 154 }
Chris@148 155 m_columnBitset = m_columnBitsets[m_fileName];
Chris@148 156
Chris@148 157 m_columnBitsetWriteMutex.unlock();
Chris@148 158
Chris@148 159 QMutexLocker locker(&m_refcountMutex);
Chris@148 160 ++m_refcount[fileName];
Chris@148 161
Chris@259 162 // std::cerr << "MatrixFile(" << this << "): fd " << m_fd << ", file " << fileName.toStdString() << ", ref " << m_refcount[fileName] << std::endl;
Chris@148 163
Chris@148 164 // std::cerr << "MatrixFile::MatrixFile: Done, size is " << "(" << m_width << ", " << m_height << ")" << std::endl;
Chris@148 165
Chris@148 166 ++totalCount;
Chris@148 167
Chris@148 168 }
Chris@148 169
Chris@148 170 MatrixFile::~MatrixFile()
Chris@148 171 {
Chris@148 172 char *requestData = 0;
Chris@148 173
Chris@148 174 if (m_requestToken >= 0) {
Chris@148 175 FileReadThread::Request request;
Chris@148 176 if (m_readThread->getRequest(m_requestToken, request)) {
Chris@148 177 requestData = request.data;
Chris@148 178 }
Chris@148 179 m_readThread->cancel(m_requestToken);
Chris@148 180 }
Chris@148 181
Chris@148 182 if (requestData) free(requestData);
Chris@148 183 if (m_cache.data) free(m_cache.data);
Chris@148 184 if (m_spareData) free(m_spareData);
Chris@148 185
Chris@148 186 if (m_fd >= 0) {
Chris@148 187 if (::close(m_fd) < 0) {
Chris@148 188 ::perror("MatrixFile::~MatrixFile: close failed");
Chris@148 189 }
Chris@148 190 }
Chris@148 191
Chris@148 192 if (m_fileName != "") {
Chris@148 193
Chris@148 194 QMutexLocker locker(&m_refcountMutex);
Chris@148 195
Chris@148 196 if (--m_refcount[m_fileName] == 0) {
Chris@148 197
Chris@148 198 if (::unlink(m_fileName.toLocal8Bit())) {
Chris@259 199 // ::perror("Unlink failed");
Chris@259 200 // std::cerr << "WARNING: MatrixFile::~MatrixFile: reference count reached 0, but failed to unlink file \"" << m_fileName.toStdString() << "\"" << std::endl;
Chris@148 201 } else {
Chris@259 202 // std::cerr << "deleted " << m_fileName.toStdString() << std::endl;
Chris@148 203 }
Chris@148 204
Chris@148 205 QMutexLocker locker2(&m_columnBitsetWriteMutex);
Chris@148 206 m_columnBitsets.erase(m_fileName);
Chris@148 207 delete m_columnBitset;
Chris@148 208 }
Chris@148 209 }
Chris@148 210
Chris@148 211 totalStorage -= (m_headerSize + (m_width * m_height * m_cellSize));
Chris@148 212 totalMemory -= (2 * m_defaultCacheWidth * m_height * m_cellSize);
Chris@148 213 totalCount --;
Chris@148 214
Chris@259 215 // std::cerr << "MatrixFile::~MatrixFile: " << std::endl;
Chris@259 216 // std::cerr << "Total storage now " << totalStorage/1024 << "K, theoretical max memory "
Chris@259 217 // << totalMemory/1024 << "K in " << totalCount << " instances" << std::endl;
Chris@148 218
Chris@148 219 }
Chris@148 220
Chris@148 221 void
Chris@148 222 MatrixFile::resize(size_t w, size_t h)
Chris@148 223 {
Chris@148 224 Profiler profiler("MatrixFile::resize", true);
Chris@148 225
Chris@148 226 assert(m_mode == ReadWrite);
Chris@148 227
Chris@148 228 QMutexLocker locker(&m_fdMutex);
Chris@148 229
Chris@148 230 totalStorage -= (m_headerSize + (m_width * m_height * m_cellSize));
Chris@148 231 totalMemory -= (2 * m_defaultCacheWidth * m_height * m_cellSize);
Chris@148 232
Chris@148 233 off_t off = m_headerSize + (w * h * m_cellSize);
Chris@148 234
Chris@148 235 #ifdef DEBUG_MATRIX_FILE
Chris@148 236 std::cerr << "MatrixFile::resize(" << w << ", " << h << "): resizing file" << std::endl;
Chris@148 237 #endif
Chris@148 238
Chris@148 239 if (w * h < m_width * m_height) {
Chris@148 240 if (::ftruncate(m_fd, off) < 0) {
Chris@148 241 ::perror("WARNING: MatrixFile::resize: ftruncate failed");
Chris@148 242 throw FileOperationFailed(m_fileName, "ftruncate");
Chris@148 243 }
Chris@148 244 }
Chris@148 245
Chris@148 246 m_width = 0;
Chris@148 247 m_height = 0;
Chris@148 248
Chris@148 249 if (::lseek(m_fd, 0, SEEK_SET) == (off_t)-1) {
Chris@148 250 ::perror("ERROR: MatrixFile::resize: Seek to write header failed");
Chris@148 251 throw FileOperationFailed(m_fileName, "lseek");
Chris@148 252 }
Chris@148 253
Chris@148 254 size_t header[2];
Chris@148 255 header[0] = w;
Chris@148 256 header[1] = h;
Chris@148 257 if (::write(m_fd, header, 2 * sizeof(size_t)) != 2 * sizeof(size_t)) {
Chris@148 258 ::perror("ERROR: MatrixFile::resize: Failed to write header");
Chris@148 259 throw FileOperationFailed(m_fileName, "write");
Chris@148 260 }
Chris@148 261
Chris@148 262 if (w > 0 && m_defaultCacheWidth > w) {
Chris@148 263 m_defaultCacheWidth = w;
Chris@148 264 }
Chris@148 265
Chris@148 266 static size_t maxCacheMB = 16;
Chris@148 267 if (2 * m_defaultCacheWidth * h * m_cellSize > maxCacheMB * 1024 * 1024) { //!!!
Chris@148 268 m_defaultCacheWidth = (maxCacheMB * 1024 * 1024) / (2 * h * m_cellSize);
Chris@148 269 if (m_defaultCacheWidth < 16) m_defaultCacheWidth = 16;
Chris@148 270 }
Chris@148 271
Chris@148 272 if (m_columnBitset) {
Chris@148 273 QMutexLocker locker(&m_columnBitsetWriteMutex);
Chris@148 274 m_columnBitset->resize(w);
Chris@148 275 }
Chris@148 276
Chris@148 277 if (m_cache.data) {
Chris@148 278 free(m_cache.data);
Chris@148 279 m_cache.data = 0;
Chris@148 280 }
Chris@148 281
Chris@148 282 if (m_spareData) {
Chris@148 283 free(m_spareData);
Chris@148 284 m_spareData = 0;
Chris@148 285 }
Chris@148 286
Chris@148 287 m_width = w;
Chris@148 288 m_height = h;
Chris@148 289
Chris@148 290 totalStorage += (m_headerSize + (m_width * m_height * m_cellSize));
Chris@148 291 totalMemory += (2 * m_defaultCacheWidth * m_height * m_cellSize);
Chris@148 292
Chris@148 293 #ifdef DEBUG_MATRIX_FILE
Chris@148 294 std::cerr << "MatrixFile::resize(" << w << ", " << h << "): cache width "
Chris@148 295 << m_defaultCacheWidth << ", storage "
Chris@148 296 << (m_headerSize + w * h * m_cellSize) << ", mem "
Chris@148 297 << (2 * h * m_defaultCacheWidth * m_cellSize) << std::endl;
Chris@148 298
Chris@148 299 std::cerr << "Total storage " << totalStorage/1024 << "K, theoretical max memory "
Chris@148 300 << totalMemory/1024 << "K in " << totalCount << " instances" << std::endl;
Chris@148 301 #endif
Chris@148 302
Chris@148 303 seekTo(0, 0);
Chris@148 304 }
Chris@148 305
Chris@148 306 void
Chris@148 307 MatrixFile::reset()
Chris@148 308 {
Chris@148 309 Profiler profiler("MatrixFile::reset", true);
Chris@148 310
Chris@148 311 assert (m_mode == ReadWrite);
Chris@148 312
Chris@148 313 if (m_eagerCache) {
Chris@148 314 void *emptyCol = calloc(m_height, m_cellSize);
Chris@148 315 for (size_t x = 0; x < m_width; ++x) setColumnAt(x, emptyCol);
Chris@148 316 free(emptyCol);
Chris@148 317 }
Chris@148 318
Chris@148 319 if (m_columnBitset) {
Chris@148 320 QMutexLocker locker(&m_columnBitsetWriteMutex);
Chris@148 321 m_columnBitset->resize(m_width);
Chris@148 322 }
Chris@148 323 }
Chris@148 324
Chris@148 325 void
Chris@148 326 MatrixFile::getColumnAt(size_t x, void *data)
Chris@148 327 {
Chris@148 328 // Profiler profiler("MatrixFile::getColumnAt");
Chris@148 329
Chris@148 330 // assert(haveSetColumnAt(x));
Chris@148 331
Chris@148 332 if (getFromCache(x, 0, m_height, data)) return;
Chris@148 333
Chris@148 334 // Profiler profiler2("MatrixFile::getColumnAt (uncached)");
Chris@148 335
Chris@148 336 ssize_t r = 0;
Chris@148 337
Chris@148 338 #ifdef DEBUG_MATRIX_FILE
Chris@148 339 std::cerr << "MatrixFile::getColumnAt(" << x << ")"
Chris@148 340 << ": reading the slow way";
Chris@148 341
Chris@148 342 if (m_requestToken >= 0 &&
Chris@148 343 x >= m_requestingX &&
Chris@148 344 x < m_requestingX + m_requestingWidth) {
Chris@148 345
Chris@148 346 std::cerr << " (awaiting " << m_requestingX << ", " << m_requestingWidth << " from disk)";
Chris@148 347 }
Chris@148 348
Chris@148 349 std::cerr << std::endl;
Chris@148 350 #endif
Chris@148 351
Chris@148 352 m_fdMutex.lock();
Chris@148 353
Chris@148 354 if (seekTo(x, 0)) {
Chris@148 355 r = ::read(m_fd, data, m_height * m_cellSize);
Chris@148 356 }
Chris@148 357
Chris@148 358 m_fdMutex.unlock();
Chris@148 359
Chris@148 360 if (r < 0) {
Chris@148 361 ::perror("MatrixFile::getColumnAt: read failed");
Chris@236 362 std::cerr << "ERROR: MatrixFile::getColumnAt: "
Chris@236 363 << "Failed to read column " << x << " (height " << m_height << ", cell size " << m_cellSize << ", fd " << m_fd << ", file \""
Chris@236 364 << m_fileName.toStdString() << "\")" << std::endl;
Chris@148 365 throw FileReadFailed(m_fileName);
Chris@148 366 }
Chris@148 367
Chris@148 368 return;
Chris@148 369 }
Chris@148 370
Chris@148 371 bool
Chris@148 372 MatrixFile::getFromCache(size_t x, size_t ystart, size_t ycount, void *data)
Chris@148 373 {
Chris@148 374 m_cacheMutex.lock();
Chris@148 375
Chris@148 376 if (!m_cache.data || x < m_cache.x || x >= m_cache.x + m_cache.width) {
Chris@148 377 bool left = (m_cache.data && x < m_cache.x);
Chris@148 378 m_cacheMutex.unlock();
Chris@148 379 primeCache(x, left); // this doesn't take effect until a later callback
Chris@148 380 m_prevX = x;
Chris@148 381 return false;
Chris@148 382 }
Chris@148 383
Chris@148 384 memcpy(data,
Chris@148 385 m_cache.data + m_cellSize * ((x - m_cache.x) * m_height + ystart),
Chris@148 386 ycount * m_cellSize);
Chris@148 387
Chris@148 388 m_cacheMutex.unlock();
Chris@148 389
Chris@148 390 if (m_cache.x > 0 && x < m_prevX && x < m_cache.x + m_cache.width/4) {
Chris@148 391 primeCache(x, true);
Chris@148 392 }
Chris@148 393
Chris@148 394 if (m_cache.x + m_cache.width < m_width &&
Chris@148 395 x > m_prevX &&
Chris@148 396 x > m_cache.x + (m_cache.width * 3) / 4) {
Chris@148 397 primeCache(x, false);
Chris@148 398 }
Chris@148 399
Chris@148 400 m_prevX = x;
Chris@148 401 return true;
Chris@148 402 }
Chris@148 403
Chris@148 404 void
Chris@148 405 MatrixFile::setColumnAt(size_t x, const void *data)
Chris@148 406 {
Chris@148 407 assert(m_mode == ReadWrite);
Chris@148 408
Chris@148 409 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@148 410 std::cerr << "MatrixFile::setColumnAt(" << x << ")" << std::endl;
Chris@148 411 #endif
Chris@148 412
Chris@148 413 ssize_t w = 0;
Chris@148 414 bool seekFailed = false;
Chris@148 415
Chris@148 416 m_fdMutex.lock();
Chris@148 417
Chris@148 418 if (seekTo(x, 0)) {
Chris@148 419 w = ::write(m_fd, data, m_height * m_cellSize);
Chris@148 420 } else {
Chris@148 421 seekFailed = true;
Chris@148 422 }
Chris@148 423
Chris@148 424 m_fdMutex.unlock();
Chris@148 425
Chris@148 426 if (!seekFailed && w != ssize_t(m_height * m_cellSize)) {
Chris@148 427 ::perror("WARNING: MatrixFile::setColumnAt: write failed");
Chris@148 428 throw FileOperationFailed(m_fileName, "write");
Chris@148 429 } else if (seekFailed) {
Chris@148 430 throw FileOperationFailed(m_fileName, "seek");
Chris@148 431 } else {
Chris@148 432 QMutexLocker locker(&m_columnBitsetWriteMutex);
Chris@148 433 m_columnBitset->set(x);
Chris@148 434 }
Chris@148 435 }
Chris@148 436
Chris@148 437 void
Chris@148 438 MatrixFile::suspend()
Chris@148 439 {
Chris@148 440 QMutexLocker locker(&m_fdMutex);
Chris@148 441 QMutexLocker locker2(&m_cacheMutex);
Chris@148 442
Chris@148 443 if (m_fd < 0) return; // already suspended
Chris@148 444
Chris@148 445 #ifdef DEBUG_MATRIX_FILE
Chris@148 446 std::cerr << "MatrixFile(" << this << ":" << m_fileName.toStdString() << ")::suspend(): fd was " << m_fd << std::endl;
Chris@148 447 #endif
Chris@148 448
Chris@148 449 if (m_requestToken >= 0) {
Chris@148 450 void *data = 0;
Chris@148 451 FileReadThread::Request request;
Chris@148 452 if (m_readThread->getRequest(m_requestToken, request)) {
Chris@148 453 data = request.data;
Chris@148 454 }
Chris@148 455 m_readThread->cancel(m_requestToken);
Chris@148 456 if (data) free(data);
Chris@148 457 m_requestToken = -1;
Chris@148 458 }
Chris@148 459
Chris@148 460 if (m_cache.data) {
Chris@148 461 free(m_cache.data);
Chris@148 462 m_cache.data = 0;
Chris@148 463 }
Chris@148 464
Chris@148 465 if (m_spareData) {
Chris@148 466 free(m_spareData);
Chris@148 467 m_spareData = 0;
Chris@148 468 }
Chris@148 469
Chris@148 470 if (::close(m_fd) < 0) {
Chris@148 471 ::perror("WARNING: MatrixFile::suspend: close failed");
Chris@148 472 throw FileOperationFailed(m_fileName, "close");
Chris@148 473 }
Chris@148 474
Chris@148 475 m_fd = -1;
Chris@148 476 }
Chris@148 477
Chris@148 478 void
Chris@148 479 MatrixFile::resume()
Chris@148 480 {
Chris@148 481 if (m_fd >= 0) return;
Chris@148 482
Chris@148 483 #ifdef DEBUG_MATRIX_FILE
Chris@148 484 std::cerr << "MatrixFile(" << this << ")::resume()" << std::endl;
Chris@148 485 #endif
Chris@148 486
Chris@148 487 if ((m_fd = ::open(m_fileName.toLocal8Bit(), m_flags, m_fmode)) < 0) {
Chris@148 488 ::perror("Open failed");
Chris@148 489 std::cerr << "ERROR: MatrixFile::resume: "
Chris@148 490 << "Failed to open cache file \""
Chris@148 491 << m_fileName.toStdString() << "\"";
Chris@148 492 if (m_mode == ReadWrite) std::cerr << " for writing";
Chris@148 493 std::cerr << std::endl;
Chris@148 494 throw FailedToOpenFile(m_fileName);
Chris@148 495 }
Chris@148 496
Chris@148 497 std::cerr << "MatrixFile(" << this << ":" << m_fileName.toStdString() << ")::resume(): fd is " << m_fd << std::endl;
Chris@148 498 }
Chris@148 499
Chris@148 500 void
Chris@148 501 MatrixFile::primeCache(size_t x, bool goingLeft)
Chris@148 502 {
Chris@148 503 // Profiler profiler("MatrixFile::primeCache");
Chris@148 504
Chris@148 505 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@148 506 std::cerr << "MatrixFile::primeCache(" << x << ", " << goingLeft << ")" << std::endl;
Chris@148 507 #endif
Chris@148 508
Chris@148 509 size_t rx = x;
Chris@148 510 size_t rw = m_defaultCacheWidth;
Chris@148 511
Chris@148 512 size_t left = rw / 3;
Chris@148 513 if (goingLeft) left = (rw * 2) / 3;
Chris@148 514
Chris@148 515 if (rx > left) rx -= left;
Chris@148 516 else rx = 0;
Chris@148 517
Chris@148 518 if (rx + rw > m_width) rw = m_width - rx;
Chris@148 519
Chris@148 520 if (!m_eagerCache) {
Chris@148 521
Chris@148 522 size_t ti = 0;
Chris@148 523
Chris@148 524 for (ti = 0; ti < rw; ++ti) {
Chris@148 525 if (!m_columnBitset->get(rx + ti)) break;
Chris@148 526 }
Chris@148 527
Chris@148 528 #ifdef DEBUG_MATRIX_FILE
Chris@148 529 if (ti < rw) {
Chris@148 530 std::cerr << "eagerCache is false and there's a hole at "
Chris@148 531 << rx + ti << ", reducing rw from " << rw << " to "
Chris@148 532 << ti << std::endl;
Chris@148 533 }
Chris@148 534 #endif
Chris@148 535
Chris@148 536 rw = std::min(rw, ti);
Chris@148 537 if (rw < 10 || rx + rw <= x) return;
Chris@148 538 }
Chris@148 539
Chris@148 540 QMutexLocker locker(&m_cacheMutex);
Chris@148 541
Chris@148 542 FileReadThread::Request request;
Chris@148 543
Chris@148 544 if (m_requestToken >= 0 &&
Chris@148 545 m_readThread->getRequest(m_requestToken, request)) {
Chris@148 546
Chris@148 547 if (x >= m_requestingX &&
Chris@148 548 x < m_requestingX + m_requestingWidth) {
Chris@148 549
Chris@148 550 if (m_readThread->isReady(m_requestToken)) {
Chris@148 551
Chris@148 552 if (!request.successful) {
Chris@148 553 std::cerr << "ERROR: MatrixFile::primeCache: Last request was unsuccessful" << std::endl;
Chris@148 554 throw FileReadFailed(m_fileName);
Chris@148 555 }
Chris@148 556
Chris@148 557 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@148 558 std::cerr << "last request is ready! (" << m_requestingX << ", "<< m_requestingWidth << ")" << std::endl;
Chris@148 559 #endif
Chris@148 560
Chris@148 561 m_cache.x = (request.start - m_headerSize) / (m_height * m_cellSize);
Chris@148 562 m_cache.width = request.size / (m_height * m_cellSize);
Chris@148 563
Chris@148 564 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@148 565 std::cerr << "received last request: actual size is: " << m_cache.x << ", " << m_cache.width << std::endl;
Chris@148 566 #endif
Chris@148 567
Chris@148 568 if (m_cache.data) {
Chris@148 569 if (m_spareData) free(m_spareData);
Chris@148 570 m_spareData = m_cache.data;
Chris@148 571 }
Chris@148 572 m_cache.data = request.data;
Chris@148 573
Chris@148 574 m_readThread->done(m_requestToken);
Chris@148 575 m_requestToken = -1;
Chris@148 576 }
Chris@148 577
Chris@148 578 // already requested something covering this area; wait for it
Chris@148 579 return;
Chris@148 580 }
Chris@148 581
Chris@148 582 // the current request is no longer of any use
Chris@148 583 m_readThread->cancel(m_requestToken);
Chris@148 584
Chris@148 585 // crude way to avoid leaking the data
Chris@148 586 while (!m_readThread->isCancelled(m_requestToken)) {
Chris@148 587 usleep(10000);
Chris@148 588 }
Chris@148 589
Chris@148 590 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@148 591 std::cerr << "cancelled " << m_requestToken << std::endl;
Chris@148 592 #endif
Chris@148 593
Chris@148 594 if (m_spareData) free(m_spareData);
Chris@148 595 m_spareData = request.data;
Chris@148 596 m_readThread->done(m_requestToken);
Chris@148 597
Chris@148 598 m_requestToken = -1;
Chris@148 599 }
Chris@148 600
Chris@148 601 if (m_fd < 0) {
Chris@148 602 m_fdMutex.lock();
Chris@148 603 if (m_fd < 0) resume();
Chris@148 604 m_fdMutex.unlock();
Chris@148 605 }
Chris@148 606
Chris@148 607 request.fd = m_fd;
Chris@148 608 request.mutex = &m_fdMutex;
Chris@148 609 request.start = m_headerSize + rx * m_height * m_cellSize;
Chris@148 610 request.size = rw * m_height * m_cellSize;
Chris@148 611 request.data = (char *)realloc(m_spareData, rw * m_height * m_cellSize);
Chris@148 612 MUNLOCK(request.data, rw * m_height * m_cellSize);
Chris@148 613 m_spareData = 0;
Chris@148 614
Chris@148 615 m_requestingX = rx;
Chris@148 616 m_requestingWidth = rw;
Chris@148 617
Chris@148 618 int token = m_readThread->request(request);
Chris@148 619 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@148 620 std::cerr << "MatrixFile::primeCache: request token is "
Chris@148 621 << token << " (x = [" << rx << "], w = [" << rw << "], left = [" << goingLeft << "])" << std::endl;
Chris@148 622 #endif
Chris@148 623 m_requestToken = token;
Chris@148 624 }
Chris@148 625
Chris@148 626 bool
Chris@148 627 MatrixFile::seekTo(size_t x, size_t y)
Chris@148 628 {
Chris@148 629 if (m_fd < 0) resume();
Chris@148 630
Chris@148 631 off_t off = m_headerSize + (x * m_height + y) * m_cellSize;
Chris@148 632
Chris@148 633 if (::lseek(m_fd, off, SEEK_SET) == (off_t)-1) {
Chris@148 634 ::perror("Seek failed");
Chris@148 635 std::cerr << "ERROR: MatrixFile::seekTo(" << x << ", " << y
Chris@148 636 << ") failed" << std::endl;
Chris@148 637 return false;
Chris@148 638 }
Chris@148 639
Chris@148 640 return true;
Chris@148 641 }
Chris@148 642