annotate data/fileio/MatrixFile.cpp @ 1160:ea636412f9fe 3.0-integration

Merge from the default branch
author Chris Cannam
date Thu, 04 Feb 2016 11:16:05 +0000
parents aa588c391d1a
children
rev   line source
Chris@148 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@148 2
Chris@148 3 /*
Chris@148 4 Sonic Visualiser
Chris@148 5 An audio file viewer and annotation editor.
Chris@148 6 Centre for Digital Music, Queen Mary, University of London.
Chris@537 7 This file copyright 2006-2009 Chris Cannam and QMUL.
Chris@148 8
Chris@148 9 This program is free software; you can redistribute it and/or
Chris@148 10 modify it under the terms of the GNU General Public License as
Chris@148 11 published by the Free Software Foundation; either version 2 of the
Chris@148 12 License, or (at your option) any later version. See the file
Chris@148 13 COPYING included with this distribution for more information.
Chris@148 14 */
Chris@148 15
Chris@148 16 #include "MatrixFile.h"
Chris@148 17 #include "base/TempDirectory.h"
Chris@150 18 #include "system/System.h"
Chris@148 19 #include "base/Profiler.h"
Chris@148 20 #include "base/Exceptions.h"
Chris@408 21 #include "base/Thread.h"
Chris@148 22
Chris@148 23 #include <sys/types.h>
Chris@148 24 #include <sys/stat.h>
Chris@148 25 #include <fcntl.h>
Chris@148 26 #include <unistd.h>
Chris@148 27
Chris@148 28 #include <iostream>
Chris@148 29
Chris@148 30 #include <cstdio>
Chris@148 31 #include <cassert>
Chris@148 32
Chris@405 33 #include <cstdlib>
Chris@405 34
Chris@148 35 #include <QFileInfo>
Chris@148 36 #include <QDir>
Chris@148 37
Chris@570 38 //#define DEBUG_MATRIX_FILE 1
Chris@403 39 //#define DEBUG_MATRIX_FILE_READ_SET 1
Chris@148 40
Chris@148 41 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@153 42 #ifndef DEBUG_MATRIX_FILE
Chris@148 43 #define DEBUG_MATRIX_FILE 1
Chris@148 44 #endif
Chris@153 45 #endif
Chris@148 46
Chris@148 47 std::map<QString, int> MatrixFile::m_refcount;
Chris@537 48 QMutex MatrixFile::m_createMutex;
Chris@148 49
Chris@148 50 static size_t totalStorage = 0;
Chris@148 51 static size_t totalCount = 0;
Chris@537 52 static size_t openCount = 0;
Chris@148 53
Chris@148 54 MatrixFile::MatrixFile(QString fileBase, Mode mode,
Chris@929 55 int cellSize, int width, int height) :
Chris@148 56 m_fd(-1),
Chris@148 57 m_mode(mode),
Chris@148 58 m_flags(0),
Chris@148 59 m_fmode(0),
Chris@148 60 m_cellSize(cellSize),
Chris@537 61 m_width(width),
Chris@537 62 m_height(height),
Chris@929 63 m_headerSize(2 * sizeof(int)),
Chris@550 64 m_setColumns(0),
Chris@554 65 m_autoClose(false),
Chris@554 66 m_readyToReadColumn(-1)
Chris@148 67 {
Chris@148 68 Profiler profiler("MatrixFile::MatrixFile", true);
Chris@148 69
Chris@537 70 #ifdef DEBUG_MATRIX_FILE
Chris@690 71 SVDEBUG << "MatrixFile::MatrixFile(" << fileBase << ", " << int(mode) << ", " << cellSize << ", " << width << ", " << height << ")" << endl;
Chris@537 72 #endif
Chris@148 73
Chris@548 74 m_createMutex.lock();
Chris@148 75
Chris@148 76 QDir tempDir(TempDirectory::getInstance()->getPath());
Chris@148 77 QString fileName(tempDir.filePath(QString("%1.mfc").arg(fileBase)));
Chris@148 78 bool newFile = !QFileInfo(fileName).exists();
Chris@148 79
Chris@148 80 if (newFile && m_mode == ReadOnly) {
Chris@843 81 cerr << "ERROR: MatrixFile::MatrixFile: Read-only mode "
Chris@843 82 << "specified, but cache file does not exist" << endl;
Chris@148 83 throw FileNotFound(fileName);
Chris@148 84 }
Chris@148 85
Chris@537 86 if (!newFile && m_mode == WriteOnly) {
Chris@843 87 cerr << "ERROR: MatrixFile::MatrixFile: Write-only mode "
Chris@843 88 << "specified, but file already exists" << endl;
Chris@537 89 throw FileOperationFailed(fileName, "create");
Chris@148 90 }
Chris@148 91
Chris@1078 92 // Use floating-point here to avoid integer overflow. We can be
Chris@1078 93 // approximate so long as we are on the cautious side
Chris@1078 94 if ((double(m_width) * m_height) * m_cellSize + m_headerSize + m_width >=
Chris@1078 95 pow(2, 31) - 10.0) { // bit of slack there
Chris@1078 96 cerr << "ERROR: MatrixFile::MatrixFile: width " << m_width
Chris@1078 97 << " is too large for height " << m_height << " and cell size "
Chris@1078 98 << m_cellSize << " (should be using multiple files)" << endl;
Chris@1078 99 throw FileOperationFailed(fileName, "size");
Chris@1078 100 }
Chris@1078 101
Chris@148 102 m_flags = 0;
Chris@148 103 m_fmode = S_IRUSR | S_IWUSR;
Chris@148 104
Chris@537 105 if (m_mode == WriteOnly) {
Chris@537 106 m_flags = O_WRONLY | O_CREAT;
Chris@148 107 } else {
Chris@148 108 m_flags = O_RDONLY;
Chris@148 109 }
Chris@148 110
Chris@233 111 #ifdef _WIN32
Chris@233 112 m_flags |= O_BINARY;
Chris@233 113 #endif
Chris@233 114
Chris@148 115 #ifdef DEBUG_MATRIX_FILE
Chris@843 116 cerr << "MatrixFile(" << this << ")::MatrixFile: opening " << fileName << "..." << endl;
Chris@148 117 #endif
Chris@148 118
Chris@148 119 if ((m_fd = ::open(fileName.toLocal8Bit(), m_flags, m_fmode)) < 0) {
Chris@148 120 ::perror("Open failed");
Chris@843 121 cerr << "ERROR: MatrixFile::MatrixFile: "
Chris@148 122 << "Failed to open cache file \""
Chris@686 123 << fileName << "\"";
Chris@843 124 if (m_mode == WriteOnly) cerr << " for writing";
Chris@843 125 cerr << endl;
Chris@148 126 throw FailedToOpenFile(fileName);
Chris@148 127 }
Chris@148 128
Chris@548 129 m_createMutex.unlock();
Chris@548 130
Chris@537 131 #ifdef DEBUG_MATRIX_FILE
Chris@843 132 cerr << "MatrixFile(" << this << ")::MatrixFile: fd is " << m_fd << endl;
Chris@537 133 #endif
Chris@537 134
Chris@148 135 if (newFile) {
Chris@537 136 initialise(); // write header and "unwritten" column tags
Chris@148 137 } else {
Chris@929 138 int header[2];
Chris@929 139 if (::read(m_fd, header, 2 * sizeof(int)) < 0) {
Chris@236 140 ::perror("MatrixFile::MatrixFile: read failed");
Chris@843 141 cerr << "ERROR: MatrixFile::MatrixFile: "
Chris@148 142 << "Failed to read header (fd " << m_fd << ", file \""
Chris@843 143 << fileName << "\")" << endl;
Chris@148 144 throw FileReadFailed(fileName);
Chris@148 145 }
Chris@537 146 if (header[0] != m_width || header[1] != m_height) {
Chris@843 147 cerr << "ERROR: MatrixFile::MatrixFile: "
Chris@537 148 << "Dimensions in file header (" << header[0] << "x"
Chris@537 149 << header[1] << ") differ from expected dimensions "
Chris@843 150 << m_width << "x" << m_height << endl;
Chris@537 151 throw FailedToOpenFile(fileName);
Chris@537 152 }
Chris@148 153 }
Chris@148 154
Chris@148 155 m_fileName = fileName;
Chris@148 156 ++m_refcount[fileName];
Chris@148 157
Chris@537 158 #ifdef DEBUG_MATRIX_FILE
Chris@843 159 cerr << "MatrixFile[" << m_fd << "]::MatrixFile: File " << fileName << ", ref " << m_refcount[fileName] << endl;
Chris@148 160
Chris@843 161 cerr << "MatrixFile[" << m_fd << "]::MatrixFile: Done, size is " << "(" << m_width << ", " << m_height << ")" << endl;
Chris@537 162 #endif
Chris@148 163
Chris@148 164 ++totalCount;
Chris@537 165 ++openCount;
Chris@148 166 }
Chris@148 167
Chris@148 168 MatrixFile::~MatrixFile()
Chris@148 169 {
Chris@148 170 if (m_fd >= 0) {
Chris@148 171 if (::close(m_fd) < 0) {
Chris@148 172 ::perror("MatrixFile::~MatrixFile: close failed");
Chris@148 173 }
Chris@546 174 openCount --;
Chris@148 175 }
Chris@148 176
Chris@537 177 QMutexLocker locker(&m_createMutex);
Chris@537 178
Chris@148 179 if (m_fileName != "") {
Chris@148 180
Chris@148 181 if (--m_refcount[m_fileName] == 0) {
Chris@148 182
Chris@148 183 if (::unlink(m_fileName.toLocal8Bit())) {
Chris@843 184 cerr << "WARNING: MatrixFile::~MatrixFile: reference count reached 0, but failed to unlink file \"" << m_fileName << "\"" << endl;
Chris@148 185 } else {
Chris@843 186 cerr << "deleted " << m_fileName << endl;
Chris@148 187 }
Chris@148 188 }
Chris@148 189 }
Chris@537 190
Chris@537 191 if (m_mode == WriteOnly) {
Chris@537 192 totalStorage -= (m_headerSize + (m_width * m_height * m_cellSize) + m_width);
Chris@537 193 }
Chris@148 194 totalCount --;
Chris@148 195
Chris@537 196 #ifdef DEBUG_MATRIX_FILE
Chris@843 197 cerr << "MatrixFile[" << m_fd << "]::~MatrixFile: " << endl;
Chris@843 198 cerr << "MatrixFile: Total storage now " << totalStorage/1024 << "K in " << totalCount << " instances (" << openCount << " open)" << endl;
Chris@537 199 #endif
Chris@148 200 }
Chris@148 201
Chris@148 202 void
Chris@537 203 MatrixFile::initialise()
Chris@148 204 {
Chris@537 205 Profiler profiler("MatrixFile::initialise", true);
Chris@148 206
Chris@537 207 assert(m_mode == WriteOnly);
Chris@550 208
Chris@1154 209 m_setColumns.resize(m_width, false);
Chris@148 210
Chris@537 211 off_t off = m_headerSize + (m_width * m_height * m_cellSize) + m_width;
Chris@148 212
Chris@148 213 #ifdef DEBUG_MATRIX_FILE
Chris@1078 214 cerr << "MatrixFile[" << m_fd << "]::initialise(" << m_width << ", " << m_height << "): cell size " << m_cellSize << ", header size " << m_headerSize << ", resizing fd " << m_fd << " to " << off << endl;
Chris@148 215 #endif
Chris@148 216
Chris@537 217 if (::lseek(m_fd, off - 1, SEEK_SET) < 0) {
Chris@537 218 ::perror("ERROR: MatrixFile::initialise: seek to end failed");
Chris@537 219 throw FileOperationFailed(m_fileName, "lseek");
Chris@148 220 }
Chris@148 221
Chris@537 222 unsigned char byte = 0;
Chris@537 223 if (::write(m_fd, &byte, 1) != 1) {
Chris@537 224 ::perror("ERROR: MatrixFile::initialise: write at end failed");
Chris@537 225 throw FileOperationFailed(m_fileName, "write");
Chris@537 226 }
Chris@148 227
Chris@537 228 if (::lseek(m_fd, 0, SEEK_SET) < 0) {
Chris@547 229 ::perror("ERROR: MatrixFile::initialise: Seek to write header failed");
Chris@148 230 throw FileOperationFailed(m_fileName, "lseek");
Chris@148 231 }
Chris@148 232
Chris@929 233 int header[2];
Chris@537 234 header[0] = m_width;
Chris@537 235 header[1] = m_height;
Chris@929 236 if (::write(m_fd, header, 2 * sizeof(int)) != 2 * sizeof(int)) {
Chris@547 237 ::perror("ERROR: MatrixFile::initialise: Failed to write header");
Chris@148 238 throw FileOperationFailed(m_fileName, "write");
Chris@148 239 }
Chris@148 240
Chris@537 241 if (m_mode == WriteOnly) {
Chris@537 242 totalStorage += (m_headerSize + (m_width * m_height * m_cellSize) + m_width);
Chris@148 243 }
Chris@148 244
Chris@537 245 #ifdef DEBUG_MATRIX_FILE
Chris@843 246 cerr << "MatrixFile[" << m_fd << "]::initialise(" << m_width << ", " << m_height << "): storage "
Chris@843 247 << (m_headerSize + m_width * m_height * m_cellSize + m_width) << endl;
Chris@148 248
Chris@843 249 cerr << "MatrixFile: Total storage " << totalStorage/1024 << "K" << endl;
Chris@148 250 #endif
Chris@148 251
Chris@554 252 seekTo(0);
Chris@148 253 }
Chris@148 254
Chris@148 255 void
Chris@537 256 MatrixFile::close()
Chris@148 257 {
Chris@537 258 #ifdef DEBUG_MATRIX_FILE
Chris@690 259 SVDEBUG << "MatrixFile::close()" << endl;
Chris@537 260 #endif
Chris@537 261 if (m_fd >= 0) {
Chris@537 262 if (::close(m_fd) < 0) {
Chris@537 263 ::perror("MatrixFile::close: close failed");
Chris@537 264 }
Chris@537 265 m_fd = -1;
Chris@537 266 -- openCount;
Chris@550 267 #ifdef DEBUG_MATRIX_FILE
Chris@843 268 cerr << "MatrixFile: Now " << openCount << " open instances" << endl;
Chris@550 269 #endif
Chris@148 270 }
Chris@148 271 }
Chris@148 272
Chris@148 273 void
Chris@929 274 MatrixFile::getColumnAt(int x, void *data)
Chris@148 275 {
Chris@537 276 assert(m_mode == ReadOnly);
Chris@537 277
Chris@537 278 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@843 279 cerr << "MatrixFile[" << m_fd << "]::getColumnAt(" << x << ")" << endl;
Chris@537 280 #endif
Chris@537 281
Chris@408 282 Profiler profiler("MatrixFile::getColumnAt");
Chris@148 283
Chris@554 284 ssize_t r = -1;
Chris@148 285
Chris@554 286 if (m_readyToReadColumn < 0 ||
Chris@929 287 m_readyToReadColumn != x) {
Chris@554 288
Chris@554 289 unsigned char set = 0;
Chris@554 290 if (!seekTo(x)) {
Chris@843 291 cerr << "ERROR: MatrixFile::getColumnAt(" << x << "): Seek failed" << endl;
Chris@554 292 throw FileOperationFailed(m_fileName, "seek");
Chris@554 293 }
Chris@554 294
Chris@554 295 r = ::read(m_fd, &set, 1);
Chris@554 296 if (r < 0) {
Chris@554 297 ::perror("MatrixFile::getColumnAt: read failed");
Chris@554 298 throw FileReadFailed(m_fileName);
Chris@554 299 }
Chris@554 300 if (!set) {
Chris@843 301 cerr << "MatrixFile[" << m_fd << "]::getColumnAt(" << x << "): Column has not been set" << endl;
Chris@554 302 return;
Chris@554 303 }
Chris@537 304 }
Chris@537 305
Chris@537 306 r = ::read(m_fd, data, m_height * m_cellSize);
Chris@537 307 if (r < 0) {
Chris@537 308 ::perror("MatrixFile::getColumnAt: read failed");
Chris@537 309 throw FileReadFailed(m_fileName);
Chris@537 310 }
Chris@537 311 }
Chris@537 312
Chris@537 313 bool
Chris@929 314 MatrixFile::haveSetColumnAt(int x) const
Chris@537 315 {
Chris@550 316 if (m_mode == WriteOnly) {
Chris@1154 317 return m_setColumns[x];
Chris@550 318 }
Chris@537 319
Chris@554 320 if (m_readyToReadColumn >= 0 &&
Chris@929 321 int(m_readyToReadColumn) == x) return true;
Chris@554 322
Chris@537 323 Profiler profiler("MatrixFile::haveSetColumnAt");
Chris@537 324
Chris@537 325 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@843 326 cerr << "MatrixFile[" << m_fd << "]::haveSetColumnAt(" << x << ")" << endl;
Chris@843 327 // cerr << ".";
Chris@148 328 #endif
Chris@148 329
Chris@537 330 unsigned char set = 0;
Chris@554 331 if (!seekTo(x)) {
Chris@843 332 cerr << "ERROR: MatrixFile::haveSetColumnAt(" << x << "): Seek failed" << endl;
Chris@537 333 throw FileOperationFailed(m_fileName, "seek");
Chris@537 334 }
Chris@148 335
Chris@537 336 ssize_t r = -1;
Chris@537 337 r = ::read(m_fd, &set, 1);
Chris@148 338 if (r < 0) {
Chris@537 339 ::perror("MatrixFile::haveSetColumnAt: read failed");
Chris@148 340 throw FileReadFailed(m_fileName);
Chris@148 341 }
Chris@148 342
Chris@554 343 if (set) m_readyToReadColumn = int(x);
Chris@554 344
Chris@537 345 return set;
Chris@148 346 }
Chris@148 347
Chris@148 348 void
Chris@929 349 MatrixFile::setColumnAt(int x, const void *data)
Chris@148 350 {
Chris@537 351 assert(m_mode == WriteOnly);
Chris@550 352 if (m_fd < 0) return; // closed
Chris@148 353
Chris@148 354 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@843 355 cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << ")" << endl;
Chris@843 356 // cerr << ".";
Chris@148 357 #endif
Chris@148 358
Chris@148 359 ssize_t w = 0;
Chris@148 360
Chris@554 361 if (!seekTo(x)) {
Chris@843 362 cerr << "ERROR: MatrixFile::setColumnAt(" << x << "): Seek failed" << endl;
Chris@537 363 throw FileOperationFailed(m_fileName, "seek");
Chris@148 364 }
Chris@148 365
Chris@537 366 unsigned char set = 0;
Chris@537 367 w = ::write(m_fd, &set, 1);
Chris@537 368 if (w != 1) {
Chris@537 369 ::perror("WARNING: MatrixFile::setColumnAt: write failed (1)");
Chris@148 370 throw FileOperationFailed(m_fileName, "write");
Chris@537 371 }
Chris@537 372
Chris@537 373 w = ::write(m_fd, data, m_height * m_cellSize);
Chris@537 374 if (w != ssize_t(m_height * m_cellSize)) {
Chris@537 375 ::perror("WARNING: MatrixFile::setColumnAt: write failed (2)");
Chris@537 376 throw FileOperationFailed(m_fileName, "write");
Chris@537 377 }
Chris@537 378 /*
Chris@537 379 if (x == 0) {
Chris@843 380 cerr << "Wrote " << m_height * m_cellSize << " bytes, as follows:" << endl;
Chris@537 381 for (int i = 0; i < m_height * m_cellSize; ++i) {
Chris@843 382 cerr << (int)(((char *)data)[i]) << " ";
Chris@537 383 }
Chris@843 384 cerr << endl;
Chris@537 385 }
Chris@537 386 */
Chris@554 387 if (!seekTo(x)) {
Chris@843 388 cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << "): Seek failed" << endl;
Chris@148 389 throw FileOperationFailed(m_fileName, "seek");
Chris@537 390 }
Chris@537 391
Chris@537 392 set = 1;
Chris@537 393 w = ::write(m_fd, &set, 1);
Chris@537 394 if (w != 1) {
Chris@537 395 ::perror("WARNING: MatrixFile::setColumnAt: write failed (3)");
Chris@537 396 throw FileOperationFailed(m_fileName, "write");
Chris@148 397 }
Chris@550 398
Chris@1154 399 m_setColumns[x] = true;
Chris@550 400 if (m_autoClose) {
Chris@1154 401 if (std::all_of(m_setColumns.begin(), m_setColumns.end(),
Chris@1154 402 [](bool c) { return c; })) {
Chris@571 403 #ifdef DEBUG_MATRIX_FILE
Chris@843 404 cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << "): All columns set: auto-closing" << endl;
Chris@571 405 #endif
Chris@550 406 close();
Chris@550 407 /*
Chris@550 408 } else {
Chris@550 409 int set = 0;
Chris@550 410 for (int i = 0; i < m_width; ++i) {
Chris@550 411 if (m_setColumns->get(i)) ++set;
Chris@550 412 }
Chris@843 413 cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << "): Auto-close on, but not all columns set yet (" << set << " of " << m_width << ")" << endl;
Chris@550 414 */
Chris@550 415 }
Chris@550 416 }
Chris@148 417 }
Chris@148 418
Chris@537 419 bool
Chris@929 420 MatrixFile::seekTo(int x) const
Chris@148 421 {
Chris@537 422 if (m_fd < 0) {
Chris@843 423 cerr << "ERROR: MatrixFile::seekTo: File not open" << endl;
Chris@537 424 return false;
Chris@148 425 }
Chris@148 426
Chris@554 427 m_readyToReadColumn = -1; // not ready, unless this is subsequently re-set
Chris@554 428
Chris@554 429 off_t off = m_headerSize + x * m_height * m_cellSize + x;
Chris@554 430
Chris@555 431 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@554 432 if (m_mode == ReadOnly) {
Chris@843 433 cerr << "MatrixFile[" << m_fd << "]::seekTo(" << x << "): off = " << off << endl;
Chris@554 434 }
Chris@555 435 #endif
Chris@148 436
Chris@148 437 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@843 438 cerr << "MatrixFile[" << m_fd << "]::seekTo(" << x << "): off = " << off << endl;
Chris@148 439 #endif
Chris@148 440
Chris@148 441 if (::lseek(m_fd, off, SEEK_SET) == (off_t)-1) {
Chris@148 442 ::perror("Seek failed");
Chris@843 443 cerr << "ERROR: MatrixFile::seekTo(" << x
Chris@843 444 << ") = " << off << " failed" << endl;
Chris@148 445 return false;
Chris@148 446 }
Chris@148 447
Chris@148 448 return true;
Chris@148 449 }
Chris@148 450