annotate data/fileio/MatrixFile.cpp @ 1119:e22bfe8ca248 tonioni

fixes the problem that the last note was not exported to csv
author Matthias Mauch <mail@matthiasmauch.net>
date Fri, 14 Aug 2015 18:16:14 +0100
parents 59e7fe1b1003
children ce82bcdc95d0
rev   line source
Chris@148 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@148 2
Chris@148 3 /*
Chris@148 4 Sonic Visualiser
Chris@148 5 An audio file viewer and annotation editor.
Chris@148 6 Centre for Digital Music, Queen Mary, University of London.
Chris@537 7 This file copyright 2006-2009 Chris Cannam and QMUL.
Chris@148 8
Chris@148 9 This program is free software; you can redistribute it and/or
Chris@148 10 modify it under the terms of the GNU General Public License as
Chris@148 11 published by the Free Software Foundation; either version 2 of the
Chris@148 12 License, or (at your option) any later version. See the file
Chris@148 13 COPYING included with this distribution for more information.
Chris@148 14 */
Chris@148 15
Chris@148 16 #include "MatrixFile.h"
Chris@148 17 #include "base/TempDirectory.h"
Chris@150 18 #include "system/System.h"
Chris@148 19 #include "base/Profiler.h"
Chris@148 20 #include "base/Exceptions.h"
Chris@408 21 #include "base/Thread.h"
Chris@148 22
Chris@148 23 #include <sys/types.h>
Chris@148 24 #include <sys/stat.h>
Chris@148 25 #include <fcntl.h>
Chris@148 26 #include <unistd.h>
Chris@148 27
Chris@148 28 #include <iostream>
Chris@148 29
Chris@148 30 #include <cstdio>
Chris@148 31 #include <cassert>
Chris@148 32
Chris@405 33 #include <cstdlib>
Chris@405 34
Chris@148 35 #include <QFileInfo>
Chris@148 36 #include <QDir>
Chris@148 37
Chris@570 38 //#define DEBUG_MATRIX_FILE 1
Chris@403 39 //#define DEBUG_MATRIX_FILE_READ_SET 1
Chris@148 40
Chris@148 41 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@153 42 #ifndef DEBUG_MATRIX_FILE
Chris@148 43 #define DEBUG_MATRIX_FILE 1
Chris@148 44 #endif
Chris@153 45 #endif
Chris@148 46
Chris@148 47 std::map<QString, int> MatrixFile::m_refcount;
Chris@537 48 QMutex MatrixFile::m_createMutex;
Chris@148 49
Chris@148 50 static size_t totalStorage = 0;
Chris@148 51 static size_t totalCount = 0;
Chris@537 52 static size_t openCount = 0;
Chris@148 53
Chris@148 54 MatrixFile::MatrixFile(QString fileBase, Mode mode,
Chris@929 55 int cellSize, int width, int height) :
Chris@148 56 m_fd(-1),
Chris@148 57 m_mode(mode),
Chris@148 58 m_flags(0),
Chris@148 59 m_fmode(0),
Chris@148 60 m_cellSize(cellSize),
Chris@537 61 m_width(width),
Chris@537 62 m_height(height),
Chris@929 63 m_headerSize(2 * sizeof(int)),
Chris@550 64 m_setColumns(0),
Chris@554 65 m_autoClose(false),
Chris@554 66 m_readyToReadColumn(-1)
Chris@148 67 {
Chris@148 68 Profiler profiler("MatrixFile::MatrixFile", true);
Chris@148 69
Chris@537 70 #ifdef DEBUG_MATRIX_FILE
Chris@690 71 SVDEBUG << "MatrixFile::MatrixFile(" << fileBase << ", " << int(mode) << ", " << cellSize << ", " << width << ", " << height << ")" << endl;
Chris@537 72 #endif
Chris@148 73
Chris@548 74 m_createMutex.lock();
Chris@148 75
Chris@148 76 QDir tempDir(TempDirectory::getInstance()->getPath());
Chris@148 77 QString fileName(tempDir.filePath(QString("%1.mfc").arg(fileBase)));
Chris@148 78 bool newFile = !QFileInfo(fileName).exists();
Chris@148 79
Chris@148 80 if (newFile && m_mode == ReadOnly) {
Chris@843 81 cerr << "ERROR: MatrixFile::MatrixFile: Read-only mode "
Chris@843 82 << "specified, but cache file does not exist" << endl;
Chris@148 83 throw FileNotFound(fileName);
Chris@148 84 }
Chris@148 85
Chris@537 86 if (!newFile && m_mode == WriteOnly) {
Chris@843 87 cerr << "ERROR: MatrixFile::MatrixFile: Write-only mode "
Chris@843 88 << "specified, but file already exists" << endl;
Chris@537 89 throw FileOperationFailed(fileName, "create");
Chris@148 90 }
Chris@148 91
Chris@148 92 m_flags = 0;
Chris@148 93 m_fmode = S_IRUSR | S_IWUSR;
Chris@148 94
Chris@537 95 if (m_mode == WriteOnly) {
Chris@537 96 m_flags = O_WRONLY | O_CREAT;
Chris@148 97 } else {
Chris@148 98 m_flags = O_RDONLY;
Chris@148 99 }
Chris@148 100
Chris@233 101 #ifdef _WIN32
Chris@233 102 m_flags |= O_BINARY;
Chris@233 103 #endif
Chris@233 104
Chris@148 105 #ifdef DEBUG_MATRIX_FILE
Chris@843 106 cerr << "MatrixFile(" << this << ")::MatrixFile: opening " << fileName << "..." << endl;
Chris@148 107 #endif
Chris@148 108
Chris@148 109 if ((m_fd = ::open(fileName.toLocal8Bit(), m_flags, m_fmode)) < 0) {
Chris@148 110 ::perror("Open failed");
Chris@843 111 cerr << "ERROR: MatrixFile::MatrixFile: "
Chris@148 112 << "Failed to open cache file \""
Chris@686 113 << fileName << "\"";
Chris@843 114 if (m_mode == WriteOnly) cerr << " for writing";
Chris@843 115 cerr << endl;
Chris@148 116 throw FailedToOpenFile(fileName);
Chris@148 117 }
Chris@148 118
Chris@548 119 m_createMutex.unlock();
Chris@548 120
Chris@537 121 #ifdef DEBUG_MATRIX_FILE
Chris@843 122 cerr << "MatrixFile(" << this << ")::MatrixFile: fd is " << m_fd << endl;
Chris@537 123 #endif
Chris@537 124
Chris@148 125 if (newFile) {
Chris@537 126 initialise(); // write header and "unwritten" column tags
Chris@148 127 } else {
Chris@929 128 int header[2];
Chris@929 129 if (::read(m_fd, header, 2 * sizeof(int)) < 0) {
Chris@236 130 ::perror("MatrixFile::MatrixFile: read failed");
Chris@843 131 cerr << "ERROR: MatrixFile::MatrixFile: "
Chris@148 132 << "Failed to read header (fd " << m_fd << ", file \""
Chris@843 133 << fileName << "\")" << endl;
Chris@148 134 throw FileReadFailed(fileName);
Chris@148 135 }
Chris@537 136 if (header[0] != m_width || header[1] != m_height) {
Chris@843 137 cerr << "ERROR: MatrixFile::MatrixFile: "
Chris@537 138 << "Dimensions in file header (" << header[0] << "x"
Chris@537 139 << header[1] << ") differ from expected dimensions "
Chris@843 140 << m_width << "x" << m_height << endl;
Chris@537 141 throw FailedToOpenFile(fileName);
Chris@537 142 }
Chris@148 143 }
Chris@148 144
Chris@148 145 m_fileName = fileName;
Chris@148 146 ++m_refcount[fileName];
Chris@148 147
Chris@537 148 #ifdef DEBUG_MATRIX_FILE
Chris@843 149 cerr << "MatrixFile[" << m_fd << "]::MatrixFile: File " << fileName << ", ref " << m_refcount[fileName] << endl;
Chris@148 150
Chris@843 151 cerr << "MatrixFile[" << m_fd << "]::MatrixFile: Done, size is " << "(" << m_width << ", " << m_height << ")" << endl;
Chris@537 152 #endif
Chris@148 153
Chris@148 154 ++totalCount;
Chris@537 155 ++openCount;
Chris@148 156 }
Chris@148 157
Chris@148 158 MatrixFile::~MatrixFile()
Chris@148 159 {
Chris@148 160 if (m_fd >= 0) {
Chris@148 161 if (::close(m_fd) < 0) {
Chris@148 162 ::perror("MatrixFile::~MatrixFile: close failed");
Chris@148 163 }
Chris@546 164 openCount --;
Chris@148 165 }
Chris@148 166
Chris@537 167 QMutexLocker locker(&m_createMutex);
Chris@537 168
Chris@550 169 delete m_setColumns;
Chris@550 170
Chris@148 171 if (m_fileName != "") {
Chris@148 172
Chris@148 173 if (--m_refcount[m_fileName] == 0) {
Chris@148 174
Chris@148 175 if (::unlink(m_fileName.toLocal8Bit())) {
Chris@843 176 cerr << "WARNING: MatrixFile::~MatrixFile: reference count reached 0, but failed to unlink file \"" << m_fileName << "\"" << endl;
Chris@148 177 } else {
Chris@843 178 cerr << "deleted " << m_fileName << endl;
Chris@148 179 }
Chris@148 180 }
Chris@148 181 }
Chris@537 182
Chris@537 183 if (m_mode == WriteOnly) {
Chris@537 184 totalStorage -= (m_headerSize + (m_width * m_height * m_cellSize) + m_width);
Chris@537 185 }
Chris@148 186 totalCount --;
Chris@148 187
Chris@537 188 #ifdef DEBUG_MATRIX_FILE
Chris@843 189 cerr << "MatrixFile[" << m_fd << "]::~MatrixFile: " << endl;
Chris@843 190 cerr << "MatrixFile: Total storage now " << totalStorage/1024 << "K in " << totalCount << " instances (" << openCount << " open)" << endl;
Chris@537 191 #endif
Chris@148 192 }
Chris@148 193
Chris@148 194 void
Chris@537 195 MatrixFile::initialise()
Chris@148 196 {
Chris@537 197 Profiler profiler("MatrixFile::initialise", true);
Chris@148 198
Chris@537 199 assert(m_mode == WriteOnly);
Chris@550 200
Chris@550 201 m_setColumns = new ResizeableBitset(m_width);
Chris@148 202
Chris@537 203 off_t off = m_headerSize + (m_width * m_height * m_cellSize) + m_width;
Chris@148 204
Chris@148 205 #ifdef DEBUG_MATRIX_FILE
Chris@843 206 cerr << "MatrixFile[" << m_fd << "]::initialise(" << m_width << ", " << m_height << "): cell size " << m_cellSize << ", header size " << m_headerSize << ", resizing file" << endl;
Chris@148 207 #endif
Chris@148 208
Chris@537 209 if (::lseek(m_fd, off - 1, SEEK_SET) < 0) {
Chris@537 210 ::perror("ERROR: MatrixFile::initialise: seek to end failed");
Chris@537 211 throw FileOperationFailed(m_fileName, "lseek");
Chris@148 212 }
Chris@148 213
Chris@537 214 unsigned char byte = 0;
Chris@537 215 if (::write(m_fd, &byte, 1) != 1) {
Chris@537 216 ::perror("ERROR: MatrixFile::initialise: write at end failed");
Chris@537 217 throw FileOperationFailed(m_fileName, "write");
Chris@537 218 }
Chris@148 219
Chris@537 220 if (::lseek(m_fd, 0, SEEK_SET) < 0) {
Chris@547 221 ::perror("ERROR: MatrixFile::initialise: Seek to write header failed");
Chris@148 222 throw FileOperationFailed(m_fileName, "lseek");
Chris@148 223 }
Chris@148 224
Chris@929 225 int header[2];
Chris@537 226 header[0] = m_width;
Chris@537 227 header[1] = m_height;
Chris@929 228 if (::write(m_fd, header, 2 * sizeof(int)) != 2 * sizeof(int)) {
Chris@547 229 ::perror("ERROR: MatrixFile::initialise: Failed to write header");
Chris@148 230 throw FileOperationFailed(m_fileName, "write");
Chris@148 231 }
Chris@148 232
Chris@537 233 if (m_mode == WriteOnly) {
Chris@537 234 totalStorage += (m_headerSize + (m_width * m_height * m_cellSize) + m_width);
Chris@148 235 }
Chris@148 236
Chris@537 237 #ifdef DEBUG_MATRIX_FILE
Chris@843 238 cerr << "MatrixFile[" << m_fd << "]::initialise(" << m_width << ", " << m_height << "): storage "
Chris@843 239 << (m_headerSize + m_width * m_height * m_cellSize + m_width) << endl;
Chris@148 240
Chris@843 241 cerr << "MatrixFile: Total storage " << totalStorage/1024 << "K" << endl;
Chris@148 242 #endif
Chris@148 243
Chris@554 244 seekTo(0);
Chris@148 245 }
Chris@148 246
Chris@148 247 void
Chris@537 248 MatrixFile::close()
Chris@148 249 {
Chris@537 250 #ifdef DEBUG_MATRIX_FILE
Chris@690 251 SVDEBUG << "MatrixFile::close()" << endl;
Chris@537 252 #endif
Chris@537 253 if (m_fd >= 0) {
Chris@537 254 if (::close(m_fd) < 0) {
Chris@537 255 ::perror("MatrixFile::close: close failed");
Chris@537 256 }
Chris@537 257 m_fd = -1;
Chris@537 258 -- openCount;
Chris@550 259 #ifdef DEBUG_MATRIX_FILE
Chris@843 260 cerr << "MatrixFile: Now " << openCount << " open instances" << endl;
Chris@550 261 #endif
Chris@148 262 }
Chris@148 263 }
Chris@148 264
Chris@148 265 void
Chris@929 266 MatrixFile::getColumnAt(int x, void *data)
Chris@148 267 {
Chris@537 268 assert(m_mode == ReadOnly);
Chris@537 269
Chris@537 270 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@843 271 cerr << "MatrixFile[" << m_fd << "]::getColumnAt(" << x << ")" << endl;
Chris@537 272 #endif
Chris@537 273
Chris@408 274 Profiler profiler("MatrixFile::getColumnAt");
Chris@148 275
Chris@554 276 ssize_t r = -1;
Chris@148 277
Chris@554 278 if (m_readyToReadColumn < 0 ||
Chris@929 279 m_readyToReadColumn != x) {
Chris@554 280
Chris@554 281 unsigned char set = 0;
Chris@554 282 if (!seekTo(x)) {
Chris@843 283 cerr << "ERROR: MatrixFile::getColumnAt(" << x << "): Seek failed" << endl;
Chris@554 284 throw FileOperationFailed(m_fileName, "seek");
Chris@554 285 }
Chris@554 286
Chris@554 287 r = ::read(m_fd, &set, 1);
Chris@554 288 if (r < 0) {
Chris@554 289 ::perror("MatrixFile::getColumnAt: read failed");
Chris@554 290 throw FileReadFailed(m_fileName);
Chris@554 291 }
Chris@554 292 if (!set) {
Chris@843 293 cerr << "MatrixFile[" << m_fd << "]::getColumnAt(" << x << "): Column has not been set" << endl;
Chris@554 294 return;
Chris@554 295 }
Chris@537 296 }
Chris@537 297
Chris@537 298 r = ::read(m_fd, data, m_height * m_cellSize);
Chris@537 299 if (r < 0) {
Chris@537 300 ::perror("MatrixFile::getColumnAt: read failed");
Chris@537 301 throw FileReadFailed(m_fileName);
Chris@537 302 }
Chris@537 303 }
Chris@537 304
Chris@537 305 bool
Chris@929 306 MatrixFile::haveSetColumnAt(int x) const
Chris@537 307 {
Chris@550 308 if (m_mode == WriteOnly) {
Chris@550 309 return m_setColumns->get(x);
Chris@550 310 }
Chris@537 311
Chris@554 312 if (m_readyToReadColumn >= 0 &&
Chris@929 313 int(m_readyToReadColumn) == x) return true;
Chris@554 314
Chris@537 315 Profiler profiler("MatrixFile::haveSetColumnAt");
Chris@537 316
Chris@537 317 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@843 318 cerr << "MatrixFile[" << m_fd << "]::haveSetColumnAt(" << x << ")" << endl;
Chris@843 319 // cerr << ".";
Chris@148 320 #endif
Chris@148 321
Chris@537 322 unsigned char set = 0;
Chris@554 323 if (!seekTo(x)) {
Chris@843 324 cerr << "ERROR: MatrixFile::haveSetColumnAt(" << x << "): Seek failed" << endl;
Chris@537 325 throw FileOperationFailed(m_fileName, "seek");
Chris@537 326 }
Chris@148 327
Chris@537 328 ssize_t r = -1;
Chris@537 329 r = ::read(m_fd, &set, 1);
Chris@148 330 if (r < 0) {
Chris@537 331 ::perror("MatrixFile::haveSetColumnAt: read failed");
Chris@148 332 throw FileReadFailed(m_fileName);
Chris@148 333 }
Chris@148 334
Chris@554 335 if (set) m_readyToReadColumn = int(x);
Chris@554 336
Chris@537 337 return set;
Chris@148 338 }
Chris@148 339
Chris@148 340 void
Chris@929 341 MatrixFile::setColumnAt(int x, const void *data)
Chris@148 342 {
Chris@537 343 assert(m_mode == WriteOnly);
Chris@550 344 if (m_fd < 0) return; // closed
Chris@148 345
Chris@148 346 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@843 347 cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << ")" << endl;
Chris@843 348 // cerr << ".";
Chris@148 349 #endif
Chris@148 350
Chris@148 351 ssize_t w = 0;
Chris@148 352
Chris@554 353 if (!seekTo(x)) {
Chris@843 354 cerr << "ERROR: MatrixFile::setColumnAt(" << x << "): Seek failed" << endl;
Chris@537 355 throw FileOperationFailed(m_fileName, "seek");
Chris@148 356 }
Chris@148 357
Chris@537 358 unsigned char set = 0;
Chris@537 359 w = ::write(m_fd, &set, 1);
Chris@537 360 if (w != 1) {
Chris@537 361 ::perror("WARNING: MatrixFile::setColumnAt: write failed (1)");
Chris@148 362 throw FileOperationFailed(m_fileName, "write");
Chris@537 363 }
Chris@537 364
Chris@537 365 w = ::write(m_fd, data, m_height * m_cellSize);
Chris@537 366 if (w != ssize_t(m_height * m_cellSize)) {
Chris@537 367 ::perror("WARNING: MatrixFile::setColumnAt: write failed (2)");
Chris@537 368 throw FileOperationFailed(m_fileName, "write");
Chris@537 369 }
Chris@537 370 /*
Chris@537 371 if (x == 0) {
Chris@843 372 cerr << "Wrote " << m_height * m_cellSize << " bytes, as follows:" << endl;
Chris@537 373 for (int i = 0; i < m_height * m_cellSize; ++i) {
Chris@843 374 cerr << (int)(((char *)data)[i]) << " ";
Chris@537 375 }
Chris@843 376 cerr << endl;
Chris@537 377 }
Chris@537 378 */
Chris@554 379 if (!seekTo(x)) {
Chris@843 380 cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << "): Seek failed" << endl;
Chris@148 381 throw FileOperationFailed(m_fileName, "seek");
Chris@537 382 }
Chris@537 383
Chris@537 384 set = 1;
Chris@537 385 w = ::write(m_fd, &set, 1);
Chris@537 386 if (w != 1) {
Chris@537 387 ::perror("WARNING: MatrixFile::setColumnAt: write failed (3)");
Chris@537 388 throw FileOperationFailed(m_fileName, "write");
Chris@148 389 }
Chris@550 390
Chris@550 391 m_setColumns->set(x);
Chris@550 392 if (m_autoClose) {
Chris@550 393 if (m_setColumns->isAllOn()) {
Chris@571 394 #ifdef DEBUG_MATRIX_FILE
Chris@843 395 cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << "): All columns set: auto-closing" << endl;
Chris@571 396 #endif
Chris@550 397 close();
Chris@550 398 /*
Chris@550 399 } else {
Chris@550 400 int set = 0;
Chris@550 401 for (int i = 0; i < m_width; ++i) {
Chris@550 402 if (m_setColumns->get(i)) ++set;
Chris@550 403 }
Chris@843 404 cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << "): Auto-close on, but not all columns set yet (" << set << " of " << m_width << ")" << endl;
Chris@550 405 */
Chris@550 406 }
Chris@550 407 }
Chris@148 408 }
Chris@148 409
Chris@537 410 bool
Chris@929 411 MatrixFile::seekTo(int x) const
Chris@148 412 {
Chris@537 413 if (m_fd < 0) {
Chris@843 414 cerr << "ERROR: MatrixFile::seekTo: File not open" << endl;
Chris@537 415 return false;
Chris@148 416 }
Chris@148 417
Chris@554 418 m_readyToReadColumn = -1; // not ready, unless this is subsequently re-set
Chris@554 419
Chris@554 420 off_t off = m_headerSize + x * m_height * m_cellSize + x;
Chris@554 421
Chris@555 422 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@554 423 if (m_mode == ReadOnly) {
Chris@843 424 cerr << "MatrixFile[" << m_fd << "]::seekTo(" << x << "): off = " << off << endl;
Chris@554 425 }
Chris@555 426 #endif
Chris@148 427
Chris@148 428 #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@843 429 cerr << "MatrixFile[" << m_fd << "]::seekTo(" << x << "): off = " << off << endl;
Chris@148 430 #endif
Chris@148 431
Chris@148 432 if (::lseek(m_fd, off, SEEK_SET) == (off_t)-1) {
Chris@148 433 ::perror("Seek failed");
Chris@843 434 cerr << "ERROR: MatrixFile::seekTo(" << x
Chris@843 435 << ") = " << off << " failed" << endl;
Chris@148 436 return false;
Chris@148 437 }
Chris@148 438
Chris@148 439 return true;
Chris@148 440 }
Chris@148 441