Chris@148: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
Chris@148: 
Chris@148: /*
Chris@148:     Sonic Visualiser
Chris@148:     An audio file viewer and annotation editor.
Chris@148:     Centre for Digital Music, Queen Mary, University of London.
Chris@537:     This file copyright 2006-2009 Chris Cannam and QMUL.
Chris@148:     
Chris@148:     This program is free software; you can redistribute it and/or
Chris@148:     modify it under the terms of the GNU General Public License as
Chris@148:     published by the Free Software Foundation; either version 2 of the
Chris@148:     License, or (at your option) any later version.  See the file
Chris@148:     COPYING included with this distribution for more information.
Chris@148: */
Chris@148: 
Chris@148: #include "MatrixFile.h"
Chris@148: #include "base/TempDirectory.h"
Chris@150: #include "system/System.h"
Chris@148: #include "base/Profiler.h"
Chris@148: #include "base/Exceptions.h"
Chris@408: #include "base/Thread.h"
Chris@148: 
Chris@148: #include <sys/types.h>
Chris@148: #include <sys/stat.h>
Chris@148: #include <fcntl.h>
Chris@148: #include <unistd.h>
Chris@148: 
Chris@148: #include <iostream>
Chris@148: 
Chris@148: #include <cstdio>
Chris@148: #include <cassert>
Chris@148: 
Chris@405: #include <cstdlib>
Chris@405: 
Chris@148: #include <QFileInfo>
Chris@148: #include <QDir>
Chris@148: 
Chris@570: //#define DEBUG_MATRIX_FILE 1
Chris@403: //#define DEBUG_MATRIX_FILE_READ_SET 1
Chris@148: 
Chris@148: #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@153: #ifndef DEBUG_MATRIX_FILE
Chris@148: #define DEBUG_MATRIX_FILE 1
Chris@148: #endif
Chris@153: #endif
Chris@148: 
Chris@148: std::map<QString, int> MatrixFile::m_refcount;
Chris@537: QMutex MatrixFile::m_createMutex;
Chris@148: 
Chris@148: static size_t totalStorage = 0;
Chris@148: static size_t totalCount = 0;
Chris@537: static size_t openCount = 0;
Chris@148: 
Chris@148: MatrixFile::MatrixFile(QString fileBase, Mode mode,
Chris@537:                        size_t cellSize, size_t width, size_t height) :
Chris@148:     m_fd(-1),
Chris@148:     m_mode(mode),
Chris@148:     m_flags(0),
Chris@148:     m_fmode(0),
Chris@148:     m_cellSize(cellSize),
Chris@537:     m_width(width),
Chris@537:     m_height(height),
Chris@550:     m_headerSize(2 * sizeof(size_t)),
Chris@550:     m_setColumns(0),
Chris@554:     m_autoClose(false),
Chris@554:     m_readyToReadColumn(-1)
Chris@148: {
Chris@148:     Profiler profiler("MatrixFile::MatrixFile", true);
Chris@148: 
Chris@537: #ifdef DEBUG_MATRIX_FILE
Chris@537:     std::cerr << "MatrixFile::MatrixFile(" << fileBase.toStdString() << ", " << int(mode) << ", " << cellSize << ", " << width << ", " << height << ")" << std::endl;
Chris@537: #endif
Chris@148: 
Chris@548:     m_createMutex.lock();
Chris@148: 
Chris@148:     QDir tempDir(TempDirectory::getInstance()->getPath());
Chris@148:     QString fileName(tempDir.filePath(QString("%1.mfc").arg(fileBase)));
Chris@148:     bool newFile = !QFileInfo(fileName).exists();
Chris@148: 
Chris@148:     if (newFile && m_mode == ReadOnly) {
Chris@148:         std::cerr << "ERROR: MatrixFile::MatrixFile: Read-only mode "
Chris@148:                   << "specified, but cache file does not exist" << std::endl;
Chris@148:         throw FileNotFound(fileName);
Chris@148:     }
Chris@148: 
Chris@537:     if (!newFile && m_mode == WriteOnly) {
Chris@537:         std::cerr << "ERROR: MatrixFile::MatrixFile: Write-only mode "
Chris@537:                   << "specified, but file already exists" << std::endl;
Chris@537:         throw FileOperationFailed(fileName, "create");
Chris@148:     }
Chris@148: 
Chris@148:     m_flags = 0;
Chris@148:     m_fmode = S_IRUSR | S_IWUSR;
Chris@148: 
Chris@537:     if (m_mode == WriteOnly) {
Chris@537:         m_flags = O_WRONLY | O_CREAT;
Chris@148:     } else {
Chris@148:         m_flags = O_RDONLY;
Chris@148:     }
Chris@148: 
Chris@233: #ifdef _WIN32
Chris@233:     m_flags |= O_BINARY;
Chris@233: #endif
Chris@233: 
Chris@148: #ifdef DEBUG_MATRIX_FILE
Chris@537:     std::cerr << "MatrixFile(" << this << ")::MatrixFile: opening " << fileName.toStdString() << "..." << std::endl;
Chris@148: #endif
Chris@148: 
Chris@148:     if ((m_fd = ::open(fileName.toLocal8Bit(), m_flags, m_fmode)) < 0) {
Chris@148:         ::perror("Open failed");
Chris@148:         std::cerr << "ERROR: MatrixFile::MatrixFile: "
Chris@148:                   << "Failed to open cache file \""
Chris@148:                   << fileName.toStdString() << "\"";
Chris@537:         if (m_mode == WriteOnly) std::cerr << " for writing";
Chris@148:         std::cerr << std::endl;
Chris@148:         throw FailedToOpenFile(fileName);
Chris@148:     }
Chris@148: 
Chris@548:     m_createMutex.unlock();
Chris@548: 
Chris@537: #ifdef DEBUG_MATRIX_FILE
Chris@537:     std::cerr << "MatrixFile(" << this << ")::MatrixFile: fd is " << m_fd << std::endl;
Chris@537: #endif
Chris@537: 
Chris@148:     if (newFile) {
Chris@537:         initialise(); // write header and "unwritten" column tags
Chris@148:     } else {
Chris@148:         size_t header[2];
Chris@148:         if (::read(m_fd, header, 2 * sizeof(size_t)) < 0) {
Chris@236:             ::perror("MatrixFile::MatrixFile: read failed");
Chris@148:             std::cerr << "ERROR: MatrixFile::MatrixFile: "
Chris@148:                       << "Failed to read header (fd " << m_fd << ", file \""
Chris@148:                       << fileName.toStdString() << "\")" << std::endl;
Chris@148:             throw FileReadFailed(fileName);
Chris@148:         }
Chris@537:         if (header[0] != m_width || header[1] != m_height) {
Chris@537:             std::cerr << "ERROR: MatrixFile::MatrixFile: "
Chris@537:                       << "Dimensions in file header (" << header[0] << "x"
Chris@537:                       << header[1] << ") differ from expected dimensions "
Chris@537:                       << m_width << "x" << m_height << std::endl;
Chris@537:             throw FailedToOpenFile(fileName);
Chris@537:         }
Chris@148:     }
Chris@148: 
Chris@148:     m_fileName = fileName;
Chris@148:     ++m_refcount[fileName];
Chris@148: 
Chris@537: #ifdef DEBUG_MATRIX_FILE
Chris@537:     std::cerr << "MatrixFile[" << m_fd << "]::MatrixFile: File " << fileName.toStdString() << ", ref " << m_refcount[fileName] << std::endl;
Chris@148: 
Chris@537:     std::cerr << "MatrixFile[" << m_fd << "]::MatrixFile: Done, size is " << "(" << m_width << ", " << m_height << ")" << std::endl;
Chris@537: #endif
Chris@148: 
Chris@148:     ++totalCount;
Chris@537:     ++openCount;
Chris@148: }
Chris@148: 
Chris@148: MatrixFile::~MatrixFile()
Chris@148: {
Chris@148:     if (m_fd >= 0) {
Chris@148:         if (::close(m_fd) < 0) {
Chris@148:             ::perror("MatrixFile::~MatrixFile: close failed");
Chris@148:         }
Chris@546:         openCount --;
Chris@148:     }
Chris@148: 
Chris@537:     QMutexLocker locker(&m_createMutex);
Chris@537: 
Chris@550:     delete m_setColumns;
Chris@550: 
Chris@148:     if (m_fileName != "") {
Chris@148: 
Chris@148:         if (--m_refcount[m_fileName] == 0) {
Chris@148: 
Chris@148:             if (::unlink(m_fileName.toLocal8Bit())) {
Chris@537:                 std::cerr << "WARNING: MatrixFile::~MatrixFile: reference count reached 0, but failed to unlink file \"" << m_fileName.toStdString() << "\"" << std::endl;
Chris@148:             } else {
Chris@537:                 std::cerr << "deleted " << m_fileName.toStdString() << std::endl;
Chris@148:             }
Chris@148:         }
Chris@148:     }
Chris@537: 
Chris@537:     if (m_mode == WriteOnly) {
Chris@537:         totalStorage -= (m_headerSize + (m_width * m_height * m_cellSize) + m_width);
Chris@537:     }
Chris@148:     totalCount --;
Chris@148: 
Chris@537: #ifdef DEBUG_MATRIX_FILE
Chris@537:     std::cerr << "MatrixFile[" << m_fd << "]::~MatrixFile: " << std::endl;
Chris@537:     std::cerr << "MatrixFile: Total storage now " << totalStorage/1024 << "K in " << totalCount << " instances (" << openCount << " open)" << std::endl;
Chris@537: #endif
Chris@148: }
Chris@148: 
Chris@148: void
Chris@537: MatrixFile::initialise()
Chris@148: {
Chris@537:     Profiler profiler("MatrixFile::initialise", true);
Chris@148: 
Chris@537:     assert(m_mode == WriteOnly);
Chris@550: 
Chris@550:     m_setColumns = new ResizeableBitset(m_width);
Chris@148:     
Chris@537:     off_t off = m_headerSize + (m_width * m_height * m_cellSize) + m_width;
Chris@148: 
Chris@148: #ifdef DEBUG_MATRIX_FILE
Chris@537:     std::cerr << "MatrixFile[" << m_fd << "]::initialise(" << m_width << ", " << m_height << "): cell size " << m_cellSize << ", header size " << m_headerSize << ", resizing file" << std::endl;
Chris@148: #endif
Chris@148: 
Chris@537:     if (::lseek(m_fd, off - 1, SEEK_SET) < 0) {
Chris@537:         ::perror("ERROR: MatrixFile::initialise: seek to end failed");
Chris@537:         throw FileOperationFailed(m_fileName, "lseek");
Chris@148:     }
Chris@148: 
Chris@537:     unsigned char byte = 0;
Chris@537:     if (::write(m_fd, &byte, 1) != 1) {
Chris@537:         ::perror("ERROR: MatrixFile::initialise: write at end failed");
Chris@537:         throw FileOperationFailed(m_fileName, "write");
Chris@537:     }
Chris@148: 
Chris@537:     if (::lseek(m_fd, 0, SEEK_SET) < 0) {
Chris@547:         ::perror("ERROR: MatrixFile::initialise: Seek to write header failed");
Chris@148:         throw FileOperationFailed(m_fileName, "lseek");
Chris@148:     }
Chris@148: 
Chris@148:     size_t header[2];
Chris@537:     header[0] = m_width;
Chris@537:     header[1] = m_height;
Chris@148:     if (::write(m_fd, header, 2 * sizeof(size_t)) != 2 * sizeof(size_t)) {
Chris@547:         ::perror("ERROR: MatrixFile::initialise: Failed to write header");
Chris@148:         throw FileOperationFailed(m_fileName, "write");
Chris@148:     }
Chris@148: 
Chris@537:     if (m_mode == WriteOnly) {
Chris@537:         totalStorage += (m_headerSize + (m_width * m_height * m_cellSize) + m_width);
Chris@148:     }
Chris@148: 
Chris@537: #ifdef DEBUG_MATRIX_FILE
Chris@547:     std::cerr << "MatrixFile[" << m_fd << "]::initialise(" << m_width << ", " << m_height << "): storage "
Chris@537:               << (m_headerSize + m_width * m_height * m_cellSize + m_width) << std::endl;
Chris@148: 
Chris@537:     std::cerr << "MatrixFile: Total storage " << totalStorage/1024 << "K" << std::endl;
Chris@148: #endif
Chris@148: 
Chris@554:     seekTo(0);
Chris@148: }
Chris@148: 
Chris@148: void
Chris@537: MatrixFile::close()
Chris@148: {
Chris@537: #ifdef DEBUG_MATRIX_FILE
Chris@537:     std::cerr << "MatrixFile::close()" << std::endl;
Chris@537: #endif
Chris@537:     if (m_fd >= 0) {
Chris@537:         if (::close(m_fd) < 0) {
Chris@537:             ::perror("MatrixFile::close: close failed");
Chris@537:         }
Chris@537:         m_fd = -1;
Chris@537:         -- openCount;
Chris@550: #ifdef DEBUG_MATRIX_FILE
Chris@550:         std::cerr << "MatrixFile: Now " << openCount << " open instances" << std::endl;
Chris@550: #endif
Chris@148:     }
Chris@148: }
Chris@148: 
Chris@148: void
Chris@148: MatrixFile::getColumnAt(size_t x, void *data)
Chris@148: {
Chris@537:     assert(m_mode == ReadOnly);
Chris@537:     
Chris@537: #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@537:     std::cerr << "MatrixFile[" << m_fd << "]::getColumnAt(" << x << ")" << std::endl;
Chris@537: #endif
Chris@537: 
Chris@408:     Profiler profiler("MatrixFile::getColumnAt");
Chris@148: 
Chris@554:     ssize_t r = -1;
Chris@148: 
Chris@554:     if (m_readyToReadColumn < 0 ||
Chris@554:         size_t(m_readyToReadColumn) != x) {
Chris@554: 
Chris@554:         unsigned char set = 0;
Chris@554:         if (!seekTo(x)) {
Chris@554:             std::cerr << "ERROR: MatrixFile::getColumnAt(" << x << "): Seek failed" << std::endl;
Chris@554:             throw FileOperationFailed(m_fileName, "seek");
Chris@554:         }
Chris@554: 
Chris@554:         r = ::read(m_fd, &set, 1);
Chris@554:         if (r < 0) {
Chris@554:             ::perror("MatrixFile::getColumnAt: read failed");
Chris@554:             throw FileReadFailed(m_fileName);
Chris@554:         }
Chris@554:         if (!set) {
Chris@554:             std::cerr << "MatrixFile[" << m_fd << "]::getColumnAt(" << x << "): Column has not been set" << std::endl;
Chris@554:             return;
Chris@554:         }
Chris@537:     }
Chris@537: 
Chris@537:     r = ::read(m_fd, data, m_height * m_cellSize);
Chris@537:     if (r < 0) {
Chris@537:         ::perror("MatrixFile::getColumnAt: read failed");
Chris@537:         throw FileReadFailed(m_fileName);
Chris@537:     }
Chris@537: }
Chris@537: 
Chris@537: bool
Chris@537: MatrixFile::haveSetColumnAt(size_t x) const
Chris@537: {
Chris@550:     if (m_mode == WriteOnly) {
Chris@550:         return m_setColumns->get(x);
Chris@550:     }
Chris@537: 
Chris@554:     if (m_readyToReadColumn >= 0 &&
Chris@554:         size_t(m_readyToReadColumn) == x) return true;
Chris@554:     
Chris@537:     Profiler profiler("MatrixFile::haveSetColumnAt");
Chris@537: 
Chris@537: #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@537:     std::cerr << "MatrixFile[" << m_fd << "]::haveSetColumnAt(" << x << ")" << std::endl;
Chris@537: //    std::cerr << ".";
Chris@148: #endif
Chris@148: 
Chris@537:     unsigned char set = 0;
Chris@554:     if (!seekTo(x)) {
Chris@537:         std::cerr << "ERROR: MatrixFile::haveSetColumnAt(" << x << "): Seek failed" << std::endl;
Chris@537:         throw FileOperationFailed(m_fileName, "seek");
Chris@537:     }
Chris@148: 
Chris@537:     ssize_t r = -1;
Chris@537:     r = ::read(m_fd, &set, 1);
Chris@148:     if (r < 0) {
Chris@537:         ::perror("MatrixFile::haveSetColumnAt: read failed");
Chris@148:         throw FileReadFailed(m_fileName);
Chris@148:     }
Chris@148: 
Chris@554:     if (set) m_readyToReadColumn = int(x);
Chris@554: 
Chris@537:     return set;
Chris@148: }
Chris@148: 
Chris@148: void
Chris@148: MatrixFile::setColumnAt(size_t x, const void *data)
Chris@148: {
Chris@537:     assert(m_mode == WriteOnly);
Chris@550:     if (m_fd < 0) return; // closed
Chris@148: 
Chris@148: #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@537:     std::cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << ")" << std::endl;
Chris@537: //    std::cerr << ".";
Chris@148: #endif
Chris@148: 
Chris@148:     ssize_t w = 0;
Chris@148: 
Chris@554:     if (!seekTo(x)) {
Chris@537:         std::cerr << "ERROR: MatrixFile::setColumnAt(" << x << "): Seek failed" << std::endl;
Chris@537:         throw FileOperationFailed(m_fileName, "seek");
Chris@148:     }
Chris@148: 
Chris@537:     unsigned char set = 0;
Chris@537:     w = ::write(m_fd, &set, 1);
Chris@537:     if (w != 1) {
Chris@537:         ::perror("WARNING: MatrixFile::setColumnAt: write failed (1)");
Chris@148:         throw FileOperationFailed(m_fileName, "write");
Chris@537:     }
Chris@537: 
Chris@537:     w = ::write(m_fd, data, m_height * m_cellSize);
Chris@537:     if (w != ssize_t(m_height * m_cellSize)) {
Chris@537:         ::perror("WARNING: MatrixFile::setColumnAt: write failed (2)");
Chris@537:         throw FileOperationFailed(m_fileName, "write");
Chris@537:     }
Chris@537: /*
Chris@537:     if (x == 0) {
Chris@537:         std::cerr << "Wrote " << m_height * m_cellSize << " bytes, as follows:" << std::endl;
Chris@537:         for (int i = 0; i < m_height * m_cellSize; ++i) {
Chris@537:             std::cerr << (int)(((char *)data)[i]) << " ";
Chris@537:         }
Chris@537:         std::cerr << std::endl;
Chris@537:     }
Chris@537: */
Chris@554:     if (!seekTo(x)) {
Chris@537:         std::cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << "): Seek failed" << std::endl;
Chris@148:         throw FileOperationFailed(m_fileName, "seek");
Chris@537:     }
Chris@537: 
Chris@537:     set = 1;
Chris@537:     w = ::write(m_fd, &set, 1);
Chris@537:     if (w != 1) {
Chris@537:         ::perror("WARNING: MatrixFile::setColumnAt: write failed (3)");
Chris@537:         throw FileOperationFailed(m_fileName, "write");
Chris@148:     }
Chris@550: 
Chris@550:     m_setColumns->set(x);
Chris@550:     if (m_autoClose) {
Chris@550:         if (m_setColumns->isAllOn()) {
Chris@571: #ifdef DEBUG_MATRIX_FILE
Chris@550:             std::cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << "): All columns set: auto-closing" << std::endl;
Chris@571: #endif
Chris@550:             close();
Chris@550: /*
Chris@550:         } else {
Chris@550:             int set = 0;
Chris@550:             for (int i = 0; i < m_width; ++i) {
Chris@550:                 if (m_setColumns->get(i)) ++set;
Chris@550:             }
Chris@550:             std::cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << "): Auto-close on, but not all columns set yet (" << set << " of " << m_width << ")" << std::endl;
Chris@550: */
Chris@550:         }
Chris@550:     }
Chris@148: }
Chris@148: 
Chris@537: bool
Chris@554: MatrixFile::seekTo(size_t x) const
Chris@148: {
Chris@537:     if (m_fd < 0) {
Chris@537:         std::cerr << "ERROR: MatrixFile::seekTo: File not open" << std::endl;
Chris@537:         return false;
Chris@148:     }
Chris@148: 
Chris@554:     m_readyToReadColumn = -1; // not ready, unless this is subsequently re-set
Chris@554: 
Chris@554:     off_t off = m_headerSize + x * m_height * m_cellSize + x;
Chris@554: 
Chris@555: #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@554:     if (m_mode == ReadOnly) {
Chris@554:         std::cerr << "MatrixFile[" << m_fd << "]::seekTo(" << x << "): off = " << off << std::endl;
Chris@554:     }
Chris@555: #endif
Chris@148: 
Chris@148: #ifdef DEBUG_MATRIX_FILE_READ_SET
Chris@554:     std::cerr << "MatrixFile[" << m_fd << "]::seekTo(" << x << "): off = " << off << std::endl;
Chris@148: #endif
Chris@148: 
Chris@148:     if (::lseek(m_fd, off, SEEK_SET) == (off_t)-1) {
Chris@148:         ::perror("Seek failed");
Chris@554:         std::cerr << "ERROR: MatrixFile::seekTo(" << x 
Chris@537:                   << ") = " << off << " failed" << std::endl;
Chris@148:         return false;
Chris@148:     }
Chris@148: 
Chris@148:     return true;
Chris@148: }
Chris@148: