# HG changeset patch
# User Chris Cannam
# Date 1476795846 -3600
# Node ID 046f05fa31f3bf8e4358280b656ff76400f8f9cf
# Parent  c811991a5efafa9fee2966795c6ea6a2d1211422# Parent  f50ded4b951c9ba4938b881ccefa2d9fa8e82030
Merge from branch "3.0-integration"

diff -r c811991a5efa -r 046f05fa31f3 data/fileio/MatrixFile.cpp
--- a/data/fileio/MatrixFile.cpp	Tue Oct 18 14:03:55 2016 +0100
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,450 +0,0 @@
-/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
-
-/*
-    Sonic Visualiser
-    An audio file viewer and annotation editor.
-    Centre for Digital Music, Queen Mary, University of London.
-    This file copyright 2006-2009 Chris Cannam and QMUL.
-    
-    This program is free software; you can redistribute it and/or
-    modify it under the terms of the GNU General Public License as
-    published by the Free Software Foundation; either version 2 of the
-    License, or (at your option) any later version.  See the file
-    COPYING included with this distribution for more information.
-*/
-
-#include "MatrixFile.h"
-#include "base/TempDirectory.h"
-#include "system/System.h"
-#include "base/Profiler.h"
-#include "base/Exceptions.h"
-#include "base/Thread.h"
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-
-#include <iostream>
-
-#include <cstdio>
-#include <cassert>
-
-#include <cstdlib>
-
-#include <QFileInfo>
-#include <QDir>
-
-//#define DEBUG_MATRIX_FILE 1
-//#define DEBUG_MATRIX_FILE_READ_SET 1
-
-#ifdef DEBUG_MATRIX_FILE_READ_SET
-#ifndef DEBUG_MATRIX_FILE
-#define DEBUG_MATRIX_FILE 1
-#endif
-#endif
-
-std::map<QString, int> MatrixFile::m_refcount;
-QMutex MatrixFile::m_createMutex;
-
-static size_t totalStorage = 0;
-static size_t totalCount = 0;
-static size_t openCount = 0;
-
-MatrixFile::MatrixFile(QString fileBase, Mode mode,
-                       int cellSize, int width, int height) :
-    m_fd(-1),
-    m_mode(mode),
-    m_flags(0),
-    m_fmode(0),
-    m_cellSize(cellSize),
-    m_width(width),
-    m_height(height),
-    m_headerSize(2 * sizeof(int)),
-    m_setColumns(0),
-    m_autoClose(false),
-    m_readyToReadColumn(-1)
-{
-    Profiler profiler("MatrixFile::MatrixFile", true);
-
-#ifdef DEBUG_MATRIX_FILE
-    SVDEBUG << "MatrixFile::MatrixFile(" << fileBase << ", " << int(mode) << ", " << cellSize << ", " << width << ", " << height << ")" << endl;
-#endif
-
-    m_createMutex.lock();
-
-    QDir tempDir(TempDirectory::getInstance()->getPath());
-    QString fileName(tempDir.filePath(QString("%1.mfc").arg(fileBase)));
-    bool newFile = !QFileInfo(fileName).exists();
-
-    if (newFile && m_mode == ReadOnly) {
-        cerr << "ERROR: MatrixFile::MatrixFile: Read-only mode "
-                  << "specified, but cache file does not exist" << endl;
-        throw FileNotFound(fileName);
-    }
-
-    if (!newFile && m_mode == WriteOnly) {
-        cerr << "ERROR: MatrixFile::MatrixFile: Write-only mode "
-                  << "specified, but file already exists" << endl;
-        throw FileOperationFailed(fileName, "create");
-    }
-
-    // Use floating-point here to avoid integer overflow. We can be
-    // approximate so long as we are on the cautious side
-    if ((double(m_width) * m_height) * m_cellSize + m_headerSize + m_width >=
-        pow(2, 31) - 10.0) { // bit of slack there
-        cerr << "ERROR: MatrixFile::MatrixFile: width " << m_width
-             << " is too large for height " << m_height << " and cell size "
-             << m_cellSize << " (should be using multiple files)" << endl;
-        throw FileOperationFailed(fileName, "size");
-    }
-    
-    m_flags = 0;
-    m_fmode = S_IRUSR | S_IWUSR;
-
-    if (m_mode == WriteOnly) {
-        m_flags = O_WRONLY | O_CREAT;
-    } else {
-        m_flags = O_RDONLY;
-    }
-
-#ifdef _WIN32
-    m_flags |= O_BINARY;
-#endif
-
-#ifdef DEBUG_MATRIX_FILE
-    cerr << "MatrixFile(" << this << ")::MatrixFile: opening " << fileName << "..." << endl;
-#endif
-
-    if ((m_fd = ::open(fileName.toLocal8Bit(), m_flags, m_fmode)) < 0) {
-        ::perror("Open failed");
-        cerr << "ERROR: MatrixFile::MatrixFile: "
-                  << "Failed to open cache file \""
-                  << fileName << "\"";
-        if (m_mode == WriteOnly) cerr << " for writing";
-        cerr << endl;
-        throw FailedToOpenFile(fileName);
-    }
-
-    m_createMutex.unlock();
-
-#ifdef DEBUG_MATRIX_FILE
-    cerr << "MatrixFile(" << this << ")::MatrixFile: fd is " << m_fd << endl;
-#endif
-
-    if (newFile) {
-        initialise(); // write header and "unwritten" column tags
-    } else {
-        int header[2];
-        if (::read(m_fd, header, 2 * sizeof(int)) < 0) {
-            ::perror("MatrixFile::MatrixFile: read failed");
-            cerr << "ERROR: MatrixFile::MatrixFile: "
-                      << "Failed to read header (fd " << m_fd << ", file \""
-                      << fileName << "\")" << endl;
-            throw FileReadFailed(fileName);
-        }
-        if (header[0] != m_width || header[1] != m_height) {
-            cerr << "ERROR: MatrixFile::MatrixFile: "
-                      << "Dimensions in file header (" << header[0] << "x"
-                      << header[1] << ") differ from expected dimensions "
-                      << m_width << "x" << m_height << endl;
-            throw FailedToOpenFile(fileName);
-        }
-    }
-
-    m_fileName = fileName;
-    ++m_refcount[fileName];
-
-#ifdef DEBUG_MATRIX_FILE
-    cerr << "MatrixFile[" << m_fd << "]::MatrixFile: File " << fileName << ", ref " << m_refcount[fileName] << endl;
-
-    cerr << "MatrixFile[" << m_fd << "]::MatrixFile: Done, size is " << "(" << m_width << ", " << m_height << ")" << endl;
-#endif
-
-    ++totalCount;
-    ++openCount;
-}
-
-MatrixFile::~MatrixFile()
-{
-    if (m_fd >= 0) {
-        if (::close(m_fd) < 0) {
-            ::perror("MatrixFile::~MatrixFile: close failed");
-        }
-        openCount --;
-    }
-
-    QMutexLocker locker(&m_createMutex);
-
-    if (m_fileName != "") {
-
-        if (--m_refcount[m_fileName] == 0) {
-
-            if (::unlink(m_fileName.toLocal8Bit())) {
-                cerr << "WARNING: MatrixFile::~MatrixFile: reference count reached 0, but failed to unlink file \"" << m_fileName << "\"" << endl;
-            } else {
-                cerr << "deleted " << m_fileName << endl;
-            }
-        }
-    }
-
-    if (m_mode == WriteOnly) {
-        totalStorage -= (m_headerSize + (m_width * m_height * m_cellSize) + m_width);
-    }
-    totalCount --;
-
-#ifdef DEBUG_MATRIX_FILE
-    cerr << "MatrixFile[" << m_fd << "]::~MatrixFile: " << endl;
-    cerr << "MatrixFile: Total storage now " << totalStorage/1024 << "K in " << totalCount << " instances (" << openCount << " open)" << endl;
-#endif
-}
-
-void
-MatrixFile::initialise()
-{
-    Profiler profiler("MatrixFile::initialise", true);
-
-    assert(m_mode == WriteOnly);
-
-    m_setColumns.resize(m_width, false);
-    
-    off_t off = m_headerSize + (m_width * m_height * m_cellSize) + m_width;
-
-#ifdef DEBUG_MATRIX_FILE
-    cerr << "MatrixFile[" << m_fd << "]::initialise(" << m_width << ", " << m_height << "): cell size " << m_cellSize << ", header size " << m_headerSize << ", resizing fd " << m_fd << " to " << off << endl;
-#endif
-
-    if (::lseek(m_fd, off - 1, SEEK_SET) < 0) {
-        ::perror("ERROR: MatrixFile::initialise: seek to end failed");
-        throw FileOperationFailed(m_fileName, "lseek");
-    }
-
-    unsigned char byte = 0;
-    if (::write(m_fd, &byte, 1) != 1) {
-        ::perror("ERROR: MatrixFile::initialise: write at end failed");
-        throw FileOperationFailed(m_fileName, "write");
-    }
-
-    if (::lseek(m_fd, 0, SEEK_SET) < 0) {
-        ::perror("ERROR: MatrixFile::initialise: Seek to write header failed");
-        throw FileOperationFailed(m_fileName, "lseek");
-    }
-
-    int header[2];
-    header[0] = m_width;
-    header[1] = m_height;
-    if (::write(m_fd, header, 2 * sizeof(int)) != 2 * sizeof(int)) {
-        ::perror("ERROR: MatrixFile::initialise: Failed to write header");
-        throw FileOperationFailed(m_fileName, "write");
-    }
-
-    if (m_mode == WriteOnly) {
-        totalStorage += (m_headerSize + (m_width * m_height * m_cellSize) + m_width);
-    }
-
-#ifdef DEBUG_MATRIX_FILE
-    cerr << "MatrixFile[" << m_fd << "]::initialise(" << m_width << ", " << m_height << "): storage "
-              << (m_headerSize + m_width * m_height * m_cellSize + m_width) << endl;
-
-    cerr << "MatrixFile: Total storage " << totalStorage/1024 << "K" << endl;
-#endif
-
-    seekTo(0);
-}
-
-void
-MatrixFile::close()
-{
-#ifdef DEBUG_MATRIX_FILE
-    SVDEBUG << "MatrixFile::close()" << endl;
-#endif
-    if (m_fd >= 0) {
-        if (::close(m_fd) < 0) {
-            ::perror("MatrixFile::close: close failed");
-        }
-        m_fd = -1;
-        -- openCount;
-#ifdef DEBUG_MATRIX_FILE
-        cerr << "MatrixFile: Now " << openCount << " open instances" << endl;
-#endif
-    }
-}
-
-void
-MatrixFile::getColumnAt(int x, void *data)
-{
-    assert(m_mode == ReadOnly);
-    
-#ifdef DEBUG_MATRIX_FILE_READ_SET
-    cerr << "MatrixFile[" << m_fd << "]::getColumnAt(" << x << ")" << endl;
-#endif
-
-    Profiler profiler("MatrixFile::getColumnAt");
-
-    ssize_t r = -1;
-
-    if (m_readyToReadColumn < 0 ||
-        m_readyToReadColumn != x) {
-
-        unsigned char set = 0;
-        if (!seekTo(x)) {
-            cerr << "ERROR: MatrixFile::getColumnAt(" << x << "): Seek failed" << endl;
-            throw FileOperationFailed(m_fileName, "seek");
-        }
-
-        r = ::read(m_fd, &set, 1);
-        if (r < 0) {
-            ::perror("MatrixFile::getColumnAt: read failed");
-            throw FileReadFailed(m_fileName);
-        }
-        if (!set) {
-            cerr << "MatrixFile[" << m_fd << "]::getColumnAt(" << x << "): Column has not been set" << endl;
-            return;
-        }
-    }
-
-    r = ::read(m_fd, data, m_height * m_cellSize);
-    if (r < 0) {
-        ::perror("MatrixFile::getColumnAt: read failed");
-        throw FileReadFailed(m_fileName);
-    }
-}
-
-bool
-MatrixFile::haveSetColumnAt(int x) const
-{
-    if (m_mode == WriteOnly) {
-        return m_setColumns[x];
-    }
-
-    if (m_readyToReadColumn >= 0 &&
-        int(m_readyToReadColumn) == x) return true;
-    
-    Profiler profiler("MatrixFile::haveSetColumnAt");
-
-#ifdef DEBUG_MATRIX_FILE_READ_SET
-    cerr << "MatrixFile[" << m_fd << "]::haveSetColumnAt(" << x << ")" << endl;
-//    cerr << ".";
-#endif
-
-    unsigned char set = 0;
-    if (!seekTo(x)) {
-        cerr << "ERROR: MatrixFile::haveSetColumnAt(" << x << "): Seek failed" << endl;
-        throw FileOperationFailed(m_fileName, "seek");
-    }
-
-    ssize_t r = -1;
-    r = ::read(m_fd, &set, 1);
-    if (r < 0) {
-        ::perror("MatrixFile::haveSetColumnAt: read failed");
-        throw FileReadFailed(m_fileName);
-    }
-
-    if (set) m_readyToReadColumn = int(x);
-
-    return set;
-}
-
-void
-MatrixFile::setColumnAt(int x, const void *data)
-{
-    assert(m_mode == WriteOnly);
-    if (m_fd < 0) return; // closed
-
-#ifdef DEBUG_MATRIX_FILE_READ_SET
-    cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << ")" << endl;
-//    cerr << ".";
-#endif
-
-    ssize_t w = 0;
-
-    if (!seekTo(x)) {
-        cerr << "ERROR: MatrixFile::setColumnAt(" << x << "): Seek failed" << endl;
-        throw FileOperationFailed(m_fileName, "seek");
-    }
-
-    unsigned char set = 0;
-    w = ::write(m_fd, &set, 1);
-    if (w != 1) {
-        ::perror("WARNING: MatrixFile::setColumnAt: write failed (1)");
-        throw FileOperationFailed(m_fileName, "write");
-    }
-
-    w = ::write(m_fd, data, m_height * m_cellSize);
-    if (w != ssize_t(m_height * m_cellSize)) {
-        ::perror("WARNING: MatrixFile::setColumnAt: write failed (2)");
-        throw FileOperationFailed(m_fileName, "write");
-    }
-/*
-    if (x == 0) {
-        cerr << "Wrote " << m_height * m_cellSize << " bytes, as follows:" << endl;
-        for (int i = 0; i < m_height * m_cellSize; ++i) {
-            cerr << (int)(((char *)data)[i]) << " ";
-        }
-        cerr << endl;
-    }
-*/
-    if (!seekTo(x)) {
-        cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << "): Seek failed" << endl;
-        throw FileOperationFailed(m_fileName, "seek");
-    }
-
-    set = 1;
-    w = ::write(m_fd, &set, 1);
-    if (w != 1) {
-        ::perror("WARNING: MatrixFile::setColumnAt: write failed (3)");
-        throw FileOperationFailed(m_fileName, "write");
-    }
-
-    m_setColumns[x] = true;
-    if (m_autoClose) {
-        if (std::all_of(m_setColumns.begin(), m_setColumns.end(),
-                        [](bool c) { return c; })) {
-#ifdef DEBUG_MATRIX_FILE
-            cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << "): All columns set: auto-closing" << endl;
-#endif
-            close();
-/*
-        } else {
-            int set = 0;
-            for (int i = 0; i < m_width; ++i) {
-                if (m_setColumns->get(i)) ++set;
-            }
-            cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << "): Auto-close on, but not all columns set yet (" << set << " of " << m_width << ")" << endl;
-*/
-        }
-    }
-}
-
-bool
-MatrixFile::seekTo(int x) const
-{
-    if (m_fd < 0) {
-        cerr << "ERROR: MatrixFile::seekTo: File not open" << endl;
-        return false;
-    }
-
-    m_readyToReadColumn = -1; // not ready, unless this is subsequently re-set
-
-    off_t off = m_headerSize + x * m_height * m_cellSize + x;
-
-#ifdef DEBUG_MATRIX_FILE_READ_SET
-    if (m_mode == ReadOnly) {
-        cerr << "MatrixFile[" << m_fd << "]::seekTo(" << x << "): off = " << off << endl;
-    }
-#endif
-
-#ifdef DEBUG_MATRIX_FILE_READ_SET
-    cerr << "MatrixFile[" << m_fd << "]::seekTo(" << x << "): off = " << off << endl;
-#endif
-
-    if (::lseek(m_fd, off, SEEK_SET) == (off_t)-1) {
-        ::perror("Seek failed");
-        cerr << "ERROR: MatrixFile::seekTo(" << x 
-                  << ") = " << off << " failed" << endl;
-        return false;
-    }
-
-    return true;
-}
-
diff -r c811991a5efa -r 046f05fa31f3 data/fileio/MatrixFile.h
--- a/data/fileio/MatrixFile.h	Tue Oct 18 14:03:55 2016 +0100
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,107 +0,0 @@
-/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
-
-/*
-    Sonic Visualiser
-    An audio file viewer and annotation editor.
-    Centre for Digital Music, Queen Mary, University of London.
-    This file copyright 2006-2009 Chris Cannam and QMUL.
-    
-    This program is free software; you can redistribute it and/or
-    modify it under the terms of the GNU General Public License as
-    published by the Free Software Foundation; either version 2 of the
-    License, or (at your option) any later version.  See the file
-    COPYING included with this distribution for more information.
-*/
-
-#ifndef MATRIX_FILE_H
-#define MATRIX_FILE_H
-
-#include "FileReadThread.h"
-
-#include <sys/types.h>
-#include <QString>
-#include <QMutex>
-#include <map>
-
-class MatrixFile : public QObject
-{
-    Q_OBJECT
-
-public:
-    enum Mode { ReadOnly, WriteOnly };
-
-    /**
-     * Construct a MatrixFile object reading from and/or writing to
-     * the matrix file with the given base name in the application's
-     * temporary directory.
-     *
-     * If mode is ReadOnly, the file must exist and be readable.
-     *
-     * If mode is WriteOnly, the file must not exist.
-     *
-     * cellSize specifies the size in bytes of the object type stored
-     * in the matrix.  For example, use cellSize = sizeof(float) for a
-     * matrix of floats.  The MatrixFile object doesn't care about the
-     * objects themselves, it just deals with raw data of a given size.
-     *
-     * width and height specify the dimensions of the file.  These
-     * cannot be changed after construction.
-     *
-     * MatrixFiles are reference counted by name.  When the last
-     * MatrixFile with a given name is destroyed, the file is removed.
-     * These are temporary files; the normal usage is to have one
-     * MatrixFile of WriteOnly type creating the file and then
-     * persisting until all readers are complete.
-     *
-     * MatrixFile has no built-in cache and is not thread-safe.  Use a
-     * separate MatrixFile in each thread.
-     */
-    MatrixFile(QString fileBase, Mode mode, int cellSize,
-               int width, int height);
-    virtual ~MatrixFile();
-
-    Mode getMode() const { return m_mode; }
-
-    int getWidth() const { return m_width; }
-    int getHeight() const { return m_height; }
-    int getCellSize() const { return m_cellSize; }
-
-    /**
-     * If this is set true on a write-mode MatrixFile, then the file
-     * will close() itself when all columns have been written.
-     */
-    void setAutoClose(bool a) { m_autoClose = a; }
-
-    void close(); // does not decrement ref count; that happens in dtor
-
-    bool haveSetColumnAt(int x) const;
-    void getColumnAt(int x, void *data); // may throw FileReadFailed
-    void setColumnAt(int x, const void *data);
-
-protected:
-    int     m_fd;
-    Mode    m_mode;
-    int     m_flags;
-    mode_t  m_fmode;
-    int     m_cellSize;
-    int     m_width;
-    int     m_height;
-    int     m_headerSize;
-    QString m_fileName;
-
-    std::vector<bool> m_setColumns; // only populated in writer
-    bool m_autoClose;
-
-    // In reader: if this is >= 0, we can read that column directly
-    // without seeking (and we know that the column exists)
-    mutable int m_readyToReadColumn;
-
-    static std::map<QString, int> m_refcount;
-    static QMutex m_createMutex;
-
-    void initialise();
-    bool seekTo(int col) const;
-};
-
-#endif
-
diff -r c811991a5efa -r 046f05fa31f3 svcore.pro
--- a/svcore.pro	Tue Oct 18 14:03:55 2016 +0100
+++ b/svcore.pro	Tue Oct 18 14:04:06 2016 +0100
@@ -144,7 +144,6 @@
            data/fileio/FileFinder.h \
            data/fileio/FileReadThread.h \
            data/fileio/FileSource.h \
-           data/fileio/MatrixFile.h \
            data/fileio/MIDIFileReader.h \
            data/fileio/MIDIFileWriter.h \
            data/fileio/MP3FileReader.h \
@@ -202,7 +201,6 @@
            data/fileio/DataFileReaderFactory.cpp \
            data/fileio/FileReadThread.cpp \
            data/fileio/FileSource.cpp \
-           data/fileio/MatrixFile.cpp \
            data/fileio/MIDIFileReader.cpp \
            data/fileio/MIDIFileWriter.cpp \
            data/fileio/MP3FileReader.cpp \