diff data/fileio/MatrixFile.cpp @ 537:3cc4b7cd2aa5

* Merge from one-fftdataserver-per-fftmodel branch.  This bit of
reworking (which is not described very accurately by the title of
the branch) turns the MatrixFile object into something that either
reads or writes, but not both, and separates the FFT file cache
reader and writer implementations separately.  This allows the
FFT data server to have a single thread owning writers and one reader
per "customer" thread, and for all locking to be vastly simplified
and concentrated in the data server alone (because none of the
classes it makes use of is used in more than one thread at a time).
The result is faster and more trustworthy code.

--- a/data/fileio/MatrixFile.cpp	Mon Jan 26 15:18:32 2009 +0000
+++ b/data/fileio/MatrixFile.cpp	Tue Jan 27 13:25:10 2009 +0000
@@ -4,7 +4,7 @@
     Sonic Visualiser
     An audio file viewer and annotation editor.
     Centre for Digital Music, Queen Mary, University of London.
-    This file copyright 2006 Chris Cannam.
+    This file copyright 2006-2009 Chris Cannam and QMUL.
     
     This program is free software; you can redistribute it and/or
     modify it under the terms of the GNU General Public License as
@@ -35,7 +35,7 @@
 #include <QFileInfo>
 #include <QDir>
 
-//#define DEBUG_MATRIX_FILE 1
+#define DEBUG_MATRIX_FILE 1
 //#define DEBUG_MATRIX_FILE_READ_SET 1
 
 #ifdef DEBUG_MATRIX_FILE_READ_SET
@@ -45,42 +45,30 @@
 #endif
 
 std::map<QString, int> MatrixFile::m_refcount;
-QMutex MatrixFile::m_refcountMutex;
-
-MatrixFile::ResizeableBitsetMap MatrixFile::m_columnBitsets;
-QMutex MatrixFile::m_columnBitsetWriteMutex;
-
-FileReadThread *MatrixFile::m_readThread = 0;
+QMutex MatrixFile::m_createMutex;
 
 static size_t totalStorage = 0;
-static size_t totalMemory = 0;
 static size_t totalCount = 0;
+static size_t openCount = 0;
 
 MatrixFile::MatrixFile(QString fileBase, Mode mode,
-                       size_t cellSize, bool eagerCache) :
+                       size_t cellSize, size_t width, size_t height) :
     m_fd(-1),
     m_mode(mode),
     m_flags(0),
     m_fmode(0),
     m_cellSize(cellSize),
-    m_width(0),
-    m_height(0),
-    m_headerSize(2 * sizeof(size_t)),
-    m_defaultCacheWidth(1024),
-    m_prevX(0),
-    m_eagerCache(eagerCache),
-    m_requestToken(-1),
-    m_spareData(0),
-    m_columnBitset(0)
+    m_width(width),
+    m_height(height),
+    m_headerSize(2 * sizeof(size_t))
 {
     Profiler profiler("MatrixFile::MatrixFile", true);
 
-    if (!m_readThread) {
-        m_readThread = new FileReadThread;
-        m_readThread->start();
-    }
+#ifdef DEBUG_MATRIX_FILE
+    std::cerr << "MatrixFile::MatrixFile(" << fileBase.toStdString() << ", " << int(mode) << ", " << cellSize << ", " << width << ", " << height << ")" << std::endl;
+#endif
 
-    m_cache.data = 0;
+    QMutexLocker locker(&m_createMutex);
 
     QDir tempDir(TempDirectory::getInstance()->getPath());
     QString fileName(tempDir.filePath(QString("%1.mfc").arg(fileBase)));
@@ -92,24 +80,17 @@
         throw FileNotFound(fileName);
     }
 
-    if (!newFile && m_mode == ReadWrite) {
-        std::cerr << "Note: MatrixFile::MatrixFile: Read/write mode "
-                  << "specified, but file already exists; falling back to "
-                  << "read-only mode" << std::endl;
-        m_mode = ReadOnly;
-    }
-
-    if (!eagerCache && m_mode == ReadOnly) {
-        std::cerr << "WARNING: MatrixFile::MatrixFile: Eager cacheing not "
-                  << "specified, but file is open in read-only mode -- cache "
-                  << "will not be used" << std::endl;
+    if (!newFile && m_mode == WriteOnly) {
+        std::cerr << "ERROR: MatrixFile::MatrixFile: Write-only mode "
+                  << "specified, but file already exists" << std::endl;
+        throw FileOperationFailed(fileName, "create");
     }
 
     m_flags = 0;
     m_fmode = S_IRUSR | S_IWUSR;
 
-    if (m_mode == ReadWrite) {
-        m_flags = O_RDWR | O_CREAT;
+    if (m_mode == WriteOnly) {
+        m_flags = O_WRONLY | O_CREAT;
     } else {
         m_flags = O_RDONLY;
     }
@@ -119,7 +100,7 @@
 #endif
 
 #ifdef DEBUG_MATRIX_FILE
-    std::cerr << "MatrixFile::MatrixFile: opening " << fileName.toStdString() << "..." << std::endl;
+    std::cerr << "MatrixFile(" << this << ")::MatrixFile: opening " << fileName.toStdString() << "..." << std::endl;
 #endif
 
     if ((m_fd = ::open(fileName.toLocal8Bit(), m_flags, m_fmode)) < 0) {
@@ -127,13 +108,17 @@
         std::cerr << "ERROR: MatrixFile::MatrixFile: "
                   << "Failed to open cache file \""
                   << fileName.toStdString() << "\"";
-        if (m_mode == ReadWrite) std::cerr << " for writing";
+        if (m_mode == WriteOnly) std::cerr << " for writing";
         std::cerr << std::endl;
         throw FailedToOpenFile(fileName);
     }
 
+#ifdef DEBUG_MATRIX_FILE
+    std::cerr << "MatrixFile(" << this << ")::MatrixFile: fd is " << m_fd << std::endl;
+#endif
+
     if (newFile) {
-        resize(0, 0); // write header
+        initialise(); // write header and "unwritten" column tags
     } else {
         size_t header[2];
         if (::read(m_fd, header, 2 * sizeof(size_t)) < 0) {
@@ -143,555 +128,260 @@
                       << fileName.toStdString() << "\")" << std::endl;
             throw FileReadFailed(fileName);
         }
-        m_width = header[0];
-        m_height = header[1];
-        seekTo(0, 0);
+        if (header[0] != m_width || header[1] != m_height) {
+            std::cerr << "ERROR: MatrixFile::MatrixFile: "
+                      << "Dimensions in file header (" << header[0] << "x"
+                      << header[1] << ") differ from expected dimensions "
+                      << m_width << "x" << m_height << std::endl;
+            throw FailedToOpenFile(fileName);
+        }
     }
 
     m_fileName = fileName;
-
-    {
-        MutexLocker locker
-            (&m_columnBitsetWriteMutex,
-             "MatrixFile::MatrixFile::m_columnBitsetWriteMutex");
-
-        if (m_columnBitsets.find(m_fileName) == m_columnBitsets.end()) {
-            m_columnBitsets[m_fileName] = new ResizeableBitset;
-        }
-        m_columnBitset = m_columnBitsets[m_fileName];
-    }
-
-    MutexLocker locker(&m_refcountMutex,
-                       "MatrixFile::MatrixFile::m_refcountMutex");
     ++m_refcount[fileName];
 
-//    std::cerr << "MatrixFile(" << this << "): fd " << m_fd << ", file " << fileName.toStdString() << ", ref " << m_refcount[fileName] << std::endl;
+#ifdef DEBUG_MATRIX_FILE
+    std::cerr << "MatrixFile[" << m_fd << "]::MatrixFile: File " << fileName.toStdString() << ", ref " << m_refcount[fileName] << std::endl;
 
-//    std::cerr << "MatrixFile::MatrixFile: Done, size is " << "(" << m_width << ", " << m_height << ")" << std::endl;
+    std::cerr << "MatrixFile[" << m_fd << "]::MatrixFile: Done, size is " << "(" << m_width << ", " << m_height << ")" << std::endl;
+#endif
 
     ++totalCount;
-
+    ++openCount;
 }
 
 MatrixFile::~MatrixFile()
 {
-    char *requestData = 0;
-
-    if (m_requestToken >= 0) {
-        FileReadThread::Request request;
-        if (m_readThread->getRequest(m_requestToken, request)) {
-            requestData = request.data;
-        }
-        m_readThread->cancel(m_requestToken);
-    }
-
-    if (requestData) free(requestData);
-    if (m_cache.data) free(m_cache.data);
-    if (m_spareData) free(m_spareData);
-
     if (m_fd >= 0) {
         if (::close(m_fd) < 0) {
             ::perror("MatrixFile::~MatrixFile: close failed");
         }
     }
 
+    QMutexLocker locker(&m_createMutex);
+
     if (m_fileName != "") {
 
-        MutexLocker locker(&m_refcountMutex,
-                           "MatrixFile::~MatrixFile::m_refcountMutex");
-
         if (--m_refcount[m_fileName] == 0) {
 
             if (::unlink(m_fileName.toLocal8Bit())) {
-//                ::perror("Unlink failed");
-//                std::cerr << "WARNING: MatrixFile::~MatrixFile: reference count reached 0, but failed to unlink file \"" << m_fileName.toStdString() << "\"" << std::endl;
+                std::cerr << "WARNING: MatrixFile::~MatrixFile: reference count reached 0, but failed to unlink file \"" << m_fileName.toStdString() << "\"" << std::endl;
             } else {
-//                std::cerr << "deleted " << m_fileName.toStdString() << std::endl;
+                std::cerr << "deleted " << m_fileName.toStdString() << std::endl;
             }
-
-            MutexLocker locker2
-                (&m_columnBitsetWriteMutex,
-                 "MatrixFile::~MatrixFile::m_columnBitsetWriteMutex");
-            m_columnBitsets.erase(m_fileName);
-            delete m_columnBitset;
         }
     }
-    
-    totalStorage -= (m_headerSize + (m_width * m_height * m_cellSize));
-    totalMemory -= (2 * m_defaultCacheWidth * m_height * m_cellSize);
+
+    if (m_mode == WriteOnly) {
+        totalStorage -= (m_headerSize + (m_width * m_height * m_cellSize) + m_width);
+    }
     totalCount --;
+    openCount --;
 
-//    std::cerr << "MatrixFile::~MatrixFile: " << std::endl;
-//    std::cerr << "Total storage now " << totalStorage/1024 << "K, theoretical max memory "
-//              << totalMemory/1024 << "K in " << totalCount << " instances" << std::endl;
-
+#ifdef DEBUG_MATRIX_FILE
+    std::cerr << "MatrixFile[" << m_fd << "]::~MatrixFile: " << std::endl;
+    std::cerr << "MatrixFile: Total storage now " << totalStorage/1024 << "K in " << totalCount << " instances (" << openCount << " open)" << std::endl;
+#endif
 }
 
 void
-MatrixFile::resize(size_t w, size_t h)
+MatrixFile::initialise()
 {
-    Profiler profiler("MatrixFile::resize", true);
+    Profiler profiler("MatrixFile::initialise", true);
 
-    assert(m_mode == ReadWrite);
-
-    MutexLocker locker(&m_fdMutex, "MatrixFile::resize::m_fdMutex");
+    assert(m_mode == WriteOnly);
     
-    totalStorage -= (m_headerSize + (m_width * m_height * m_cellSize));
-    totalMemory -= (2 * m_defaultCacheWidth * m_height * m_cellSize);
-
-    off_t off = m_headerSize + (w * h * m_cellSize);
+    off_t off = m_headerSize + (m_width * m_height * m_cellSize) + m_width;
 
 #ifdef DEBUG_MATRIX_FILE
-    std::cerr << "MatrixFile::resize(" << w << ", " << h << "): resizing file" << std::endl;
+    std::cerr << "MatrixFile[" << m_fd << "]::initialise(" << m_width << ", " << m_height << "): cell size " << m_cellSize << ", header size " << m_headerSize << ", resizing file" << std::endl;
 #endif
 
-    if (w * h < m_width * m_height) {
-        if (::ftruncate(m_fd, off) < 0) {
-            ::perror("WARNING: MatrixFile::resize: ftruncate failed");
-            throw FileOperationFailed(m_fileName, "ftruncate");
-        }
+    if (::lseek(m_fd, off - 1, SEEK_SET) < 0) {
+        ::perror("ERROR: MatrixFile::initialise: seek to end failed");
+        throw FileOperationFailed(m_fileName, "lseek");
     }
 
-    m_width = 0;
-    m_height = 0;
+    unsigned char byte = 0;
+    if (::write(m_fd, &byte, 1) != 1) {
+        ::perror("ERROR: MatrixFile::initialise: write at end failed");
+        throw FileOperationFailed(m_fileName, "write");
+    }
 
-    if (::lseek(m_fd, 0, SEEK_SET) == (off_t)-1) {
+    if (::lseek(m_fd, 0, SEEK_SET) < 0) {
         ::perror("ERROR: MatrixFile::resize: Seek to write header failed");
         throw FileOperationFailed(m_fileName, "lseek");
     }
 
     size_t header[2];
-    header[0] = w;
-    header[1] = h;
+    header[0] = m_width;
+    header[1] = m_height;
     if (::write(m_fd, header, 2 * sizeof(size_t)) != 2 * sizeof(size_t)) {
         ::perror("ERROR: MatrixFile::resize: Failed to write header");
         throw FileOperationFailed(m_fileName, "write");
     }
 
-    if (w > 0 && m_defaultCacheWidth > w) {
-        m_defaultCacheWidth = w;
+    if (m_mode == WriteOnly) {
+        totalStorage += (m_headerSize + (m_width * m_height * m_cellSize) + m_width);
     }
 
-//!!!    static size_t maxCacheMB = 16;
-    static size_t maxCacheMB = 4;
-    if (2 * m_defaultCacheWidth * h * m_cellSize > maxCacheMB * 1024 * 1024) { //!!!
-        m_defaultCacheWidth = (maxCacheMB * 1024 * 1024) / (2 * h * m_cellSize);
-        if (m_defaultCacheWidth < 16) m_defaultCacheWidth = 16;
-    }
+#ifdef DEBUG_MATRIX_FILE
+    std::cerr << "MatrixFile[" << m_fd << "]::resize(" << m_width << ", " << m_height << "): storage "
+              << (m_headerSize + m_width * m_height * m_cellSize + m_width) << std::endl;
 
-    if (m_columnBitset) {
-        MutexLocker locker(&m_columnBitsetWriteMutex,
-                           "MatrixFile::resize::m_columnBitsetWriteMutex");
-        m_columnBitset->resize(w);
-    }
-
-    if (m_cache.data) {
-        free(m_cache.data);
-        m_cache.data = 0;
-    }
-
-    if (m_spareData) {
-        free(m_spareData);
-        m_spareData = 0;
-    }
-    
-    m_width = w;
-    m_height = h;
-
-    totalStorage += (m_headerSize + (m_width * m_height * m_cellSize));
-    totalMemory += (2 * m_defaultCacheWidth * m_height * m_cellSize);
-
-#ifdef DEBUG_MATRIX_FILE
-    std::cerr << "MatrixFile::resize(" << w << ", " << h << "): cache width "
-              << m_defaultCacheWidth << ", storage "
-              << (m_headerSize + w * h * m_cellSize) << ", mem "
-              << (2 * h * m_defaultCacheWidth * m_cellSize) << std::endl;
-
-    std::cerr << "Total storage " << totalStorage/1024 << "K, theoretical max memory "
-              << totalMemory/1024 << "K in " << totalCount << " instances" << std::endl;
+    std::cerr << "MatrixFile: Total storage " << totalStorage/1024 << "K" << std::endl;
 #endif
 
     seekTo(0, 0);
 }
 
 void
-MatrixFile::reset()
+MatrixFile::close()
 {
-    Profiler profiler("MatrixFile::reset", true);
-
-    assert (m_mode == ReadWrite);
-    
-    if (m_eagerCache) {
-        void *emptyCol = calloc(m_height, m_cellSize);
-        for (size_t x = 0; x < m_width; ++x) setColumnAt(x, emptyCol);
-        free(emptyCol);
-    }
-    
-    if (m_columnBitset) {
-        MutexLocker locker(&m_columnBitsetWriteMutex,
-                           "MatrixFile::reset::m_columnBitsetWriteMutex");
-        m_columnBitset->resize(m_width);
+#ifdef DEBUG_MATRIX_FILE
+    std::cerr << "MatrixFile::close()" << std::endl;
+#endif
+    if (m_fd >= 0) {
+        if (::close(m_fd) < 0) {
+            ::perror("MatrixFile::close: close failed");
+        }
+        m_fd = -1;
+        -- openCount;
     }
 }
 
 void
 MatrixFile::getColumnAt(size_t x, void *data)
 {
+    assert(m_mode == ReadOnly);
+    
+#ifdef DEBUG_MATRIX_FILE_READ_SET
+    std::cerr << "MatrixFile[" << m_fd << "]::getColumnAt(" << x << ")" << std::endl;
+#endif
+
     Profiler profiler("MatrixFile::getColumnAt");
 
-//    assert(haveSetColumnAt(x));
-
-    if (getFromCache(x, 0, m_height, data)) return;
-
-    Profiler profiler2("MatrixFile::getColumnAt (uncached)");
-
-    ssize_t r = 0;
-
-#ifdef DEBUG_MATRIX_FILE
-    std::cerr << "MatrixFile::getColumnAt(" << x << ")"
-              << ": reading the slow way";
-
-    if (m_requestToken >= 0 &&
-        x >= m_requestingX &&
-        x <  m_requestingX + m_requestingWidth) {
-        
-        std::cerr << " (awaiting " << m_requestingX << ", " << m_requestingWidth << " from disk)";
+    unsigned char set = 0;
+    if (!seekTo(x, 0)) {
+        std::cerr << "ERROR: MatrixFile::getColumnAt(" << x << "): Seek failed" << std::endl;
+        throw FileOperationFailed(m_fileName, "seek");
     }
 
-    std::cerr << std::endl;
+    ssize_t r = -1;
+    r = ::read(m_fd, &set, 1);
+    if (r < 0) {
+        ::perror("MatrixFile::getColumnAt: read failed");
+        throw FileReadFailed(m_fileName);
+    }
+    if (!set) {
+        std::cerr << "MatrixFile[" << m_fd << "]::getColumnAt(" << x << "): Column has not been set" << std::endl;
+        return;
+    }
+
+    r = ::read(m_fd, data, m_height * m_cellSize);
+    if (r < 0) {
+        ::perror("MatrixFile::getColumnAt: read failed");
+        throw FileReadFailed(m_fileName);
+    }
+}
+
+bool
+MatrixFile::haveSetColumnAt(size_t x) const
+{
+    assert(m_mode == ReadOnly);
+
+    Profiler profiler("MatrixFile::haveSetColumnAt");
+
+#ifdef DEBUG_MATRIX_FILE_READ_SET
+    std::cerr << "MatrixFile[" << m_fd << "]::haveSetColumnAt(" << x << ")" << std::endl;
+//    std::cerr << ".";
 #endif
 
-    {
-        MutexLocker locker(&m_fdMutex, "MatrixFile::getColumnAt::m_fdMutex");
+    unsigned char set = 0;
+    if (!seekTo(x, 0)) {
+        std::cerr << "ERROR: MatrixFile::haveSetColumnAt(" << x << "): Seek failed" << std::endl;
+        throw FileOperationFailed(m_fileName, "seek");
+    }
 
-        if (seekTo(x, 0)) {
-            r = ::read(m_fd, data, m_height * m_cellSize);
-        }
-    }
-    
+    ssize_t r = -1;
+    r = ::read(m_fd, &set, 1);
     if (r < 0) {
-        ::perror("MatrixFile::getColumnAt: read failed");
-        std::cerr << "ERROR: MatrixFile::getColumnAt: "
-                  << "Failed to read column " << x << " (height " << m_height << ", cell size " << m_cellSize << ", fd " << m_fd << ", file \""
-                  << m_fileName.toStdString() << "\")" << std::endl;
+        ::perror("MatrixFile::haveSetColumnAt: read failed");
         throw FileReadFailed(m_fileName);
     }
 
-    return;
-}
-
-bool
-MatrixFile::getFromCache(size_t x, size_t ystart, size_t ycount, void *data)
-{
-    bool fail = false;
-    bool primeLeft = false;
-
-    {
-        MutexLocker locker(&m_cacheMutex,
-                           "MatrixFile::getFromCache::m_cacheMutex");
-
-        if (!m_cache.data || x < m_cache.x || x >= m_cache.x + m_cache.width) {
-            fail = true;
-            primeLeft = (m_cache.data && x < m_cache.x);
-        } else {
-            memcpy(data,
-                   m_cache.data + m_cellSize * ((x - m_cache.x) * m_height + ystart),
-                   ycount * m_cellSize);
-        }            
-    }
-
-    if (fail) {
-        primeCache(x, primeLeft); // this doesn't take effect until a later callback
-        m_prevX = x;
-        return false;
-    }
-
-    if (m_cache.x > 0 && x < m_prevX && x < m_cache.x + m_cache.width/4) {
-        primeCache(x, true);
-    }
-
-    if (m_cache.x + m_cache.width < m_width &&
-        x > m_prevX &&
-        x > m_cache.x + (m_cache.width * 3) / 4) {
-        primeCache(x, false);
-    }
-
-    m_prevX = x;
-    return true;
+    return set;
 }
 
 void
 MatrixFile::setColumnAt(size_t x, const void *data)
 {
-    assert(m_mode == ReadWrite);
+    assert(m_mode == WriteOnly);
 
 #ifdef DEBUG_MATRIX_FILE_READ_SET
-//    std::cerr << "MatrixFile::setColumnAt(" << x << ")" << std::endl;
-    std::cerr << ".";
+    std::cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << ")" << std::endl;
+//    std::cerr << ".";
 #endif
 
     ssize_t w = 0;
-    bool seekFailed = false;
-    
-    {
-        MutexLocker locker(&m_fdMutex, "MatrixFile::setColumnAt::m_fdMutex");
 
-        if (seekTo(x, 0)) {
-            w = ::write(m_fd, data, m_height * m_cellSize);
-        } else {
-            seekFailed = true;
-        }
+    if (!seekTo(x, 0)) {
+        std::cerr << "ERROR: MatrixFile::setColumnAt(" << x << "): Seek failed" << std::endl;
+        throw FileOperationFailed(m_fileName, "seek");
     }
 
-    if (!seekFailed && w != ssize_t(m_height * m_cellSize)) {
-        ::perror("WARNING: MatrixFile::setColumnAt: write failed");
+    unsigned char set = 0;
+    w = ::write(m_fd, &set, 1);
+    if (w != 1) {
+        ::perror("WARNING: MatrixFile::setColumnAt: write failed (1)");
         throw FileOperationFailed(m_fileName, "write");
-    } else if (seekFailed) {
+    }
+
+    w = ::write(m_fd, data, m_height * m_cellSize);
+    if (w != ssize_t(m_height * m_cellSize)) {
+        ::perror("WARNING: MatrixFile::setColumnAt: write failed (2)");
+        throw FileOperationFailed(m_fileName, "write");
+    }
+/*
+    if (x == 0) {
+        std::cerr << "Wrote " << m_height * m_cellSize << " bytes, as follows:" << std::endl;
+        for (int i = 0; i < m_height * m_cellSize; ++i) {
+            std::cerr << (int)(((char *)data)[i]) << " ";
+        }
+        std::cerr << std::endl;
+    }
+*/
+    if (!seekTo(x, 0)) {
+        std::cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << "): Seek failed" << std::endl;
         throw FileOperationFailed(m_fileName, "seek");
-    } else {
-        MutexLocker locker
-            (&m_columnBitsetWriteMutex,
-             "MatrixFile::setColumnAt::m_columnBitsetWriteMutex");
-        m_columnBitset->set(x);
+    }
+
+    set = 1;
+    w = ::write(m_fd, &set, 1);
+    if (w != 1) {
+        ::perror("WARNING: MatrixFile::setColumnAt: write failed (3)");
+        throw FileOperationFailed(m_fileName, "write");
     }
 }
 
-void
-MatrixFile::suspend()
+bool
+MatrixFile::seekTo(size_t x, size_t y) const
 {
-    MutexLocker locker(&m_fdMutex, "MatrixFile::suspend::m_fdMutex");
-    MutexLocker locker2(&m_cacheMutex, "MatrixFile::suspend::m_cacheMutex");
-
-    if (m_fd < 0) return; // already suspended
-
-#ifdef DEBUG_MATRIX_FILE
-    std::cerr << "MatrixFile(" << this << ":" << m_fileName.toStdString() << ")::suspend(): fd was " << m_fd << std::endl;
-#endif
-
-    if (m_requestToken >= 0) {
-        void *data = 0;
-        FileReadThread::Request request;
-        if (m_readThread->getRequest(m_requestToken, request)) {
-            data = request.data;
-        }
-        m_readThread->cancel(m_requestToken);
-        if (data) free(data);
-        m_requestToken = -1;
+    if (m_fd < 0) {
+        std::cerr << "ERROR: MatrixFile::seekTo: File not open" << std::endl;
+        return false;
     }
 
-    if (m_cache.data) {
-        free(m_cache.data);
-        m_cache.data = 0;
-    }
-
-    if (m_spareData) {
-        free(m_spareData);
-        m_spareData = 0;
-    }
-    
-    if (::close(m_fd) < 0) {
-        ::perror("WARNING: MatrixFile::suspend: close failed");
-        throw FileOperationFailed(m_fileName, "close");
-    }
-
-    m_fd = -1;
-}
-
-void
-MatrixFile::resume()
-{
-    if (m_fd >= 0) return;
-
-#ifdef DEBUG_MATRIX_FILE    
-    std::cerr << "MatrixFile(" << this << ")::resume()" << std::endl;
-#endif
-
-    if ((m_fd = ::open(m_fileName.toLocal8Bit(), m_flags, m_fmode)) < 0) {
-        ::perror("Open failed");
-        std::cerr << "ERROR: MatrixFile::resume: "
-                  << "Failed to open cache file \""
-                  << m_fileName.toStdString() << "\"";
-        if (m_mode == ReadWrite) std::cerr << " for writing";
-        std::cerr << std::endl;
-        throw FailedToOpenFile(m_fileName);
-    }
-
-#ifdef DEBUG_MATRIX_FILE
-    std::cerr << "MatrixFile(" << this << ":" << m_fileName.toStdString() << ")::resume(): fd is " << m_fd << std::endl;
-#endif
-}
-
-void
-MatrixFile::primeCache(size_t x, bool goingLeft)
-{
-//    Profiler profiler("MatrixFile::primeCache");
+    off_t off = m_headerSize + x * m_height * m_cellSize + x + y * m_cellSize;
 
 #ifdef DEBUG_MATRIX_FILE_READ_SET
-    std::cerr << "MatrixFile::primeCache(" << x << ", " << goingLeft << ")" << std::endl;
+    std::cerr << "MatrixFile[" << m_fd << "]::seekTo(" << x << "," << y << "): off = " << off << std::endl;
 #endif
 
-    size_t rx = x;
-    size_t rw = m_defaultCacheWidth;
-
-    size_t left = rw / 3;
-    if (goingLeft) left = (rw * 2) / 3;
-
-    if (rx > left) rx -= left;
-    else rx = 0;
-
-    if (rx + rw > m_width) rw = m_width - rx;
-
-    if (!m_eagerCache) {
-
-        size_t ti = 0;
-
-        for (ti = 0; ti < rw; ++ti) {
-            if (!m_columnBitset->get(rx + ti)) break;
-        }
-        
-#ifdef DEBUG_MATRIX_FILE
-        if (ti < rw) {
-            std::cerr << "eagerCache is false and there's a hole at "
-                      << rx + ti << ", reducing rw from " << rw << " to "
-                      << ti << std::endl;
-        }
-#endif
-
-        rw = std::min(rw, ti);
-        if (rw < 10 || rx + rw <= x) return;
-    }
-
-    MutexLocker locker(&m_cacheMutex, "MatrixFile::primeCache::m_cacheMutex");
-
-    // Check for the existence of the request first; if it exists,
-    // check whether it's ready.  Only when we know it's ready do we
-    // retrieve the actual request, because the reason we need the
-    // request is to check whether it was successful or not and
-    // extract data from it, and none of that can be relied upon if we
-    // retrieve the request before it's ready.  (There used to be a
-    // race condition here, where we retrieved the request and only
-    // afterwards checked the ready status, pulling data from the
-    // request if it was found to be ready then.)
-
-    if (m_requestToken >= 0 &&
-        m_readThread->haveRequest(m_requestToken)) {
-
-        if (x >= m_requestingX &&
-            x <  m_requestingX + m_requestingWidth) {
-
-            if (m_readThread->isReady(m_requestToken)) {
-
-                FileReadThread::Request request;
-                if (!m_readThread->getRequest(m_requestToken, request)) {
-                    std::cerr << "ERROR: MatrixFile::primeCache: File read thread has lost our request!" << std::endl;
-                    throw FileReadFailed(m_fileName);
-                }
-
-                if (!request.successful) {
-                    std::cerr << "ERROR: MatrixFile::primeCache: Last request was unsuccessful" << std::endl;
-                    throw FileReadFailed(m_fileName);
-                }
-                
-#ifdef DEBUG_MATRIX_FILE_READ_SET
-                std::cerr << "last request is ready! (" << m_requestingX << ", "<< m_requestingWidth << ")"  << std::endl;
-#endif
-
-                m_cache.x = (request.start - m_headerSize) / (m_height * m_cellSize);
-                m_cache.width = request.size / (m_height * m_cellSize);
-      
-#ifdef DEBUG_MATRIX_FILE_READ_SET
-                std::cerr << "received last request: actual size is: " << m_cache.x << ", " << m_cache.width << std::endl;
-#endif
-
-                if (m_cache.data) {
-                    if (m_spareData) {
-//                        std::cerr << this << ": Freeing spare data" << std::endl;
-                        free(m_spareData);
-                    }
-//                    std::cerr << this << ": Moving old cache data to spare" << std::endl;
-                    m_spareData = m_cache.data;
-                }
-//                std::cerr << this << ": Moving request data to cache" << std::endl;
-                m_cache.data = request.data;
-
-                m_readThread->done(m_requestToken);
-                m_requestToken = -1;
-            }
-
-            // already requested something covering this area; wait for it
-            return;
-        }
-
-        FileReadThread::Request dud;
-
-        if (!m_readThread->getRequest(m_requestToken, dud)) {
-
-            std::cerr << "ERROR: MatrixFile::primeCache: Inconsistent replies from FileReadThread" << std::endl;
-
-        } else {
-
-            // current request is for the wrong area, so no longer of any use
-            m_readThread->cancel(m_requestToken);
-
-            // crude way to avoid leaking the data
-            while (!m_readThread->isCancelled(m_requestToken)) {
-                usleep(10000);
-            }
-
-#ifdef DEBUG_MATRIX_FILE_READ_SET
-            std::cerr << "cancelled " << m_requestToken << std::endl;
-#endif
-
-            if (m_spareData) {
-                free(m_spareData);
-            }
-            m_spareData = dud.data;
-            m_readThread->done(m_requestToken);
-        }
-
-        m_requestToken = -1;
-    }
-
-    if (m_fd < 0) {
-        MutexLocker locker(&m_fdMutex, "MatrixFile::primeCache::m_fdMutex");
-        if (m_fd < 0) resume();
-    }
-
-    FileReadThread::Request request;
-
-    request.fd = m_fd;
-    request.mutex = &m_fdMutex;
-    request.start = m_headerSize + rx * m_height * m_cellSize;
-    request.size = rw * m_height * m_cellSize;
-
-//    std::cerr << this << ": Moving spare data to request, and resizing to " << rw * m_height * m_cellSize << std::endl;
-
-    request.data = (char *)realloc(m_spareData, rw * m_height * m_cellSize);
-    MUNLOCK(request.data, rw * m_height * m_cellSize);
-    m_spareData = 0;
-
-    m_requestingX = rx;
-    m_requestingWidth = rw;
-
-    int token = m_readThread->request(request);
-#ifdef DEBUG_MATRIX_FILE_READ_SET
-    std::cerr << "MatrixFile::primeCache: request token is "
-              << token << " (x = [" << rx << "], w = [" << rw << "], left = [" << goingLeft << "])" << std::endl;
-#endif
-    m_requestToken = token;
-}
-
-bool
-MatrixFile::seekTo(size_t x, size_t y)
-{
-    if (m_fd < 0) resume();
-
-    off_t off = m_headerSize + (x * m_height + y) * m_cellSize;
-
     if (::lseek(m_fd, off, SEEK_SET) == (off_t)-1) {
         ::perror("Seek failed");
         std::cerr << "ERROR: MatrixFile::seekTo(" << x << ", " << y
-                  << ") failed" << std::endl;
+                  << ") = " << off << " failed" << std::endl;
         return false;
     }