diff base/MatrixFileCache.cpp @ 90:c4e163f911dd

* Switch spectrogram layer over to using the new rudimentary disk-backed
  FFT cache

--- a/base/MatrixFileCache.cpp	Wed May 03 11:15:46 2006 +0000
+++ b/base/MatrixFileCache.cpp	Wed May 03 14:26:26 2006 +0000
@@ -15,6 +15,7 @@
 
 #include "MatrixFileCache.h"
 #include "base/TempDirectory.h"
+#include "base/System.h"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -28,17 +29,38 @@
 #include <QFileInfo>
 #include <QDir>
 
+#define HAVE_MMAP 1
+
+#ifdef HAVE_MMAP
+#include <sys/mman.h>
+#endif
+
+//!!! This class is a work in progress -- it does only as much as we
+// need for the current SpectrogramLayer.  Slated for substantial
+// refactoring and extension.
+
 MatrixFileCache::MatrixFileCache(QString fileBase, Mode mode) :
     m_fd(-1),
-    m_off(-1),
     m_mode(mode),
     m_width(0),
     m_height(0),
+    m_headerSize(2 * sizeof(size_t)),
+    m_autoRegionWidth(2048),
+    m_off(-1),
     m_rx(0),
     m_rw(0),
-    m_range(0),
-    m_headerSize(2 * sizeof(size_t))
+    m_userRegion(false),
+    m_region(0),
+    m_mmapped(false),
+    m_mmapSize(0),
+    m_mmapOff(0),
+    m_preferMmap(true)
 {
+    // Ensure header size is a multiple of the size of our data (for
+    // alignment purposes)
+    size_t hs = ((m_headerSize / sizeof(float)) * sizeof(float));
+    if (hs != m_headerSize) m_headerSize = hs + sizeof(float);
+
     QDir tempDir(TempDirectory::instance()->getPath());
     QString fileName(tempDir.filePath(QString("%1.mfc").arg(fileBase)));
     bool newFile = !QFileInfo(fileName).exists();
@@ -58,7 +80,7 @@
         flags = O_RDONLY;
     }
 
-    if ((m_fd = ::open(fileName.toLocal8Bit(), flags, mode)) < 0) {
+    if ((m_fd = ::open(fileName.toLocal8Bit(), flags, fmode)) < 0) {
         ::perror("Open failed");
         std::cerr << "ERROR: MatrixFileCache::MatrixFileCache: "
                   << "Failed to open cache file \""
@@ -71,10 +93,11 @@
         resize(0, 0); // write header
     } else {
         size_t header[2];
-        if (::read(m_fd, header, 2 * sizeof(size_t))) {
+        if (::read(m_fd, header, 2 * sizeof(size_t)) < 0) {
             perror("Read failed");
             std::cerr << "ERROR: MatrixFileCache::MatrixFileCache: "
-                      << "Failed to read header" << std::endl;
+                      << "Failed to read header (fd " << m_fd << ", file \""
+                      << fileName.toStdString() << "\")" << std::endl;
             return;
         }
         m_width = header[0];
@@ -82,17 +105,29 @@
         seekTo(0, 0);
     }
 
-    std::cerr << "MatrixFileCache::MatrixFileCache: Done, size is " << m_width << "x" << m_height << std::endl;
+    std::cerr << "MatrixFileCache::MatrixFileCache: Done, size is " << "(" << m_width << ", " << m_height << ")" << std::endl;
 
 }
 
 MatrixFileCache::~MatrixFileCache()
 {
+    if (m_rw > 0) {
+        if (m_mmapped) {
+#ifdef HAVE_MMAP
+            ::munmap(m_region, m_mmapSize);
+#endif
+        } else {
+            delete[] m_region;
+        }
+    }
+
     if (m_fd >= 0) {
         if (::close(m_fd) < 0) {
             ::perror("MatrixFileCache::~MatrixFileCache: close failed");
         }
     }
+
+    //!!! refcount and unlink
 }
 
 size_t 
@@ -120,24 +155,32 @@
 
     if (w * h > m_width * m_height) {
 
-        if (::lseek(m_fd, off - sizeof(float), SEEK_SET) == (off_t)-1) {
-            ::perror("Seek failed");
-            std::cerr << "ERROR: MatrixFileCache::resize(" << w << ", "
-                      << h << "): seek failed, cannot resize" << std::endl;
-            return;
+#ifdef HAVE_MMAP
+        // If we're going to mmap the file, we need to ensure it's long
+        // enough beforehand
+        
+        if (m_preferMmap) {
+        
+            if (::lseek(m_fd, off - sizeof(float), SEEK_SET) == (off_t)-1) {
+                ::perror("Seek failed");
+                std::cerr << "ERROR: MatrixFileCache::resize(" << w << ", "
+                          << h << "): seek failed, cannot resize" << std::endl;
+                return;
+            }
+            
+            // guess this requires efficient support for sparse files
+            
+            float f(0);
+            if (::write(m_fd, &f, sizeof(float)) != sizeof(float)) {
+                ::perror("WARNING: MatrixFileCache::resize: write failed");
+            }
         }
-
-        // guess this requires efficient support for sparse files
-        
-        float f(0);
-        if (::write(m_fd, &f, sizeof(float)) != sizeof(float)) {
-            ::perror("WARNING: MatrixFileCache::resize: write failed");
-        }
+#endif
 
     } else {
         
         if (::ftruncate(m_fd, off) < 0) {
-            ::perror("MatrixFileCache::resize: ftruncate failed");
+            ::perror("WARNING: MatrixFileCache::resize: ftruncate failed");
         }
     }
 
@@ -173,27 +216,48 @@
         return;
     }
     
-    //...
+    float *emptyCol = new float[m_height];
+    for (size_t y = 0; y < m_height; ++y) emptyCol[y] = 0.f;
+
+    seekTo(0, 0);
+    for (size_t x = 0; x < m_width; ++x) setColumnAt(x, emptyCol);
+    
+    delete[] emptyCol;
 }
 
 void
-MatrixFileCache::setRangeOfInterest(size_t x, size_t width)
+MatrixFileCache::setRegionOfInterest(size_t x, size_t width)
 {
+    setRegion(x, width, true);
+}
+
+void
+MatrixFileCache::clearRegionOfInterest()
+{
+    m_userRegion = false;
 }
 
 float
 MatrixFileCache::getValueAt(size_t x, size_t y) const
 {
     if (m_rw > 0 && x >= m_rx && x < m_rx + m_rw) {
-        return m_range[x - m_rx][y];
+        float *rp = getRegionPtr(x, y);
+        if (rp) return *rp;
+    } else if (!m_userRegion) {
+        if (autoSetRegion(x)) {
+            float *rp = getRegionPtr(x, y);
+            if (rp) return *rp;
+        }
     }
 
     if (!seekTo(x, y)) return 0.f;
     float value;
-    if (::read(m_fd, &value, sizeof(float)) != sizeof(float)) {
+    ssize_t r = ::read(m_fd, &value, sizeof(float));
+    if (r != sizeof(float)) {
         ::perror("MatrixFileCache::getValueAt: read failed");
-        return 0.f;
+        value = 0.f;
     }
+    if (r > 0) m_off += r;
     return value;
 }
 
@@ -201,16 +265,31 @@
 MatrixFileCache::getColumnAt(size_t x, float *values) const
 {
     if (m_rw > 0 && x >= m_rx && x < m_rx + m_rw) {
-        for (size_t y = 0; y < m_height; ++y) {
-            values[y] = m_range[x - m_rx][y];
+        float *rp = getRegionPtr(x, 0);
+        if (rp) {
+            for (size_t y = 0; y < m_height; ++y) {
+                values[y] = rp[y];
+            }
+            return;
+        }
+    } else if (!m_userRegion) {
+        if (autoSetRegion(x)) {
+            float *rp = getRegionPtr(x, 0);
+            if (rp) {
+                for (size_t y = 0; y < m_height; ++y) {
+                    values[y] = rp[y];
+                }
+                return;
+            }
         }
     }
 
     if (!seekTo(x, 0)) return;
-    if (::read(m_fd, values, m_height * sizeof(float)) != m_height * sizeof(float)) {
+    ssize_t r = ::read(m_fd, values, m_height * sizeof(float));
+    if (r != m_height * sizeof(float)) {
         ::perror("MatrixFileCache::getColumnAt: read failed");
     }
-    return;
+    if (r > 0) m_off += r;
 }
 
 void
@@ -223,11 +302,13 @@
     }
 
     if (!seekTo(x, y)) return;
-    if (::write(m_fd, &value, sizeof(float)) != sizeof(float)) {
+    ssize_t w = ::write(m_fd, &value, sizeof(float));
+    if (w != sizeof(float)) {
         ::perror("WARNING: MatrixFileCache::setValueAt: write failed");
     }
+    if (w > 0) m_off += w;
 
-    //... update range as appropriate
+    //... update region as appropriate
 }
 
 void
@@ -240,11 +321,149 @@
     }
 
     if (!seekTo(x, 0)) return;
-    if (::write(m_fd, values, m_height * sizeof(float)) != m_height * sizeof(float)) {
+    ssize_t w = ::write(m_fd, values, m_height * sizeof(float));
+    if (w != m_height * sizeof(float)) {
         ::perror("WARNING: MatrixFileCache::setColumnAt: write failed");
     }
+    if (w > 0) m_off += w;
 
-    //... update range as appropriate
+    //... update region as appropriate
+}
+
+float *
+MatrixFileCache::getRegionPtr(size_t x, size_t y) const
+{
+    if (m_rw == 0) return 0;
+
+    float *region = m_region;
+
+    if (m_mmapOff > 0) {
+        char *cr = (char *)m_region;
+        cr += m_mmapOff;
+        region = (float *)cr;
+    }
+
+    float *ptr = &(region[(x - m_rx) * m_height + y]);
+    
+//    std::cerr << "getRegionPtr(" << x << "," << y << "): region is " << m_region << ", returning " << ptr << std::endl;
+    return ptr;
+}
+
+bool
+MatrixFileCache::autoSetRegion(size_t x) const
+{
+    size_t rx = x;
+    size_t rw = m_autoRegionWidth;
+    size_t left = rw / 4;
+    if (x < m_rx) left = (rw * 3) / 4;
+    if (rx > left) rx -= left;
+    else rx = 0;
+    if (rx + rw > m_width) rw = m_width - rx;
+    return setRegion(rx, rw, false);
+}
+
+bool
+MatrixFileCache::setRegion(size_t x, size_t width, bool user) const
+{
+    if (!user && m_userRegion) return false;
+    if (m_rw > 0 && x >= m_rx && x + width <= m_rx + m_rw) return true;
+
+    if (m_rw > 0) {
+        if (m_mmapped) {
+#ifdef HAVE_MMAP
+            ::munmap(m_region, m_mmapSize);
+            std::cerr << "unmapped " << m_mmapSize << " at " << m_region << std::endl;
+#endif
+        } else {
+            delete[] m_region;
+        }
+        m_region = 0;
+        m_mmapped = false;
+        m_mmapSize = 0;
+        m_mmapOff = 0;
+        m_rw = 0;
+    }
+
+    if (width == 0) {
+        return true;
+    }
+
+#ifdef HAVE_MMAP
+
+    if (m_preferMmap) {
+
+        size_t mmapSize = m_height * width * sizeof(float);
+        off_t offset = m_headerSize + (x * m_height) * sizeof(float);
+        int pagesize = getpagesize();
+        off_t aligned = (offset / pagesize) * pagesize;
+        size_t mmapOff = offset - aligned;
+        mmapSize += mmapOff;
+        
+        m_region = (float *)
+            ::mmap(0, mmapSize, PROT_READ, MAP_PRIVATE, m_fd, aligned);
+        
+        if (m_region == MAP_FAILED) {
+            
+            ::perror("Mmap failed");
+            std::cerr << "ERROR: MatrixFileCache::setRegion(" << x << ", "
+                      << width << "): Mmap(0, " << mmapSize
+                      << ", " << PROT_READ << ", " << MAP_SHARED << ", " << m_fd 
+                      << ", " << aligned << ") failed, falling back to "
+                      << "non-mmapping code for this cache" << std::endl;
+            m_preferMmap = false;
+            
+        } else {
+
+            std::cerr << "mmap succeeded (offset " << aligned << ", size " << mmapSize << ", m_mmapOff " << mmapOff << ") = " << m_region << std::endl;
+            
+            m_mmapped = true;
+            m_mmapSize = mmapSize;
+            m_mmapOff = mmapOff;
+            m_rx = x;
+            m_rw = width;
+            if (user) m_userRegion = true;
+//            MUNLOCK(m_region, m_mmapSize);
+            return true;
+        }
+    }
+#endif
+
+    if (!seekTo(x, 0)) return false;
+
+    m_region = new float[width * m_height];
+
+    ssize_t r = ::read(m_fd, m_region, width * m_height * sizeof(float));
+    if (r < 0) {
+        ::perror("Read failed");
+        std::cerr << "ERROR: MatrixFileCache::setRegion(" << x << ", " << width
+                  << ") failed" << std::endl;
+        delete[] m_region;
+        m_region = 0;
+        return false;
+    }
+    
+    m_off += r;
+
+    if (r < width * m_height * sizeof(float)) {
+        // didn't manage to read the whole thing, but did get something
+        std::cerr << "WARNING: MatrixFileCache::setRegion(" << x << ", " << width
+                  << "): ";
+        width = r / (m_height * sizeof(float));
+        std::cerr << "Only got " << width << " columns" << std::endl;
+    }
+
+    m_rx = x;
+    m_rw = width;
+    if (m_rw == 0) {
+        delete[] m_region;
+        m_region = 0;
+    }
+
+    std::cerr << "MatrixFileCache::setRegion: set region to " << x << ", " << width << std::endl;
+
+    if (user) m_userRegion = true;
+    if (m_rw > 0) MUNLOCK(m_region, m_rw * m_height);
+    return true;
 }
 
 bool
@@ -253,6 +472,11 @@
     off_t off = m_headerSize + (x * m_height + y) * sizeof(float);
     if (off == m_off) return true;
 
+    if (m_mode == ReadWrite) {
+        std::cerr << "writer: ";
+        std::cerr << "seek required (from " << m_off << " to " << off << ")" << std::endl;
+    }
+
     if (::lseek(m_fd, off, SEEK_SET) == (off_t)-1) {
         ::perror("Seek failed");
         std::cerr << "ERROR: MatrixFileCache::seekTo(" << x << ", " << y