Chris@87: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
Chris@87: 
Chris@87: /*
Chris@87:     Sonic Visualiser
Chris@87:     An audio file viewer and annotation editor.
Chris@87:     Centre for Digital Music, Queen Mary, University of London.
Chris@87:     This file copyright 2006 Chris Cannam.
Chris@87:     
Chris@87:     This program is free software; you can redistribute it and/or
Chris@87:     modify it under the terms of the GNU General Public License as
Chris@87:     published by the Free Software Foundation; either version 2 of the
Chris@87:     License, or (at your option) any later version.  See the file
Chris@87:     COPYING included with this distribution for more information.
Chris@87: */
Chris@87: 
Chris@87: #include "MatrixFileCache.h"
Chris@87: #include "base/TempDirectory.h"
Chris@90: #include "base/System.h"
Chris@87: 
Chris@87: #include <sys/types.h>
Chris@87: #include <sys/stat.h>
Chris@87: #include <fcntl.h>
Chris@87: #include <unistd.h>
Chris@87: 
Chris@87: #include <iostream>
Chris@87: 
Chris@87: #include <cstdio>
Chris@87: 
Chris@87: #include <QFileInfo>
Chris@91: #include <QFile>
Chris@87: #include <QDir>
Chris@87: 
Chris@91: //#define HAVE_MMAP 1
Chris@90: 
Chris@90: #ifdef HAVE_MMAP
Chris@90: #include <sys/mman.h>
Chris@90: #endif
Chris@90: 
Chris@91: std::map<QString, int> MatrixFileCache::m_refcount;
Chris@91: QMutex MatrixFileCache::m_refcountMutex;
Chris@91: 
Chris@90: //!!! This class is a work in progress -- it does only as much as we
Chris@90: // need for the current SpectrogramLayer.  Slated for substantial
Chris@90: // refactoring and extension.
Chris@90: 
Chris@87: MatrixFileCache::MatrixFileCache(QString fileBase, Mode mode) :
Chris@87:     m_fd(-1),
Chris@87:     m_mode(mode),
Chris@87:     m_width(0),
Chris@87:     m_height(0),
Chris@90:     m_headerSize(2 * sizeof(size_t)),
Chris@91:     m_autoRegionWidth(256),
Chris@90:     m_off(-1),
Chris@87:     m_rx(0),
Chris@87:     m_rw(0),
Chris@90:     m_userRegion(false),
Chris@90:     m_region(0),
Chris@90:     m_mmapped(false),
Chris@90:     m_mmapSize(0),
Chris@90:     m_mmapOff(0),
Chris@91:     m_preferMmap(false)
Chris@87: {
Chris@90:     // Ensure header size is a multiple of the size of our data (for
Chris@90:     // alignment purposes)
Chris@90:     size_t hs = ((m_headerSize / sizeof(float)) * sizeof(float));
Chris@90:     if (hs != m_headerSize) m_headerSize = hs + sizeof(float);
Chris@90: 
Chris@87:     QDir tempDir(TempDirectory::instance()->getPath());
Chris@87:     QString fileName(tempDir.filePath(QString("%1.mfc").arg(fileBase)));
Chris@87:     bool newFile = !QFileInfo(fileName).exists();
Chris@87: 
Chris@87:     if (newFile && mode == ReadOnly) {
Chris@87:         std::cerr << "ERROR: MatrixFileCache::MatrixFileCache: Read-only mode "
Chris@87:                   << "specified, but cache file does not exist" << std::endl;
Chris@87:         return;
Chris@87:     }
Chris@87: 
Chris@87:     int flags = 0;
Chris@87:     mode_t fmode = S_IRUSR | S_IWUSR;
Chris@87: 
Chris@87:     if (mode == ReadWrite) {
Chris@87:         flags = O_RDWR | O_CREAT;
Chris@87:     } else {
Chris@87:         flags = O_RDONLY;
Chris@87:     }
Chris@87: 
Chris@90:     if ((m_fd = ::open(fileName.toLocal8Bit(), flags, fmode)) < 0) {
Chris@87:         ::perror("Open failed");
Chris@87:         std::cerr << "ERROR: MatrixFileCache::MatrixFileCache: "
Chris@87:                   << "Failed to open cache file \""
Chris@87:                   << fileName.toStdString() << "\"";
Chris@87:         if (mode == ReadWrite) std::cerr << " for writing";
Chris@87:         std::cerr << std::endl;
Chris@91:         return;
Chris@87:     }
Chris@87: 
Chris@87:     if (newFile) {
Chris@87:         resize(0, 0); // write header
Chris@87:     } else {
Chris@87:         size_t header[2];
Chris@90:         if (::read(m_fd, header, 2 * sizeof(size_t)) < 0) {
Chris@87:             perror("Read failed");
Chris@87:             std::cerr << "ERROR: MatrixFileCache::MatrixFileCache: "
Chris@90:                       << "Failed to read header (fd " << m_fd << ", file \""
Chris@90:                       << fileName.toStdString() << "\")" << std::endl;
Chris@87:             return;
Chris@87:         }
Chris@87:         m_width = header[0];
Chris@87:         m_height = header[1];
Chris@87:         seekTo(0, 0);
Chris@87:     }
Chris@87: 
Chris@91:     m_fileName = fileName;
Chris@91:     QMutexLocker locker(&m_refcountMutex);
Chris@91:     ++m_refcount[fileName];
Chris@91: 
Chris@90:     std::cerr << "MatrixFileCache::MatrixFileCache: Done, size is " << "(" << m_width << ", " << m_height << ")" << std::endl;
Chris@87: 
Chris@87: }
Chris@87: 
Chris@87: MatrixFileCache::~MatrixFileCache()
Chris@87: {
Chris@90:     if (m_rw > 0) {
Chris@90:         if (m_mmapped) {
Chris@90: #ifdef HAVE_MMAP
Chris@90:             ::munmap(m_region, m_mmapSize);
Chris@90: #endif
Chris@90:         } else {
Chris@90:             delete[] m_region;
Chris@90:         }
Chris@90:     }
Chris@90: 
Chris@87:     if (m_fd >= 0) {
Chris@87:         if (::close(m_fd) < 0) {
Chris@87:             ::perror("MatrixFileCache::~MatrixFileCache: close failed");
Chris@87:         }
Chris@87:     }
Chris@90: 
Chris@91:     if (m_fileName != "") {
Chris@91:         QMutexLocker locker(&m_refcountMutex);
Chris@91:         if (--m_refcount[m_fileName] == 0) {
Chris@91:             if (!QFile(m_fileName).remove()) {
Chris@91:                 std::cerr << "WARNING: MatrixFileCache::~MatrixFileCache: reference count reached 0, but failed to unlink file \"" << m_fileName.toStdString() << "\"" << std::endl;
Chris@91:             } else {
Chris@91:                 std::cerr << "deleted " << m_fileName.toStdString() << std::endl;
Chris@91:             }
Chris@91:         }
Chris@91:     }
Chris@87: }
Chris@87: 
Chris@87: size_t 
Chris@87: MatrixFileCache::getWidth() const
Chris@87: {
Chris@87:     return m_width;
Chris@87: }
Chris@87: 
Chris@87: size_t
Chris@87: MatrixFileCache::getHeight() const
Chris@87: {
Chris@87:     return m_height;
Chris@87: }
Chris@87: 
Chris@87: void
Chris@87: MatrixFileCache::resize(size_t w, size_t h)
Chris@87: {
Chris@87:     if (m_mode != ReadWrite) {
Chris@87:         std::cerr << "ERROR: MatrixFileCache::resize called on read-only cache"
Chris@87:                   << std::endl;
Chris@87:         return;
Chris@87:     }
Chris@87: 
Chris@87:     off_t off = m_headerSize + (w * h * sizeof(float));
Chris@87: 
Chris@87:     if (w * h > m_width * m_height) {
Chris@87: 
Chris@90: #ifdef HAVE_MMAP
Chris@90:         // If we're going to mmap the file, we need to ensure it's long
Chris@90:         // enough beforehand
Chris@90:         
Chris@90:         if (m_preferMmap) {
Chris@90:         
Chris@90:             if (::lseek(m_fd, off - sizeof(float), SEEK_SET) == (off_t)-1) {
Chris@90:                 ::perror("Seek failed");
Chris@90:                 std::cerr << "ERROR: MatrixFileCache::resize(" << w << ", "
Chris@90:                           << h << "): seek failed, cannot resize" << std::endl;
Chris@90:                 return;
Chris@90:             }
Chris@90:             
Chris@90:             // guess this requires efficient support for sparse files
Chris@90:             
Chris@90:             float f(0);
Chris@90:             if (::write(m_fd, &f, sizeof(float)) != sizeof(float)) {
Chris@90:                 ::perror("WARNING: MatrixFileCache::resize: write failed");
Chris@90:             }
Chris@87:         }
Chris@90: #endif
Chris@87: 
Chris@87:     } else {
Chris@87:         
Chris@87:         if (::ftruncate(m_fd, off) < 0) {
Chris@90:             ::perror("WARNING: MatrixFileCache::resize: ftruncate failed");
Chris@87:         }
Chris@87:     }
Chris@87: 
Chris@87:     m_width = 0;
Chris@87:     m_height = 0;
Chris@87:     m_off = 0;
Chris@87: 
Chris@87:     if (::lseek(m_fd, 0, SEEK_SET) == (off_t)-1) {
Chris@87:         ::perror("ERROR: MatrixFileCache::resize: Seek to write header failed");
Chris@87:         return;
Chris@87:     }
Chris@87: 
Chris@87:     size_t header[2];
Chris@87:     header[0] = w;
Chris@87:     header[1] = h;
Chris@87:     if (::write(m_fd, header, 2 * sizeof(size_t)) != 2 * sizeof(size_t)) {
Chris@87:         ::perror("ERROR: MatrixFileCache::resize: Failed to write header");
Chris@87:         return;
Chris@87:     }
Chris@87: 
Chris@87:     m_width = w;
Chris@87:     m_height = h;
Chris@87: 
Chris@87:     seekTo(0, 0);
Chris@87: }
Chris@87: 
Chris@87: void
Chris@87: MatrixFileCache::reset()
Chris@87: {
Chris@87:     if (m_mode != ReadWrite) {
Chris@87:         std::cerr << "ERROR: MatrixFileCache::reset called on read-only cache"
Chris@87:                   << std::endl;
Chris@87:         return;
Chris@87:     }
Chris@87:     
Chris@90:     float *emptyCol = new float[m_height];
Chris@90:     for (size_t y = 0; y < m_height; ++y) emptyCol[y] = 0.f;
Chris@90: 
Chris@90:     seekTo(0, 0);
Chris@90:     for (size_t x = 0; x < m_width; ++x) setColumnAt(x, emptyCol);
Chris@90:     
Chris@90:     delete[] emptyCol;
Chris@87: }
Chris@87: 
Chris@87: void
Chris@90: MatrixFileCache::setRegionOfInterest(size_t x, size_t width)
Chris@87: {
Chris@90:     setRegion(x, width, true);
Chris@90: }
Chris@90: 
Chris@90: void
Chris@90: MatrixFileCache::clearRegionOfInterest()
Chris@90: {
Chris@90:     m_userRegion = false;
Chris@87: }
Chris@87: 
Chris@87: float
Chris@87: MatrixFileCache::getValueAt(size_t x, size_t y) const
Chris@87: {
Chris@87:     if (m_rw > 0 && x >= m_rx && x < m_rx + m_rw) {
Chris@90:         float *rp = getRegionPtr(x, y);
Chris@90:         if (rp) return *rp;
Chris@90:     } else if (!m_userRegion) {
Chris@90:         if (autoSetRegion(x)) {
Chris@90:             float *rp = getRegionPtr(x, y);
Chris@90:             if (rp) return *rp;
Chris@91:             else return 0.f;
Chris@90:         }
Chris@87:     }
Chris@87: 
Chris@87:     if (!seekTo(x, y)) return 0.f;
Chris@87:     float value;
Chris@90:     ssize_t r = ::read(m_fd, &value, sizeof(float));
Chris@91:     if (r < 0) {
Chris@91:         ::perror("MatrixFileCache::getValueAt: Read failed");
Chris@91:     }
Chris@90:     if (r != sizeof(float)) {
Chris@90:         value = 0.f;
Chris@87:     }
Chris@90:     if (r > 0) m_off += r;
Chris@87:     return value;
Chris@87: }
Chris@87: 
Chris@87: void
Chris@87: MatrixFileCache::getColumnAt(size_t x, float *values) const
Chris@87: {
Chris@87:     if (m_rw > 0 && x >= m_rx && x < m_rx + m_rw) {
Chris@90:         float *rp = getRegionPtr(x, 0);
Chris@90:         if (rp) {
Chris@90:             for (size_t y = 0; y < m_height; ++y) {
Chris@90:                 values[y] = rp[y];
Chris@90:             }
Chris@90:         }
Chris@91:         return;
Chris@90:     } else if (!m_userRegion) {
Chris@90:         if (autoSetRegion(x)) {
Chris@90:             float *rp = getRegionPtr(x, 0);
Chris@90:             if (rp) {
Chris@90:                 for (size_t y = 0; y < m_height; ++y) {
Chris@90:                     values[y] = rp[y];
Chris@90:                 }
Chris@90:                 return;
Chris@90:             }
Chris@87:         }
Chris@87:     }
Chris@87: 
Chris@87:     if (!seekTo(x, 0)) return;
Chris@90:     ssize_t r = ::read(m_fd, values, m_height * sizeof(float));
Chris@91:     if (r < 0) {
Chris@87:         ::perror("MatrixFileCache::getColumnAt: read failed");
Chris@87:     }
Chris@90:     if (r > 0) m_off += r;
Chris@87: }
Chris@87: 
Chris@87: void
Chris@87: MatrixFileCache::setValueAt(size_t x, size_t y, float value)
Chris@87: {
Chris@87:     if (m_mode != ReadWrite) {
Chris@87:         std::cerr << "ERROR: MatrixFileCache::setValueAt called on read-only cache"
Chris@87:                   << std::endl;
Chris@87:         return;
Chris@87:     }
Chris@87: 
Chris@87:     if (!seekTo(x, y)) return;
Chris@90:     ssize_t w = ::write(m_fd, &value, sizeof(float));
Chris@90:     if (w != sizeof(float)) {
Chris@87:         ::perror("WARNING: MatrixFileCache::setValueAt: write failed");
Chris@87:     }
Chris@90:     if (w > 0) m_off += w;
Chris@87: 
Chris@90:     //... update region as appropriate
Chris@87: }
Chris@87: 
Chris@87: void
Chris@87: MatrixFileCache::setColumnAt(size_t x, float *values)
Chris@87: {
Chris@87:     if (m_mode != ReadWrite) {
Chris@87:         std::cerr << "ERROR: MatrixFileCache::setColumnAt called on read-only cache"
Chris@87:                   << std::endl;
Chris@87:         return;
Chris@87:     }
Chris@87: 
Chris@87:     if (!seekTo(x, 0)) return;
Chris@90:     ssize_t w = ::write(m_fd, values, m_height * sizeof(float));
Chris@91:     if (w != ssize_t(m_height * sizeof(float))) {
Chris@87:         ::perror("WARNING: MatrixFileCache::setColumnAt: write failed");
Chris@87:     }
Chris@90:     if (w > 0) m_off += w;
Chris@87: 
Chris@90:     //... update region as appropriate
Chris@90: }
Chris@90: 
Chris@90: float *
Chris@90: MatrixFileCache::getRegionPtr(size_t x, size_t y) const
Chris@90: {
Chris@90:     if (m_rw == 0) return 0;
Chris@90: 
Chris@90:     float *region = m_region;
Chris@90: 
Chris@90:     if (m_mmapOff > 0) {
Chris@90:         char *cr = (char *)m_region;
Chris@90:         cr += m_mmapOff;
Chris@90:         region = (float *)cr;
Chris@90:     }
Chris@90: 
Chris@90:     float *ptr = &(region[(x - m_rx) * m_height + y]);
Chris@90:     
Chris@90: //    std::cerr << "getRegionPtr(" << x << "," << y << "): region is " << m_region << ", returning " << ptr << std::endl;
Chris@90:     return ptr;
Chris@90: }
Chris@90: 
Chris@90: bool
Chris@90: MatrixFileCache::autoSetRegion(size_t x) const
Chris@90: {
Chris@90:     size_t rx = x;
Chris@90:     size_t rw = m_autoRegionWidth;
Chris@90:     size_t left = rw / 4;
Chris@90:     if (x < m_rx) left = (rw * 3) / 4;
Chris@90:     if (rx > left) rx -= left;
Chris@90:     else rx = 0;
Chris@90:     if (rx + rw > m_width) rw = m_width - rx;
Chris@90:     return setRegion(rx, rw, false);
Chris@90: }
Chris@90: 
Chris@90: bool
Chris@90: MatrixFileCache::setRegion(size_t x, size_t width, bool user) const
Chris@90: {
Chris@90:     if (!user && m_userRegion) return false;
Chris@90:     if (m_rw > 0 && x >= m_rx && x + width <= m_rx + m_rw) return true;
Chris@90: 
Chris@90:     if (m_rw > 0) {
Chris@90:         if (m_mmapped) {
Chris@90: #ifdef HAVE_MMAP
Chris@90:             ::munmap(m_region, m_mmapSize);
Chris@90:             std::cerr << "unmapped " << m_mmapSize << " at " << m_region << std::endl;
Chris@90: #endif
Chris@90:         } else {
Chris@90:             delete[] m_region;
Chris@90:         }
Chris@90:         m_region = 0;
Chris@90:         m_mmapped = false;
Chris@90:         m_mmapSize = 0;
Chris@90:         m_mmapOff = 0;
Chris@90:         m_rw = 0;
Chris@90:     }
Chris@90: 
Chris@90:     if (width == 0) {
Chris@90:         return true;
Chris@90:     }
Chris@90: 
Chris@90: #ifdef HAVE_MMAP
Chris@90: 
Chris@90:     if (m_preferMmap) {
Chris@90: 
Chris@90:         size_t mmapSize = m_height * width * sizeof(float);
Chris@90:         off_t offset = m_headerSize + (x * m_height) * sizeof(float);
Chris@90:         int pagesize = getpagesize();
Chris@90:         off_t aligned = (offset / pagesize) * pagesize;
Chris@90:         size_t mmapOff = offset - aligned;
Chris@90:         mmapSize += mmapOff;
Chris@90:         
Chris@90:         m_region = (float *)
Chris@90:             ::mmap(0, mmapSize, PROT_READ, MAP_PRIVATE, m_fd, aligned);
Chris@90:         
Chris@90:         if (m_region == MAP_FAILED) {
Chris@90:             
Chris@90:             ::perror("Mmap failed");
Chris@90:             std::cerr << "ERROR: MatrixFileCache::setRegion(" << x << ", "
Chris@90:                       << width << "): Mmap(0, " << mmapSize
Chris@90:                       << ", " << PROT_READ << ", " << MAP_SHARED << ", " << m_fd 
Chris@90:                       << ", " << aligned << ") failed, falling back to "
Chris@90:                       << "non-mmapping code for this cache" << std::endl;
Chris@90:             m_preferMmap = false;
Chris@90:             
Chris@90:         } else {
Chris@90: 
Chris@90:             std::cerr << "mmap succeeded (offset " << aligned << ", size " << mmapSize << ", m_mmapOff " << mmapOff << ") = " << m_region << std::endl;
Chris@90:             
Chris@90:             m_mmapped = true;
Chris@90:             m_mmapSize = mmapSize;
Chris@90:             m_mmapOff = mmapOff;
Chris@90:             m_rx = x;
Chris@90:             m_rw = width;
Chris@90:             if (user) m_userRegion = true;
Chris@91:             MUNLOCK(m_region, m_mmapSize);
Chris@90:             return true;
Chris@90:         }
Chris@90:     }
Chris@90: #endif
Chris@90: 
Chris@90:     if (!seekTo(x, 0)) return false;
Chris@90: 
Chris@90:     m_region = new float[width * m_height];
Chris@91:     MUNLOCK(m_region, width * m_height * sizeof(float));
Chris@90: 
Chris@90:     ssize_t r = ::read(m_fd, m_region, width * m_height * sizeof(float));
Chris@90:     if (r < 0) {
Chris@90:         ::perror("Read failed");
Chris@90:         std::cerr << "ERROR: MatrixFileCache::setRegion(" << x << ", " << width
Chris@90:                   << ") failed" << std::endl;
Chris@90:         delete[] m_region;
Chris@90:         m_region = 0;
Chris@90:         return false;
Chris@90:     }
Chris@90:     
Chris@90:     m_off += r;
Chris@90: 
Chris@91:     if (r < ssize_t(width * m_height * sizeof(float))) {
Chris@90:         // didn't manage to read the whole thing, but did get something
Chris@90:         std::cerr << "WARNING: MatrixFileCache::setRegion(" << x << ", " << width
Chris@90:                   << "): ";
Chris@90:         width = r / (m_height * sizeof(float));
Chris@90:         std::cerr << "Only got " << width << " columns" << std::endl;
Chris@90:     }
Chris@90: 
Chris@90:     m_rx = x;
Chris@90:     m_rw = width;
Chris@90:     if (m_rw == 0) {
Chris@90:         delete[] m_region;
Chris@90:         m_region = 0;
Chris@90:     }
Chris@90: 
Chris@90:     std::cerr << "MatrixFileCache::setRegion: set region to " << x << ", " << width << std::endl;
Chris@90: 
Chris@90:     if (user) m_userRegion = true;
Chris@90:     return true;
Chris@87: }
Chris@87: 
Chris@87: bool
Chris@87: MatrixFileCache::seekTo(size_t x, size_t y) const
Chris@87: {
Chris@87:     off_t off = m_headerSize + (x * m_height + y) * sizeof(float);
Chris@87:     if (off == m_off) return true;
Chris@87: 
Chris@90:     if (m_mode == ReadWrite) {
Chris@90:         std::cerr << "writer: ";
Chris@90:         std::cerr << "seek required (from " << m_off << " to " << off << ")" << std::endl;
Chris@90:     }
Chris@90: 
Chris@87:     if (::lseek(m_fd, off, SEEK_SET) == (off_t)-1) {
Chris@87:         ::perror("Seek failed");
Chris@87:         std::cerr << "ERROR: MatrixFileCache::seekTo(" << x << ", " << y
Chris@87:                   << ") failed" << std::endl;
Chris@87:         return false;
Chris@87:     }
Chris@87: 
Chris@87:     m_off = off;
Chris@87:     return true;
Chris@87: }
Chris@87: