# HG changeset patch # User Chris Cannam # Date 1146666386 0 # Node ID c4e163f911ddca9e0c60b0578ee1dc03b612a313 # Parent 6a1803d578e0e126a41ba5efb9c9cba1c91af845 * Switch spectrogram layer over to using the new rudimentary disk-backed FFT cache diff -r 6a1803d578e0 -r c4e163f911dd base/FFTCache.cpp --- a/base/FFTCache.cpp Wed May 03 11:15:46 2006 +0000 +++ b/base/FFTCache.cpp Wed May 03 14:26:26 2006 +0000 @@ -18,6 +18,10 @@ #include +//!!! This class is a work in progress -- it does only as much as we +// need for the current SpectrogramLayer. Slated for substantial +// refactoring and extension. + FFTMemoryCache::FFTMemoryCache() : m_width(0), m_height(0), diff -r 6a1803d578e0 -r c4e163f911dd base/FFTCache.h --- a/base/FFTCache.h Wed May 03 11:15:46 2006 +0000 +++ b/base/FFTCache.h Wed May 03 14:26:26 2006 +0000 @@ -16,11 +16,11 @@ #ifndef _FFT_CACHE_H_ #define _FFT_CACHE_H_ -#include +#include +#include + #include -#define M_PI (3.14159265358979232846) - class FFTCacheBase { public: @@ -36,16 +36,22 @@ virtual float getNormalizedMagnitudeAt(size_t x, size_t y) const = 0; virtual float getPhaseAt(size_t x, size_t y) const = 0; - virtual bool isLocalPeak(size_t x, size_t y) const = 0; - virtual bool isOverThreshold(size_t x, size_t y, float threshold) const = 0; - virtual void setNormalizationFactor(size_t x, float factor) = 0; virtual void setMagnitudeAt(size_t x, size_t y, float mag) = 0; virtual void setNormalizedMagnitudeAt(size_t x, size_t y, float norm) = 0; virtual void setPhaseAt(size_t x, size_t y, float phase) = 0; - virtual QColor getColour(unsigned char index) const = 0; - virtual void setColour(unsigned char index, QColor colour) = 0; + virtual void setColumnAt(size_t x, float *mags, float *phases, float factor) = 0; + + bool isLocalPeak(size_t x, size_t y) const { + float mag = getMagnitudeAt(x, y); + if (y > 0 && mag < getMagnitudeAt(x, y - 1)) return false; + if (y < getHeight()-1 && mag < getMagnitudeAt(x, y + 1)) return false; + return true; + } + bool isOverThreshold(size_t x, size_t y, float threshold) const { + return getMagnitudeAt(x, y) > threshold; + } protected: FFTCacheBase() { } @@ -96,17 +102,6 @@ return (float(i) / 32767.0) * M_PI; } - virtual bool isLocalPeak(size_t x, size_t y) const { - if (y > 0 && m_magnitude[x][y] < m_magnitude[x][y-1]) return false; - if (y < m_height-1 && m_magnitude[x][y] < m_magnitude[x][y+1]) return false; - return true; - } - - virtual bool isOverThreshold(size_t x, size_t y, float threshold) const { - if (threshold == 0.0) return true; - return getMagnitudeAt(x, y) > threshold; - } - virtual void setNormalizationFactor(size_t x, float factor) { if (x < m_width) m_factor[x] = factor; } @@ -129,22 +124,21 @@ } } - virtual QColor getColour(unsigned char index) const { - return m_colours[index]; + virtual void setColumnAt(size_t x, float *mags, float *phases, float factor) { + setNormalizationFactor(x, factor); + for (size_t y = 0; y < m_height; ++y) { + setMagnitudeAt(x, y, mags[y]); + setPhaseAt(x, y, phases[y]); + } } - - virtual void setColour(unsigned char index, QColor colour) { - m_colours[index] = colour; - } - + private: size_t m_width; size_t m_height; uint16_t **m_magnitude; uint16_t **m_phase; float *m_factor; - QColor m_colours[256]; - + void resize(uint16_t **&, size_t, size_t); }; diff -r 6a1803d578e0 -r c4e163f911dd base/FFTFileCache.cpp --- a/base/FFTFileCache.cpp Wed May 03 11:15:46 2006 +0000 +++ b/base/FFTFileCache.cpp Wed May 03 14:26:26 2006 +0000 @@ -17,8 +17,112 @@ #include "MatrixFileCache.h" -FFTFileCache::FFTFileCache() +#include + +//!!! This class is a work in progress -- it does only as much as we +// need for the current SpectrogramLayer. Slated for substantial +// refactoring and extension. + +// The underlying matrix has height (m_height * 2 + 1). In each +// column we store magnitude at [0], [2] etc and phase at [1], [3] +// etc, and then store the normalization factor (maximum magnitude) at +// [m_height * 2]. + +FFTFileCache::FFTFileCache(QString fileBase, MatrixFileCache::Mode mode) : + m_colbuf(0), + m_mfc(new MatrixFileCache(fileBase, mode)) { - //... - } + +FFTFileCache::~FFTFileCache() +{ + delete m_colbuf; + delete m_mfc; +} + +size_t +FFTFileCache::getWidth() const +{ + return m_mfc->getWidth(); +} + +size_t +FFTFileCache::getHeight() const +{ + size_t mh = m_mfc->getHeight(); + if (mh > 0) return (mh - 1) / 2; + else return 0; +} + +void +FFTFileCache::resize(size_t width, size_t height) +{ + m_mfc->resize(width, height * 2 + 1); + delete m_colbuf; + m_colbuf = new float[height * 2 + 1]; +} + +void +FFTFileCache::reset() +{ + m_mfc->reset(); +} + +float +FFTFileCache::getMagnitudeAt(size_t x, size_t y) const +{ + return m_mfc->getValueAt(x, y * 2); +} + +float +FFTFileCache::getNormalizedMagnitudeAt(size_t x, size_t y) const +{ + float factor = m_mfc->getValueAt(x, m_mfc->getHeight() - 1); + float mag = m_mfc->getValueAt(x, y * 2); + if (factor != 0) return mag / factor; + else return 0.f; +} + +float +FFTFileCache::getPhaseAt(size_t x, size_t y) const +{ + return m_mfc->getValueAt(x, y * 2 + 1); +} + +void +FFTFileCache::setNormalizationFactor(size_t x, float factor) +{ + m_mfc->setValueAt(x, m_mfc->getHeight() - 1, factor); +} + +void +FFTFileCache::setMagnitudeAt(size_t x, size_t y, float mag) +{ + m_mfc->setValueAt(x, y * 2, mag); +} + +void +FFTFileCache::setNormalizedMagnitudeAt(size_t x, size_t y, float norm) +{ + float factor = m_mfc->getValueAt(x, m_mfc->getHeight() - 1); + m_mfc->setValueAt(x, y * 2, norm * factor); +} + +void +FFTFileCache::setPhaseAt(size_t x, size_t y, float phase) +{ + m_mfc->setValueAt(x, y * 2 + 1, phase); +} + +void +FFTFileCache::setColumnAt(size_t x, float *mags, float *phases, float factor) +{ + size_t h = getHeight(); + for (size_t y = 0; y < h; ++y) { + m_colbuf[y * 2] = mags[y]; + m_colbuf[y * 2 + 1] = phases[y]; + } + m_colbuf[h * 2] = factor; + m_mfc->setColumnAt(x, m_colbuf); +} + diff -r 6a1803d578e0 -r c4e163f911dd base/FFTFileCache.h --- a/base/FFTFileCache.h Wed May 03 11:15:46 2006 +0000 +++ b/base/FFTFileCache.h Wed May 03 14:26:26 2006 +0000 @@ -17,27 +17,27 @@ #define _FFT_FILE_CACHE_H_ #include "FFTCache.h" - -class MatrixFileCache; +#include "MatrixFileCache.h" class FFTFileCache : public FFTCacheBase { public: - //!!! + //!!! This is very much a work in progress. + // // Initially, make this take a string for the filename, // and make the spectrogram layer have two, one for the main // thread and one for the fill thread, one RO and one RW, both // using the same string based off spectrogram layer address // or export ID. - // Subsequently factor out into reader and writer classes? - // Make take arguments to ctor describing FFT parameters and + // Subsequently factor out into reader and writer; + // make take arguments to ctor describing FFT parameters and // calculate its own string and eventually do its own FFT as // well. Intention is to make it able ultimately to write // its own cache so it can do it in the background while e.g. - // plugins read from it -- need the reader thread to be able + // plugins get data from it -- need the reader thread to be able // to block waiting for the writer thread as appropriate. - FFTFileCache(); + FFTFileCache(QString fileBase, MatrixFileCache::Mode mode); virtual ~FFTFileCache(); virtual size_t getWidth() const; @@ -50,19 +50,16 @@ virtual float getNormalizedMagnitudeAt(size_t x, size_t y) const; virtual float getPhaseAt(size_t x, size_t y) const; - virtual bool isLocalPeak(size_t x, size_t y) const; - virtual bool isOverThreshold(size_t x, size_t y, float threshold) const; - virtual void setNormalizationFactor(size_t x, float factor); virtual void setMagnitudeAt(size_t x, size_t y, float mag); virtual void setNormalizedMagnitudeAt(size_t x, size_t y, float norm); virtual void setPhaseAt(size_t x, size_t y, float phase); - virtual QColor getColour(unsigned char index) const; - virtual void setColour(unsigned char index, QColor colour); + //!!! not thread safe (but then neither is m_mfc) + virtual void setColumnAt(size_t x, float *mags, float *phases, float factor); protected: - size_t m_height; + float *m_colbuf; MatrixFileCache *m_mfc; }; diff -r 6a1803d578e0 -r c4e163f911dd base/MatrixFileCache.cpp --- a/base/MatrixFileCache.cpp Wed May 03 11:15:46 2006 +0000 +++ b/base/MatrixFileCache.cpp Wed May 03 14:26:26 2006 +0000 @@ -15,6 +15,7 @@ #include "MatrixFileCache.h" #include "base/TempDirectory.h" +#include "base/System.h" #include #include @@ -28,17 +29,38 @@ #include #include +#define HAVE_MMAP 1 + +#ifdef HAVE_MMAP +#include +#endif + +//!!! This class is a work in progress -- it does only as much as we +// need for the current SpectrogramLayer. Slated for substantial +// refactoring and extension. + MatrixFileCache::MatrixFileCache(QString fileBase, Mode mode) : m_fd(-1), - m_off(-1), m_mode(mode), m_width(0), m_height(0), + m_headerSize(2 * sizeof(size_t)), + m_autoRegionWidth(2048), + m_off(-1), m_rx(0), m_rw(0), - m_range(0), - m_headerSize(2 * sizeof(size_t)) + m_userRegion(false), + m_region(0), + m_mmapped(false), + m_mmapSize(0), + m_mmapOff(0), + m_preferMmap(true) { + // Ensure header size is a multiple of the size of our data (for + // alignment purposes) + size_t hs = ((m_headerSize / sizeof(float)) * sizeof(float)); + if (hs != m_headerSize) m_headerSize = hs + sizeof(float); + QDir tempDir(TempDirectory::instance()->getPath()); QString fileName(tempDir.filePath(QString("%1.mfc").arg(fileBase))); bool newFile = !QFileInfo(fileName).exists(); @@ -58,7 +80,7 @@ flags = O_RDONLY; } - if ((m_fd = ::open(fileName.toLocal8Bit(), flags, mode)) < 0) { + if ((m_fd = ::open(fileName.toLocal8Bit(), flags, fmode)) < 0) { ::perror("Open failed"); std::cerr << "ERROR: MatrixFileCache::MatrixFileCache: " << "Failed to open cache file \"" @@ -71,10 +93,11 @@ resize(0, 0); // write header } else { size_t header[2]; - if (::read(m_fd, header, 2 * sizeof(size_t))) { + if (::read(m_fd, header, 2 * sizeof(size_t)) < 0) { perror("Read failed"); std::cerr << "ERROR: MatrixFileCache::MatrixFileCache: " - << "Failed to read header" << std::endl; + << "Failed to read header (fd " << m_fd << ", file \"" + << fileName.toStdString() << "\")" << std::endl; return; } m_width = header[0]; @@ -82,17 +105,29 @@ seekTo(0, 0); } - std::cerr << "MatrixFileCache::MatrixFileCache: Done, size is " << m_width << "x" << m_height << std::endl; + std::cerr << "MatrixFileCache::MatrixFileCache: Done, size is " << "(" << m_width << ", " << m_height << ")" << std::endl; } MatrixFileCache::~MatrixFileCache() { + if (m_rw > 0) { + if (m_mmapped) { +#ifdef HAVE_MMAP + ::munmap(m_region, m_mmapSize); +#endif + } else { + delete[] m_region; + } + } + if (m_fd >= 0) { if (::close(m_fd) < 0) { ::perror("MatrixFileCache::~MatrixFileCache: close failed"); } } + + //!!! refcount and unlink } size_t @@ -120,24 +155,32 @@ if (w * h > m_width * m_height) { - if (::lseek(m_fd, off - sizeof(float), SEEK_SET) == (off_t)-1) { - ::perror("Seek failed"); - std::cerr << "ERROR: MatrixFileCache::resize(" << w << ", " - << h << "): seek failed, cannot resize" << std::endl; - return; +#ifdef HAVE_MMAP + // If we're going to mmap the file, we need to ensure it's long + // enough beforehand + + if (m_preferMmap) { + + if (::lseek(m_fd, off - sizeof(float), SEEK_SET) == (off_t)-1) { + ::perror("Seek failed"); + std::cerr << "ERROR: MatrixFileCache::resize(" << w << ", " + << h << "): seek failed, cannot resize" << std::endl; + return; + } + + // guess this requires efficient support for sparse files + + float f(0); + if (::write(m_fd, &f, sizeof(float)) != sizeof(float)) { + ::perror("WARNING: MatrixFileCache::resize: write failed"); + } } - - // guess this requires efficient support for sparse files - - float f(0); - if (::write(m_fd, &f, sizeof(float)) != sizeof(float)) { - ::perror("WARNING: MatrixFileCache::resize: write failed"); - } +#endif } else { if (::ftruncate(m_fd, off) < 0) { - ::perror("MatrixFileCache::resize: ftruncate failed"); + ::perror("WARNING: MatrixFileCache::resize: ftruncate failed"); } } @@ -173,27 +216,48 @@ return; } - //... + float *emptyCol = new float[m_height]; + for (size_t y = 0; y < m_height; ++y) emptyCol[y] = 0.f; + + seekTo(0, 0); + for (size_t x = 0; x < m_width; ++x) setColumnAt(x, emptyCol); + + delete[] emptyCol; } void -MatrixFileCache::setRangeOfInterest(size_t x, size_t width) +MatrixFileCache::setRegionOfInterest(size_t x, size_t width) { + setRegion(x, width, true); +} + +void +MatrixFileCache::clearRegionOfInterest() +{ + m_userRegion = false; } float MatrixFileCache::getValueAt(size_t x, size_t y) const { if (m_rw > 0 && x >= m_rx && x < m_rx + m_rw) { - return m_range[x - m_rx][y]; + float *rp = getRegionPtr(x, y); + if (rp) return *rp; + } else if (!m_userRegion) { + if (autoSetRegion(x)) { + float *rp = getRegionPtr(x, y); + if (rp) return *rp; + } } if (!seekTo(x, y)) return 0.f; float value; - if (::read(m_fd, &value, sizeof(float)) != sizeof(float)) { + ssize_t r = ::read(m_fd, &value, sizeof(float)); + if (r != sizeof(float)) { ::perror("MatrixFileCache::getValueAt: read failed"); - return 0.f; + value = 0.f; } + if (r > 0) m_off += r; return value; } @@ -201,16 +265,31 @@ MatrixFileCache::getColumnAt(size_t x, float *values) const { if (m_rw > 0 && x >= m_rx && x < m_rx + m_rw) { - for (size_t y = 0; y < m_height; ++y) { - values[y] = m_range[x - m_rx][y]; + float *rp = getRegionPtr(x, 0); + if (rp) { + for (size_t y = 0; y < m_height; ++y) { + values[y] = rp[y]; + } + return; + } + } else if (!m_userRegion) { + if (autoSetRegion(x)) { + float *rp = getRegionPtr(x, 0); + if (rp) { + for (size_t y = 0; y < m_height; ++y) { + values[y] = rp[y]; + } + return; + } } } if (!seekTo(x, 0)) return; - if (::read(m_fd, values, m_height * sizeof(float)) != m_height * sizeof(float)) { + ssize_t r = ::read(m_fd, values, m_height * sizeof(float)); + if (r != m_height * sizeof(float)) { ::perror("MatrixFileCache::getColumnAt: read failed"); } - return; + if (r > 0) m_off += r; } void @@ -223,11 +302,13 @@ } if (!seekTo(x, y)) return; - if (::write(m_fd, &value, sizeof(float)) != sizeof(float)) { + ssize_t w = ::write(m_fd, &value, sizeof(float)); + if (w != sizeof(float)) { ::perror("WARNING: MatrixFileCache::setValueAt: write failed"); } + if (w > 0) m_off += w; - //... update range as appropriate + //... update region as appropriate } void @@ -240,11 +321,149 @@ } if (!seekTo(x, 0)) return; - if (::write(m_fd, values, m_height * sizeof(float)) != m_height * sizeof(float)) { + ssize_t w = ::write(m_fd, values, m_height * sizeof(float)); + if (w != m_height * sizeof(float)) { ::perror("WARNING: MatrixFileCache::setColumnAt: write failed"); } + if (w > 0) m_off += w; - //... update range as appropriate + //... update region as appropriate +} + +float * +MatrixFileCache::getRegionPtr(size_t x, size_t y) const +{ + if (m_rw == 0) return 0; + + float *region = m_region; + + if (m_mmapOff > 0) { + char *cr = (char *)m_region; + cr += m_mmapOff; + region = (float *)cr; + } + + float *ptr = &(region[(x - m_rx) * m_height + y]); + +// std::cerr << "getRegionPtr(" << x << "," << y << "): region is " << m_region << ", returning " << ptr << std::endl; + return ptr; +} + +bool +MatrixFileCache::autoSetRegion(size_t x) const +{ + size_t rx = x; + size_t rw = m_autoRegionWidth; + size_t left = rw / 4; + if (x < m_rx) left = (rw * 3) / 4; + if (rx > left) rx -= left; + else rx = 0; + if (rx + rw > m_width) rw = m_width - rx; + return setRegion(rx, rw, false); +} + +bool +MatrixFileCache::setRegion(size_t x, size_t width, bool user) const +{ + if (!user && m_userRegion) return false; + if (m_rw > 0 && x >= m_rx && x + width <= m_rx + m_rw) return true; + + if (m_rw > 0) { + if (m_mmapped) { +#ifdef HAVE_MMAP + ::munmap(m_region, m_mmapSize); + std::cerr << "unmapped " << m_mmapSize << " at " << m_region << std::endl; +#endif + } else { + delete[] m_region; + } + m_region = 0; + m_mmapped = false; + m_mmapSize = 0; + m_mmapOff = 0; + m_rw = 0; + } + + if (width == 0) { + return true; + } + +#ifdef HAVE_MMAP + + if (m_preferMmap) { + + size_t mmapSize = m_height * width * sizeof(float); + off_t offset = m_headerSize + (x * m_height) * sizeof(float); + int pagesize = getpagesize(); + off_t aligned = (offset / pagesize) * pagesize; + size_t mmapOff = offset - aligned; + mmapSize += mmapOff; + + m_region = (float *) + ::mmap(0, mmapSize, PROT_READ, MAP_PRIVATE, m_fd, aligned); + + if (m_region == MAP_FAILED) { + + ::perror("Mmap failed"); + std::cerr << "ERROR: MatrixFileCache::setRegion(" << x << ", " + << width << "): Mmap(0, " << mmapSize + << ", " << PROT_READ << ", " << MAP_SHARED << ", " << m_fd + << ", " << aligned << ") failed, falling back to " + << "non-mmapping code for this cache" << std::endl; + m_preferMmap = false; + + } else { + + std::cerr << "mmap succeeded (offset " << aligned << ", size " << mmapSize << ", m_mmapOff " << mmapOff << ") = " << m_region << std::endl; + + m_mmapped = true; + m_mmapSize = mmapSize; + m_mmapOff = mmapOff; + m_rx = x; + m_rw = width; + if (user) m_userRegion = true; +// MUNLOCK(m_region, m_mmapSize); + return true; + } + } +#endif + + if (!seekTo(x, 0)) return false; + + m_region = new float[width * m_height]; + + ssize_t r = ::read(m_fd, m_region, width * m_height * sizeof(float)); + if (r < 0) { + ::perror("Read failed"); + std::cerr << "ERROR: MatrixFileCache::setRegion(" << x << ", " << width + << ") failed" << std::endl; + delete[] m_region; + m_region = 0; + return false; + } + + m_off += r; + + if (r < width * m_height * sizeof(float)) { + // didn't manage to read the whole thing, but did get something + std::cerr << "WARNING: MatrixFileCache::setRegion(" << x << ", " << width + << "): "; + width = r / (m_height * sizeof(float)); + std::cerr << "Only got " << width << " columns" << std::endl; + } + + m_rx = x; + m_rw = width; + if (m_rw == 0) { + delete[] m_region; + m_region = 0; + } + + std::cerr << "MatrixFileCache::setRegion: set region to " << x << ", " << width << std::endl; + + if (user) m_userRegion = true; + if (m_rw > 0) MUNLOCK(m_region, m_rw * m_height); + return true; } bool @@ -253,6 +472,11 @@ off_t off = m_headerSize + (x * m_height + y) * sizeof(float); if (off == m_off) return true; + if (m_mode == ReadWrite) { + std::cerr << "writer: "; + std::cerr << "seek required (from " << m_off << " to " << off << ")" << std::endl; + } + if (::lseek(m_fd, off, SEEK_SET) == (off_t)-1) { ::perror("Seek failed"); std::cerr << "ERROR: MatrixFileCache::seekTo(" << x << ", " << y diff -r 6a1803d578e0 -r c4e163f911dd base/MatrixFileCache.h --- a/base/MatrixFileCache.h Wed May 03 11:15:46 2006 +0000 +++ b/base/MatrixFileCache.h Wed May 03 14:26:26 2006 +0000 @@ -37,12 +37,11 @@ void resize(size_t width, size_t height); void reset(); - void setRangeOfInterest(size_t x, size_t width); + void setRegionOfInterest(size_t x, size_t width); + void clearRegionOfInterest(); float getValueAt(size_t x, size_t y) const; void getColumnAt(size_t x, float *values) const; -// float getColumnMaximum(size_t x) const; -// float getColumnMinimum(size_t x) const; void setValueAt(size_t x, size_t y, float value); void setColumnAt(size_t x, float *values); @@ -52,12 +51,22 @@ Mode m_mode; size_t m_width; size_t m_height; - size_t m_rx; - size_t m_rw; - float **m_range; size_t m_headerSize; + size_t m_autoRegionWidth; - mutable off_t m_off; + mutable off_t m_off; + mutable size_t m_rx; + mutable size_t m_rw; + mutable bool m_userRegion; + mutable float *m_region; + mutable bool m_mmapped; + mutable size_t m_mmapSize; + mutable size_t m_mmapOff; + mutable bool m_preferMmap; + float *getRegionPtr(size_t x, size_t y) const; + + bool autoSetRegion(size_t x) const; + bool setRegion(size_t x, size_t width, bool user) const; bool seekTo(size_t x, size_t y) const; };