Mercurial > hg > svcore
changeset 537:3cc4b7cd2aa5
* Merge from one-fftdataserver-per-fftmodel branch. This bit of
reworking (which is not described very accurately by the title of
the branch) turns the MatrixFile object into something that either
reads or writes, but not both, and separates the FFT file cache
reader and writer implementations separately. This allows the
FFT data server to have a single thread owning writers and one reader
per "customer" thread, and for all locking to be vastly simplified
and concentrated in the data server alone (because none of the
classes it makes use of is used in more than one thread at a time).
The result is faster and more trustworthy code.
| author | Chris Cannam |
|---|---|
| date | Tue, 27 Jan 2009 13:25:10 +0000 |
| parents | beb51f558e9c |
| children | 87aef350f1dc |
| files | data/data.pro data/fft/FFTCache.h data/fft/FFTCacheReader.h data/fft/FFTCacheStorageType.h data/fft/FFTCacheWriter.h data/fft/FFTDataServer.cpp data/fft/FFTDataServer.h data/fft/FFTFileCache.cpp data/fft/FFTFileCache.h data/fft/FFTFileCacheReader.cpp data/fft/FFTFileCacheReader.h data/fft/FFTFileCacheWriter.cpp data/fft/FFTFileCacheWriter.h data/fft/FFTMemoryCache.cpp data/fft/FFTMemoryCache.h data/fileio/MatrixFile.cpp data/fileio/MatrixFile.h data/model/EditableDenseThreeDimensionalModel.cpp |
| diffstat | 18 files changed, 1256 insertions(+), 1425 deletions(-) [+] |
line wrap: on
line diff
--- a/data/data.pro Mon Jan 26 15:18:32 2009 +0000 +++ b/data/data.pro Tue Jan 27 13:25:10 2009 +0000 @@ -16,9 +16,12 @@ # Input HEADERS += fft/FFTapi.h \ - fft/FFTCache.h \ + fft/FFTCacheReader.h \ + fft/FFTCacheStorageType.h \ + fft/FFTCacheWriter.h \ fft/FFTDataServer.h \ - fft/FFTFileCache.h \ + fft/FFTFileCacheReader.h \ + fft/FFTFileCacheWriter.h \ fft/FFTMemoryCache.h \ fileio/AudioFileReader.h \ fileio/AudioFileReaderFactory.h \ @@ -73,7 +76,8 @@ osc/OSCQueue.h SOURCES += fft/FFTapi.cpp \ fft/FFTDataServer.cpp \ - fft/FFTFileCache.cpp \ + fft/FFTFileCacheReader.cpp \ + fft/FFTFileCacheWriter.cpp \ fft/FFTMemoryCache.cpp \ fileio/AudioFileReader.cpp \ fileio/AudioFileReaderFactory.cpp \
--- a/data/fft/FFTCache.h Mon Jan 26 15:18:32 2009 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,68 +0,0 @@ -/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ - -/* - Sonic Visualiser - An audio file viewer and annotation editor. - Centre for Digital Music, Queen Mary, University of London. - This file copyright 2006 Chris Cannam. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation; either version 2 of the - License, or (at your option) any later version. See the file - COPYING included with this distribution for more information. -*/ - -#ifndef _FFT_CACHE_H_ -#define _FFT_CACHE_H_ - -#include <cstdlib> -#include <cmath> - -#include <stdint.h> - -class FFTCache -{ -public: - virtual ~FFTCache() { } - - virtual size_t getWidth() const = 0; - virtual size_t getHeight() const = 0; - - virtual void resize(size_t width, size_t height) = 0; - virtual void reset() = 0; // zero-fill or 1-fill as appropriate without changing size - - virtual float getMagnitudeAt(size_t x, size_t y) const = 0; - virtual float getNormalizedMagnitudeAt(size_t x, size_t y) const = 0; - virtual float getMaximumMagnitudeAt(size_t x) const = 0; - virtual float getPhaseAt(size_t x, size_t y) const = 0; - - virtual void getValuesAt(size_t x, size_t y, float &real, float &imaginary) const = 0; - virtual void getMagnitudesAt(size_t x, float *values, size_t minbin, size_t count, size_t step) const = 0; - - virtual bool haveSetColumnAt(size_t x) const = 0; - - // may modify argument arrays - virtual void setColumnAt(size_t x, float *mags, float *phases, float factor) = 0; - - // may modify argument arrays - virtual void setColumnAt(size_t x, float *reals, float *imags) = 0; - - virtual void suspend() { } - - enum StorageType { - Compact, // 16 bits normalized polar - Rectangular, // floating point real+imag - Polar // floating point mag+phase - }; - virtual StorageType getStorageType() = 0; - - enum Type { MemoryCache, FileCache }; - virtual Type getType() = 0; - -protected: - FFTCache() { } -}; - - -#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data/fft/FFTCacheReader.h Tue Jan 27 13:25:10 2009 +0000 @@ -0,0 +1,41 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Sonic Visualiser + An audio file viewer and annotation editor. + Centre for Digital Music, Queen Mary, University of London. + This file copyright 2006-2009 Chris Cannam and QMUL. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef _FFT_CACHE_READER_H_ +#define _FFT_CACHE_READER_H_ + +#include "FFTCacheStorageType.h" +#include <stddef.h> + +class FFTCacheReader +{ +public: + virtual size_t getWidth() const = 0; + virtual size_t getHeight() const = 0; + + virtual float getMagnitudeAt(size_t x, size_t y) const = 0; + virtual float getNormalizedMagnitudeAt(size_t x, size_t y) const = 0; + virtual float getMaximumMagnitudeAt(size_t x) const = 0; + virtual float getPhaseAt(size_t x, size_t y) const = 0; + + virtual void getValuesAt(size_t x, size_t y, float &real, float &imag) const = 0; + virtual void getMagnitudesAt(size_t x, float *values, size_t minbin, size_t count, size_t step) const = 0; + + virtual bool haveSetColumnAt(size_t x) const = 0; + + virtual FFTCache::StorageType getStorageType() const = 0; +}; + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data/fft/FFTCacheStorageType.h Tue Jan 27 13:25:10 2009 +0000 @@ -0,0 +1,27 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Sonic Visualiser + An audio file viewer and annotation editor. + Centre for Digital Music, Queen Mary, University of London. + This file copyright 2006-2009 Chris Cannam and QMUL. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef _FFT_CACHE_STORAGE_TYPE_H_ +#define _FFT_CACHE_STORAGE_TYPE_H_ + +namespace FFTCache { +enum StorageType { //!!! dup + Compact, // 16 bits normalized polar + Rectangular, // floating point real+imag + Polar // floating point mag+phase +}; +} + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data/fft/FFTCacheWriter.h Tue Jan 27 13:25:10 2009 +0000 @@ -0,0 +1,36 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Sonic Visualiser + An audio file viewer and annotation editor. + Centre for Digital Music, Queen Mary, University of London. + This file copyright 2006-2009 Chris Cannam and QMUL. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef _FFT_CACHE_WRITER_H_ +#define _FFT_CACHE_WRITER_H_ + +#include <stddef.h> + +class FFTCacheWriter +{ +public: + virtual size_t getWidth() const = 0; + virtual size_t getHeight() const = 0; + + virtual void setColumnAt(size_t x, float *mags, float *phases, float factor) = 0; + virtual void setColumnAt(size_t x, float *reals, float *imags) = 0; + + virtual void allColumnsWritten() = 0; + + virtual FFTCache::StorageType getStorageType() const = 0; +}; + +#endif +
--- a/data/fft/FFTDataServer.cpp Mon Jan 26 15:18:32 2009 +0000 +++ b/data/fft/FFTDataServer.cpp Tue Jan 27 13:25:10 2009 +0000 @@ -15,7 +15,8 @@ #include "FFTDataServer.h" -#include "FFTFileCache.h" +#include "FFTFileCacheReader.h" +#include "FFTFileCacheWriter.h" #include "FFTMemoryCache.h" #include "model/DenseTimeValueModel.h" @@ -27,7 +28,9 @@ #include "base/Profiler.h" #include "base/Thread.h" // for debug mutex locker -//#define DEBUG_FFT_SERVER 1 +#include <QWriteLocker> + +#define DEBUG_FFT_SERVER 1 //#define DEBUG_FFT_SERVER_FILL 1 #ifdef DEBUG_FFT_SERVER_FILL @@ -326,6 +329,7 @@ std::cerr << "ERROR: FFTDataServer::releaseInstance(" << server << "): instance not allocated" << std::endl; } else if (--i->second.second == 0) { +/*!!! if (server->m_lastUsedCache == -1) { // never used #ifdef DEBUG_FFT_SERVER std::cerr << "FFTDataServer::releaseInstance: instance " @@ -335,6 +339,7 @@ delete server; m_servers.erase(i); } else { +*/ #ifdef DEBUG_FFT_SERVER std::cerr << "FFTDataServer::releaseInstance: instance " << server << " no longer in use, marking for possible collection" @@ -353,7 +358,7 @@ if (!found) m_releasedServers.push_back(server); server->suspend(); purgeLimbo(); - } +//!!! } } else { #ifdef DEBUG_FFT_SERVER std::cerr << "FFTDataServer::releaseInstance: instance " @@ -497,7 +502,6 @@ m_cacheWidth(0), m_cacheWidthPower(0), m_cacheWidthMask(0), - m_lastUsedCache(-1), m_criteria(criteria), m_fftInput(0), m_exiting(false), @@ -637,9 +641,6 @@ MutexLocker locker(&m_writeMutex, "FFTDataServer::suspend::m_writeMutex"); m_suspended = true; - for (CacheVector::iterator i = m_caches.begin(); i != m_caches.end(); ++i) { - if (*i) (*i)->suspend(); - } } void @@ -721,7 +722,7 @@ if ((recommendation & StorageAdviser::UseMemory) || (recommendation & StorageAdviser::PreferMemory)) { - memoryCache = true; +//!!! memoryCache = true; } compactCache = canCompact && @@ -734,63 +735,21 @@ #endif } -FFTCache * -FFTDataServer::getCacheAux(size_t c) +bool +FFTDataServer::makeCache(int c) { - Profiler profiler("FFTDataServer::getCacheAux", false); -#ifdef DEBUG_FFT_SERVER - std::cerr << "FFTDataServer(" << this << " [" << (void *)QThread::currentThreadId() << "])::getCacheAux" << std::endl; -#endif + QWriteLocker locker(&m_cacheVectorLock); - MutexLocker locker(&m_writeMutex, - "FFTDataServer::getCacheAux::m_writeMutex"); - - if (m_lastUsedCache == -1) { - m_fillThread->start(); + if (m_caches[c]) { + // someone else must have created the cache between our + // testing for it and taking the write lock + return true; } - if (int(c) != m_lastUsedCache) { - -#ifdef DEBUG_FFT_SERVER - std::cerr << "switch from " << m_lastUsedCache << " to " << c << std::endl; -#endif - - for (IntQueue::iterator i = m_dormantCaches.begin(); - i != m_dormantCaches.end(); ++i) { - if (*i == int(c)) { - m_dormantCaches.erase(i); - break; - } - } - - if (m_lastUsedCache >= 0) { - bool inDormant = false; - for (size_t i = 0; i < m_dormantCaches.size(); ++i) { - if (m_dormantCaches[i] == m_lastUsedCache) { - inDormant = true; - break; - } - } - if (!inDormant) { - m_dormantCaches.push_back(m_lastUsedCache); - } - while (m_dormantCaches.size() > 4) { - int dc = m_dormantCaches.front(); - m_dormantCaches.pop_front(); - m_caches[dc]->suspend(); - } - } - } - - if (m_caches[c]) { - m_lastUsedCache = c; - return m_caches[c]; - } + CacheBlock *cb = new CacheBlock; QString name = QString("%1-%2").arg(m_fileBaseName).arg(c); - FFTCache *cache = 0; - size_t width = m_cacheWidth; if (c * m_cacheWidth + width > m_width) { width = m_width - c * m_cacheWidth; @@ -801,73 +760,111 @@ getStorageAdvice(width, m_height, memoryCache, compactCache); - try { + bool success = false; - if (memoryCache) { + if (memoryCache) { - cache = new FFTMemoryCache - (compactCache ? FFTMemoryCache::Compact : - m_polar ? FFTMemoryCache::Polar : - FFTMemoryCache::Rectangular); + try { - } else { + cb->memoryCache = new FFTMemoryCache + (compactCache ? FFTCache::Compact : + m_polar ? FFTCache::Polar : + FFTCache::Rectangular, + width, m_height); - cache = new FFTFileCache - (name, - MatrixFile::ReadWrite, - compactCache ? FFTFileCache::Compact : - m_polar ? FFTFileCache::Polar : - FFTFileCache::Rectangular); - } + success = true; - cache->resize(width, m_height); - cache->reset(); + } catch (std::bad_alloc) { - } catch (std::bad_alloc) { - - delete cache; - cache = 0; - - if (memoryCache) { + delete cb->memoryCache; + cb->memoryCache = 0; - std::cerr << "WARNING: Memory allocation failed when resizing" - << " FFT memory cache no. " << c << " to " << width + std::cerr << "WARNING: Memory allocation failed when creating" + << " FFT memory cache no. " << c << " of " << width << "x" << m_height << " (of total width " << m_width << "): falling back to disc cache" << std::endl; - try { - - purgeLimbo(0); - - cache = new FFTFileCache(name, - MatrixFile::ReadWrite, - FFTFileCache::Compact); - - cache->resize(width, m_height); - cache->reset(); - - } catch (std::bad_alloc) { - - delete cache; - cache = 0; - } - } - - if (!cache) { - std::cerr << "ERROR: Memory allocation failed when resizing" - << " FFT file cache no. " << c << " to " << width - << "x" << m_height << " (of total width " << m_width - << "): abandoning this cache" << std::endl; - - throw AllocationFailed("Failed to create or resize an FFT model slice"); + memoryCache = false; } } - m_caches[c] = cache; - m_lastUsedCache = c; - return cache; + if (!memoryCache) { + + try { + + cb->fileCacheWriter = new FFTFileCacheWriter + (name, + compactCache ? FFTCache::Compact : + m_polar ? FFTCache::Polar : + FFTCache::Rectangular, + width, m_height); + + success = true; + + } catch (std::exception e) { + + delete cb->fileCacheWriter; + cb->fileCacheWriter = 0; + + std::cerr << "ERROR: Failed to construct disc cache for FFT data: " + << e.what() << std::endl; + } + } + + m_caches[c] = cb; + + return success; } + +bool +FFTDataServer::makeCacheReader(int c) +{ + // preconditions: m_caches[c] exists and contains a file writer; + // m_cacheVectorLock is not locked by this thread +#ifdef DEBUG_FFT_SERVER + std::cerr << "FFTDataServer::makeCacheReader(" << c << ")" << std::endl; +#endif + QThread *me = QThread::currentThread(); + QWriteLocker locker(&m_cacheVectorLock); + CacheBlock *cb(m_caches.at(c)); + if (!cb || !cb->fileCacheWriter) return false; + + try { + + cb->fileCacheReader[me] = new FFTFileCacheReader(cb->fileCacheWriter); + + } catch (std::exception e) { + + delete cb->fileCacheReader[me]; + cb->fileCacheReader.erase(me); + + std::cerr << "ERROR: Failed to construct disc cache reader for FFT data: " + << e.what() << std::endl; + return false; + } + + // erase a reader that looks like it may no longer going to be + // used by this thread for a while (leaving alone the current + // and previous cache readers) + int deleteCandidate = c - 2; + if (deleteCandidate < 0) deleteCandidate = c + 2; + if (deleteCandidate >= m_caches.size()) { + return true; + } + + cb = m_caches.at(deleteCandidate); + if (cb && cb->fileCacheReader.find(me) != cb->fileCacheReader.end()) { +#ifdef DEBUG_FFT_SERVER + std::cerr << "FFTDataServer::makeCacheReader: Deleting probably unpopular reader " << deleteCandidate << " for this thread (as I create reader " << c << ")" << std::endl; +#endif + delete cb->fileCacheReader[me]; + cb->fileCacheReader.erase(me); + } + + return true; +} + float FFTDataServer::getMagnitudeAt(size_t x, size_t y) { @@ -876,9 +873,10 @@ if (x >= m_width || y >= m_height) return 0; size_t col; - FFTCache *cache = getCache(x, col); + FFTCacheReader *cache = getCacheReader(x, col); if (!cache) return 0; + //!!! n.b. can throw if (!cache->haveSetColumnAt(col)) { Profiler profiler("FFTDataServer::getMagnitudeAt: filling"); #ifdef DEBUG_FFT_SERVER @@ -908,9 +906,10 @@ } size_t col; - FFTCache *cache = getCache(x, col); + FFTCacheReader *cache = getCacheReader(x, col); if (!cache) return false; + //!!! n.b. can throw if (!cache->haveSetColumnAt(col)) { Profiler profiler("FFTDataServer::getMagnitudesAt: filling"); MutexLocker locker(&m_writeMutex, @@ -931,9 +930,10 @@ if (x >= m_width || y >= m_height) return 0; size_t col; - FFTCache *cache = getCache(x, col); + FFTCacheReader *cache = getCacheReader(x, col); if (!cache) return 0; + //!!! n.b. can throw if (!cache->haveSetColumnAt(col)) { Profiler profiler("FFTDataServer::getNormalizedMagnitudeAt: filling"); // hold mutex so that write thread doesn't mess with class @@ -959,9 +959,10 @@ } size_t col; - FFTCache *cache = getCache(x, col); + FFTCacheReader *cache = getCacheReader(x, col); if (!cache) return false; + //!!! n.b. can throw if (!cache->haveSetColumnAt(col)) { Profiler profiler("FFTDataServer::getNormalizedMagnitudesAt: filling"); MutexLocker locker(&m_writeMutex, @@ -984,9 +985,10 @@ if (x >= m_width) return 0; size_t col; - FFTCache *cache = getCache(x, col); + FFTCacheReader *cache = getCacheReader(x, col); if (!cache) return 0; + //!!! n.b. can throw if (!cache->haveSetColumnAt(col)) { Profiler profiler("FFTDataServer::getMaximumMagnitudeAt: filling"); // hold mutex so that write thread doesn't mess with class @@ -1006,9 +1008,10 @@ if (x >= m_width || y >= m_height) return 0; size_t col; - FFTCache *cache = getCache(x, col); + FFTCacheReader *cache = getCacheReader(x, col); if (!cache) return 0; + //!!! n.b. can throw if (!cache->haveSetColumnAt(col)) { Profiler profiler("FFTDataServer::getPhaseAt: filling"); // hold mutex so that write thread doesn't mess with class @@ -1034,9 +1037,10 @@ } size_t col; - FFTCache *cache = getCache(x, col); + FFTCacheReader *cache = getCacheReader(x, col); if (!cache) return false; + //!!! n.b. can throw if (!cache->haveSetColumnAt(col)) { Profiler profiler("FFTDataServer::getPhasesAt: filling"); MutexLocker locker(&m_writeMutex, @@ -1063,7 +1067,7 @@ } size_t col; - FFTCache *cache = getCache(x, col); + FFTCacheReader *cache = getCacheReader(x, col); if (!cache) { real = 0; @@ -1071,6 +1075,7 @@ return; } + //!!! n.b. can throw if (!cache->haveSetColumnAt(col)) { Profiler profiler("FFTDataServer::getValuesAt: filling"); #ifdef DEBUG_FFT_SERVER @@ -1094,6 +1099,7 @@ if (x >= m_width) return true; if (!haveCache(x)) { +/*!!! if (m_lastUsedCache == -1) { if (m_suspended) { std::cerr << "FFTDataServer::isColumnReady(" << x << "): no cache, calling resume" << std::endl; @@ -1101,13 +1107,15 @@ } m_fillThread->start(); } +*/ return false; } size_t col; - FFTCache *cache = getCache(x, col); + FFTCacheReader *cache = getCacheReader(x, col); if (!cache) return true; + //!!! n.b. can throw return cache->haveSetColumnAt(col); } @@ -1133,7 +1141,6 @@ std::cerr << "WARNING: FFTDataServer::fillColumn(" << x << "): " << "x > width (" << x << " > " << m_width << ")" << std::endl; -// abort(); //!!! return; } @@ -1141,16 +1148,9 @@ #ifdef DEBUG_FFT_SERVER_FILL std::cout << "FFTDataServer::fillColumn(" << x << ")" << std::endl; #endif - FFTCache *cache = getCache(x, col); + FFTCacheWriter *cache = getCacheWriter(x, col); if (!cache) return; - { - MutexLocker locker(lockHeld ? 0 : &m_writeMutex, - "FFTDataServer::fillColumn::m_writeMutex [1]"); - - if (cache->haveSetColumnAt(col)) return; - } - int winsize = m_windowSize; int fftsize = m_fftSize; int hs = fftsize/2; @@ -1271,6 +1271,19 @@ } } +void +FFTDataServer::fillComplete() +{ + for (int i = 0; i < int(m_caches.size()); ++i) { + if (m_caches[i]->memoryCache) { + m_caches[i]->memoryCache->allColumnsWritten(); + } + if (m_caches[i]->fileCacheWriter) { + m_caches[i]->fileCacheWriter->allColumnsWritten(); + } + } +} + size_t FFTDataServer::getFillCompletion() const { @@ -1418,7 +1431,12 @@ } } + m_server.fillComplete(); m_completion = 100; m_extent = end; + +#ifdef DEBUG_FFT_SERVER + std::cerr << "FFTDataServer::FillThread::run exiting" << std::endl; +#endif }
--- a/data/fft/FFTDataServer.h Mon Jan 26 15:18:32 2009 +0000 +++ b/data/fft/FFTDataServer.h Tue Jan 27 13:25:10 2009 +0000 @@ -21,8 +21,13 @@ #include "base/StorageAdviser.h" #include "FFTapi.h" +#include "FFTFileCacheReader.h" +#include "FFTFileCacheWriter.h" +#include "FFTMemoryCache.h" #include <QMutex> +#include <QReadWriteLock> +#include <QReadLocker> #include <QWaitCondition> #include <QString> @@ -31,7 +36,6 @@ class DenseTimeValueModel; class Model; -class FFTCache; class FFTDataServer { @@ -140,33 +144,86 @@ size_t m_cacheWidthPower; size_t m_cacheWidthMask; - int m_lastUsedCache; - FFTCache *getCache(size_t x, size_t &col) { - col = x & m_cacheWidthMask; + struct CacheBlock { + FFTMemoryCache *memoryCache; + typedef std::map<QThread *, FFTFileCacheReader *> ThreadReaderMap; + ThreadReaderMap fileCacheReader; + FFTFileCacheWriter *fileCacheWriter; + CacheBlock() : memoryCache(0), fileCacheWriter(0) { } + ~CacheBlock() { + delete memoryCache; + while (!fileCacheReader.empty()) { + delete fileCacheReader.begin()->second; + fileCacheReader.erase(fileCacheReader.begin()); + } + delete fileCacheWriter; + } + }; + + typedef std::vector<CacheBlock *> CacheVector; + CacheVector m_caches; + QReadWriteLock m_cacheVectorLock; // locks cache lookup, not use + + FFTCacheReader *getCacheReader(size_t x, size_t &col) { + Profiler profiler("FFTDataServer::getCacheReader"); + col = x & m_cacheWidthMask; int c = x >> m_cacheWidthPower; - // The only use of m_lastUsedCache without a lock is to - // establish whether a cache has been created at all (they're - // created on demand, but not destroyed until the server is). - if (c == m_lastUsedCache) return m_caches[c]; - else return getCacheAux(c); + m_cacheVectorLock.lockForRead(); + CacheBlock *cb(m_caches.at(c)); + if (cb) { + if (cb->memoryCache) return cb->memoryCache; + if (cb->fileCacheWriter) { + QThread *me = QThread::currentThread(); + CacheBlock::ThreadReaderMap &map = cb->fileCacheReader; + if (map.find(me) == map.end()) { + m_cacheVectorLock.unlock(); + if (!makeCacheReader(c)) return 0; + return getCacheReader(x, col); + } + FFTCacheReader *reader = cb->fileCacheReader.at(me); + m_cacheVectorLock.unlock(); + return reader; + } + // if cb exists but cb->fileCacheWriter doesn't, creation + // must have failed: don't try again + m_cacheVectorLock.unlock(); + return 0; + } + m_cacheVectorLock.unlock(); + if (!makeCache(c)) return 0; + return getCacheReader(x, col); } + + FFTCacheWriter *getCacheWriter(size_t x, size_t &col) { + Profiler profiler("FFTDataServer::getCacheWriter"); + col = x & m_cacheWidthMask; + int c = x >> m_cacheWidthPower; + { + QReadLocker locker(&m_cacheVectorLock); + CacheBlock *cb(m_caches.at(c)); + if (cb) { + if (cb->memoryCache) return cb->memoryCache; + if (cb->fileCacheWriter) return cb->fileCacheWriter; + // if cb exists, creation must have failed: don't try again + return 0; + } + } + if (!makeCache(c)) return 0; + return getCacheWriter(x, col); + } + bool haveCache(size_t x) { int c = x >> m_cacheWidthPower; - if (c == m_lastUsedCache) return true; - else return (m_caches[c] != 0); + return (m_caches[c] != 0); } - - typedef std::vector<FFTCache *> CacheVector; - CacheVector m_caches; - typedef std::deque<int> IntQueue; - IntQueue m_dormantCaches; - + bool makeCache(int c); + bool makeCacheReader(int c); + StorageAdviser::Criteria m_criteria; void getStorageAdvice(size_t w, size_t h, bool &memory, bool &compact); - FFTCache *getCacheAux(size_t c); QMutex m_writeMutex; QWaitCondition m_condition; @@ -199,6 +256,7 @@ void deleteProcessingData(); void fillColumn(size_t x, bool lockHeld); + void fillComplete(); QString generateFileBasename() const; static QString generateFileBasename(const DenseTimeValueModel *model,
--- a/data/fft/FFTFileCache.cpp Mon Jan 26 15:18:32 2009 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,375 +0,0 @@ -/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ - -/* - Sonic Visualiser - An audio file viewer and annotation editor. - Centre for Digital Music, Queen Mary, University of London. - This file copyright 2006 Chris Cannam and QMUL. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation; either version 2 of the - License, or (at your option) any later version. See the file - COPYING included with this distribution for more information. -*/ - -#include "FFTFileCache.h" - -#include "fileio/MatrixFile.h" - -#include "base/Profiler.h" -#include "base/Thread.h" -#include "base/Exceptions.h" - -#include <iostream> - - -// The underlying matrix has height (m_height * 2 + 1). In each -// column we store magnitude at [0], [2] etc and phase at [1], [3] -// etc, and then store the normalization factor (maximum magnitude) at -// [m_height * 2]. In compact mode, the factor takes two cells. - -FFTFileCache::FFTFileCache(QString fileBase, MatrixFile::Mode mode, - StorageType storageType) : - m_writebuf(0), - m_readbuf(0), - m_readbufCol(0), - m_readbufWidth(0), - m_mfc(new MatrixFile - (fileBase, mode, - storageType == Compact ? sizeof(uint16_t) : sizeof(float), - mode == MatrixFile::ReadOnly)), - m_storageType(storageType), - m_factorSize(storageType == Compact ? 2 : 1) -{ -// std::cerr << "FFTFileCache: storage type is " << (storageType == Compact ? "Compact" : storageType == Polar ? "Polar" : "Rectangular") << std::endl; -} - -FFTFileCache::~FFTFileCache() -{ - if (m_readbuf) delete[] m_readbuf; - if (m_writebuf) delete[] m_writebuf; - delete m_mfc; -} - -size_t -FFTFileCache::getWidth() const -{ - return m_mfc->getWidth(); -} - -size_t -FFTFileCache::getHeight() const -{ - size_t mh = m_mfc->getHeight(); - if (mh > m_factorSize) return (mh - m_factorSize) / 2; - else return 0; -} - -void -FFTFileCache::resize(size_t width, size_t height) -{ - MutexLocker locker(&m_writeMutex, "FFTFileCache::resize::m_writeMutex"); - - m_mfc->resize(width, height * 2 + m_factorSize); - - { - MutexLocker locker(&m_readbufMutex, "FFTFileCache::resize::m_readMutex"); - if (m_readbuf) { - delete[] m_readbuf; - m_readbuf = 0; - } - } - - if (m_writebuf) { - delete[] m_writebuf; - } - m_writebuf = new char[(height * 2 + m_factorSize) * m_mfc->getCellSize()]; -} - -void -FFTFileCache::reset() -{ - m_mfc->reset(); -} - -float -FFTFileCache::getMagnitudeAt(size_t x, size_t y) const -{ - Profiler profiler("FFTFileCache::getMagnitudeAt", false); - - float value = 0.f; - - switch (m_storageType) { - - case Compact: - value = (getFromReadBufCompactUnsigned(x, y * 2, true) / 65535.0) - * getNormalizationFactor(x, true); - break; - - case Rectangular: - { - float real, imag; - getValuesAt(x, y, real, imag); - value = sqrtf(real * real + imag * imag); - break; - } - - case Polar: - value = getFromReadBufStandard(x, y * 2, true); - break; - } - - return value; -} - -float -FFTFileCache::getNormalizedMagnitudeAt(size_t x, size_t y) const -{ - float value = 0.f; - - switch (m_storageType) { - - case Compact: - value = getFromReadBufCompactUnsigned(x, y * 2, true) / 65535.0; - break; - - default: - { - float mag = getMagnitudeAt(x, y); - float factor = getNormalizationFactor(x, true); - if (factor != 0) value = mag / factor; - else value = 0.f; - break; - } - } - - return value; -} - -float -FFTFileCache::getMaximumMagnitudeAt(size_t x) const -{ - return getNormalizationFactor(x, true); -} - -float -FFTFileCache::getPhaseAt(size_t x, size_t y) const -{ - float value = 0.f; - - switch (m_storageType) { - - case Compact: - value = (getFromReadBufCompactSigned(x, y * 2 + 1, true) / 32767.0) * M_PI; - break; - - case Rectangular: - { - float real, imag; - getValuesAt(x, y, real, imag); - value = atan2f(imag, real); - break; - } - - case Polar: - value = getFromReadBufStandard(x, y * 2 + 1, true); - break; - } - - return value; -} - -void -FFTFileCache::getValuesAt(size_t x, size_t y, float &real, float &imag) const -{ - switch (m_storageType) { - - case Rectangular: - m_readbufMutex.lock(); - real = getFromReadBufStandard(x, y * 2, false); - imag = getFromReadBufStandard(x, y * 2 + 1, false); - m_readbufMutex.unlock(); - return; - - default: - float mag = getMagnitudeAt(x, y); - float phase = getPhaseAt(x, y); - real = mag * cosf(phase); - imag = mag * sinf(phase); - return; - } -} - -void -FFTFileCache::getMagnitudesAt(size_t x, float *values, size_t minbin, size_t count, size_t step) const -{ - Profiler profiler("FFTFileCache::getMagnitudesAt"); - - m_readbufMutex.lock(); - - switch (m_storageType) { - - case Compact: - for (size_t i = 0; i < count; ++i) { - size_t y = minbin + i * step; - values[i] = (getFromReadBufCompactUnsigned(x, y * 2, false) / 65535.0) - * getNormalizationFactor(x, false); - } - break; - - case Rectangular: - { - float real, imag; - for (size_t i = 0; i < count; ++i) { - size_t y = minbin + i * step; - real = getFromReadBufStandard(x, y * 2, false); - imag = getFromReadBufStandard(x, y * 2 + 1, false); - values[i] = sqrtf(real * real + imag * imag); - } - break; - } - - case Polar: - for (size_t i = 0; i < count; ++i) { - size_t y = minbin + i * step; - values[i] = getFromReadBufStandard(x, y * 2, false); - } - break; - } - - m_readbufMutex.unlock(); -} - -bool -FFTFileCache::haveSetColumnAt(size_t x) const -{ - return m_mfc->haveSetColumnAt(x); -} - -void -FFTFileCache::setColumnAt(size_t x, float *mags, float *phases, float factor) -{ - MutexLocker locker(&m_writeMutex, "FFTFileCache::setColumnAt::m_writeMutex"); - - size_t h = getHeight(); - - switch (m_storageType) { - - case Compact: - for (size_t y = 0; y < h; ++y) { - ((uint16_t *)m_writebuf)[y * 2] = uint16_t((mags[y] / factor) * 65535.0); - ((uint16_t *)m_writebuf)[y * 2 + 1] = uint16_t(int16_t((phases[y] * 32767) / M_PI)); - } - break; - - case Rectangular: - for (size_t y = 0; y < h; ++y) { - ((float *)m_writebuf)[y * 2] = mags[y] * cosf(phases[y]); - ((float *)m_writebuf)[y * 2 + 1] = mags[y] * sinf(phases[y]); - } - break; - - case Polar: - for (size_t y = 0; y < h; ++y) { - ((float *)m_writebuf)[y * 2] = mags[y]; - ((float *)m_writebuf)[y * 2 + 1] = phases[y]; - } - break; - } - -// static float maxFactor = 0; -// if (factor > maxFactor) maxFactor = factor; -// std::cerr << "Column " << x << ": normalization factor: " << factor << ", max " << maxFactor << " (height " << getHeight() << ")" << std::endl; - - setNormalizationFactorToWritebuf(factor); - - m_mfc->setColumnAt(x, m_writebuf); -} - -void -FFTFileCache::setColumnAt(size_t x, float *real, float *imag) -{ - MutexLocker locker(&m_writeMutex, "FFTFileCache::setColumnAt::m_writeMutex"); - - size_t h = getHeight(); - - float factor = 0.0f; - - switch (m_storageType) { - - case Compact: - for (size_t y = 0; y < h; ++y) { - float mag = sqrtf(real[y] * real[y] + imag[y] * imag[y]); - if (mag > factor) factor = mag; - } - for (size_t y = 0; y < h; ++y) { - float mag = sqrtf(real[y] * real[y] + imag[y] * imag[y]); - float phase = atan2f(imag[y], real[y]); - ((uint16_t *)m_writebuf)[y * 2] = uint16_t((mag / factor) * 65535.0); - ((uint16_t *)m_writebuf)[y * 2 + 1] = uint16_t(int16_t((phase * 32767) / M_PI)); - } - break; - - case Rectangular: - for (size_t y = 0; y < h; ++y) { - ((float *)m_writebuf)[y * 2] = real[y]; - ((float *)m_writebuf)[y * 2 + 1] = imag[y]; - float mag = sqrtf(real[y] * real[y] + imag[y] * imag[y]); - if (mag > factor) factor = mag; - } - break; - - case Polar: - for (size_t y = 0; y < h; ++y) { - float mag = sqrtf(real[y] * real[y] + imag[y] * imag[y]); - if (mag > factor) factor = mag; - ((float *)m_writebuf)[y * 2] = mag; - float phase = atan2f(imag[y], real[y]); - ((float *)m_writebuf)[y * 2 + 1] = phase; - } - break; - } - -// static float maxFactor = 0; -// if (factor > maxFactor) maxFactor = factor; -// std::cerr << "[RI] Column " << x << ": normalization factor: " << factor << ", max " << maxFactor << " (height " << getHeight() << ")" << std::endl; - - setNormalizationFactorToWritebuf(factor); - - m_mfc->setColumnAt(x, m_writebuf); -} - -size_t -FFTFileCache::getCacheSize(size_t width, size_t height, StorageType type) -{ - return (height * 2 + (type == Compact ? 2 : 1)) * width * - (type == Compact ? sizeof(uint16_t) : sizeof(float)) + - 2 * sizeof(size_t); // matrix file header size -} - -void -FFTFileCache::populateReadBuf(size_t x) const // m_readbufMutex already held -{ - Profiler profiler("FFTFileCache::populateReadBuf", false); - - if (!m_readbuf) { - m_readbuf = new char[m_mfc->getHeight() * 2 * m_mfc->getCellSize()]; - } - try { - m_mfc->getColumnAt(x, m_readbuf); - if (m_mfc->haveSetColumnAt(x + 1)) { - m_mfc->getColumnAt - (x + 1, m_readbuf + m_mfc->getCellSize() * m_mfc->getHeight()); - m_readbufWidth = 2; - } else { - m_readbufWidth = 1; - } - } catch (FileReadFailed f) { - std::cerr << "ERROR: FFTFileCache::populateReadBuf: File read failed: " - << f.what() << std::endl; - memset(m_readbuf, 0, m_mfc->getHeight() * 2 * m_mfc->getCellSize()); - } - m_readbufCol = x; -} -
--- a/data/fft/FFTFileCache.h Mon Jan 26 15:18:32 2009 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,162 +0,0 @@ -/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ - -/* - Sonic Visualiser - An audio file viewer and annotation editor. - Centre for Digital Music, Queen Mary, University of London. - This file copyright 2006 Chris Cannam and QMUL. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation; either version 2 of the - License, or (at your option) any later version. See the file - COPYING included with this distribution for more information. -*/ - -#ifndef _FFT_FILE_CACHE_H_ -#define _FFT_FILE_CACHE_H_ - -#include "FFTCache.h" -#include "fileio/MatrixFile.h" - -#include <QMutex> - -class FFTFileCache : public FFTCache -{ -public: - FFTFileCache(QString fileBase, MatrixFile::Mode mode, - StorageType storageType); - virtual ~FFTFileCache(); - - MatrixFile::Mode getMode() const { return m_mfc->getMode(); } - - virtual size_t getWidth() const; - virtual size_t getHeight() const; - - virtual void resize(size_t width, size_t height); - virtual void reset(); // zero-fill or 1-fill as appropriate without changing size - - virtual float getMagnitudeAt(size_t x, size_t y) const; - virtual float getNormalizedMagnitudeAt(size_t x, size_t y) const; - virtual float getMaximumMagnitudeAt(size_t x) const; - virtual float getPhaseAt(size_t x, size_t y) const; - - virtual void getValuesAt(size_t x, size_t y, float &real, float &imag) const; - virtual void getMagnitudesAt(size_t x, float *values, size_t minbin, size_t count, size_t step) const; - - virtual bool haveSetColumnAt(size_t x) const; - - virtual void setColumnAt(size_t x, float *mags, float *phases, float factor); - virtual void setColumnAt(size_t x, float *reals, float *imags); - - virtual void suspend() { m_mfc->suspend(); } - - static size_t getCacheSize(size_t width, size_t height, StorageType type); - - virtual StorageType getStorageType() { return m_storageType; } - virtual Type getType() { return FileCache; } - -protected: - char *m_writebuf; - mutable char *m_readbuf; - mutable size_t m_readbufCol; - mutable size_t m_readbufWidth; - - float getFromReadBufStandard(size_t x, size_t y, bool lock) const { - if (lock) m_readbufMutex.lock(); - float v; - if (m_readbuf && - (m_readbufCol == x || (m_readbufWidth > 1 && m_readbufCol+1 == x))) { - v = ((float *)m_readbuf)[(x - m_readbufCol) * m_mfc->getHeight() + y]; - if (lock) m_readbufMutex.unlock(); - return v; - } else { - populateReadBuf(x); - v = getFromReadBufStandard(x, y, false); - if (lock) m_readbufMutex.unlock(); - return v; - } - } - - float getFromReadBufCompactUnsigned(size_t x, size_t y, bool lock) const { - if (lock) m_readbufMutex.lock(); - float v; - if (m_readbuf && - (m_readbufCol == x || (m_readbufWidth > 1 && m_readbufCol+1 == x))) { - v = ((uint16_t *)m_readbuf)[(x - m_readbufCol) * m_mfc->getHeight() + y]; - if (lock) m_readbufMutex.unlock(); - return v; - } else { - populateReadBuf(x); - v = getFromReadBufCompactUnsigned(x, y, false); - if (lock) m_readbufMutex.unlock(); - return v; - } - } - - float getFromReadBufCompactSigned(size_t x, size_t y, bool lock) const { - if (lock) m_readbufMutex.lock(); - float v; - if (m_readbuf && - (m_readbufCol == x || (m_readbufWidth > 1 && m_readbufCol+1 == x))) { - v = ((int16_t *)m_readbuf)[(x - m_readbufCol) * m_mfc->getHeight() + y]; - if (lock) m_readbufMutex.unlock(); - return v; - } else { - populateReadBuf(x); - v = getFromReadBufCompactSigned(x, y, false); - if (lock) m_readbufMutex.unlock(); - return v; - } - } - - void populateReadBuf(size_t x) const; - - float getNormalizationFactor(size_t col, bool lock) const { - size_t h = m_mfc->getHeight(); - if (h < m_factorSize) return 0; - if (m_storageType != Compact) { - return getFromReadBufStandard(col, h - 1, lock); - } else { - if (lock) m_readbufMutex.lock(); - union { - float f; - uint16_t u[2]; - } factor; - if (!m_readbuf || - !(m_readbufCol == col || - (m_readbufWidth > 1 && m_readbufCol+1 == col))) { - populateReadBuf(col); - } - size_t ix = (col - m_readbufCol) * m_mfc->getHeight() + h; - factor.u[0] = ((uint16_t *)m_readbuf)[ix - 2]; - factor.u[1] = ((uint16_t *)m_readbuf)[ix - 1]; - if (lock) m_readbufMutex.unlock(); - return factor.f; - } - } - - void setNormalizationFactorToWritebuf(float newfactor) { - size_t h = m_mfc->getHeight(); - if (h < m_factorSize) return; - if (m_storageType != Compact) { - ((float *)m_writebuf)[h - 1] = newfactor; - } else { - union { - float f; - uint16_t u[2]; - } factor; - factor.f = newfactor; - ((uint16_t *)m_writebuf)[h - 2] = factor.u[0]; - ((uint16_t *)m_writebuf)[h - 1] = factor.u[1]; - } - } - - MatrixFile *m_mfc; - QMutex m_writeMutex; - mutable QMutex m_readbufMutex; - StorageType m_storageType; - size_t m_factorSize; -}; - -#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data/fft/FFTFileCacheReader.cpp Tue Jan 27 13:25:10 2009 +0000 @@ -0,0 +1,251 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Sonic Visualiser + An audio file viewer and annotation editor. + Centre for Digital Music, Queen Mary, University of London. + This file copyright 2006-2009 Chris Cannam and QMUL. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#include "FFTFileCacheReader.h" +#include "FFTFileCacheWriter.h" + +#include "fileio/MatrixFile.h" + +#include "base/Profiler.h" +#include "base/Thread.h" +#include "base/Exceptions.h" + +#include <iostream> + + +// The underlying matrix has height (m_height * 2 + 1). In each +// column we store magnitude at [0], [2] etc and phase at [1], [3] +// etc, and then store the normalization factor (maximum magnitude) at +// [m_height * 2]. In compact mode, the factor takes two cells. + +FFTFileCacheReader::FFTFileCacheReader(FFTFileCacheWriter *writer) : + m_readbuf(0), + m_readbufCol(0), + m_readbufWidth(0), + m_storageType(writer->getStorageType()), + m_factorSize(m_storageType == FFTCache::Compact ? 2 : 1), + m_mfc(new MatrixFile + (writer->getFileBase(), + MatrixFile::ReadOnly, + m_storageType == FFTCache::Compact ? sizeof(uint16_t) : sizeof(float), + writer->getWidth(), + writer->getHeight() * 2 + m_factorSize)) +{ +// std::cerr << "FFTFileCacheReader: storage type is " << (storageType == FFTCache::Compact ? "Compact" : storageType == Polar ? "Polar" : "Rectangular") << std::endl; +} + +FFTFileCacheReader::~FFTFileCacheReader() +{ + if (m_readbuf) delete[] m_readbuf; + delete m_mfc; +} + +size_t +FFTFileCacheReader::getWidth() const +{ + return m_mfc->getWidth(); +} + +size_t +FFTFileCacheReader::getHeight() const +{ + size_t mh = m_mfc->getHeight(); + if (mh > m_factorSize) return (mh - m_factorSize) / 2; + else return 0; +} + +float +FFTFileCacheReader::getMagnitudeAt(size_t x, size_t y) const +{ + Profiler profiler("FFTFileCacheReader::getMagnitudeAt", false); + + float value = 0.f; + + switch (m_storageType) { + + case FFTCache::Compact: + value = (getFromReadBufCompactUnsigned(x, y * 2) / 65535.0) + * getNormalizationFactor(x); + break; + + case FFTCache::Rectangular: + { + float real, imag; + getValuesAt(x, y, real, imag); + value = sqrtf(real * real + imag * imag); + break; + } + + case FFTCache::Polar: + value = getFromReadBufStandard(x, y * 2); + break; + } + + return value; +} + +float +FFTFileCacheReader::getNormalizedMagnitudeAt(size_t x, size_t y) const +{ + float value = 0.f; + + switch (m_storageType) { + + case FFTCache::Compact: + value = getFromReadBufCompactUnsigned(x, y * 2) / 65535.0; + break; + + default: + { + float mag = getMagnitudeAt(x, y); + float factor = getNormalizationFactor(x); + if (factor != 0) value = mag / factor; + else value = 0.f; + break; + } + } + + return value; +} + +float +FFTFileCacheReader::getMaximumMagnitudeAt(size_t x) const +{ + return getNormalizationFactor(x); +} + +float +FFTFileCacheReader::getPhaseAt(size_t x, size_t y) const +{ + float value = 0.f; + + switch (m_storageType) { + + case FFTCache::Compact: + value = (getFromReadBufCompactSigned(x, y * 2 + 1) / 32767.0) * M_PI; + break; + + case FFTCache::Rectangular: + { + float real, imag; + getValuesAt(x, y, real, imag); + value = atan2f(imag, real); + break; + } + + case FFTCache::Polar: + value = getFromReadBufStandard(x, y * 2 + 1); + break; + } + + return value; +} + +void +FFTFileCacheReader::getValuesAt(size_t x, size_t y, float &real, float &imag) const +{ + switch (m_storageType) { + + case FFTCache::Rectangular: + real = getFromReadBufStandard(x, y * 2); + imag = getFromReadBufStandard(x, y * 2 + 1); + return; + + default: + float mag = getMagnitudeAt(x, y); + float phase = getPhaseAt(x, y); + real = mag * cosf(phase); + imag = mag * sinf(phase); + return; + } +} + +void +FFTFileCacheReader::getMagnitudesAt(size_t x, float *values, size_t minbin, size_t count, size_t step) const +{ + Profiler profiler("FFTFileCacheReader::getMagnitudesAt"); + + switch (m_storageType) { + + case FFTCache::Compact: + for (size_t i = 0; i < count; ++i) { + size_t y = minbin + i * step; + values[i] = (getFromReadBufCompactUnsigned(x, y * 2) / 65535.0) + * getNormalizationFactor(x); + } + break; + + case FFTCache::Rectangular: + { + float real, imag; + for (size_t i = 0; i < count; ++i) { + size_t y = minbin + i * step; + real = getFromReadBufStandard(x, y * 2); + imag = getFromReadBufStandard(x, y * 2 + 1); + values[i] = sqrtf(real * real + imag * imag); + } + break; + } + + case FFTCache::Polar: + for (size_t i = 0; i < count; ++i) { + size_t y = minbin + i * step; + values[i] = getFromReadBufStandard(x, y * 2); + } + break; + } +} + +bool +FFTFileCacheReader::haveSetColumnAt(size_t x) const +{ + return m_mfc->haveSetColumnAt(x); +} + +size_t +FFTFileCacheReader::getCacheSize(size_t width, size_t height, + FFTCache::StorageType type) +{ + return (height * 2 + (type == FFTCache::Compact ? 2 : 1)) * width * + (type == FFTCache::Compact ? sizeof(uint16_t) : sizeof(float)) + + 2 * sizeof(size_t); // matrix file header size +} + +void +FFTFileCacheReader::populateReadBuf(size_t x) const +{ + Profiler profiler("FFTFileCacheReader::populateReadBuf", false); + + if (!m_readbuf) { + m_readbuf = new char[m_mfc->getHeight() * 2 * m_mfc->getCellSize()]; + } + + try { + m_mfc->getColumnAt(x, m_readbuf); + if (m_mfc->haveSetColumnAt(x + 1)) { + m_mfc->getColumnAt + (x + 1, m_readbuf + m_mfc->getCellSize() * m_mfc->getHeight()); + m_readbufWidth = 2; + } else { + m_readbufWidth = 1; + } + } catch (FileReadFailed f) { + std::cerr << "ERROR: FFTFileCacheReader::populateReadBuf: File read failed: " + << f.what() << std::endl; + memset(m_readbuf, 0, m_mfc->getHeight() * 2 * m_mfc->getCellSize()); + } + m_readbufCol = x; +} +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data/fft/FFTFileCacheReader.h Tue Jan 27 13:25:10 2009 +0000 @@ -0,0 +1,122 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Sonic Visualiser + An audio file viewer and annotation editor. + Centre for Digital Music, Queen Mary, University of London. + This file copyright 2006-2009 Chris Cannam and QMUL. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef _FFT_FILE_CACHE_READER_H_ +#define _FFT_FILE_CACHE_READER_H_ + +#include "data/fileio/MatrixFile.h" +#include "FFTCacheReader.h" +#include "FFTCacheStorageType.h" + +class FFTFileCacheWriter; + +class FFTFileCacheReader : public FFTCacheReader +{ +public: + FFTFileCacheReader(FFTFileCacheWriter *); + ~FFTFileCacheReader(); + + size_t getWidth() const; + size_t getHeight() const; + + float getMagnitudeAt(size_t x, size_t y) const; + float getNormalizedMagnitudeAt(size_t x, size_t y) const; + float getMaximumMagnitudeAt(size_t x) const; + float getPhaseAt(size_t x, size_t y) const; + + void getValuesAt(size_t x, size_t y, float &real, float &imag) const; + void getMagnitudesAt(size_t x, float *values, size_t minbin, size_t count, size_t step) const; + + bool haveSetColumnAt(size_t x) const; + + static size_t getCacheSize(size_t width, size_t height, + FFTCache::StorageType type); + + FFTCache::StorageType getStorageType() const { return m_storageType; } + +protected: + mutable char *m_readbuf; + mutable size_t m_readbufCol; + mutable size_t m_readbufWidth; + + float getFromReadBufStandard(size_t x, size_t y) const { + float v; + if (m_readbuf && + (m_readbufCol == x || (m_readbufWidth > 1 && m_readbufCol+1 == x))) { + v = ((float *)m_readbuf)[(x - m_readbufCol) * m_mfc->getHeight() + y]; + return v; + } else { + populateReadBuf(x); + v = getFromReadBufStandard(x, y); + return v; + } + } + + float getFromReadBufCompactUnsigned(size_t x, size_t y) const { + float v; + if (m_readbuf && + (m_readbufCol == x || (m_readbufWidth > 1 && m_readbufCol+1 == x))) { + v = ((uint16_t *)m_readbuf)[(x - m_readbufCol) * m_mfc->getHeight() + y]; + return v; + } else { + populateReadBuf(x); + v = getFromReadBufCompactUnsigned(x, y); + return v; + } + } + + float getFromReadBufCompactSigned(size_t x, size_t y) const { + float v; + if (m_readbuf && + (m_readbufCol == x || (m_readbufWidth > 1 && m_readbufCol+1 == x))) { + v = ((int16_t *)m_readbuf)[(x - m_readbufCol) * m_mfc->getHeight() + y]; + return v; + } else { + populateReadBuf(x); + v = getFromReadBufCompactSigned(x, y); + return v; + } + } + + void populateReadBuf(size_t x) const; + + float getNormalizationFactor(size_t col) const { + size_t h = m_mfc->getHeight(); + if (h < m_factorSize) return 0; + if (m_storageType != FFTCache::Compact) { + return getFromReadBufStandard(col, h - 1); + } else { + union { + float f; + uint16_t u[2]; + } factor; + if (!m_readbuf || + !(m_readbufCol == col || + (m_readbufWidth > 1 && m_readbufCol+1 == col))) { + populateReadBuf(col); + } + size_t ix = (col - m_readbufCol) * m_mfc->getHeight() + h; + factor.u[0] = ((uint16_t *)m_readbuf)[ix - 2]; + factor.u[1] = ((uint16_t *)m_readbuf)[ix - 1]; + return factor.f; + } + } + + FFTCache::StorageType m_storageType; + size_t m_factorSize; + MatrixFile *m_mfc; +}; + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data/fft/FFTFileCacheWriter.cpp Tue Jan 27 13:25:10 2009 +0000 @@ -0,0 +1,186 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Sonic Visualiser + An audio file viewer and annotation editor. + Centre for Digital Music, Queen Mary, University of London. + This file copyright 2006-2009 Chris Cannam and QMUL. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#include "FFTFileCacheWriter.h" + +#include "fileio/MatrixFile.h" + +#include "base/Profiler.h" +#include "base/Thread.h" +#include "base/Exceptions.h" + +#include <iostream> + +//#define DEBUG_FFT_FILE_CACHE_WRITER 1 + + +// The underlying matrix has height (m_height * 2 + 1). In each +// column we store magnitude at [0], [2] etc and phase at [1], [3] +// etc, and then store the normalization factor (maximum magnitude) at +// [m_height * 2]. In compact mode, the factor takes two cells. + +FFTFileCacheWriter::FFTFileCacheWriter(QString fileBase, + FFTCache::StorageType storageType, + size_t width, size_t height) : + m_writebuf(0), + m_fileBase(fileBase), + m_storageType(storageType), + m_factorSize(storageType == FFTCache::Compact ? 2 : 1), + m_mfc(new MatrixFile + (fileBase, MatrixFile::WriteOnly, + storageType == FFTCache::Compact ? sizeof(uint16_t) : sizeof(float), + width, height * 2 + m_factorSize)) +{ + std::cerr << "FFTFileCacheWriter: storage type is " << (storageType == FFTCache::Compact ? "Compact" : storageType == FFTCache::Polar ? "Polar" : "Rectangular") << ", size " << width << "x" << height << std::endl; + m_writebuf = new char[(height * 2 + m_factorSize) * m_mfc->getCellSize()]; +} + +FFTFileCacheWriter::~FFTFileCacheWriter() +{ + if (m_writebuf) delete[] m_writebuf; + delete m_mfc; +} + +QString +FFTFileCacheWriter::getFileBase() const +{ + return m_fileBase; +} + +size_t +FFTFileCacheWriter::getWidth() const +{ + return m_mfc->getWidth(); +} + +size_t +FFTFileCacheWriter::getHeight() const +{ + size_t mh = m_mfc->getHeight(); + if (mh > m_factorSize) return (mh - m_factorSize) / 2; + else return 0; +} + +void +FFTFileCacheWriter::setColumnAt(size_t x, float *mags, float *phases, float factor) +{ + size_t h = getHeight(); + + switch (m_storageType) { + + case FFTCache::Compact: + for (size_t y = 0; y < h; ++y) { + ((uint16_t *)m_writebuf)[y * 2] = uint16_t((mags[y] / factor) * 65535.0); + ((uint16_t *)m_writebuf)[y * 2 + 1] = uint16_t(int16_t((phases[y] * 32767) / M_PI)); + } + break; + + case FFTCache::Rectangular: + for (size_t y = 0; y < h; ++y) { + ((float *)m_writebuf)[y * 2] = mags[y] * cosf(phases[y]); + ((float *)m_writebuf)[y * 2 + 1] = mags[y] * sinf(phases[y]); + } + break; + + case FFTCache::Polar: + for (size_t y = 0; y < h; ++y) { + ((float *)m_writebuf)[y * 2] = mags[y]; + ((float *)m_writebuf)[y * 2 + 1] = phases[y]; + } + break; + } + + static float maxFactor = 0; + if (factor > maxFactor) maxFactor = factor; +#ifdef DEBUG_FFT_FILE_CACHE_WRITER + std::cerr << "Column " << x << ": normalization factor: " << factor << ", max " << maxFactor << " (height " << getHeight() << ")" << std::endl; +#endif + + setNormalizationFactorToWritebuf(factor); + + m_mfc->setColumnAt(x, m_writebuf); +} + +void +FFTFileCacheWriter::setColumnAt(size_t x, float *real, float *imag) +{ + size_t h = getHeight(); + + float factor = 0.0f; + + switch (m_storageType) { + + case FFTCache::Compact: + for (size_t y = 0; y < h; ++y) { + float mag = sqrtf(real[y] * real[y] + imag[y] * imag[y]); + if (mag > factor) factor = mag; + } + for (size_t y = 0; y < h; ++y) { + float mag = sqrtf(real[y] * real[y] + imag[y] * imag[y]); + float phase = atan2f(imag[y], real[y]); + ((uint16_t *)m_writebuf)[y * 2] = uint16_t((mag / factor) * 65535.0); + ((uint16_t *)m_writebuf)[y * 2 + 1] = uint16_t(int16_t((phase * 32767) / M_PI)); + } + break; + + case FFTCache::Rectangular: + for (size_t y = 0; y < h; ++y) { + ((float *)m_writebuf)[y * 2] = real[y]; + ((float *)m_writebuf)[y * 2 + 1] = imag[y]; + float mag = sqrtf(real[y] * real[y] + imag[y] * imag[y]); + if (mag > factor) factor = mag; + } + break; + + case FFTCache::Polar: + for (size_t y = 0; y < h; ++y) { + float mag = sqrtf(real[y] * real[y] + imag[y] * imag[y]); + if (mag > factor) factor = mag; + ((float *)m_writebuf)[y * 2] = mag; + float phase = atan2f(imag[y], real[y]); + ((float *)m_writebuf)[y * 2 + 1] = phase; + } + break; + } + + static float maxFactor = 0; + if (factor > maxFactor) maxFactor = factor; +#ifdef DEBUG_FFT_FILE_CACHE_WRITER + std::cerr << "[RI] Column " << x << ": normalization factor: " << factor << ", max " << maxFactor << " (height " << getHeight() << ")" << std::endl; +#endif + + setNormalizationFactorToWritebuf(factor); + + m_mfc->setColumnAt(x, m_writebuf); +} + +size_t +FFTFileCacheWriter::getCacheSize(size_t width, size_t height, + FFTCache::StorageType type) +{ + return (height * 2 + (type == FFTCache::Compact ? 2 : 1)) * width * + (type == FFTCache::Compact ? sizeof(uint16_t) : sizeof(float)) + + 2 * sizeof(size_t); // matrix file header size +} + +void +FFTFileCacheWriter::allColumnsWritten() +{ +#ifdef DEBUG_FFT_FILE_CACHE_WRITER + std::cerr << "FFTFileCacheWriter::allColumnsWritten" << std::endl; +#endif + m_mfc->close(); +} +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data/fft/FFTFileCacheWriter.h Tue Jan 27 13:25:10 2009 +0000 @@ -0,0 +1,70 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Sonic Visualiser + An audio file viewer and annotation editor. + Centre for Digital Music, Queen Mary, University of London. + This file copyright 2006-2009 Chris Cannam and QMUL. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef _FFT_FILE_CACHE_WRITER_H_ +#define _FFT_FILE_CACHE_WRITER_H_ + +#include "FFTCacheStorageType.h" +#include "FFTCacheWriter.h" +#include "data/fileio/MatrixFile.h" + +class FFTFileCacheWriter : public FFTCacheWriter +{ +public: + FFTFileCacheWriter(QString fileBase, + FFTCache::StorageType storageType, + size_t width, size_t height); + ~FFTFileCacheWriter(); + + size_t getWidth() const; + size_t getHeight() const; + + void setColumnAt(size_t x, float *mags, float *phases, float factor); + void setColumnAt(size_t x, float *reals, float *imags); + + static size_t getCacheSize(size_t width, size_t height, + FFTCache::StorageType type); + + void allColumnsWritten(); + + QString getFileBase() const; + FFTCache::StorageType getStorageType() const { return m_storageType; } + +protected: + char *m_writebuf; + + void setNormalizationFactorToWritebuf(float newfactor) { + size_t h = m_mfc->getHeight(); + if (h < m_factorSize) return; + if (m_storageType != FFTCache::Compact) { + ((float *)m_writebuf)[h - 1] = newfactor; + } else { + union { + float f; + uint16_t u[2]; + } factor; + factor.f = newfactor; + ((uint16_t *)m_writebuf)[h - 2] = factor.u[0]; + ((uint16_t *)m_writebuf)[h - 1] = factor.u[1]; + } + } + + QString m_fileBase; + FFTCache::StorageType m_storageType; + size_t m_factorSize; + MatrixFile *m_mfc; +}; + +#endif
--- a/data/fft/FFTMemoryCache.cpp Mon Jan 26 15:18:32 2009 +0000 +++ b/data/fft/FFTMemoryCache.cpp Tue Jan 27 13:25:10 2009 +0000 @@ -17,12 +17,14 @@ #include "system/System.h" #include <iostream> +#include <cstdlib> //#define DEBUG_FFT_MEMORY_CACHE 1 -FFTMemoryCache::FFTMemoryCache(StorageType storageType) : - m_width(0), - m_height(0), +FFTMemoryCache::FFTMemoryCache(FFTCache::StorageType storageType, + size_t width, size_t height) : + m_width(width), + m_height(height), m_magnitude(0), m_phase(0), m_fmagnitude(0), @@ -34,8 +36,10 @@ { #ifdef DEBUG_FFT_MEMORY_CACHE std::cerr << "FFTMemoryCache[" << this << "]::FFTMemoryCache (type " - << m_storageType << ")" << std::endl; + << m_storageType << "), size " << m_width << "x" << m_height << std::endl; #endif + + initialise(); } FFTMemoryCache::~FFTMemoryCache() @@ -63,25 +67,25 @@ } void -FFTMemoryCache::resize(size_t width, size_t height) +FFTMemoryCache::initialise() { - Profiler profiler("FFTMemoryCache::resize"); + Profiler profiler("FFTMemoryCache::initialise"); + + size_t width = m_width, height = m_height; #ifdef DEBUG_FFT_MEMORY_CACHE - std::cerr << "FFTMemoryCache[" << this << "]::resize(" << width << "x" << height << " = " << width*height << ")" << std::endl; + std::cerr << "FFTMemoryCache[" << this << "]::initialise(" << width << "x" << height << " = " << width*height << ")" << std::endl; #endif - - if (m_width == width && m_height == height) return; - if (m_storageType == Compact) { - resize(m_magnitude, width, height); - resize(m_phase, width, height); - } else if (m_storageType == Polar) { - resize(m_fmagnitude, width, height); - resize(m_fphase, width, height); + if (m_storageType == FFTCache::Compact) { + initialise(m_magnitude); + initialise(m_phase); + } else if (m_storageType == FFTCache::Polar) { + initialise(m_fmagnitude); + initialise(m_fphase); } else { - resize(m_freal, width, height); - resize(m_fimag, width, height); + initialise(m_freal); + initialise(m_fimag); } m_colset.resize(width); @@ -97,98 +101,41 @@ } void -FFTMemoryCache::resize(uint16_t **&array, size_t width, size_t height) +FFTMemoryCache::initialise(uint16_t **&array) { - for (size_t i = width; i < m_width; ++i) { - free(array[i]); - } + array = (uint16_t **)malloc(m_width * sizeof(uint16_t *)); + if (!array) throw std::bad_alloc(); + MUNLOCK(array, m_width * sizeof(uint16_t *)); - if (width != m_width) { - array = (uint16_t **)realloc(array, width * sizeof(uint16_t *)); - if (!array) throw std::bad_alloc(); - MUNLOCK(array, width * sizeof(uint16_t *)); - } - - for (size_t i = m_width; i < width; ++i) { - array[i] = 0; - } - - for (size_t i = 0; i < width; ++i) { - array[i] = (uint16_t *)realloc(array[i], height * sizeof(uint16_t)); + for (size_t i = 0; i < m_width; ++i) { + array[i] = (uint16_t *)malloc(m_height * sizeof(uint16_t)); if (!array[i]) throw std::bad_alloc(); - MUNLOCK(array[i], height * sizeof(uint16_t)); + MUNLOCK(array[i], m_height * sizeof(uint16_t)); } } void -FFTMemoryCache::resize(float **&array, size_t width, size_t height) +FFTMemoryCache::initialise(float **&array) { - for (size_t i = width; i < m_width; ++i) { - free(array[i]); - } + array = (float **)malloc(m_width * sizeof(float *)); + if (!array) throw std::bad_alloc(); + MUNLOCK(array, m_width * sizeof(float *)); - if (width != m_width) { - array = (float **)realloc(array, width * sizeof(float *)); - if (!array) throw std::bad_alloc(); - MUNLOCK(array, width * sizeof(float *)); - } - - for (size_t i = m_width; i < width; ++i) { - array[i] = 0; - } - - for (size_t i = 0; i < width; ++i) { - array[i] = (float *)realloc(array[i], height * sizeof(float)); + for (size_t i = 0; i < m_width; ++i) { + array[i] = (float *)malloc(m_height * sizeof(float)); if (!array[i]) throw std::bad_alloc(); - MUNLOCK(array[i], height * sizeof(float)); + MUNLOCK(array[i], m_height * sizeof(float)); } } void -FFTMemoryCache::reset() -{ - switch (m_storageType) { - - case Compact: - for (size_t x = 0; x < m_width; ++x) { - for (size_t y = 0; y < m_height; ++y) { - m_magnitude[x][y] = 0; - m_phase[x][y] = 0; - } - m_factor[x] = 1.0; - } - break; - - case Polar: - for (size_t x = 0; x < m_width; ++x) { - for (size_t y = 0; y < m_height; ++y) { - m_fmagnitude[x][y] = 0; - m_fphase[x][y] = 0; - } - m_factor[x] = 1.0; - } - break; - - case Rectangular: - for (size_t x = 0; x < m_width; ++x) { - for (size_t y = 0; y < m_height; ++y) { - m_freal[x][y] = 0; - m_fimag[x][y] = 0; - } - m_factor[x] = 1.0; - } - break; - } -} - -void FFTMemoryCache::setColumnAt(size_t x, float *mags, float *phases, float factor) { Profiler profiler("FFTMemoryCache::setColumnAt: from polar"); setNormalizationFactor(x, factor); - if (m_storageType == Rectangular) { + if (m_storageType == FFTCache::Rectangular) { Profiler subprof("FFTMemoryCache::setColumnAt: polar to cart"); for (size_t y = 0; y < m_height; ++y) { m_freal[x][y] = mags[y] * cosf(phases[y]); @@ -201,7 +148,9 @@ } } + m_colsetMutex.lock(); m_colset.set(x); + m_colsetMutex.unlock(); } void @@ -213,7 +162,7 @@ switch (m_storageType) { - case Rectangular: + case FFTCache::Rectangular: for (size_t y = 0; y < m_height; ++y) { m_freal[x][y] = reals[y]; m_fimag[x][y] = imags[y]; @@ -222,8 +171,8 @@ } break; - case Compact: - case Polar: + case FFTCache::Compact: + case FFTCache::Polar: { Profiler subprof("FFTMemoryCache::setColumnAt: cart to polar"); for (size_t y = 0; y < m_height; ++y) { @@ -237,26 +186,28 @@ } }; - if (m_storageType == Rectangular) { + if (m_storageType == FFTCache::Rectangular) { m_factor[x] = max; + m_colsetMutex.lock(); m_colset.set(x); + m_colsetMutex.unlock(); } else { setColumnAt(x, reals, imags, max); } } size_t -FFTMemoryCache::getCacheSize(size_t width, size_t height, StorageType type) +FFTMemoryCache::getCacheSize(size_t width, size_t height, FFTCache::StorageType type) { size_t sz = 0; switch (type) { - case Compact: + case FFTCache::Compact: sz = (height * 2 + 1) * width * sizeof(uint16_t); - case Polar: - case Rectangular: + case FFTCache::Polar: + case FFTCache::Rectangular: sz = (height * 2 + 1) * width * sizeof(float); }
--- a/data/fft/FFTMemoryCache.h Mon Jan 26 15:18:32 2009 +0000 +++ b/data/fft/FFTMemoryCache.h Tue Jan 27 13:25:10 2009 +0000 @@ -16,11 +16,14 @@ #ifndef _FFT_MEMORY_CACHE_H_ #define _FFT_MEMORY_CACHE_H_ -#include "FFTCache.h" - +#include "FFTCacheReader.h" +#include "FFTCacheWriter.h" +#include "FFTCacheStorageType.h" #include "base/ResizeableBitset.h" #include "base/Profiler.h" +#include <QMutex> + /** * In-memory FFT cache. For this we want to cache magnitude with * enough resolution to have gain applied afterwards and determine @@ -40,20 +43,18 @@ * set appropriately. */ -class FFTMemoryCache : public FFTCache +class FFTMemoryCache : public FFTCacheReader, public FFTCacheWriter { public: - FFTMemoryCache(StorageType storageType); // of size zero, call resize() before using - virtual ~FFTMemoryCache(); + FFTMemoryCache(FFTCache::StorageType storageType, + size_t width, size_t height); + ~FFTMemoryCache(); - virtual size_t getWidth() const { return m_width; } - virtual size_t getHeight() const { return m_height; } + size_t getWidth() const { return m_width; } + size_t getHeight() const { return m_height; } - virtual void resize(size_t width, size_t height); - virtual void reset(); // zero-fill or 1-fill as appropriate without changing size - - virtual float getMagnitudeAt(size_t x, size_t y) const { - if (m_storageType == Rectangular) { + float getMagnitudeAt(size_t x, size_t y) const { + if (m_storageType == FFTCache::Rectangular) { Profiler profiler("FFTMemoryCache::getMagnitudeAt: cart to polar"); return sqrtf(m_freal[x][y] * m_freal[x][y] + m_fimag[x][y] * m_fimag[x][y]); @@ -62,21 +63,21 @@ } } - virtual float getNormalizedMagnitudeAt(size_t x, size_t y) const { - if (m_storageType == Rectangular) return getMagnitudeAt(x, y) / m_factor[x]; - else if (m_storageType == Polar) return m_fmagnitude[x][y]; + float getNormalizedMagnitudeAt(size_t x, size_t y) const { + if (m_storageType == FFTCache::Rectangular) return getMagnitudeAt(x, y) / m_factor[x]; + else if (m_storageType == FFTCache::Polar) return m_fmagnitude[x][y]; else return float(m_magnitude[x][y]) / 65535.0; } - virtual float getMaximumMagnitudeAt(size_t x) const { + float getMaximumMagnitudeAt(size_t x) const { return m_factor[x]; } - virtual float getPhaseAt(size_t x, size_t y) const { - if (m_storageType == Rectangular) { + float getPhaseAt(size_t x, size_t y) const { + if (m_storageType == FFTCache::Rectangular) { Profiler profiler("FFTMemoryCache::getValuesAt: cart to polar"); return atan2f(m_fimag[x][y], m_freal[x][y]); - } else if (m_storageType == Polar) { + } else if (m_storageType == FFTCache::Polar) { return m_fphase[x][y]; } else { int16_t i = (int16_t)m_phase[x][y]; @@ -84,8 +85,8 @@ } } - virtual void getValuesAt(size_t x, size_t y, float &real, float &imag) const { - if (m_storageType == Rectangular) { + void getValuesAt(size_t x, size_t y, float &real, float &imag) const { + if (m_storageType == FFTCache::Rectangular) { real = m_freal[x][y]; imag = m_fimag[x][y]; } else { @@ -97,15 +98,15 @@ } } - virtual void getMagnitudesAt(size_t x, float *values, size_t minbin, size_t count, size_t step) const + void getMagnitudesAt(size_t x, float *values, size_t minbin, size_t count, size_t step) const { - if (m_storageType == Rectangular) { + if (m_storageType == FFTCache::Rectangular) { for (size_t i = 0; i < count; ++i) { size_t y = i * step + minbin; values[i] = sqrtf(m_freal[x][y] * m_freal[x][y] + m_fimag[x][y] * m_fimag[x][y]); } - } else if (m_storageType == Polar) { + } else if (m_storageType == FFTCache::Polar) { for (size_t i = 0; i < count; ++i) { size_t y = i * step + minbin; values[i] = m_fmagnitude[x][y] * m_factor[x]; @@ -118,18 +119,23 @@ } } - virtual bool haveSetColumnAt(size_t x) const { - return m_colset.get(x); + bool haveSetColumnAt(size_t x) const { + m_colsetMutex.lock(); + bool have = m_colset.get(x); + m_colsetMutex.unlock(); + return have; } - virtual void setColumnAt(size_t x, float *mags, float *phases, float factor); + void setColumnAt(size_t x, float *mags, float *phases, float factor); - virtual void setColumnAt(size_t x, float *reals, float *imags); + void setColumnAt(size_t x, float *reals, float *imags); - static size_t getCacheSize(size_t width, size_t height, StorageType type); + void allColumnsWritten() { } - virtual StorageType getStorageType() { return m_storageType; } - virtual Type getType() { return MemoryCache; } + static size_t getCacheSize(size_t width, size_t height, + FFTCache::StorageType type); + + FFTCache::StorageType getStorageType() const { return m_storageType; } private: size_t m_width; @@ -141,35 +147,38 @@ float **m_freal; float **m_fimag; float *m_factor; - StorageType m_storageType; + FFTCache::StorageType m_storageType; ResizeableBitset m_colset; + mutable QMutex m_colsetMutex; - virtual void setNormalizationFactor(size_t x, float factor) { + void initialise(); + + void setNormalizationFactor(size_t x, float factor) { if (x < m_width) m_factor[x] = factor; } - virtual void setMagnitudeAt(size_t x, size_t y, float mag) { + void setMagnitudeAt(size_t x, size_t y, float mag) { // norm factor must already be set setNormalizedMagnitudeAt(x, y, mag / m_factor[x]); } - virtual void setNormalizedMagnitudeAt(size_t x, size_t y, float norm) { + void setNormalizedMagnitudeAt(size_t x, size_t y, float norm) { if (x < m_width && y < m_height) { - if (m_storageType == Polar) m_fmagnitude[x][y] = norm; + if (m_storageType == FFTCache::Polar) m_fmagnitude[x][y] = norm; else m_magnitude[x][y] = uint16_t(norm * 65535.0); } } - virtual void setPhaseAt(size_t x, size_t y, float phase) { + void setPhaseAt(size_t x, size_t y, float phase) { // phase in range -pi -> pi if (x < m_width && y < m_height) { - if (m_storageType == Polar) m_fphase[x][y] = phase; + if (m_storageType == FFTCache::Polar) m_fphase[x][y] = phase; else m_phase[x][y] = uint16_t(int16_t((phase * 32767) / M_PI)); } } - void resize(uint16_t **&, size_t, size_t); - void resize(float **&, size_t, size_t); + void initialise(uint16_t **&); + void initialise(float **&); };
--- a/data/fileio/MatrixFile.cpp Mon Jan 26 15:18:32 2009 +0000 +++ b/data/fileio/MatrixFile.cpp Tue Jan 27 13:25:10 2009 +0000 @@ -4,7 +4,7 @@ Sonic Visualiser An audio file viewer and annotation editor. Centre for Digital Music, Queen Mary, University of London. - This file copyright 2006 Chris Cannam. + This file copyright 2006-2009 Chris Cannam and QMUL. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as @@ -35,7 +35,7 @@ #include <QFileInfo> #include <QDir> -//#define DEBUG_MATRIX_FILE 1 +#define DEBUG_MATRIX_FILE 1 //#define DEBUG_MATRIX_FILE_READ_SET 1 #ifdef DEBUG_MATRIX_FILE_READ_SET @@ -45,42 +45,30 @@ #endif std::map<QString, int> MatrixFile::m_refcount; -QMutex MatrixFile::m_refcountMutex; - -MatrixFile::ResizeableBitsetMap MatrixFile::m_columnBitsets; -QMutex MatrixFile::m_columnBitsetWriteMutex; - -FileReadThread *MatrixFile::m_readThread = 0; +QMutex MatrixFile::m_createMutex; static size_t totalStorage = 0; -static size_t totalMemory = 0; static size_t totalCount = 0; +static size_t openCount = 0; MatrixFile::MatrixFile(QString fileBase, Mode mode, - size_t cellSize, bool eagerCache) : + size_t cellSize, size_t width, size_t height) : m_fd(-1), m_mode(mode), m_flags(0), m_fmode(0), m_cellSize(cellSize), - m_width(0), - m_height(0), - m_headerSize(2 * sizeof(size_t)), - m_defaultCacheWidth(1024), - m_prevX(0), - m_eagerCache(eagerCache), - m_requestToken(-1), - m_spareData(0), - m_columnBitset(0) + m_width(width), + m_height(height), + m_headerSize(2 * sizeof(size_t)) { Profiler profiler("MatrixFile::MatrixFile", true); - if (!m_readThread) { - m_readThread = new FileReadThread; - m_readThread->start(); - } +#ifdef DEBUG_MATRIX_FILE + std::cerr << "MatrixFile::MatrixFile(" << fileBase.toStdString() << ", " << int(mode) << ", " << cellSize << ", " << width << ", " << height << ")" << std::endl; +#endif - m_cache.data = 0; + QMutexLocker locker(&m_createMutex); QDir tempDir(TempDirectory::getInstance()->getPath()); QString fileName(tempDir.filePath(QString("%1.mfc").arg(fileBase))); @@ -92,24 +80,17 @@ throw FileNotFound(fileName); } - if (!newFile && m_mode == ReadWrite) { - std::cerr << "Note: MatrixFile::MatrixFile: Read/write mode " - << "specified, but file already exists; falling back to " - << "read-only mode" << std::endl; - m_mode = ReadOnly; - } - - if (!eagerCache && m_mode == ReadOnly) { - std::cerr << "WARNING: MatrixFile::MatrixFile: Eager cacheing not " - << "specified, but file is open in read-only mode -- cache " - << "will not be used" << std::endl; + if (!newFile && m_mode == WriteOnly) { + std::cerr << "ERROR: MatrixFile::MatrixFile: Write-only mode " + << "specified, but file already exists" << std::endl; + throw FileOperationFailed(fileName, "create"); } m_flags = 0; m_fmode = S_IRUSR | S_IWUSR; - if (m_mode == ReadWrite) { - m_flags = O_RDWR | O_CREAT; + if (m_mode == WriteOnly) { + m_flags = O_WRONLY | O_CREAT; } else { m_flags = O_RDONLY; } @@ -119,7 +100,7 @@ #endif #ifdef DEBUG_MATRIX_FILE - std::cerr << "MatrixFile::MatrixFile: opening " << fileName.toStdString() << "..." << std::endl; + std::cerr << "MatrixFile(" << this << ")::MatrixFile: opening " << fileName.toStdString() << "..." << std::endl; #endif if ((m_fd = ::open(fileName.toLocal8Bit(), m_flags, m_fmode)) < 0) { @@ -127,13 +108,17 @@ std::cerr << "ERROR: MatrixFile::MatrixFile: " << "Failed to open cache file \"" << fileName.toStdString() << "\""; - if (m_mode == ReadWrite) std::cerr << " for writing"; + if (m_mode == WriteOnly) std::cerr << " for writing"; std::cerr << std::endl; throw FailedToOpenFile(fileName); } +#ifdef DEBUG_MATRIX_FILE + std::cerr << "MatrixFile(" << this << ")::MatrixFile: fd is " << m_fd << std::endl; +#endif + if (newFile) { - resize(0, 0); // write header + initialise(); // write header and "unwritten" column tags } else { size_t header[2]; if (::read(m_fd, header, 2 * sizeof(size_t)) < 0) { @@ -143,555 +128,260 @@ << fileName.toStdString() << "\")" << std::endl; throw FileReadFailed(fileName); } - m_width = header[0]; - m_height = header[1]; - seekTo(0, 0); + if (header[0] != m_width || header[1] != m_height) { + std::cerr << "ERROR: MatrixFile::MatrixFile: " + << "Dimensions in file header (" << header[0] << "x" + << header[1] << ") differ from expected dimensions " + << m_width << "x" << m_height << std::endl; + throw FailedToOpenFile(fileName); + } } m_fileName = fileName; - - { - MutexLocker locker - (&m_columnBitsetWriteMutex, - "MatrixFile::MatrixFile::m_columnBitsetWriteMutex"); - - if (m_columnBitsets.find(m_fileName) == m_columnBitsets.end()) { - m_columnBitsets[m_fileName] = new ResizeableBitset; - } - m_columnBitset = m_columnBitsets[m_fileName]; - } - - MutexLocker locker(&m_refcountMutex, - "MatrixFile::MatrixFile::m_refcountMutex"); ++m_refcount[fileName]; -// std::cerr << "MatrixFile(" << this << "): fd " << m_fd << ", file " << fileName.toStdString() << ", ref " << m_refcount[fileName] << std::endl; +#ifdef DEBUG_MATRIX_FILE + std::cerr << "MatrixFile[" << m_fd << "]::MatrixFile: File " << fileName.toStdString() << ", ref " << m_refcount[fileName] << std::endl; -// std::cerr << "MatrixFile::MatrixFile: Done, size is " << "(" << m_width << ", " << m_height << ")" << std::endl; + std::cerr << "MatrixFile[" << m_fd << "]::MatrixFile: Done, size is " << "(" << m_width << ", " << m_height << ")" << std::endl; +#endif ++totalCount; - + ++openCount; } MatrixFile::~MatrixFile() { - char *requestData = 0; - - if (m_requestToken >= 0) { - FileReadThread::Request request; - if (m_readThread->getRequest(m_requestToken, request)) { - requestData = request.data; - } - m_readThread->cancel(m_requestToken); - } - - if (requestData) free(requestData); - if (m_cache.data) free(m_cache.data); - if (m_spareData) free(m_spareData); - if (m_fd >= 0) { if (::close(m_fd) < 0) { ::perror("MatrixFile::~MatrixFile: close failed"); } } + QMutexLocker locker(&m_createMutex); + if (m_fileName != "") { - MutexLocker locker(&m_refcountMutex, - "MatrixFile::~MatrixFile::m_refcountMutex"); - if (--m_refcount[m_fileName] == 0) { if (::unlink(m_fileName.toLocal8Bit())) { -// ::perror("Unlink failed"); -// std::cerr << "WARNING: MatrixFile::~MatrixFile: reference count reached 0, but failed to unlink file \"" << m_fileName.toStdString() << "\"" << std::endl; + std::cerr << "WARNING: MatrixFile::~MatrixFile: reference count reached 0, but failed to unlink file \"" << m_fileName.toStdString() << "\"" << std::endl; } else { -// std::cerr << "deleted " << m_fileName.toStdString() << std::endl; + std::cerr << "deleted " << m_fileName.toStdString() << std::endl; } - - MutexLocker locker2 - (&m_columnBitsetWriteMutex, - "MatrixFile::~MatrixFile::m_columnBitsetWriteMutex"); - m_columnBitsets.erase(m_fileName); - delete m_columnBitset; } } - - totalStorage -= (m_headerSize + (m_width * m_height * m_cellSize)); - totalMemory -= (2 * m_defaultCacheWidth * m_height * m_cellSize); + + if (m_mode == WriteOnly) { + totalStorage -= (m_headerSize + (m_width * m_height * m_cellSize) + m_width); + } totalCount --; + openCount --; -// std::cerr << "MatrixFile::~MatrixFile: " << std::endl; -// std::cerr << "Total storage now " << totalStorage/1024 << "K, theoretical max memory " -// << totalMemory/1024 << "K in " << totalCount << " instances" << std::endl; - +#ifdef DEBUG_MATRIX_FILE + std::cerr << "MatrixFile[" << m_fd << "]::~MatrixFile: " << std::endl; + std::cerr << "MatrixFile: Total storage now " << totalStorage/1024 << "K in " << totalCount << " instances (" << openCount << " open)" << std::endl; +#endif } void -MatrixFile::resize(size_t w, size_t h) +MatrixFile::initialise() { - Profiler profiler("MatrixFile::resize", true); + Profiler profiler("MatrixFile::initialise", true); - assert(m_mode == ReadWrite); - - MutexLocker locker(&m_fdMutex, "MatrixFile::resize::m_fdMutex"); + assert(m_mode == WriteOnly); - totalStorage -= (m_headerSize + (m_width * m_height * m_cellSize)); - totalMemory -= (2 * m_defaultCacheWidth * m_height * m_cellSize); - - off_t off = m_headerSize + (w * h * m_cellSize); + off_t off = m_headerSize + (m_width * m_height * m_cellSize) + m_width; #ifdef DEBUG_MATRIX_FILE - std::cerr << "MatrixFile::resize(" << w << ", " << h << "): resizing file" << std::endl; + std::cerr << "MatrixFile[" << m_fd << "]::initialise(" << m_width << ", " << m_height << "): cell size " << m_cellSize << ", header size " << m_headerSize << ", resizing file" << std::endl; #endif - if (w * h < m_width * m_height) { - if (::ftruncate(m_fd, off) < 0) { - ::perror("WARNING: MatrixFile::resize: ftruncate failed"); - throw FileOperationFailed(m_fileName, "ftruncate"); - } + if (::lseek(m_fd, off - 1, SEEK_SET) < 0) { + ::perror("ERROR: MatrixFile::initialise: seek to end failed"); + throw FileOperationFailed(m_fileName, "lseek"); } - m_width = 0; - m_height = 0; + unsigned char byte = 0; + if (::write(m_fd, &byte, 1) != 1) { + ::perror("ERROR: MatrixFile::initialise: write at end failed"); + throw FileOperationFailed(m_fileName, "write"); + } - if (::lseek(m_fd, 0, SEEK_SET) == (off_t)-1) { + if (::lseek(m_fd, 0, SEEK_SET) < 0) { ::perror("ERROR: MatrixFile::resize: Seek to write header failed"); throw FileOperationFailed(m_fileName, "lseek"); } size_t header[2]; - header[0] = w; - header[1] = h; + header[0] = m_width; + header[1] = m_height; if (::write(m_fd, header, 2 * sizeof(size_t)) != 2 * sizeof(size_t)) { ::perror("ERROR: MatrixFile::resize: Failed to write header"); throw FileOperationFailed(m_fileName, "write"); } - if (w > 0 && m_defaultCacheWidth > w) { - m_defaultCacheWidth = w; + if (m_mode == WriteOnly) { + totalStorage += (m_headerSize + (m_width * m_height * m_cellSize) + m_width); } -//!!! static size_t maxCacheMB = 16; - static size_t maxCacheMB = 4; - if (2 * m_defaultCacheWidth * h * m_cellSize > maxCacheMB * 1024 * 1024) { //!!! - m_defaultCacheWidth = (maxCacheMB * 1024 * 1024) / (2 * h * m_cellSize); - if (m_defaultCacheWidth < 16) m_defaultCacheWidth = 16; - } +#ifdef DEBUG_MATRIX_FILE + std::cerr << "MatrixFile[" << m_fd << "]::resize(" << m_width << ", " << m_height << "): storage " + << (m_headerSize + m_width * m_height * m_cellSize + m_width) << std::endl; - if (m_columnBitset) { - MutexLocker locker(&m_columnBitsetWriteMutex, - "MatrixFile::resize::m_columnBitsetWriteMutex"); - m_columnBitset->resize(w); - } - - if (m_cache.data) { - free(m_cache.data); - m_cache.data = 0; - } - - if (m_spareData) { - free(m_spareData); - m_spareData = 0; - } - - m_width = w; - m_height = h; - - totalStorage += (m_headerSize + (m_width * m_height * m_cellSize)); - totalMemory += (2 * m_defaultCacheWidth * m_height * m_cellSize); - -#ifdef DEBUG_MATRIX_FILE - std::cerr << "MatrixFile::resize(" << w << ", " << h << "): cache width " - << m_defaultCacheWidth << ", storage " - << (m_headerSize + w * h * m_cellSize) << ", mem " - << (2 * h * m_defaultCacheWidth * m_cellSize) << std::endl; - - std::cerr << "Total storage " << totalStorage/1024 << "K, theoretical max memory " - << totalMemory/1024 << "K in " << totalCount << " instances" << std::endl; + std::cerr << "MatrixFile: Total storage " << totalStorage/1024 << "K" << std::endl; #endif seekTo(0, 0); } void -MatrixFile::reset() +MatrixFile::close() { - Profiler profiler("MatrixFile::reset", true); - - assert (m_mode == ReadWrite); - - if (m_eagerCache) { - void *emptyCol = calloc(m_height, m_cellSize); - for (size_t x = 0; x < m_width; ++x) setColumnAt(x, emptyCol); - free(emptyCol); - } - - if (m_columnBitset) { - MutexLocker locker(&m_columnBitsetWriteMutex, - "MatrixFile::reset::m_columnBitsetWriteMutex"); - m_columnBitset->resize(m_width); +#ifdef DEBUG_MATRIX_FILE + std::cerr << "MatrixFile::close()" << std::endl; +#endif + if (m_fd >= 0) { + if (::close(m_fd) < 0) { + ::perror("MatrixFile::close: close failed"); + } + m_fd = -1; + -- openCount; } } void MatrixFile::getColumnAt(size_t x, void *data) { + assert(m_mode == ReadOnly); + +#ifdef DEBUG_MATRIX_FILE_READ_SET + std::cerr << "MatrixFile[" << m_fd << "]::getColumnAt(" << x << ")" << std::endl; +#endif + Profiler profiler("MatrixFile::getColumnAt"); -// assert(haveSetColumnAt(x)); - - if (getFromCache(x, 0, m_height, data)) return; - - Profiler profiler2("MatrixFile::getColumnAt (uncached)"); - - ssize_t r = 0; - -#ifdef DEBUG_MATRIX_FILE - std::cerr << "MatrixFile::getColumnAt(" << x << ")" - << ": reading the slow way"; - - if (m_requestToken >= 0 && - x >= m_requestingX && - x < m_requestingX + m_requestingWidth) { - - std::cerr << " (awaiting " << m_requestingX << ", " << m_requestingWidth << " from disk)"; + unsigned char set = 0; + if (!seekTo(x, 0)) { + std::cerr << "ERROR: MatrixFile::getColumnAt(" << x << "): Seek failed" << std::endl; + throw FileOperationFailed(m_fileName, "seek"); } - std::cerr << std::endl; + ssize_t r = -1; + r = ::read(m_fd, &set, 1); + if (r < 0) { + ::perror("MatrixFile::getColumnAt: read failed"); + throw FileReadFailed(m_fileName); + } + if (!set) { + std::cerr << "MatrixFile[" << m_fd << "]::getColumnAt(" << x << "): Column has not been set" << std::endl; + return; + } + + r = ::read(m_fd, data, m_height * m_cellSize); + if (r < 0) { + ::perror("MatrixFile::getColumnAt: read failed"); + throw FileReadFailed(m_fileName); + } +} + +bool +MatrixFile::haveSetColumnAt(size_t x) const +{ + assert(m_mode == ReadOnly); + + Profiler profiler("MatrixFile::haveSetColumnAt"); + +#ifdef DEBUG_MATRIX_FILE_READ_SET + std::cerr << "MatrixFile[" << m_fd << "]::haveSetColumnAt(" << x << ")" << std::endl; +// std::cerr << "."; #endif - { - MutexLocker locker(&m_fdMutex, "MatrixFile::getColumnAt::m_fdMutex"); + unsigned char set = 0; + if (!seekTo(x, 0)) { + std::cerr << "ERROR: MatrixFile::haveSetColumnAt(" << x << "): Seek failed" << std::endl; + throw FileOperationFailed(m_fileName, "seek"); + } - if (seekTo(x, 0)) { - r = ::read(m_fd, data, m_height * m_cellSize); - } - } - + ssize_t r = -1; + r = ::read(m_fd, &set, 1); if (r < 0) { - ::perror("MatrixFile::getColumnAt: read failed"); - std::cerr << "ERROR: MatrixFile::getColumnAt: " - << "Failed to read column " << x << " (height " << m_height << ", cell size " << m_cellSize << ", fd " << m_fd << ", file \"" - << m_fileName.toStdString() << "\")" << std::endl; + ::perror("MatrixFile::haveSetColumnAt: read failed"); throw FileReadFailed(m_fileName); } - return; -} - -bool -MatrixFile::getFromCache(size_t x, size_t ystart, size_t ycount, void *data) -{ - bool fail = false; - bool primeLeft = false; - - { - MutexLocker locker(&m_cacheMutex, - "MatrixFile::getFromCache::m_cacheMutex"); - - if (!m_cache.data || x < m_cache.x || x >= m_cache.x + m_cache.width) { - fail = true; - primeLeft = (m_cache.data && x < m_cache.x); - } else { - memcpy(data, - m_cache.data + m_cellSize * ((x - m_cache.x) * m_height + ystart), - ycount * m_cellSize); - } - } - - if (fail) { - primeCache(x, primeLeft); // this doesn't take effect until a later callback - m_prevX = x; - return false; - } - - if (m_cache.x > 0 && x < m_prevX && x < m_cache.x + m_cache.width/4) { - primeCache(x, true); - } - - if (m_cache.x + m_cache.width < m_width && - x > m_prevX && - x > m_cache.x + (m_cache.width * 3) / 4) { - primeCache(x, false); - } - - m_prevX = x; - return true; + return set; } void MatrixFile::setColumnAt(size_t x, const void *data) { - assert(m_mode == ReadWrite); + assert(m_mode == WriteOnly); #ifdef DEBUG_MATRIX_FILE_READ_SET -// std::cerr << "MatrixFile::setColumnAt(" << x << ")" << std::endl; - std::cerr << "."; + std::cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << ")" << std::endl; +// std::cerr << "."; #endif ssize_t w = 0; - bool seekFailed = false; - - { - MutexLocker locker(&m_fdMutex, "MatrixFile::setColumnAt::m_fdMutex"); - if (seekTo(x, 0)) { - w = ::write(m_fd, data, m_height * m_cellSize); - } else { - seekFailed = true; - } + if (!seekTo(x, 0)) { + std::cerr << "ERROR: MatrixFile::setColumnAt(" << x << "): Seek failed" << std::endl; + throw FileOperationFailed(m_fileName, "seek"); } - if (!seekFailed && w != ssize_t(m_height * m_cellSize)) { - ::perror("WARNING: MatrixFile::setColumnAt: write failed"); + unsigned char set = 0; + w = ::write(m_fd, &set, 1); + if (w != 1) { + ::perror("WARNING: MatrixFile::setColumnAt: write failed (1)"); throw FileOperationFailed(m_fileName, "write"); - } else if (seekFailed) { + } + + w = ::write(m_fd, data, m_height * m_cellSize); + if (w != ssize_t(m_height * m_cellSize)) { + ::perror("WARNING: MatrixFile::setColumnAt: write failed (2)"); + throw FileOperationFailed(m_fileName, "write"); + } +/* + if (x == 0) { + std::cerr << "Wrote " << m_height * m_cellSize << " bytes, as follows:" << std::endl; + for (int i = 0; i < m_height * m_cellSize; ++i) { + std::cerr << (int)(((char *)data)[i]) << " "; + } + std::cerr << std::endl; + } +*/ + if (!seekTo(x, 0)) { + std::cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << "): Seek failed" << std::endl; throw FileOperationFailed(m_fileName, "seek"); - } else { - MutexLocker locker - (&m_columnBitsetWriteMutex, - "MatrixFile::setColumnAt::m_columnBitsetWriteMutex"); - m_columnBitset->set(x); + } + + set = 1; + w = ::write(m_fd, &set, 1); + if (w != 1) { + ::perror("WARNING: MatrixFile::setColumnAt: write failed (3)"); + throw FileOperationFailed(m_fileName, "write"); } } -void -MatrixFile::suspend() +bool +MatrixFile::seekTo(size_t x, size_t y) const { - MutexLocker locker(&m_fdMutex, "MatrixFile::suspend::m_fdMutex"); - MutexLocker locker2(&m_cacheMutex, "MatrixFile::suspend::m_cacheMutex"); - - if (m_fd < 0) return; // already suspended - -#ifdef DEBUG_MATRIX_FILE - std::cerr << "MatrixFile(" << this << ":" << m_fileName.toStdString() << ")::suspend(): fd was " << m_fd << std::endl; -#endif - - if (m_requestToken >= 0) { - void *data = 0; - FileReadThread::Request request; - if (m_readThread->getRequest(m_requestToken, request)) { - data = request.data; - } - m_readThread->cancel(m_requestToken); - if (data) free(data); - m_requestToken = -1; + if (m_fd < 0) { + std::cerr << "ERROR: MatrixFile::seekTo: File not open" << std::endl; + return false; } - if (m_cache.data) { - free(m_cache.data); - m_cache.data = 0; - } - - if (m_spareData) { - free(m_spareData); - m_spareData = 0; - } - - if (::close(m_fd) < 0) { - ::perror("WARNING: MatrixFile::suspend: close failed"); - throw FileOperationFailed(m_fileName, "close"); - } - - m_fd = -1; -} - -void -MatrixFile::resume() -{ - if (m_fd >= 0) return; - -#ifdef DEBUG_MATRIX_FILE - std::cerr << "MatrixFile(" << this << ")::resume()" << std::endl; -#endif - - if ((m_fd = ::open(m_fileName.toLocal8Bit(), m_flags, m_fmode)) < 0) { - ::perror("Open failed"); - std::cerr << "ERROR: MatrixFile::resume: " - << "Failed to open cache file \"" - << m_fileName.toStdString() << "\""; - if (m_mode == ReadWrite) std::cerr << " for writing"; - std::cerr << std::endl; - throw FailedToOpenFile(m_fileName); - } - -#ifdef DEBUG_MATRIX_FILE - std::cerr << "MatrixFile(" << this << ":" << m_fileName.toStdString() << ")::resume(): fd is " << m_fd << std::endl; -#endif -} - -void -MatrixFile::primeCache(size_t x, bool goingLeft) -{ -// Profiler profiler("MatrixFile::primeCache"); + off_t off = m_headerSize + x * m_height * m_cellSize + x + y * m_cellSize; #ifdef DEBUG_MATRIX_FILE_READ_SET - std::cerr << "MatrixFile::primeCache(" << x << ", " << goingLeft << ")" << std::endl; + std::cerr << "MatrixFile[" << m_fd << "]::seekTo(" << x << "," << y << "): off = " << off << std::endl; #endif - size_t rx = x; - size_t rw = m_defaultCacheWidth; - - size_t left = rw / 3; - if (goingLeft) left = (rw * 2) / 3; - - if (rx > left) rx -= left; - else rx = 0; - - if (rx + rw > m_width) rw = m_width - rx; - - if (!m_eagerCache) { - - size_t ti = 0; - - for (ti = 0; ti < rw; ++ti) { - if (!m_columnBitset->get(rx + ti)) break; - } - -#ifdef DEBUG_MATRIX_FILE - if (ti < rw) { - std::cerr << "eagerCache is false and there's a hole at " - << rx + ti << ", reducing rw from " << rw << " to " - << ti << std::endl; - } -#endif - - rw = std::min(rw, ti); - if (rw < 10 || rx + rw <= x) return; - } - - MutexLocker locker(&m_cacheMutex, "MatrixFile::primeCache::m_cacheMutex"); - - // Check for the existence of the request first; if it exists, - // check whether it's ready. Only when we know it's ready do we - // retrieve the actual request, because the reason we need the - // request is to check whether it was successful or not and - // extract data from it, and none of that can be relied upon if we - // retrieve the request before it's ready. (There used to be a - // race condition here, where we retrieved the request and only - // afterwards checked the ready status, pulling data from the - // request if it was found to be ready then.) - - if (m_requestToken >= 0 && - m_readThread->haveRequest(m_requestToken)) { - - if (x >= m_requestingX && - x < m_requestingX + m_requestingWidth) { - - if (m_readThread->isReady(m_requestToken)) { - - FileReadThread::Request request; - if (!m_readThread->getRequest(m_requestToken, request)) { - std::cerr << "ERROR: MatrixFile::primeCache: File read thread has lost our request!" << std::endl; - throw FileReadFailed(m_fileName); - } - - if (!request.successful) { - std::cerr << "ERROR: MatrixFile::primeCache: Last request was unsuccessful" << std::endl; - throw FileReadFailed(m_fileName); - } - -#ifdef DEBUG_MATRIX_FILE_READ_SET - std::cerr << "last request is ready! (" << m_requestingX << ", "<< m_requestingWidth << ")" << std::endl; -#endif - - m_cache.x = (request.start - m_headerSize) / (m_height * m_cellSize); - m_cache.width = request.size / (m_height * m_cellSize); - -#ifdef DEBUG_MATRIX_FILE_READ_SET - std::cerr << "received last request: actual size is: " << m_cache.x << ", " << m_cache.width << std::endl; -#endif - - if (m_cache.data) { - if (m_spareData) { -// std::cerr << this << ": Freeing spare data" << std::endl; - free(m_spareData); - } -// std::cerr << this << ": Moving old cache data to spare" << std::endl; - m_spareData = m_cache.data; - } -// std::cerr << this << ": Moving request data to cache" << std::endl; - m_cache.data = request.data; - - m_readThread->done(m_requestToken); - m_requestToken = -1; - } - - // already requested something covering this area; wait for it - return; - } - - FileReadThread::Request dud; - - if (!m_readThread->getRequest(m_requestToken, dud)) { - - std::cerr << "ERROR: MatrixFile::primeCache: Inconsistent replies from FileReadThread" << std::endl; - - } else { - - // current request is for the wrong area, so no longer of any use - m_readThread->cancel(m_requestToken); - - // crude way to avoid leaking the data - while (!m_readThread->isCancelled(m_requestToken)) { - usleep(10000); - } - -#ifdef DEBUG_MATRIX_FILE_READ_SET - std::cerr << "cancelled " << m_requestToken << std::endl; -#endif - - if (m_spareData) { - free(m_spareData); - } - m_spareData = dud.data; - m_readThread->done(m_requestToken); - } - - m_requestToken = -1; - } - - if (m_fd < 0) { - MutexLocker locker(&m_fdMutex, "MatrixFile::primeCache::m_fdMutex"); - if (m_fd < 0) resume(); - } - - FileReadThread::Request request; - - request.fd = m_fd; - request.mutex = &m_fdMutex; - request.start = m_headerSize + rx * m_height * m_cellSize; - request.size = rw * m_height * m_cellSize; - -// std::cerr << this << ": Moving spare data to request, and resizing to " << rw * m_height * m_cellSize << std::endl; - - request.data = (char *)realloc(m_spareData, rw * m_height * m_cellSize); - MUNLOCK(request.data, rw * m_height * m_cellSize); - m_spareData = 0; - - m_requestingX = rx; - m_requestingWidth = rw; - - int token = m_readThread->request(request); -#ifdef DEBUG_MATRIX_FILE_READ_SET - std::cerr << "MatrixFile::primeCache: request token is " - << token << " (x = [" << rx << "], w = [" << rw << "], left = [" << goingLeft << "])" << std::endl; -#endif - m_requestToken = token; -} - -bool -MatrixFile::seekTo(size_t x, size_t y) -{ - if (m_fd < 0) resume(); - - off_t off = m_headerSize + (x * m_height + y) * m_cellSize; - if (::lseek(m_fd, off, SEEK_SET) == (off_t)-1) { ::perror("Seek failed"); std::cerr << "ERROR: MatrixFile::seekTo(" << x << ", " << y - << ") failed" << std::endl; + << ") = " << off << " failed" << std::endl; return false; }
--- a/data/fileio/MatrixFile.h Mon Jan 26 15:18:32 2009 +0000 +++ b/data/fileio/MatrixFile.h Tue Jan 27 13:25:10 2009 +0000 @@ -4,7 +4,7 @@ Sonic Visualiser An audio file viewer and annotation editor. Centre for Digital Music, Queen Mary, University of London. - This file copyright 2006 Chris Cannam. + This file copyright 2006-2009 Chris Cannam and QMUL. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as @@ -30,7 +30,7 @@ Q_OBJECT public: - enum Mode { ReadOnly, ReadWrite }; + enum Mode { ReadOnly, WriteOnly }; /** * Construct a MatrixFile object reading from and/or writing to @@ -39,23 +39,27 @@ * * If mode is ReadOnly, the file must exist and be readable. * - * If mode is ReadWrite and the file does not exist, it will be - * created. If mode is ReadWrite and the file does exist, the - * existing file will be used and the mode will be reset to - * ReadOnly. Call getMode() to check whether this has occurred - * after construction. + * If mode is WriteOnly, the file must not exist. * * cellSize specifies the size in bytes of the object type stored * in the matrix. For example, use cellSize = sizeof(float) for a * matrix of floats. The MatrixFile object doesn't care about the * objects themselves, it just deals with raw data of a given size. * - * If eagerCache is true, blocks from the file will be cached for - * read. If eagerCache is false, only columns that have been set - * by calling setColumnAt on this MatrixFile (i.e. columns for - * which haveSetColumnAt returns true) will be cached. + * width and height specify the dimensions of the file. These + * cannot be changed after construction. + * + * MatrixFiles are reference counted by name. When the last + * MatrixFile with a given name is destroyed, the file is removed. + * These are temporary files; the normal usage is to have one + * MatrixFile of WriteOnly type creating the file and then + * persisting until all readers are complete. + * + * MatrixFile has no built-in cache and is not thread-safe. Use a + * separate MatrixFile in each thread. */ - MatrixFile(QString fileBase, Mode mode, size_t cellSize, bool eagerCache); + MatrixFile(QString fileBase, Mode mode, size_t cellSize, + size_t width, size_t height); virtual ~MatrixFile(); Mode getMode() const { return m_mode; } @@ -64,15 +68,12 @@ size_t getHeight() const { return m_height; } size_t getCellSize() const { return m_cellSize; } - void resize(size_t width, size_t height); - void reset(); + void close(); // does not decrement ref count; that happens in dtor - bool haveSetColumnAt(size_t x) const { return m_columnBitset->get(x); } + bool haveSetColumnAt(size_t x) const; void getColumnAt(size_t x, void *data); // may throw FileReadFailed void setColumnAt(size_t x, const void *data); - void suspend(); - protected: int m_fd; Mode m_mode; @@ -83,41 +84,12 @@ size_t m_height; size_t m_headerSize; QString m_fileName; - size_t m_defaultCacheWidth; - size_t m_prevX; - - struct Cache { - size_t x; - size_t width; - char *data; - }; - - Cache m_cache; - bool m_eagerCache; - - bool getFromCache(size_t x, size_t ystart, size_t ycount, void *data); - void primeCache(size_t x, bool left); - - void resume(); - - bool seekTo(size_t x, size_t y); - - static FileReadThread *m_readThread; - int m_requestToken; - - size_t m_requestingX; - size_t m_requestingWidth; - char *m_spareData; static std::map<QString, int> m_refcount; - static QMutex m_refcountMutex; - QMutex m_fdMutex; - QMutex m_cacheMutex; + static QMutex m_createMutex; - typedef std::map<QString, ResizeableBitset *> ResizeableBitsetMap; - static ResizeableBitsetMap m_columnBitsets; - static QMutex m_columnBitsetWriteMutex; - ResizeableBitset *m_columnBitset; + void initialise(); + bool seekTo(size_t x, size_t y) const; }; #endif
--- a/data/model/EditableDenseThreeDimensionalModel.cpp Mon Jan 26 15:18:32 2009 +0000 +++ b/data/model/EditableDenseThreeDimensionalModel.cpp Tue Jan 27 13:25:10 2009 +0000 @@ -291,15 +291,16 @@ int tdist = trunc; if (trunc < 0) { top = false; tdist = -trunc; } Column p = expandAndRetrieve(index - tdist); - if (p.size() != m_yBinCount) { + int psize = p.size(), csize = c.size(); + if (psize != m_yBinCount) { std::cerr << "WARNING: EditableDenseThreeDimensionalModel::expandAndRetrieve: Trying to expand from incorrectly sized column" << std::endl; } if (top) { - for (int i = c.size(); i < p.size(); ++i) { + for (int i = csize; i < psize; ++i) { c.push_back(p.at(i)); } } else { - for (int i = int(p.size()) - int(c.size()); i >= 0; --i) { + for (int i = psize - csize - 1; i >= 0; --i) { c.push_front(p.at(i)); } }