changeset 359:824ee993ca8d

* merge revision 842 from spectrogram-rejig -- attempt to improve disk/memory
  allocation in ffts

--- a/base/StorageAdviser.cpp	Wed Jan 09 17:09:44 2008 +0000
+++ b/base/StorageAdviser.cpp	Tue Jan 22 23:11:48 2008 +0000
@@ -77,6 +77,7 @@
     else if (minmb > (memoryFree / 3)) memoryStatus = Marginal;
     else if (memoryTotal == -1 ||
              minmb > (memoryTotal / 10)) memoryStatus = Marginal;
+    else if (memoryFree < memoryTotal / 4) memoryStatus = Marginal;
     else memoryStatus = Sufficient;
 
     if (discFree == -1) discStatus = Unknown;

--- a/data/fft/FFTCache.h	Wed Jan 09 17:09:44 2008 +0000
+++ b/data/fft/FFTCache.h	Tue Jan 22 23:11:48 2008 +0000
@@ -49,6 +49,9 @@
 
     virtual void suspend() { }
 
+    enum Type { MemoryCache, FileCache };
+    virtual Type getType() = 0;
+
 protected:
     FFTCache() { }
 };

--- a/data/fft/FFTDataServer.cpp	Wed Jan 09 17:09:44 2008 +0000
+++ b/data/fft/FFTDataServer.cpp	Tue Jan 22 23:11:48 2008 +0000
@@ -30,7 +30,7 @@
 #include <QMessageBox>
 #include <QApplication>
 
-//#define DEBUG_FFT_SERVER 1
+#define DEBUG_FFT_SERVER 1
 //#define DEBUG_FFT_SERVER_FILL 1
 
 #ifdef DEBUG_FFT_SERVER_FILL
@@ -496,9 +496,10 @@
     m_width(0),
     m_height(0),
     m_cacheWidth(0),
-    m_memoryCache(false),
-    m_compactCache(false),
+    m_cacheWidthPower(0),
+    m_cacheWidthMask(0),
     m_lastUsedCache(-1),
+    m_criteria(criteria),
     m_fftInput(0),
     m_exiting(false),
     m_suspended(true), //!!! or false?
@@ -526,84 +527,38 @@
     if (m_width * columnSize < maxCacheSize * 2) m_cacheWidth = m_width;
     else m_cacheWidth = maxCacheSize / columnSize;
     
+#ifdef DEBUG_FFT_SERVER
+    std::cerr << "FFTDataServer(" << this << "): cache width nominal "
+              << m_cacheWidth << ", actual ";
+#endif
+    
     int bits = 0;
-    while (m_cacheWidth) { m_cacheWidth >>= 1; ++bits; }
+    while (m_cacheWidth > 1) { m_cacheWidth >>= 1; ++bits; }
+    m_cacheWidthPower = bits + 1;
     m_cacheWidth = 2;
     while (bits) { m_cacheWidth <<= 1; --bits; }
+    m_cacheWidthMask = m_cacheWidth - 1;
 
-    if (criteria == StorageAdviser::NoCriteria) {
+#ifdef DEBUG_FFT_SERVER
+    std::cerr << m_cacheWidth << " (power " << m_cacheWidthPower << ", mask "
+              << m_cacheWidthMask << ")" << std::endl;
+#endif
+
+    if (m_criteria == StorageAdviser::NoCriteria) {
 
         // assume "spectrogram" criteria for polar ffts, and "feature
         // extraction" criteria for rectangular ones.
 
         if (m_polar) {
-            criteria = StorageAdviser::Criteria
+            m_criteria = StorageAdviser::Criteria
                 (StorageAdviser::SpeedCritical |
                  StorageAdviser::LongRetentionLikely);
         } else {
-            criteria = StorageAdviser::Criteria
+            m_criteria = StorageAdviser::Criteria
                 (StorageAdviser::PrecisionCritical);
         }
     }
 
-    int cells = m_width * m_height;
-    int minimumSize = (cells / 1024) * sizeof(uint16_t); // kb
-    int maximumSize = (cells / 1024) * sizeof(float); // kb
-
-    // We don't have a compact rectangular representation, and compact
-    // of course is never precision-critical
-    bool canCompact = true;
-    if ((criteria & StorageAdviser::PrecisionCritical) || !m_polar) {
-        canCompact = false;
-        minimumSize = maximumSize; // don't use compact
-    }
-    
-    StorageAdviser::Recommendation recommendation;
-
-    try {
-
-        recommendation =
-            StorageAdviser::recommend(criteria, minimumSize, maximumSize);
-
-    } catch (InsufficientDiscSpace s) {
-
-        // Delete any unused servers we may have been leaving around
-        // in case we wanted them again
-
-        purgeLimbo(0);
-
-        // This time we don't catch InsufficientDiscSpace -- we
-        // haven't allocated anything yet and can safely let the
-        // exception out to indicate to the caller that we can't
-        // handle it.
-
-        recommendation =
-            StorageAdviser::recommend(criteria, minimumSize, maximumSize);
-    }
-
-    std::cerr << "Recommendation was: " << recommendation << std::endl;
-
-    m_memoryCache = false;
-
-    if ((recommendation & StorageAdviser::UseMemory) ||
-        (recommendation & StorageAdviser::PreferMemory)) {
-        m_memoryCache = true;
-    }
-
-    m_compactCache = canCompact &&
-        (recommendation & StorageAdviser::ConserveSpace);
-
-    std::cerr << "FFTDataServer: memory cache = " << m_memoryCache << ", compact cache = " << m_compactCache << std::endl;
-    
-#ifdef DEBUG_FFT_SERVER
-    std::cerr << "Width " << m_width << ", cache width " << m_cacheWidth << " (size " << m_cacheWidth * columnSize << ")" << std::endl;
-#endif
-
-    StorageAdviser::notifyPlannedAllocation
-        (m_memoryCache ? StorageAdviser::MemoryAllocation :
-                         StorageAdviser::DiscAllocation,
-         m_compactCache ? minimumSize : maximumSize);
-
     for (size_t i = 0; i <= m_width / m_cacheWidth; ++i) {
         m_caches.push_back(0);
     }
@@ -648,14 +603,9 @@
                        "FFTDataServer::m_writeMutex[~FFTDataServer]");
 
     for (CacheVector::iterator i = m_caches.begin(); i != m_caches.end(); ++i) {
+
         if (*i) {
             delete *i;
-        } else {
-            StorageAdviser::notifyDoneAllocation
-                (m_memoryCache ? StorageAdviser::MemoryAllocation :
-                                 StorageAdviser::DiscAllocation,
-                 m_cacheWidth * m_height *
-                 (m_compactCache ? sizeof(uint16_t) : sizeof(float)) / 1024 + 1);
         }
     }
 
@@ -724,6 +674,65 @@
     }
 }
 
+void
+FFTDataServer::getStorageAdvice(size_t w, size_t h,
+                                bool &memoryCache, bool &compactCache)
+{
+    int cells = w * h;
+    int minimumSize = (cells / 1024) * sizeof(uint16_t); // kb
+    int maximumSize = (cells / 1024) * sizeof(float); // kb
+
+    // We don't have a compact rectangular representation, and compact
+    // of course is never precision-critical
+
+    bool canCompact = true;
+    if ((m_criteria & StorageAdviser::PrecisionCritical) || !m_polar) {
+        canCompact = false;
+        minimumSize = maximumSize; // don't use compact
+    }
+    
+    StorageAdviser::Recommendation recommendation;
+
+    try {
+
+        recommendation =
+            StorageAdviser::recommend(m_criteria, minimumSize, maximumSize);
+
+    } catch (InsufficientDiscSpace s) {
+
+        // Delete any unused servers we may have been leaving around
+        // in case we wanted them again
+
+        purgeLimbo(0);
+
+        // This time we don't catch InsufficientDiscSpace -- we
+        // haven't allocated anything yet and can safely let the
+        // exception out to indicate to the caller that we can't
+        // handle it.
+
+        recommendation =
+            StorageAdviser::recommend(m_criteria, minimumSize, maximumSize);
+    }
+
+    std::cerr << "Recommendation was: " << recommendation << std::endl;
+
+    memoryCache = false;
+
+    if ((recommendation & StorageAdviser::UseMemory) ||
+        (recommendation & StorageAdviser::PreferMemory)) {
+        memoryCache = true;
+    }
+
+    compactCache = canCompact &&
+        (recommendation & StorageAdviser::ConserveSpace);
+
+    std::cerr << "FFTDataServer: memory cache = " << memoryCache << ", compact cache = " << compactCache << std::endl;
+    
+#ifdef DEBUG_FFT_SERVER
+    std::cerr << "Width " << w << " of " << m_width << ", height " << h << ", size " << w * h << std::endl;
+#endif
+}
+
 FFTCache *
 FFTDataServer::getCacheAux(size_t c)
 {
@@ -784,30 +793,28 @@
         width = m_width - c * m_cacheWidth;
     }
 
+    bool memoryCache = false;
+    bool compactCache = false;
+
+    getStorageAdvice(width, m_height, memoryCache, compactCache);
+
     try {
 
-        if (m_memoryCache) {
+        if (memoryCache) {
 
             cache = new FFTMemoryCache
-                (m_compactCache ? FFTMemoryCache::Compact :
-//                                  FFTMemoryCache::Polar);
-                 m_polar ? FFTMemoryCache::Polar :
-                           FFTMemoryCache::Rectangular);
-
-        } else if (m_compactCache) {
-
-            cache = new FFTFileCache
-                (name,
-                 MatrixFile::ReadWrite,
-                 FFTFileCache::Compact);
+                (compactCache ? FFTMemoryCache::Compact :
+                      m_polar ? FFTMemoryCache::Polar :
+                                FFTMemoryCache::Rectangular);
 
         } else {
 
             cache = new FFTFileCache
                 (name,
                  MatrixFile::ReadWrite,
-                 m_polar ? FFTFileCache::Polar :
-                           FFTFileCache::Rectangular);
+                 compactCache ? FFTFileCache::Compact :
+                      m_polar ? FFTFileCache::Polar :
+                                FFTFileCache::Rectangular);
         }
 
         cache->resize(width, m_height);
@@ -818,7 +825,7 @@
         delete cache;
         cache = 0;
 
-        if (m_memoryCache) {
+        if (memoryCache) {
             
             std::cerr << "WARNING: Memory allocation failed when resizing"
                       << " FFT memory cache no. " << c << " to " << width 
@@ -827,7 +834,10 @@
 
             try {
 
-                cache = new FFTFileCache(name, MatrixFile::ReadWrite,
+                purgeLimbo(0);
+
+                cache = new FFTFileCache(name,
+                                         MatrixFile::ReadWrite,
                                          FFTFileCache::Compact);
 
                 cache->resize(width, m_height);
@@ -855,12 +865,6 @@
               "There may be insufficient memory or disc space to continue."));
     }
 
-    StorageAdviser::notifyDoneAllocation
-        (m_memoryCache ? StorageAdviser::MemoryAllocation :
-         StorageAdviser::DiscAllocation,
-         width * m_height *
-         (m_compactCache ? sizeof(uint16_t) : sizeof(float)) / 1024 + 1);
-
     m_caches[c] = cache;
     m_lastUsedCache = c;
     return cache;

--- a/data/fft/FFTDataServer.h	Wed Jan 09 17:09:44 2008 +0000
+++ b/data/fft/FFTDataServer.h	Tue Jan 22 23:11:48 2008 +0000
@@ -135,8 +135,22 @@
     size_t m_cacheWidth;
     size_t m_cacheWidthPower;
     size_t m_cacheWidthMask;
-    bool m_memoryCache;
-    bool m_compactCache;
+
+    int m_lastUsedCache;
+    FFTCache *getCache(size_t x, size_t &col) {
+        col   = x & m_cacheWidthMask;
+        int c = x >> m_cacheWidthPower;
+        // The only use of m_lastUsedCache without a lock is to
+        // establish whether a cache has been created at all (they're
+        // created on demand, but not destroyed until the server is).
+        if (c == m_lastUsedCache) return m_caches[c];
+        else return getCacheAux(c);
+    }
+    bool haveCache(size_t x) {
+        int c = x >> m_cacheWidthPower;
+        if (c == m_lastUsedCache) return true;
+        else return (m_caches[c] != 0);
+    }
 
     typedef std::vector<FFTCache *> CacheVector;
     CacheVector m_caches;
@@ -144,21 +158,9 @@
     typedef std::deque<int> IntQueue;
     IntQueue m_dormantCaches;
 
-    int m_lastUsedCache;
-    FFTCache *getCache(size_t x, size_t &col) {
-        col   = x % m_cacheWidth;
-        int c = x / m_cacheWidth;
-        // The only use of m_lastUsedCache without a lock is to
-        // establish whether a cache has been created at all (they're
-        // created on demand, but not destroyed until the server is).
-        if (c == m_lastUsedCache) return m_caches[c];
-        else return getCacheAux(c);
-    }
-    bool haveCache(size_t x) {
-        int c = x / m_cacheWidth;
-        if (c == m_lastUsedCache) return true;
-        else return (m_caches[c] != 0);
-    }
+    StorageAdviser::Criteria m_criteria;
+
+    void getStorageAdvice(size_t w, size_t h, bool &memory, bool &compact);
         
     FFTCache *getCacheAux(size_t c);
     QMutex m_writeMutex;

--- a/data/fft/FFTFileCache.h	Wed Jan 09 17:09:44 2008 +0000
+++ b/data/fft/FFTFileCache.h	Tue Jan 22 23:11:48 2008 +0000
@@ -58,6 +58,8 @@
 
     static size_t getCacheSize(size_t width, size_t height, StorageType type);
 
+    virtual Type getType() { return FileCache; }
+
 protected:
     char *m_writebuf;
     mutable char *m_readbuf;

--- a/data/fft/FFTMemoryCache.cpp	Wed Jan 09 17:09:44 2008 +0000
+++ b/data/fft/FFTMemoryCache.cpp	Tue Jan 22 23:11:48 2008 +0000
@@ -59,7 +59,7 @@
 void
 FFTMemoryCache::resize(size_t width, size_t height)
 {
-//    std::cerr << "FFTMemoryCache[" << this << "]::resize(" << width << "x" << height << " = " << width*height << ")" << std::endl;
+    std::cerr << "FFTMemoryCache[" << this << "]::resize(" << width << "x" << height << " = " << width*height << ")" << std::endl;
     
     if (m_width == width && m_height == height) return;
 

--- a/data/fft/FFTMemoryCache.h	Wed Jan 09 17:09:44 2008 +0000
+++ b/data/fft/FFTMemoryCache.h	Tue Jan 22 23:11:48 2008 +0000
@@ -109,6 +109,8 @@
 
     static size_t getCacheSize(size_t width, size_t height, StorageType type);
 
+    virtual Type getType() { return MemoryCache; }
+
 private:
     size_t m_width;
     size_t m_height;