# HG changeset patch
# User Chris Cannam
# Date 1194962050 0
# Node ID 02d2ad95ea5204ccad22f135c51fa8d943d14684
# Parent  aa8dbac62024479b4dd326103d6943905026affc
* Get storage advice for each cache in an FFT data server.  Allows us to be
  more confident about the actual memory situation and cut over from memory
  to disc part way through an FFT calculation if necessary.  StorageAdviser
  is now a bit too optimistic though (it's too keen to allocate large numbers
  of small blocks in memory).

diff -r aa8dbac62024 -r 02d2ad95ea52 base/StorageAdviser.cpp
--- a/base/StorageAdviser.cpp	Sun Nov 11 20:31:12 2007 +0000
+++ b/base/StorageAdviser.cpp	Tue Nov 13 13:54:10 2007 +0000
@@ -77,6 +77,7 @@
     else if (minmb > (memoryFree / 3)) memoryStatus = Marginal;
     else if (memoryTotal == -1 ||
              minmb > (memoryTotal / 10)) memoryStatus = Marginal;
+    else if (memoryFree < memoryTotal / 4) memoryStatus = Marginal;
     else memoryStatus = Sufficient;
 
     if (discFree == -1) discStatus = Unknown;
diff -r aa8dbac62024 -r 02d2ad95ea52 data/fft/FFTCache.h
--- a/data/fft/FFTCache.h	Sun Nov 11 20:31:12 2007 +0000
+++ b/data/fft/FFTCache.h	Tue Nov 13 13:54:10 2007 +0000
@@ -49,6 +49,9 @@
 
     virtual void suspend() { }
 
+    enum Type { MemoryCache, FileCache };
+    virtual Type getType() = 0;
+
 protected:
     FFTCache() { }
 };
diff -r aa8dbac62024 -r 02d2ad95ea52 data/fft/FFTDataServer.cpp
--- a/data/fft/FFTDataServer.cpp	Sun Nov 11 20:31:12 2007 +0000
+++ b/data/fft/FFTDataServer.cpp	Tue Nov 13 13:54:10 2007 +0000
@@ -30,7 +30,7 @@
 #include <QMessageBox>
 #include <QApplication>
 
-//#define DEBUG_FFT_SERVER 1
+#define DEBUG_FFT_SERVER 1
 //#define DEBUG_FFT_SERVER_FILL 1
 
 #ifdef DEBUG_FFT_SERVER_FILL
@@ -496,9 +496,10 @@
     m_width(0),
     m_height(0),
     m_cacheWidth(0),
-    m_memoryCache(false),
-    m_compactCache(false),
+    m_cacheWidthPower(0),
+    m_cacheWidthMask(0),
     m_lastUsedCache(-1),
+    m_criteria(criteria),
     m_fftInput(0),
     m_exiting(false),
     m_suspended(true), //!!! or false?
@@ -526,84 +527,38 @@
     if (m_width * columnSize < maxCacheSize * 2) m_cacheWidth = m_width;
     else m_cacheWidth = maxCacheSize / columnSize;
     
+#ifdef DEBUG_FFT_SERVER
+    std::cerr << "FFTDataServer(" << this << "): cache width nominal "
+              << m_cacheWidth << ", actual ";
+#endif
+    
     int bits = 0;
-    while (m_cacheWidth) { m_cacheWidth >>= 1; ++bits; }
+    while (m_cacheWidth > 1) { m_cacheWidth >>= 1; ++bits; }
+    m_cacheWidthPower = bits + 1;
     m_cacheWidth = 2;
     while (bits) { m_cacheWidth <<= 1; --bits; }
+    m_cacheWidthMask = m_cacheWidth - 1;
 
-    if (criteria == StorageAdviser::NoCriteria) {
+#ifdef DEBUG_FFT_SERVER
+    std::cerr << m_cacheWidth << " (power " << m_cacheWidthPower << ", mask "
+              << m_cacheWidthMask << ")" << std::endl;
+#endif
+
+    if (m_criteria == StorageAdviser::NoCriteria) {
 
         // assume "spectrogram" criteria for polar ffts, and "feature
         // extraction" criteria for rectangular ones.
 
         if (m_polar) {
-            criteria = StorageAdviser::Criteria
+            m_criteria = StorageAdviser::Criteria
                 (StorageAdviser::SpeedCritical |
                  StorageAdviser::LongRetentionLikely);
         } else {
-            criteria = StorageAdviser::Criteria
+            m_criteria = StorageAdviser::Criteria
                 (StorageAdviser::PrecisionCritical);
         }
     }
 
-    int cells = m_width * m_height;
-    int minimumSize = (cells / 1024) * sizeof(uint16_t); // kb
-    int maximumSize = (cells / 1024) * sizeof(float); // kb
-
-    // We don't have a compact rectangular representation, and compact
-    // of course is never precision-critical
-    bool canCompact = true;
-    if ((criteria & StorageAdviser::PrecisionCritical) || !m_polar) {
-        canCompact = false;
-        minimumSize = maximumSize; // don't use compact
-    }
-    
-    StorageAdviser::Recommendation recommendation;
-
-    try {
-
-        recommendation =
-            StorageAdviser::recommend(criteria, minimumSize, maximumSize);
-
-    } catch (InsufficientDiscSpace s) {
-
-        // Delete any unused servers we may have been leaving around
-        // in case we wanted them again
-
-        purgeLimbo(0);
-
-        // This time we don't catch InsufficientDiscSpace -- we
-        // haven't allocated anything yet and can safely let the
-        // exception out to indicate to the caller that we can't
-        // handle it.
-
-        recommendation =
-            StorageAdviser::recommend(criteria, minimumSize, maximumSize);
-    }
-
-    std::cerr << "Recommendation was: " << recommendation << std::endl;
-
-    m_memoryCache = false;
-
-    if ((recommendation & StorageAdviser::UseMemory) ||
-        (recommendation & StorageAdviser::PreferMemory)) {
-        m_memoryCache = true;
-    }
-
-    m_compactCache = canCompact &&
-        (recommendation & StorageAdviser::ConserveSpace);
-
-    std::cerr << "FFTDataServer: memory cache = " << m_memoryCache << ", compact cache = " << m_compactCache << std::endl;
-    
-#ifdef DEBUG_FFT_SERVER
-    std::cerr << "Width " << m_width << ", cache width " << m_cacheWidth << " (size " << m_cacheWidth * columnSize << ")" << std::endl;
-#endif
-
-    StorageAdviser::notifyPlannedAllocation
-        (m_memoryCache ? StorageAdviser::MemoryAllocation :
-                         StorageAdviser::DiscAllocation,
-         m_compactCache ? minimumSize : maximumSize);
-
     for (size_t i = 0; i <= m_width / m_cacheWidth; ++i) {
         m_caches.push_back(0);
     }
@@ -648,14 +603,9 @@
                        "FFTDataServer::m_writeMutex[~FFTDataServer]");
 
     for (CacheVector::iterator i = m_caches.begin(); i != m_caches.end(); ++i) {
+
         if (*i) {
             delete *i;
-        } else {
-            StorageAdviser::notifyDoneAllocation
-                (m_memoryCache ? StorageAdviser::MemoryAllocation :
-                                 StorageAdviser::DiscAllocation,
-                 m_cacheWidth * m_height *
-                 (m_compactCache ? sizeof(uint16_t) : sizeof(float)) / 1024 + 1);
         }
     }
 
@@ -724,6 +674,65 @@
     }
 }
 
+void
+FFTDataServer::getStorageAdvice(size_t w, size_t h,
+                                bool &memoryCache, bool &compactCache)
+{
+    int cells = w * h;
+    int minimumSize = (cells / 1024) * sizeof(uint16_t); // kb
+    int maximumSize = (cells / 1024) * sizeof(float); // kb
+
+    // We don't have a compact rectangular representation, and compact
+    // of course is never precision-critical
+
+    bool canCompact = true;
+    if ((m_criteria & StorageAdviser::PrecisionCritical) || !m_polar) {
+        canCompact = false;
+        minimumSize = maximumSize; // don't use compact
+    }
+    
+    StorageAdviser::Recommendation recommendation;
+
+    try {
+
+        recommendation =
+            StorageAdviser::recommend(m_criteria, minimumSize, maximumSize);
+
+    } catch (InsufficientDiscSpace s) {
+
+        // Delete any unused servers we may have been leaving around
+        // in case we wanted them again
+
+        purgeLimbo(0);
+
+        // This time we don't catch InsufficientDiscSpace -- we
+        // haven't allocated anything yet and can safely let the
+        // exception out to indicate to the caller that we can't
+        // handle it.
+
+        recommendation =
+            StorageAdviser::recommend(m_criteria, minimumSize, maximumSize);
+    }
+
+    std::cerr << "Recommendation was: " << recommendation << std::endl;
+
+    memoryCache = false;
+
+    if ((recommendation & StorageAdviser::UseMemory) ||
+        (recommendation & StorageAdviser::PreferMemory)) {
+        memoryCache = true;
+    }
+
+    compactCache = canCompact &&
+        (recommendation & StorageAdviser::ConserveSpace);
+
+    std::cerr << "FFTDataServer: memory cache = " << memoryCache << ", compact cache = " << compactCache << std::endl;
+    
+#ifdef DEBUG_FFT_SERVER
+    std::cerr << "Width " << w << " of " << m_width << ", height " << h << ", size " << w * h << std::endl;
+#endif
+}
+
 FFTCache *
 FFTDataServer::getCacheAux(size_t c)
 {
@@ -784,30 +793,28 @@
         width = m_width - c * m_cacheWidth;
     }
 
+    bool memoryCache = false;
+    bool compactCache = false;
+
+    getStorageAdvice(width, m_height, memoryCache, compactCache);
+
     try {
 
-        if (m_memoryCache) {
+        if (memoryCache) {
 
             cache = new FFTMemoryCache
-                (m_compactCache ? FFTMemoryCache::Compact :
-//                                  FFTMemoryCache::Polar);
-                 m_polar ? FFTMemoryCache::Polar :
-                           FFTMemoryCache::Rectangular);
-
-        } else if (m_compactCache) {
-
-            cache = new FFTFileCache
-                (name,
-                 MatrixFile::ReadWrite,
-                 FFTFileCache::Compact);
+                (compactCache ? FFTMemoryCache::Compact :
+                      m_polar ? FFTMemoryCache::Polar :
+                                FFTMemoryCache::Rectangular);
 
         } else {
 
             cache = new FFTFileCache
                 (name,
                  MatrixFile::ReadWrite,
-                 m_polar ? FFTFileCache::Polar :
-                           FFTFileCache::Rectangular);
+                 compactCache ? FFTFileCache::Compact :
+                      m_polar ? FFTFileCache::Polar :
+                                FFTFileCache::Rectangular);
         }
 
         cache->resize(width, m_height);
@@ -818,7 +825,7 @@
         delete cache;
         cache = 0;
 
-        if (m_memoryCache) {
+        if (memoryCache) {
             
             std::cerr << "WARNING: Memory allocation failed when resizing"
                       << " FFT memory cache no. " << c << " to " << width 
@@ -827,7 +834,10 @@
 
             try {
 
-                cache = new FFTFileCache(name, MatrixFile::ReadWrite,
+                purgeLimbo(0);
+
+                cache = new FFTFileCache(name,
+                                         MatrixFile::ReadWrite,
                                          FFTFileCache::Compact);
 
                 cache->resize(width, m_height);
@@ -855,12 +865,6 @@
               "There may be insufficient memory or disc space to continue."));
     }
 
-    StorageAdviser::notifyDoneAllocation
-        (m_memoryCache ? StorageAdviser::MemoryAllocation :
-         StorageAdviser::DiscAllocation,
-         width * m_height *
-         (m_compactCache ? sizeof(uint16_t) : sizeof(float)) / 1024 + 1);
-
     m_caches[c] = cache;
     m_lastUsedCache = c;
     return cache;
diff -r aa8dbac62024 -r 02d2ad95ea52 data/fft/FFTDataServer.h
--- a/data/fft/FFTDataServer.h	Sun Nov 11 20:31:12 2007 +0000
+++ b/data/fft/FFTDataServer.h	Tue Nov 13 13:54:10 2007 +0000
@@ -135,8 +135,22 @@
     size_t m_cacheWidth;
     size_t m_cacheWidthPower;
     size_t m_cacheWidthMask;
-    bool m_memoryCache;
-    bool m_compactCache;
+
+    int m_lastUsedCache;
+    FFTCache *getCache(size_t x, size_t &col) {
+        col   = x & m_cacheWidthMask;
+        int c = x >> m_cacheWidthPower;
+        // The only use of m_lastUsedCache without a lock is to
+        // establish whether a cache has been created at all (they're
+        // created on demand, but not destroyed until the server is).
+        if (c == m_lastUsedCache) return m_caches[c];
+        else return getCacheAux(c);
+    }
+    bool haveCache(size_t x) {
+        int c = x >> m_cacheWidthPower;
+        if (c == m_lastUsedCache) return true;
+        else return (m_caches[c] != 0);
+    }
 
     typedef std::vector<FFTCache *> CacheVector;
     CacheVector m_caches;
@@ -144,21 +158,9 @@
     typedef std::deque<int> IntQueue;
     IntQueue m_dormantCaches;
 
-    int m_lastUsedCache;
-    FFTCache *getCache(size_t x, size_t &col) {
-        col   = x % m_cacheWidth;
-        int c = x / m_cacheWidth;
-        // The only use of m_lastUsedCache without a lock is to
-        // establish whether a cache has been created at all (they're
-        // created on demand, but not destroyed until the server is).
-        if (c == m_lastUsedCache) return m_caches[c];
-        else return getCacheAux(c);
-    }
-    bool haveCache(size_t x) {
-        int c = x / m_cacheWidth;
-        if (c == m_lastUsedCache) return true;
-        else return (m_caches[c] != 0);
-    }
+    StorageAdviser::Criteria m_criteria;
+
+    void getStorageAdvice(size_t w, size_t h, bool &memory, bool &compact);
         
     FFTCache *getCacheAux(size_t c);
     QMutex m_writeMutex;
diff -r aa8dbac62024 -r 02d2ad95ea52 data/fft/FFTFileCache.h
--- a/data/fft/FFTFileCache.h	Sun Nov 11 20:31:12 2007 +0000
+++ b/data/fft/FFTFileCache.h	Tue Nov 13 13:54:10 2007 +0000
@@ -58,6 +58,8 @@
 
     static size_t getCacheSize(size_t width, size_t height, StorageType type);
 
+    virtual Type getType() { return FileCache; }
+
 protected:
     char *m_writebuf;
     mutable char *m_readbuf;
diff -r aa8dbac62024 -r 02d2ad95ea52 data/fft/FFTMemoryCache.cpp
--- a/data/fft/FFTMemoryCache.cpp	Sun Nov 11 20:31:12 2007 +0000
+++ b/data/fft/FFTMemoryCache.cpp	Tue Nov 13 13:54:10 2007 +0000
@@ -59,7 +59,7 @@
 void
 FFTMemoryCache::resize(size_t width, size_t height)
 {
-//    std::cerr << "FFTMemoryCache[" << this << "]::resize(" << width << "x" << height << " = " << width*height << ")" << std::endl;
+    std::cerr << "FFTMemoryCache[" << this << "]::resize(" << width << "x" << height << " = " << width*height << ")" << std::endl;
     
     if (m_width == width && m_height == height) return;
 
diff -r aa8dbac62024 -r 02d2ad95ea52 data/fft/FFTMemoryCache.h
--- a/data/fft/FFTMemoryCache.h	Sun Nov 11 20:31:12 2007 +0000
+++ b/data/fft/FFTMemoryCache.h	Tue Nov 13 13:54:10 2007 +0000
@@ -109,6 +109,8 @@
 
     static size_t getCacheSize(size_t width, size_t height, StorageType type);
 
+    virtual Type getType() { return MemoryCache; }
+
 private:
     size_t m_width;
     size_t m_height;