comparison data/fft/FFTDataServer.cpp @ 359:824ee993ca8d

* merge revision 842 from spectrogram-rejig -- attempt to improve disk/memory
  allocation in ffts

 #include "base/Thread.h" // for debug mutex locker
 
 #include <QMessageBox>
 #include <QApplication>
 
-//#define DEBUG_FFT_SERVER 1
+#define DEBUG_FFT_SERVER 1
 //#define DEBUG_FFT_SERVER_FILL 1
 
 #ifdef DEBUG_FFT_SERVER_FILL
 #ifndef DEBUG_FFT_SERVER
 #define DEBUG_FFT_SERVER 1

     m_fftSize(fftSize),
     m_polar(polar),
     m_width(0),
     m_height(0),
     m_cacheWidth(0),
-    m_memoryCache(false),
-    m_compactCache(false),
+    m_cacheWidthPower(0),
+    m_cacheWidthMask(0),
     m_lastUsedCache(-1),
+    m_criteria(criteria),
     m_fftInput(0),
     m_exiting(false),
     m_suspended(true), //!!! or false?
     m_fillThread(0)
 {

     size_t maxCacheSize = 20 * 1024 * 1024;
     size_t columnSize = m_height * sizeof(fftsample) * 2 + sizeof(fftsample);
     if (m_width * columnSize < maxCacheSize * 2) m_cacheWidth = m_width;
     else m_cacheWidth = maxCacheSize / columnSize;
     
+#ifdef DEBUG_FFT_SERVER
+    std::cerr << "FFTDataServer(" << this << "): cache width nominal "
+              << m_cacheWidth << ", actual ";
+#endif
+    
     int bits = 0;
-    while (m_cacheWidth) { m_cacheWidth >>= 1; ++bits; }
+    while (m_cacheWidth > 1) { m_cacheWidth >>= 1; ++bits; }
+    m_cacheWidthPower = bits + 1;
     m_cacheWidth = 2;
     while (bits) { m_cacheWidth <<= 1; --bits; }
-
-    if (criteria == StorageAdviser::NoCriteria) {
+    m_cacheWidthMask = m_cacheWidth - 1;
+
+#ifdef DEBUG_FFT_SERVER
+    std::cerr << m_cacheWidth << " (power " << m_cacheWidthPower << ", mask "
+              << m_cacheWidthMask << ")" << std::endl;
+#endif
+
+    if (m_criteria == StorageAdviser::NoCriteria) {
 
         // assume "spectrogram" criteria for polar ffts, and "feature
         // extraction" criteria for rectangular ones.
 
         if (m_polar) {
-            criteria = StorageAdviser::Criteria
+            m_criteria = StorageAdviser::Criteria
                 (StorageAdviser::SpeedCritical |
                  StorageAdviser::LongRetentionLikely);
         } else {
-            criteria = StorageAdviser::Criteria
+            m_criteria = StorageAdviser::Criteria
                 (StorageAdviser::PrecisionCritical);
         }
     }
-
-    int cells = m_width * m_height;
-    int minimumSize = (cells / 1024) * sizeof(uint16_t); // kb
-    int maximumSize = (cells / 1024) * sizeof(float); // kb
-
-    // We don't have a compact rectangular representation, and compact
-    // of course is never precision-critical
-    bool canCompact = true;
-    if ((criteria & StorageAdviser::PrecisionCritical) || !m_polar) {
-        canCompact = false;
-        minimumSize = maximumSize; // don't use compact
-    }
-    
-    StorageAdviser::Recommendation recommendation;
-
-    try {
-
-        recommendation =
-            StorageAdviser::recommend(criteria, minimumSize, maximumSize);
-
-    } catch (InsufficientDiscSpace s) {
-
-        // Delete any unused servers we may have been leaving around
-        // in case we wanted them again
-
-        purgeLimbo(0);
-
-        // This time we don't catch InsufficientDiscSpace -- we
-        // haven't allocated anything yet and can safely let the
-        // exception out to indicate to the caller that we can't
-        // handle it.
-
-        recommendation =
-            StorageAdviser::recommend(criteria, minimumSize, maximumSize);
-    }
-
-    std::cerr << "Recommendation was: " << recommendation << std::endl;
-
-    m_memoryCache = false;
-
-    if ((recommendation & StorageAdviser::UseMemory) ||
-        (recommendation & StorageAdviser::PreferMemory)) {
-        m_memoryCache = true;
-    }
-
-    m_compactCache = canCompact &&
-        (recommendation & StorageAdviser::ConserveSpace);
-
-    std::cerr << "FFTDataServer: memory cache = " << m_memoryCache << ", compact cache = " << m_compactCache << std::endl;
-    
-#ifdef DEBUG_FFT_SERVER
-    std::cerr << "Width " << m_width << ", cache width " << m_cacheWidth << " (size " << m_cacheWidth * columnSize << ")" << std::endl;
-#endif
-
-    StorageAdviser::notifyPlannedAllocation
-        (m_memoryCache ? StorageAdviser::MemoryAllocation :
-                         StorageAdviser::DiscAllocation,
-         m_compactCache ? minimumSize : maximumSize);
 
     for (size_t i = 0; i <= m_width / m_cacheWidth; ++i) {
         m_caches.push_back(0);
     }
 

 
     MutexLocker locker(&m_writeMutex,
                        "FFTDataServer::m_writeMutex[~FFTDataServer]");
 
     for (CacheVector::iterator i = m_caches.begin(); i != m_caches.end(); ++i) {
+
         if (*i) {
             delete *i;
-        } else {
-            StorageAdviser::notifyDoneAllocation
-                (m_memoryCache ? StorageAdviser::MemoryAllocation :
-                                 StorageAdviser::DiscAllocation,
-                 m_cacheWidth * m_height *
-                 (m_compactCache ? sizeof(uint16_t) : sizeof(float)) / 1024 + 1);
         }
     }
 
     deleteProcessingData();
 }

             m_condition.wakeAll();
         }
     }
 }
 
+void
+FFTDataServer::getStorageAdvice(size_t w, size_t h,
+                                bool &memoryCache, bool &compactCache)
+{
+    int cells = w * h;
+    int minimumSize = (cells / 1024) * sizeof(uint16_t); // kb
+    int maximumSize = (cells / 1024) * sizeof(float); // kb
+
+    // We don't have a compact rectangular representation, and compact
+    // of course is never precision-critical
+
+    bool canCompact = true;
+    if ((m_criteria & StorageAdviser::PrecisionCritical) || !m_polar) {
+        canCompact = false;
+        minimumSize = maximumSize; // don't use compact
+    }
+    
+    StorageAdviser::Recommendation recommendation;
+
+    try {
+
+        recommendation =
+            StorageAdviser::recommend(m_criteria, minimumSize, maximumSize);
+
+    } catch (InsufficientDiscSpace s) {
+
+        // Delete any unused servers we may have been leaving around
+        // in case we wanted them again
+
+        purgeLimbo(0);
+
+        // This time we don't catch InsufficientDiscSpace -- we
+        // haven't allocated anything yet and can safely let the
+        // exception out to indicate to the caller that we can't
+        // handle it.
+
+        recommendation =
+            StorageAdviser::recommend(m_criteria, minimumSize, maximumSize);
+    }
+
+    std::cerr << "Recommendation was: " << recommendation << std::endl;
+
+    memoryCache = false;
+
+    if ((recommendation & StorageAdviser::UseMemory) ||
+        (recommendation & StorageAdviser::PreferMemory)) {
+        memoryCache = true;
+    }
+
+    compactCache = canCompact &&
+        (recommendation & StorageAdviser::ConserveSpace);
+
+    std::cerr << "FFTDataServer: memory cache = " << memoryCache << ", compact cache = " << compactCache << std::endl;
+    
+#ifdef DEBUG_FFT_SERVER
+    std::cerr << "Width " << w << " of " << m_width << ", height " << h << ", size " << w * h << std::endl;
+#endif
+}
+
 FFTCache *
 FFTDataServer::getCacheAux(size_t c)
 {
     Profiler profiler("FFTDataServer::getCacheAux", false);
 #ifdef DEBUG_FFT_SERVER

     size_t width = m_cacheWidth;
     if (c * m_cacheWidth + width > m_width) {
         width = m_width - c * m_cacheWidth;
     }
 
+    bool memoryCache = false;
+    bool compactCache = false;
+
+    getStorageAdvice(width, m_height, memoryCache, compactCache);
+
     try {
 
-        if (m_memoryCache) {
+        if (memoryCache) {
 
             cache = new FFTMemoryCache
-                (m_compactCache ? FFTMemoryCache::Compact :
-//                                  FFTMemoryCache::Polar);
-                 m_polar ? FFTMemoryCache::Polar :
-                           FFTMemoryCache::Rectangular);
-
-        } else if (m_compactCache) {
+                (compactCache ? FFTMemoryCache::Compact :
+                      m_polar ? FFTMemoryCache::Polar :
+                                FFTMemoryCache::Rectangular);
+
+        } else {
 
             cache = new FFTFileCache
                 (name,
                  MatrixFile::ReadWrite,
-                 FFTFileCache::Compact);
-
-        } else {
-
-            cache = new FFTFileCache
-                (name,
-                 MatrixFile::ReadWrite,
-                 m_polar ? FFTFileCache::Polar :
-                           FFTFileCache::Rectangular);
+                 compactCache ? FFTFileCache::Compact :
+                      m_polar ? FFTFileCache::Polar :
+                                FFTFileCache::Rectangular);
         }
 
         cache->resize(width, m_height);
         cache->reset();
 
     } catch (std::bad_alloc) {
 
         delete cache;
         cache = 0;
 
-        if (m_memoryCache) {
+        if (memoryCache) {
             
             std::cerr << "WARNING: Memory allocation failed when resizing"
                       << " FFT memory cache no. " << c << " to " << width 
                       << "x" << m_height << " (of total width " << m_width
                       << "): falling back to disc cache" << std::endl;
 
             try {
 
-                cache = new FFTFileCache(name, MatrixFile::ReadWrite,
+                purgeLimbo(0);
+
+                cache = new FFTFileCache(name,
+                                         MatrixFile::ReadWrite,
                                          FFTFileCache::Compact);
 
                 cache->resize(width, m_height);
                 cache->reset();
 

             (0, QApplication::tr("FFT cache resize failed"),
              QApplication::tr
              ("Failed to create or resize an FFT model slice.\n"
               "There may be insufficient memory or disc space to continue."));
     }
-
-    StorageAdviser::notifyDoneAllocation
-        (m_memoryCache ? StorageAdviser::MemoryAllocation :
-         StorageAdviser::DiscAllocation,
-         width * m_height *
-         (m_compactCache ? sizeof(uint16_t) : sizeof(float)) / 1024 + 1);
 
     m_caches[c] = cache;
     m_lastUsedCache = c;
     return cache;
 }