changeset 550:107d3f3705c9

* Auto-close write-mode matrix files when all columns written

--- a/base/ResizeableBitset.h	Thu Feb 05 12:53:19 2009 +0000
+++ b/base/ResizeableBitset.h	Thu Feb 05 14:17:11 2009 +0000
@@ -71,6 +71,23 @@
         get(source) ? set(dest) : reset(dest);
     }
 
+    bool isAllOff() const {
+        for (size_t i = 0; i < m_bits->size(); ++i) {
+            if ((*m_bits)[i]) return false;
+        }
+        return true;
+    }
+
+    bool isAllOn() const {
+        for (size_t i = 0; i + 1 < m_bits->size(); ++i) {
+            if ((*m_bits)[i] != 0xff) return false;
+        }
+        for (size_t i = (m_size / 8) * 8; i < m_size; ++i) {
+            if (!get(i)) return false;
+        }
+        return true;
+    }
+
     size_t size() const {
         return m_size;
     }

--- a/data/fft/FFTCacheWriter.h	Thu Feb 05 12:53:19 2009 +0000
+++ b/data/fft/FFTCacheWriter.h	Thu Feb 05 14:17:11 2009 +0000
@@ -29,7 +29,9 @@
     virtual void setColumnAt(size_t x, float *mags, float *phases, float factor) = 0;
     virtual void setColumnAt(size_t x, float *reals, float *imags) = 0;
 
-    virtual void allColumnsWritten() = 0;
+    virtual bool haveSetColumnAt(size_t x) const = 0;
+
+    virtual void allColumnsWritten() = 0; // notify cache to close
 
     virtual FFTCache::StorageType getStorageType() const = 0;
 };

--- a/data/fft/FFTDataServer.cpp	Thu Feb 05 12:53:19 2009 +0000
+++ b/data/fft/FFTDataServer.cpp	Thu Feb 05 14:17:11 2009 +0000
@@ -900,7 +900,7 @@
         std::cerr << "FFTDataServer::getMagnitudeAt: calling fillColumn(" 
                   << x << ")" << std::endl;
 #endif
-        fillColumn(x, cache);
+        fillColumn(x);
     }
     return cache->getMagnitudeAt(col, y);
 }
@@ -925,7 +925,7 @@
     //!!! n.b. can throw
     if (!cache->haveSetColumnAt(col)) {
         Profiler profiler("FFTDataServer::getMagnitudesAt: filling");
-        fillColumn(x, cache);
+        fillColumn(x);
     }
 
     cache->getMagnitudesAt(col, values, minbin, count, step);
@@ -947,7 +947,7 @@
     //!!! n.b. can throw
     if (!cache->haveSetColumnAt(col)) {
         Profiler profiler("FFTDataServer::getNormalizedMagnitudeAt: filling");
-        fillColumn(x, cache);
+        fillColumn(x);
     }
     return cache->getNormalizedMagnitudeAt(col, y);
 }
@@ -972,7 +972,7 @@
     //!!! n.b. can throw
     if (!cache->haveSetColumnAt(col)) {
         Profiler profiler("FFTDataServer::getNormalizedMagnitudesAt: filling");
-        fillColumn(x, cache);
+        fillColumn(x);
     }
 
     for (size_t i = 0; i < count; ++i) {
@@ -996,7 +996,7 @@
     //!!! n.b. can throw
     if (!cache->haveSetColumnAt(col)) {
         Profiler profiler("FFTDataServer::getMaximumMagnitudeAt: filling");
-        fillColumn(x, cache);
+        fillColumn(x);
     }
     return cache->getMaximumMagnitudeAt(col);
 }
@@ -1015,7 +1015,7 @@
     //!!! n.b. can throw
     if (!cache->haveSetColumnAt(col)) {
         Profiler profiler("FFTDataServer::getPhaseAt: filling");
-        fillColumn(x, cache);
+        fillColumn(x);
     }
     return cache->getPhaseAt(col, y);
 }
@@ -1040,7 +1040,7 @@
     //!!! n.b. can throw
     if (!cache->haveSetColumnAt(col)) {
         Profiler profiler("FFTDataServer::getPhasesAt: filling");
-        fillColumn(x, cache);
+        fillColumn(x);
     }
 
     for (size_t i = 0; i < count; ++i) {
@@ -1076,7 +1076,7 @@
 #ifdef DEBUG_FFT_SERVER
         std::cerr << "FFTDataServer::getValuesAt(" << x << ", " << y << "): filling" << std::endl;
 #endif
-        fillColumn(x, cache);
+        fillColumn(x);
     }        
 
     cache->getValuesAt(col, y, real, imaginary);
@@ -1111,7 +1111,7 @@
 }    
 
 void
-FFTDataServer::fillColumn(size_t x, FFTCacheReader *tester)
+FFTDataServer::fillColumn(size_t x)
 {
     Profiler profiler("FFTDataServer::fillColumn", false);
 
@@ -1166,16 +1166,14 @@
 
     QMutexLocker locker(&m_fftBuffersLock);
 
-    if (tester) {
-        // We are being called from a function that wanted to obtain a
-        // column using an FFTCacheReader.  Before calling us, it
-        // checked whether the column was available already, and the
-        // reader reported that it wasn't.  Now we test again with the
-        // mutex held, to avoid a race condition in case another
-        // thread has called fillColumn at the same time.
-        if (tester->haveSetColumnAt(x & m_cacheWidthMask)) {
-            return;
-        }
+    // We may have been called from a function that wanted to obtain a
+    // column using an FFTCacheReader.  Before calling us, it checked
+    // whether the column was available already, and the reader
+    // reported that it wasn't.  Now we test again, with the mutex
+    // held, to avoid a race condition in case another thread has
+    // called fillColumn at the same time.
+    if (cache->haveSetColumnAt(x & m_cacheWidthMask)) {
+        return;
     }
 
     if (!m_fftInput) {

--- a/data/fft/FFTDataServer.h	Thu Feb 05 12:53:19 2009 +0000
+++ b/data/fft/FFTDataServer.h	Thu Feb 05 14:17:11 2009 +0000
@@ -259,7 +259,7 @@
     FillThread *m_fillThread;
 
     void deleteProcessingData();
-    void fillColumn(size_t x, FFTCacheReader *tester = 0);
+    void fillColumn(size_t x);
     void fillComplete();
 
     QString generateFileBasename() const;

--- a/data/fft/FFTFileCacheWriter.cpp	Thu Feb 05 12:53:19 2009 +0000
+++ b/data/fft/FFTFileCacheWriter.cpp	Thu Feb 05 14:17:11 2009 +0000
@@ -44,6 +44,7 @@
            width, height * 2 + m_factorSize))
 {
     std::cerr << "FFTFileCacheWriter: storage type is " << (storageType == FFTCache::Compact ? "Compact" : storageType == FFTCache::Polar ? "Polar" : "Rectangular") << ", size " << width << "x" << height << std::endl;
+    m_mfc->setAutoClose(true);
     m_writebuf = new char[(height * 2 + m_factorSize) * m_mfc->getCellSize()];
 }
 
@@ -73,6 +74,12 @@
     else return 0;
 }
 
+bool
+FFTFileCacheWriter::haveSetColumnAt(size_t x) const
+{
+    return m_mfc->haveSetColumnAt(x);
+}
+
 void
 FFTFileCacheWriter::setColumnAt(size_t x, float *mags, float *phases, float factor)
 {

--- a/data/fft/FFTFileCacheWriter.h	Thu Feb 05 12:53:19 2009 +0000
+++ b/data/fft/FFTFileCacheWriter.h	Thu Feb 05 14:17:11 2009 +0000
@@ -37,6 +37,8 @@
     static size_t getCacheSize(size_t width, size_t height,
                                FFTCache::StorageType type);
 
+    bool haveSetColumnAt(size_t x) const;
+
     void allColumnsWritten();
 
     QString getFileBase() const;

--- a/data/fileio/MatrixFile.cpp	Thu Feb 05 12:53:19 2009 +0000
+++ b/data/fileio/MatrixFile.cpp	Thu Feb 05 14:17:11 2009 +0000
@@ -60,7 +60,9 @@
     m_cellSize(cellSize),
     m_width(width),
     m_height(height),
-    m_headerSize(2 * sizeof(size_t))
+    m_headerSize(2 * sizeof(size_t)),
+    m_setColumns(0),
+    m_autoClose(false)
 {
     Profiler profiler("MatrixFile::MatrixFile", true);
 
@@ -163,6 +165,8 @@
 
     QMutexLocker locker(&m_createMutex);
 
+    delete m_setColumns;
+
     if (m_fileName != "") {
 
         if (--m_refcount[m_fileName] == 0) {
@@ -192,6 +196,8 @@
     Profiler profiler("MatrixFile::initialise", true);
 
     assert(m_mode == WriteOnly);
+
+    m_setColumns = new ResizeableBitset(m_width);
     
     off_t off = m_headerSize + (m_width * m_height * m_cellSize) + m_width;
 
@@ -249,6 +255,9 @@
         }
         m_fd = -1;
         -- openCount;
+#ifdef DEBUG_MATRIX_FILE
+        std::cerr << "MatrixFile: Now " << openCount << " open instances" << std::endl;
+#endif
     }
 }
 
@@ -290,7 +299,9 @@
 bool
 MatrixFile::haveSetColumnAt(size_t x) const
 {
-    assert(m_mode == ReadOnly);
+    if (m_mode == WriteOnly) {
+        return m_setColumns->get(x);
+    }
 
     Profiler profiler("MatrixFile::haveSetColumnAt");
 
@@ -319,6 +330,7 @@
 MatrixFile::setColumnAt(size_t x, const void *data)
 {
     assert(m_mode == WriteOnly);
+    if (m_fd < 0) return; // closed
 
 #ifdef DEBUG_MATRIX_FILE_READ_SET
     std::cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << ")" << std::endl;
@@ -364,6 +376,22 @@
         ::perror("WARNING: MatrixFile::setColumnAt: write failed (3)");
         throw FileOperationFailed(m_fileName, "write");
     }
+
+    m_setColumns->set(x);
+    if (m_autoClose) {
+        if (m_setColumns->isAllOn()) {
+            std::cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << "): All columns set: auto-closing" << std::endl;
+            close();
+/*
+        } else {
+            int set = 0;
+            for (int i = 0; i < m_width; ++i) {
+                if (m_setColumns->get(i)) ++set;
+            }
+            std::cerr << "MatrixFile[" << m_fd << "]::setColumnAt(" << x << "): Auto-close on, but not all columns set yet (" << set << " of " << m_width << ")" << std::endl;
+*/
+        }
+    }
 }
 
 bool

--- a/data/fileio/MatrixFile.h	Thu Feb 05 12:53:19 2009 +0000
+++ b/data/fileio/MatrixFile.h	Thu Feb 05 14:17:11 2009 +0000
@@ -67,7 +67,13 @@
     size_t getWidth() const { return m_width; }
     size_t getHeight() const { return m_height; }
     size_t getCellSize() const { return m_cellSize; }
-    
+
+    /**
+     * If this is set true on a write-mode MatrixFile, then the file
+     * will close() itself when all columns have been written.
+     */
+    void setAutoClose(bool a) { m_autoClose = a; }
+
     void close(); // does not decrement ref count; that happens in dtor
 
     bool haveSetColumnAt(size_t x) const;
@@ -85,6 +91,9 @@
     size_t  m_headerSize;
     QString m_fileName;
 
+    ResizeableBitset *m_setColumns; // only in writer
+    bool m_autoClose;
+
     static std::map<QString, int> m_refcount;
     static QMutex m_createMutex;