changeset 1777:d484490cdf69

Split EditableDenseThreeDimensionalModel into explicitly compressed and uncompressed variants. Simplifies the uncompressed version, and we may want to consider whether we need the compressed one at all.

--- a/data/fileio/CSVFileReader.cpp	Mon Sep 09 10:25:16 2019 +0100
+++ b/data/fileio/CSVFileReader.cpp	Tue Sep 10 16:34:47 2019 +0100
@@ -335,10 +335,7 @@
                 
                 case CSVFormat::ThreeDimensionalModel:
                     model3 = new EditableDenseThreeDimensionalModel
-                        (sampleRate,
-                         windowSize,
-                         valueColumns,
-                         EditableDenseThreeDimensionalModel::NoCompression);
+                        (sampleRate, windowSize, valueColumns);
                     model = model3;
                     break;
 

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data/model/BasicCompressedDenseThreeDimensionalModel.cpp	Tue Sep 10 16:34:47 2019 +0100
@@ -0,0 +1,576 @@
+/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
+
+/*
+    Sonic Visualiser
+    An audio file viewer and annotation editor.
+    Centre for Digital Music, Queen Mary, University of London.
+    This file copyright 2006 Chris Cannam and QMUL.
+    
+    This program is free software; you can redistribute it and/or
+    modify it under the terms of the GNU General Public License as
+    published by the Free Software Foundation; either version 2 of the
+    License, or (at your option) any later version.  See the file
+    COPYING included with this distribution for more information.
+*/
+
+#include "BasicCompressedDenseThreeDimensionalModel.h"
+
+#include "base/LogRange.h"
+
+#include <QTextStream>
+#include <QStringList>
+#include <QReadLocker>
+#include <QWriteLocker>
+
+#include <iostream>
+
+#include <cmath>
+#include <cassert>
+
+using std::vector;
+
+#include "system/System.h"
+
+BasicCompressedDenseThreeDimensionalModel::BasicCompressedDenseThreeDimensionalModel(sv_samplerate_t sampleRate,
+                                                                                     int resolution,
+                                                                                     int yBinCount,
+                                                                                     bool notifyOnAdd) :
+    m_startFrame(0),
+    m_sampleRate(sampleRate),
+    m_resolution(resolution),
+    m_yBinCount(yBinCount),
+    m_minimum(0.0),
+    m_maximum(0.0),
+    m_haveExtents(false),
+    m_notifyOnAdd(notifyOnAdd),
+    m_sinceLastNotifyMin(-1),
+    m_sinceLastNotifyMax(-1),
+    m_completion(100)
+{
+}    
+
+bool
+BasicCompressedDenseThreeDimensionalModel::isOK() const
+{
+    return true;
+}
+
+bool
+BasicCompressedDenseThreeDimensionalModel::isReady(int *completion) const
+{
+    if (completion) *completion = getCompletion();
+    return true;
+}
+
+sv_samplerate_t
+BasicCompressedDenseThreeDimensionalModel::getSampleRate() const
+{
+    return m_sampleRate;
+}
+
+sv_frame_t
+BasicCompressedDenseThreeDimensionalModel::getStartFrame() const
+{
+    return m_startFrame;
+}
+
+void
+BasicCompressedDenseThreeDimensionalModel::setStartFrame(sv_frame_t f)
+{
+    m_startFrame = f; 
+}
+
+sv_frame_t
+BasicCompressedDenseThreeDimensionalModel::getTrueEndFrame() const
+{
+    return m_resolution * m_data.size() + (m_resolution - 1);
+}
+
+int
+BasicCompressedDenseThreeDimensionalModel::getResolution() const
+{
+    return m_resolution;
+}
+
+void
+BasicCompressedDenseThreeDimensionalModel::setResolution(int sz)
+{
+    m_resolution = sz;
+}
+
+int
+BasicCompressedDenseThreeDimensionalModel::getWidth() const
+{
+    return int(m_data.size());
+}
+
+int
+BasicCompressedDenseThreeDimensionalModel::getHeight() const
+{
+    return m_yBinCount;
+}
+
+void
+BasicCompressedDenseThreeDimensionalModel::setHeight(int sz)
+{
+    m_yBinCount = sz;
+}
+
+float
+BasicCompressedDenseThreeDimensionalModel::getMinimumLevel() const
+{
+    return m_minimum;
+}
+
+void
+BasicCompressedDenseThreeDimensionalModel::setMinimumLevel(float level)
+{
+    m_minimum = level;
+}
+
+float
+BasicCompressedDenseThreeDimensionalModel::getMaximumLevel() const
+{
+    return m_maximum;
+}
+
+void
+BasicCompressedDenseThreeDimensionalModel::setMaximumLevel(float level)
+{
+    m_maximum = level;
+}
+
+BasicCompressedDenseThreeDimensionalModel::Column
+BasicCompressedDenseThreeDimensionalModel::getColumn(int index) const
+{
+    QReadLocker locker(&m_lock);
+    if (in_range_for(m_data, index)) return expandAndRetrieve(index);
+    else return Column();
+}
+
+float
+BasicCompressedDenseThreeDimensionalModel::getValueAt(int index, int n) const
+{
+    Column c = getColumn(index);
+    if (in_range_for(c, n)) return c.at(n);
+    return m_minimum;
+}
+
+//static int given = 0, stored = 0;
+
+void
+BasicCompressedDenseThreeDimensionalModel::truncateAndStore(int index,
+                                                     const Column &values)
+{
+    assert(in_range_for(m_data, index));
+
+    //cout << "truncateAndStore(" << index << ", " << values.size() << ")" << endl;
+
+    // The default case is to store the entire column at m_data[index]
+    // and place 0 at m_trunc[index] to indicate that it has not been
+    // truncated.  We only do clever stuff if one of the clever-stuff
+    // tests works out.
+
+    m_trunc[index] = 0;
+    if (index == 0 ||
+        int(values.size()) != m_yBinCount) {
+//        given += values.size();
+//        stored += values.size();
+        m_data[index] = values;
+        return;
+    }
+
+    // Maximum distance between a column and the one we refer to as
+    // the source of its truncated values.  Limited by having to fit
+    // in a signed char, but in any case small values are usually
+    // better
+    static int maxdist = 6;
+
+    bool known = false; // do we know whether to truncate at top or bottom?
+    bool top = false;   // if we do know, will we truncate at top?
+
+    // If the previous column is not truncated, then it is the only
+    // candidate for comparison.  If it is truncated, then the column
+    // that it refers to is the only candidate.  Either way, we only
+    // have one possible column to compare against here, and we are
+    // being careful to ensure it is not a truncated one (to avoid
+    // doing more work recursively when uncompressing).
+    int tdist = 1;
+    int ptrunc = m_trunc[index-1];
+    if (ptrunc < 0) {
+        top = false;
+        known = true;
+        tdist = -ptrunc + 1;
+    } else if (ptrunc > 0) {
+        top = true;
+        known = true;
+        tdist = ptrunc + 1;
+    }
+
+    Column p = expandAndRetrieve(index - tdist);
+    int h = m_yBinCount;
+
+    if (int(p.size()) == h && tdist <= maxdist) {
+
+        int bcount = 0, tcount = 0;
+        if (!known || !top) {
+            // count how many identical values there are at the bottom
+            for (int i = 0; i < h; ++i) {
+                if (values.at(i) == p.at(i)) ++bcount;
+                else break;
+            }
+        }
+        if (!known || top) {
+            // count how many identical values there are at the top
+            for (int i = h; i > 0; --i) {
+                if (values.at(i-1) == p.at(i-1)) ++tcount;
+                else break;
+            }
+        }
+        if (!known) top = (tcount > bcount);
+
+        int limit = h / 4; // don't bother unless we have at least this many
+        if ((top ? tcount : bcount) > limit) {
+        
+            if (!top) {
+                // create a new column with h - bcount values from bcount up
+                Column tcol(h - bcount);
+//                given += values.size();
+//                stored += h - bcount;
+                for (int i = bcount; i < h; ++i) {
+                    tcol[i - bcount] = values.at(i);
+                }
+                m_data[index] = tcol;
+                m_trunc[index] = (signed char)(-tdist);
+                return;
+            } else {
+                // create a new column with h - tcount values from 0 up
+                Column tcol(h - tcount);
+//                given += values.size();
+//                stored += h - tcount;
+                for (int i = 0; i < h - tcount; ++i) {
+                    tcol[i] = values.at(i);
+                }
+                m_data[index] = tcol;
+                m_trunc[index] = (signed char)(tdist);
+                return;
+            }
+        }
+    }                
+
+//    given += values.size();
+//    stored += values.size();
+//    cout << "given: " << given << ", stored: " << stored << " (" 
+//              << ((float(stored) / float(given)) * 100.f) << "%)" << endl;
+
+    // default case if nothing wacky worked out
+    m_data[index] = values;
+    return;
+}
+
+BasicCompressedDenseThreeDimensionalModel::Column
+BasicCompressedDenseThreeDimensionalModel::rightHeight(const Column &c) const
+{
+    if (int(c.size()) == m_yBinCount) return c;
+    else {
+        Column cc(c);
+        cc.resize(m_yBinCount, 0.0);
+        return cc;
+    }
+}
+
+BasicCompressedDenseThreeDimensionalModel::Column
+BasicCompressedDenseThreeDimensionalModel::expandAndRetrieve(int index) const
+{
+    // See comment above m_trunc declaration in header
+
+    assert(index >= 0 && index < int(m_data.size()));
+    Column c = m_data.at(index);
+    if (index == 0) {
+        return rightHeight(c);
+    }
+    int trunc = (int)m_trunc[index];
+    if (trunc == 0) {
+        return rightHeight(c);
+    }
+    bool top = true;
+    int tdist = trunc;
+    if (trunc < 0) { top = false; tdist = -trunc; }
+    Column p = expandAndRetrieve(index - tdist);
+    int psize = int(p.size()), csize = int(c.size());
+    if (psize != m_yBinCount) {
+        cerr << "WARNING: BasicCompressedDenseThreeDimensionalModel::expandAndRetrieve: Trying to expand from incorrectly sized column" << endl;
+    }
+    if (top) {
+        for (int i = csize; i < psize; ++i) {
+            c.push_back(p.at(i));
+        }
+    } else {
+        Column cc(psize);
+        for (int i = 0; i < psize - csize; ++i) {
+            cc[i] = p.at(i);
+        }
+        for (int i = 0; i < csize; ++i) {
+            cc[i + (psize - csize)] = c.at(i);
+        }
+        return cc;
+    }
+    return c;
+}
+
+void
+BasicCompressedDenseThreeDimensionalModel::setColumn(int index,
+                                              const Column &values)
+{
+    QWriteLocker locker(&m_lock);
+
+    while (index >= int(m_data.size())) {
+        m_data.push_back(Column());
+        m_trunc.push_back(0);
+    }
+
+    bool allChange = false;
+
+    for (int i = 0; in_range_for(values, i); ++i) {
+        float value = values[i];
+        if (ISNAN(value) || ISINF(value)) {
+            continue;
+        }
+        if (!m_haveExtents || value < m_minimum) {
+            m_minimum = value;
+            allChange = true;
+        }
+        if (!m_haveExtents || value > m_maximum) {
+            m_maximum = value;
+            allChange = true;
+        }
+        m_haveExtents = true;
+    }
+
+    truncateAndStore(index, values);
+
+//    assert(values == expandAndRetrieve(index));
+
+    sv_frame_t windowStart = index;
+    windowStart *= m_resolution;
+
+    if (m_notifyOnAdd) {
+        if (allChange) {
+            emit modelChanged(getId());
+        } else {
+            emit modelChangedWithin(getId(),
+                                    windowStart, windowStart + m_resolution);
+        }
+    } else {
+        if (allChange) {
+            m_sinceLastNotifyMin = -1;
+            m_sinceLastNotifyMax = -1;
+            emit modelChanged(getId());
+        } else {
+            if (m_sinceLastNotifyMin == -1 ||
+                windowStart < m_sinceLastNotifyMin) {
+                m_sinceLastNotifyMin = windowStart;
+            }
+            if (m_sinceLastNotifyMax == -1 ||
+                windowStart > m_sinceLastNotifyMax) {
+                m_sinceLastNotifyMax = windowStart;
+            }
+        }
+    }
+}
+
+QString
+BasicCompressedDenseThreeDimensionalModel::getBinName(int n) const
+{
+    if (n >= 0 && (int)m_binNames.size() > n) return m_binNames[n];
+    else return "";
+}
+
+void
+BasicCompressedDenseThreeDimensionalModel::setBinName(int n, QString name)
+{
+    while ((int)m_binNames.size() <= n) m_binNames.push_back("");
+    m_binNames[n] = name;
+    emit modelChanged(getId());
+}
+
+void
+BasicCompressedDenseThreeDimensionalModel::setBinNames(std::vector<QString> names)
+{
+    m_binNames = names;
+    emit modelChanged(getId());
+}
+
+bool
+BasicCompressedDenseThreeDimensionalModel::hasBinValues() const
+{
+    return !m_binValues.empty();
+}
+
+float
+BasicCompressedDenseThreeDimensionalModel::getBinValue(int n) const
+{
+    if (n < (int)m_binValues.size()) return m_binValues[n];
+    else return 0.f;
+}
+
+void
+BasicCompressedDenseThreeDimensionalModel::setBinValues(std::vector<float> values)
+{
+    m_binValues = values;
+}
+
+QString
+BasicCompressedDenseThreeDimensionalModel::getBinValueUnit() const
+{
+    return m_binValueUnit;
+}
+
+void
+BasicCompressedDenseThreeDimensionalModel::setBinValueUnit(QString unit)
+{
+    m_binValueUnit = unit;
+}
+
+bool
+BasicCompressedDenseThreeDimensionalModel::shouldUseLogValueScale() const
+{
+    QReadLocker locker(&m_lock);
+
+    vector<double> sample;
+    vector<int> n;
+    
+    for (int i = 0; i < 10; ++i) {
+        int index = i * 10;
+        if (in_range_for(m_data, index)) {
+            const Column &c = m_data.at(index);
+            while (c.size() > sample.size()) {
+                sample.push_back(0.0);
+                n.push_back(0);
+            }
+            for (int j = 0; in_range_for(c, j); ++j) {
+                sample[j] += c.at(j);
+                ++n[j];
+            }
+        }
+    }
+
+    if (sample.empty()) return false;
+    for (decltype(sample)::size_type j = 0; j < sample.size(); ++j) {
+        if (n[j]) sample[j] /= n[j];
+    }
+    
+    return LogRange::shouldUseLogScale(sample);
+}
+
+void
+BasicCompressedDenseThreeDimensionalModel::setCompletion(int completion, bool update)
+{
+    if (m_completion != completion) {
+        m_completion = completion;
+
+        if (completion == 100) {
+
+            m_notifyOnAdd = true; // henceforth
+            emit modelChanged(getId());
+
+        } else if (!m_notifyOnAdd) {
+
+            if (update &&
+                m_sinceLastNotifyMin >= 0 &&
+                m_sinceLastNotifyMax >= 0) {
+                emit modelChangedWithin(getId(),
+                                        m_sinceLastNotifyMin,
+                                        m_sinceLastNotifyMax + m_resolution);
+                m_sinceLastNotifyMin = m_sinceLastNotifyMax = -1;
+            } else {
+                emit completionChanged(getId());
+            }
+        } else {
+            emit completionChanged(getId());
+        }            
+    }
+}
+
+int
+BasicCompressedDenseThreeDimensionalModel::getCompletion() const
+{
+    return m_completion;
+}
+
+QString
+BasicCompressedDenseThreeDimensionalModel::toDelimitedDataString(QString delimiter,
+                                                          DataExportOptions,
+                                                          sv_frame_t startFrame,
+                                                          sv_frame_t duration) const
+{
+    QReadLocker locker(&m_lock);
+    QString s;
+    for (int i = 0; in_range_for(m_data, i); ++i) {
+        Column c = getColumn(i);
+        sv_frame_t fr = m_startFrame + i * m_resolution;
+        if (fr >= startFrame && fr < startFrame + duration) {
+            QStringList list;
+            for (int j = 0; in_range_for(c, j); ++j) {
+                list << QString("%1").arg(c.at(j));
+            }
+            s += list.join(delimiter) + "\n";
+        }
+    }
+    return s;
+}
+
+void
+BasicCompressedDenseThreeDimensionalModel::toXml(QTextStream &out,
+                                          QString indent,
+                                          QString extraAttributes) const
+{
+    QReadLocker locker(&m_lock);
+
+    // For historical reasons we read and write "resolution" as "windowSize".
+
+    // Our dataset doesn't have its own export ID, we just use
+    // ours. Actually any model could do that, since datasets aren't
+    // in the same id-space as models when re-read
+
+    SVDEBUG << "BasicCompressedDenseThreeDimensionalModel::toXml" << endl;
+
+    Model::toXml
+        (out, indent,
+         QString("type=\"dense\" dimensions=\"3\" windowSize=\"%1\" yBinCount=\"%2\" minimum=\"%3\" maximum=\"%4\" dataset=\"%5\" startFrame=\"%6\" %7")
+         .arg(m_resolution)
+         .arg(m_yBinCount)
+         .arg(m_minimum)
+         .arg(m_maximum)
+         .arg(getExportId())
+         .arg(m_startFrame)
+         .arg(extraAttributes));
+
+    out << indent;
+    out << QString("<dataset id=\"%1\" dimensions=\"3\" separator=\" \">\n")
+        .arg(getExportId());
+
+    for (int i = 0; in_range_for(m_binNames, i); ++i) {
+        if (m_binNames[i] != "") {
+            out << indent + "  ";
+            out << QString("<bin number=\"%1\" name=\"%2\"/>\n")
+                .arg(i).arg(m_binNames[i]);
+        }
+    }
+
+    for (int i = 0; in_range_for(m_data, i); ++i) {
+        Column c = getColumn(i);
+        out << indent + "  ";
+        out << QString("<row n=\"%1\">").arg(i);
+        for (int j = 0; in_range_for(c, j); ++j) {
+            if (j > 0) out << " ";
+            out << c.at(j);
+        }
+        out << QString("</row>\n");
+        out.flush();
+    }
+
+    out << indent + "</dataset>\n";
+}
+
+

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data/model/BasicCompressedDenseThreeDimensionalModel.h	Tue Sep 10 16:34:47 2019 +0100
@@ -0,0 +1,229 @@
+/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
+
+/*
+    Sonic Visualiser
+    An audio file viewer and annotation editor.
+    Centre for Digital Music, Queen Mary, University of London.
+    This file copyright 2006 Chris Cannam and QMUL.
+    
+    This program is free software; you can redistribute it and/or
+    modify it under the terms of the GNU General Public License as
+    published by the Free Software Foundation; either version 2 of the
+    License, or (at your option) any later version.  See the file
+    COPYING included with this distribution for more information.
+*/
+
+#ifndef SV_BASIC_COMPRESSED_DENSE_THREE_DIMENSIONAL_MODEL_H
+#define SV_BASIC_COMPRESSED_DENSE_THREE_DIMENSIONAL_MODEL_H
+
+#include "DenseThreeDimensionalModel.h"
+
+#include <QReadWriteLock>
+
+#include <vector>
+
+class BasicCompressedDenseThreeDimensionalModel : public DenseThreeDimensionalModel
+{
+    Q_OBJECT
+
+public:
+
+    // BasicCompressedDenseThreeDimensionalModel supports a basic
+    // compression method that reduces the size of multirate data
+    // (e.g. wavelet transform outputs) that are stored as plain 3d
+    // grids by about 60% or thereabouts.  However, it can only be
+    // used for models whose columns are set in order from 0 and never
+    // subsequently changed.  For a model that is actually going to be
+    // edited, you need an EditableDenseThreeDimensionalModel.
+
+    BasicCompressedDenseThreeDimensionalModel(sv_samplerate_t sampleRate,
+                                              int resolution,
+                                              int height,
+                                              bool notifyOnAdd = true);
+
+    bool isOK() const override;
+    bool isReady(int *completion = 0) const override;
+    void setCompletion(int completion, bool update = true);
+    int getCompletion() const override;
+
+    sv_samplerate_t getSampleRate() const override;
+    sv_frame_t getStartFrame() const override;
+    sv_frame_t getTrueEndFrame() const override;
+
+    /**
+     * Set the frame offset of the first column.
+     */
+    virtual void setStartFrame(sv_frame_t);
+
+    /**
+     * Return the number of sample frames covered by each set of bins.
+     */
+    int getResolution() const override;
+
+    /**
+     * Set the number of sample frames covered by each set of bins.
+     */
+    virtual void setResolution(int sz);
+
+    /**
+     * Return the number of columns.
+     */
+    int getWidth() const override;
+
+    /**
+     * Return the number of bins in each column.
+     */
+    int getHeight() const override;
+
+    /**
+     * Set the number of bins in each column.
+     *
+     * You can set (via setColumn) a vector of any length as a column,
+     * but any column being retrieved will be resized to this height
+     * (or the height that was supplied to the constructor, if this is
+     * never called) on retrieval. That is, the model owner determines
+     * the height of the model at a single stroke; the columns
+     * themselves don't have any effect on the height of the model.
+     */
+    virtual void setHeight(int sz);
+
+    /**
+     * Return the minimum value of the value in each bin.
+     */
+    float getMinimumLevel() const override;
+
+    /**
+     * Set the minimum value of the value in a bin.
+     */
+    virtual void setMinimumLevel(float sz);
+
+    /**
+     * Return the maximum value of the value in each bin.
+     */
+    float getMaximumLevel() const override;
+
+    /**
+     * Set the maximum value of the value in a bin.
+     */
+    virtual void setMaximumLevel(float sz);
+
+    /**
+     * Get the set of bin values at the given column.
+     */
+    Column getColumn(int x) const override;
+
+    /**
+     * Get a single value, from the n'th bin of the given column.
+     */
+    float getValueAt(int x, int n) const override;
+
+    /**
+     * Set the entire set of bin values at the given column.
+     */
+    virtual void setColumn(int x, const Column &values);
+
+    /**
+     * Return the name of bin n. This is a single label per bin that
+     * does not vary from one column to the next.
+     */
+    QString getBinName(int n) const override;
+
+    /**
+     * Set the name of bin n.
+     */
+    virtual void setBinName(int n, QString);
+
+    /**
+     * Set the names of all bins.
+     */
+    virtual void setBinNames(std::vector<QString> names);
+
+    /**
+     * Return true if the bins have values as well as names. (The
+     * values may have been derived from the names, e.g. by parsing
+     * numbers from them.) If this returns true, getBinValue() may be
+     * used to retrieve the values.
+     */
+    bool hasBinValues() const override;
+
+    /**
+     * Return the value of bin n, if any. This is a "vertical scale"
+     * value which does not vary from one column to the next. This is
+     * only meaningful if hasBinValues() returns true.
+     */
+    float getBinValue(int n) const override;
+
+    /**
+     * Set the values of all bins (separate from their labels). These
+     * are "vertical scale" values which do not vary from one column
+     * to the next.
+     */
+    virtual void setBinValues(std::vector<float> values);
+
+    /**
+     * Obtain the name of the unit of the values returned from
+     * getBinValue(), if any.
+     */
+    QString getBinValueUnit() const override;
+
+    /**
+     * Set the name of the unit of the values return from
+     * getBinValue() if any.
+     */
+    virtual void setBinValueUnit(QString unit);
+
+    /**
+     * Return true if the distribution of values in the bins is such
+     * as to suggest a log scale (mapping to colour etc) may be better
+     * than a linear one.
+     */
+    bool shouldUseLogValueScale() const override;
+
+    QString getTypeName() const override { return tr("Editable Dense 3-D"); }
+
+    QString toDelimitedDataString(QString delimiter,
+                                  DataExportOptions options,
+                                  sv_frame_t startFrame,
+                                  sv_frame_t duration) const override;
+
+    void toXml(QTextStream &out,
+                       QString indent = "",
+                       QString extraAttributes = "") const override;
+
+protected:
+    typedef std::vector<Column> ValueMatrix;
+    ValueMatrix m_data;
+
+    // m_trunc is used for simple compression.  If at least the top N
+    // elements of column x (for N = some proportion of the column
+    // height) are equal to those of an earlier column x', then
+    // m_trunc[x] will contain x-x' and column x will be truncated so
+    // as to remove the duplicate elements.  If the equal elements are
+    // at the bottom, then m_trunc[x] will contain x'-x (a negative
+    // value).  If m_trunc[x] is 0 then the whole of column x is
+    // stored.
+    std::vector<signed char> m_trunc;
+    void truncateAndStore(int index, const Column & values);
+    Column expandAndRetrieve(int index) const;
+    Column rightHeight(const Column &c) const;
+
+    std::vector<QString> m_binNames;
+    std::vector<float> m_binValues;
+    QString m_binValueUnit;
+
+    sv_frame_t m_startFrame;
+    sv_samplerate_t m_sampleRate;
+    int m_resolution;
+    int m_yBinCount;
+    float m_minimum;
+    float m_maximum;
+    bool m_haveExtents;
+    bool m_notifyOnAdd;
+    sv_frame_t m_sinceLastNotifyMin;
+    sv_frame_t m_sinceLastNotifyMax;
+    int m_completion;
+
+    mutable QReadWriteLock m_lock;
+};
+
+#endif

--- a/data/model/Dense3DModelPeakCache.cpp	Mon Sep 09 10:25:16 2019 +0100
+++ b/data/model/Dense3DModelPeakCache.cpp	Tue Sep 10 16:34:47 2019 +0100
@@ -35,7 +35,6 @@
                   (source->getSampleRate(),
                    source->getResolution() * m_columnsPerPeak,
                    source->getHeight(),
-                   EditableDenseThreeDimensionalModel::NoCompression,
                    false));
 
     connect(source.get(), SIGNAL(modelChanged(ModelId)),

--- a/data/model/EditableDenseThreeDimensionalModel.cpp	Mon Sep 09 10:25:16 2019 +0100
+++ b/data/model/EditableDenseThreeDimensionalModel.cpp	Tue Sep 10 16:34:47 2019 +0100
@@ -19,8 +19,7 @@
 
 #include <QTextStream>
 #include <QStringList>
-#include <QReadLocker>
-#include <QWriteLocker>
+#include <QMutexLocker>
 
 #include <iostream>
 
@@ -34,13 +33,11 @@
 EditableDenseThreeDimensionalModel::EditableDenseThreeDimensionalModel(sv_samplerate_t sampleRate,
                                                                        int resolution,
                                                                        int yBinCount,
-                                                                       CompressionType compression,
                                                                        bool notifyOnAdd) :
     m_startFrame(0),
     m_sampleRate(sampleRate),
     m_resolution(resolution),
     m_yBinCount(yBinCount),
-    m_compression(compression),
     m_minimum(0.0),
     m_maximum(0.0),
     m_haveExtents(false),
@@ -145,218 +142,82 @@
 EditableDenseThreeDimensionalModel::Column
 EditableDenseThreeDimensionalModel::getColumn(int index) const
 {
-    QReadLocker locker(&m_lock);
-    if (in_range_for(m_data, index)) return expandAndRetrieve(index);
-    else return Column();
-}
-
-float
-EditableDenseThreeDimensionalModel::getValueAt(int index, int n) const
-{
-    Column c = getColumn(index);
-    if (in_range_for(c, n)) return c.at(n);
-    return m_minimum;
-}
-
-//static int given = 0, stored = 0;
-
-void
-EditableDenseThreeDimensionalModel::truncateAndStore(int index,
-                                                     const Column &values)
-{
-    assert(in_range_for(m_data, index));
-
-    //cout << "truncateAndStore(" << index << ", " << values.size() << ")" << endl;
-
-    // The default case is to store the entire column at m_data[index]
-    // and place 0 at m_trunc[index] to indicate that it has not been
-    // truncated.  We only do clever stuff if one of the clever-stuff
-    // tests works out.
-
-    m_trunc[index] = 0;
-    if (index == 0 ||
-        m_compression == NoCompression ||
-        int(values.size()) != m_yBinCount) {
-//        given += values.size();
-//        stored += values.size();
-        m_data[index] = values;
-        return;
+    QMutexLocker locker(&m_mutex);
+    if (!in_range_for(m_data, index)) {
+        return {};
     }
-
-    // Maximum distance between a column and the one we refer to as
-    // the source of its truncated values.  Limited by having to fit
-    // in a signed char, but in any case small values are usually
-    // better
-    static int maxdist = 6;
-
-    bool known = false; // do we know whether to truncate at top or bottom?
-    bool top = false;   // if we do know, will we truncate at top?
-
-    // If the previous column is not truncated, then it is the only
-    // candidate for comparison.  If it is truncated, then the column
-    // that it refers to is the only candidate.  Either way, we only
-    // have one possible column to compare against here, and we are
-    // being careful to ensure it is not a truncated one (to avoid
-    // doing more work recursively when uncompressing).
-    int tdist = 1;
-    int ptrunc = m_trunc[index-1];
-    if (ptrunc < 0) {
-        top = false;
-        known = true;
-        tdist = -ptrunc + 1;
-    } else if (ptrunc > 0) {
-        top = true;
-        known = true;
-        tdist = ptrunc + 1;
-    }
-
-    Column p = expandAndRetrieve(index - tdist);
-    int h = m_yBinCount;
-
-    if (int(p.size()) == h && tdist <= maxdist) {
-
-        int bcount = 0, tcount = 0;
-        if (!known || !top) {
-            // count how many identical values there are at the bottom
-            for (int i = 0; i < h; ++i) {
-                if (values.at(i) == p.at(i)) ++bcount;
-                else break;
-            }
-        }
-        if (!known || top) {
-            // count how many identical values there are at the top
-            for (int i = h; i > 0; --i) {
-                if (values.at(i-1) == p.at(i-1)) ++tcount;
-                else break;
-            }
-        }
-        if (!known) top = (tcount > bcount);
-
-        int limit = h / 4; // don't bother unless we have at least this many
-        if ((top ? tcount : bcount) > limit) {
-        
-            if (!top) {
-                // create a new column with h - bcount values from bcount up
-                Column tcol(h - bcount);
-//                given += values.size();
-//                stored += h - bcount;
-                for (int i = bcount; i < h; ++i) {
-                    tcol[i - bcount] = values.at(i);
-                }
-                m_data[index] = tcol;
-                m_trunc[index] = (signed char)(-tdist);
-                return;
-            } else {
-                // create a new column with h - tcount values from 0 up
-                Column tcol(h - tcount);
-//                given += values.size();
-//                stored += h - tcount;
-                for (int i = 0; i < h - tcount; ++i) {
-                    tcol[i] = values.at(i);
-                }
-                m_data[index] = tcol;
-                m_trunc[index] = (signed char)(tdist);
-                return;
-            }
-        }
-    }                
-
-//    given += values.size();
-//    stored += values.size();
-//    cout << "given: " << given << ", stored: " << stored << " (" 
-//              << ((float(stored) / float(given)) * 100.f) << "%)" << endl;
-
-    // default case if nothing wacky worked out
-    m_data[index] = values;
-    return;
-}
-
-EditableDenseThreeDimensionalModel::Column
-EditableDenseThreeDimensionalModel::rightHeight(const Column &c) const
-{
-    if (int(c.size()) == m_yBinCount) return c;
-    else {
+    Column c = m_data.at(index);
+    if (int(c.size()) == m_yBinCount) {
+        return c;
+    } else {
         Column cc(c);
         cc.resize(m_yBinCount, 0.0);
         return cc;
     }
 }
 
-EditableDenseThreeDimensionalModel::Column
-EditableDenseThreeDimensionalModel::expandAndRetrieve(int index) const
+float
+EditableDenseThreeDimensionalModel::getValueAt(int index, int n) const
 {
-    // See comment above m_trunc declaration in header
-
-    assert(index >= 0 && index < int(m_data.size()));
-    Column c = m_data.at(index);
-    if (index == 0) {
-        return rightHeight(c);
+    QMutexLocker locker(&m_mutex);
+    if (!in_range_for(m_data, index)) {
+        return m_minimum;
     }
-    int trunc = (int)m_trunc[index];
-    if (trunc == 0) {
-        return rightHeight(c);
+    const Column &c = m_data.at(index);
+    if (!in_range_for(c, n)) {
+        return m_minimum;
     }
-    bool top = true;
-    int tdist = trunc;
-    if (trunc < 0) { top = false; tdist = -trunc; }
-    Column p = expandAndRetrieve(index - tdist);
-    int psize = int(p.size()), csize = int(c.size());
-    if (psize != m_yBinCount) {
-        cerr << "WARNING: EditableDenseThreeDimensionalModel::expandAndRetrieve: Trying to expand from incorrectly sized column" << endl;
-    }
-    if (top) {
-        for (int i = csize; i < psize; ++i) {
-            c.push_back(p.at(i));
-        }
-    } else {
-        Column cc(psize);
-        for (int i = 0; i < psize - csize; ++i) {
-            cc[i] = p.at(i);
-        }
-        for (int i = 0; i < csize; ++i) {
-            cc[i + (psize - csize)] = c.at(i);
-        }
-        return cc;
-    }
-    return c;
+    return c.at(n);
 }
 
 void
 EditableDenseThreeDimensionalModel::setColumn(int index,
                                               const Column &values)
 {
-    QWriteLocker locker(&m_lock);
-
-    while (index >= int(m_data.size())) {
-        m_data.push_back(Column());
-        m_trunc.push_back(0);
-    }
-
     bool allChange = false;
-
-    for (int i = 0; in_range_for(values, i); ++i) {
-        float value = values[i];
-        if (ISNAN(value) || ISINF(value)) {
-            continue;
-        }
-        if (!m_haveExtents || value < m_minimum) {
-            m_minimum = value;
-            allChange = true;
-        }
-        if (!m_haveExtents || value > m_maximum) {
-            m_maximum = value;
-            allChange = true;
-        }
-        m_haveExtents = true;
-    }
-
-    truncateAndStore(index, values);
-
-//    assert(values == expandAndRetrieve(index));
-
     sv_frame_t windowStart = index;
     windowStart *= m_resolution;
 
+    {
+        QMutexLocker locker(&m_mutex);
+
+        while (index >= int(m_data.size())) {
+            m_data.push_back(Column());
+        }
+
+        for (int i = 0; in_range_for(values, i); ++i) {
+            float value = values[i];
+            if (ISNAN(value) || ISINF(value)) {
+                continue;
+            }
+            if (!m_haveExtents || value < m_minimum) {
+                m_minimum = value;
+                allChange = true;
+            }
+            if (!m_haveExtents || value > m_maximum) {
+                m_maximum = value;
+                allChange = true;
+            }
+            m_haveExtents = true;
+        }
+
+        m_data[index] = values;
+
+        if (allChange) {
+            m_sinceLastNotifyMin = -1;
+            m_sinceLastNotifyMax = -1;
+        } else {
+            if (m_sinceLastNotifyMin == -1 ||
+                windowStart < m_sinceLastNotifyMin) {
+                m_sinceLastNotifyMin = windowStart;
+            }
+            if (m_sinceLastNotifyMax == -1 ||
+                windowStart > m_sinceLastNotifyMax) {
+                m_sinceLastNotifyMax = windowStart;
+            }
+        }
+    }
+
     if (m_notifyOnAdd) {
         if (allChange) {
             emit modelChanged(getId());
@@ -366,18 +227,7 @@
         }
     } else {
         if (allChange) {
-            m_sinceLastNotifyMin = -1;
-            m_sinceLastNotifyMax = -1;
             emit modelChanged(getId());
-        } else {
-            if (m_sinceLastNotifyMin == -1 ||
-                windowStart < m_sinceLastNotifyMin) {
-                m_sinceLastNotifyMin = windowStart;
-            }
-            if (m_sinceLastNotifyMax == -1 ||
-                windowStart > m_sinceLastNotifyMax) {
-                m_sinceLastNotifyMax = windowStart;
-            }
         }
     }
 }
@@ -438,7 +288,7 @@
 bool
 EditableDenseThreeDimensionalModel::shouldUseLogValueScale() const
 {
-    QReadLocker locker(&m_lock);
+    QMutexLocker locker(&m_mutex);
 
     vector<double> sample;
     vector<int> n;
@@ -507,7 +357,7 @@
                                                           sv_frame_t startFrame,
                                                           sv_frame_t duration) const
 {
-    QReadLocker locker(&m_lock);
+    QMutexLocker locker(&m_mutex);
     QString s;
     for (int i = 0; in_range_for(m_data, i); ++i) {
         sv_frame_t fr = m_startFrame + i * m_resolution;
@@ -527,7 +377,7 @@
                                           QString indent,
                                           QString extraAttributes) const
 {
-    QReadLocker locker(&m_lock);
+    QMutexLocker locker(&m_mutex);
 
     // For historical reasons we read and write "resolution" as "windowSize".
 
@@ -552,7 +402,7 @@
     out << QString("<dataset id=\"%1\" dimensions=\"3\" separator=\" \">\n")
         .arg(getExportId());
 
-    for (int i = 0; i < (int)m_binNames.size(); ++i) {
+    for (int i = 0; in_range_for(m_binNames, i); ++i) {
         if (m_binNames[i] != "") {
             out << indent + "  ";
             out << QString("<bin number=\"%1\" name=\"%2\"/>\n")
@@ -560,12 +410,13 @@
         }
     }
 
-    for (int i = 0; i < (int)m_data.size(); ++i) {
+    for (int i = 0; in_range_for(m_data, i); ++i) {
+        Column c = getColumn(i);
         out << indent + "  ";
         out << QString("<row n=\"%1\">").arg(i);
-        for (int j = 0; j < (int)m_data.at(i).size(); ++j) {
+        for (int j = 0; in_range_for(c, j); ++j) {
             if (j > 0) out << " ";
-            out << m_data.at(i).at(j);
+            out << c.at(j);
         }
         out << QString("</row>\n");
         out.flush();

--- a/data/model/EditableDenseThreeDimensionalModel.h	Mon Sep 09 10:25:16 2019 +0100
+++ b/data/model/EditableDenseThreeDimensionalModel.h	Tue Sep 10 16:34:47 2019 +0100
@@ -18,7 +18,7 @@
 
 #include "DenseThreeDimensionalModel.h"
 
-#include <QReadWriteLock>
+#include <QMutex>
 
 #include <vector>
 
@@ -27,25 +27,9 @@
     Q_OBJECT
 
 public:
-
-    // EditableDenseThreeDimensionalModel supports a basic compression
-    // method that reduces the size of multirate data (e.g. wavelet
-    // transform outputs) that are stored as plain 3d grids by about
-    // 60% or thereabouts.  However, it can only be used for models
-    // whose columns are set in order from 0 and never subsequently
-    // changed.  If the model is going to be actually edited, it must
-    // have NoCompression.
-
-    enum CompressionType
-    {
-        NoCompression,
-        BasicMultirateCompression
-    };
-
     EditableDenseThreeDimensionalModel(sv_samplerate_t sampleRate,
                                        int resolution,
                                        int height,
-                                       CompressionType compression,
                                        bool notifyOnAdd = true);
 
     bool isOK() const override;
@@ -201,19 +185,6 @@
     typedef std::vector<Column> ValueMatrix;
     ValueMatrix m_data;
 
-    // m_trunc is used for simple compression.  If at least the top N
-    // elements of column x (for N = some proportion of the column
-    // height) are equal to those of an earlier column x', then
-    // m_trunc[x] will contain x-x' and column x will be truncated so
-    // as to remove the duplicate elements.  If the equal elements are
-    // at the bottom, then m_trunc[x] will contain x'-x (a negative
-    // value).  If m_trunc[x] is 0 then the whole of column x is
-    // stored.
-    std::vector<signed char> m_trunc;
-    void truncateAndStore(int index, const Column & values);
-    Column expandAndRetrieve(int index) const;
-    Column rightHeight(const Column &c) const;
-
     std::vector<QString> m_binNames;
     std::vector<float> m_binValues;
     QString m_binValueUnit;
@@ -222,7 +193,6 @@
     sv_samplerate_t m_sampleRate;
     int m_resolution;
     int m_yBinCount;
-    CompressionType m_compression;
     float m_minimum;
     float m_maximum;
     bool m_haveExtents;
@@ -231,7 +201,7 @@
     sv_frame_t m_sinceLastNotifyMax;
     int m_completion;
 
-    mutable QReadWriteLock m_lock;
+    mutable QMutex m_mutex;
 };
 
 #endif

--- a/files.pri	Mon Sep 09 10:25:16 2019 +0100
+++ b/files.pri	Tue Sep 10 16:34:47 2019 +0100
@@ -78,6 +78,7 @@
            data/midi/rtmidi/RtMidi.h \
            data/model/AggregateWaveModel.h \
            data/model/AlignmentModel.h \
+           data/model/BasicCompressedDenseThreeDimensionalModel.h \
            data/model/Dense3DModelPeakCache.h \
            data/model/DenseThreeDimensionalModel.h \
            data/model/DenseTimeValueModel.h \
@@ -209,6 +210,7 @@
            data/midi/rtmidi/RtMidi.cpp \
            data/model/AggregateWaveModel.cpp \
            data/model/AlignmentModel.cpp \
+           data/model/BasicCompressedDenseThreeDimensionalModel.cpp \
            data/model/Dense3DModelPeakCache.cpp \
            data/model/DenseTimeValueModel.cpp \
            data/model/EditableDenseThreeDimensionalModel.cpp \

--- a/rdf/RDFImporter.cpp	Mon Sep 09 10:25:16 2019 +0100
+++ b/rdf/RDFImporter.cpp	Tue Sep 10 16:34:47 2019 +0100
@@ -359,8 +359,7 @@
         } else {
 
             auto m = std::make_shared<EditableDenseThreeDimensionalModel>
-                (sampleRate, hopSize, height, 
-                 EditableDenseThreeDimensionalModel::NoCompression, false);
+                (sampleRate, hopSize, height, false);
             
             EditableDenseThreeDimensionalModel::Column column;
 

--- a/transform/FeatureExtractionModelTransformer.cpp	Mon Sep 09 10:25:16 2019 +0100
+++ b/transform/FeatureExtractionModelTransformer.cpp	Tue Sep 10 16:34:47 2019 +0100
@@ -25,7 +25,7 @@
 #include "base/Exceptions.h"
 #include "data/model/SparseOneDimensionalModel.h"
 #include "data/model/SparseTimeValueModel.h"
-#include "data/model/EditableDenseThreeDimensionalModel.h"
+#include "data/model/BasicCompressedDenseThreeDimensionalModel.h"
 #include "data/model/DenseTimeValueModel.h"
 #include "data/model/NoteModel.h"
 #include "data/model/RegionModel.h"
@@ -504,11 +504,9 @@
         // has a fixed sample rate and more than one value per result
         // must be a dense 3D model.
 
-        EditableDenseThreeDimensionalModel *model =
-            new EditableDenseThreeDimensionalModel
-            (modelRate, modelResolution, binCount,
-             EditableDenseThreeDimensionalModel::BasicMultirateCompression,
-             false);
+        auto model =
+            new BasicCompressedDenseThreeDimensionalModel
+            (modelRate, modelResolution, binCount, false);
 
         if (!m_descriptors[n].binNames.empty()) {
             std::vector<QString> names;
@@ -1148,10 +1146,10 @@
             }
         }
 
-    } else if (isOutputType<EditableDenseThreeDimensionalModel>(n)) {
+    } else if (isOutputType<BasicCompressedDenseThreeDimensionalModel>(n)) {
 
         auto model = ModelById::getAs
-            <EditableDenseThreeDimensionalModel>(outputId);
+            <BasicCompressedDenseThreeDimensionalModel>(outputId);
         if (!model) return;
         
         DenseThreeDimensionalModel::Column values = feature.values;
@@ -1181,6 +1179,6 @@
          setOutputCompletion<SparseTimeValueModel>(n, completion) ||
          setOutputCompletion<NoteModel>(n, completion) ||
          setOutputCompletion<RegionModel>(n, completion) ||
-         setOutputCompletion<EditableDenseThreeDimensionalModel>(n, completion));
+         setOutputCompletion<BasicCompressedDenseThreeDimensionalModel>(n, completion));
 }