# HG changeset patch # User Chris Cannam # Date 1433938472 -3600 # Node ID 7fb788f891d0ec9f852f34e414a3d2f088b21a2a # Parent b6092700a73ea770fcad75b1669ea800f6ac2aa1# Parent c1b522cfd1ff934c1781ed051685fbe1a6143dca Merge diff -r c1b522cfd1ff -r 7fb788f891d0 base/ResizeableBitset.h --- a/base/ResizeableBitset.h Fri Jun 05 10:00:26 2015 +0100 +++ b/base/ResizeableBitset.h Wed Jun 10 13:14:32 2015 +0100 @@ -62,7 +62,10 @@ } void set(size_t column) { - ((*m_bits)[column >> 3]) |= uint8_t((1u << (column & 0x07)) & 0xff); + size_t ix = (column >> 3); + uint8_t prior = (*m_bits)[ix]; + uint8_t extra = ((1u << (column & 0x07)) & 0xff); + (*m_bits)[ix] = uint8_t(prior | extra); } void reset(size_t column) { diff -r c1b522cfd1ff -r 7fb788f891d0 data/fft/FFTDataServer.cpp --- a/data/fft/FFTDataServer.cpp Fri Jun 05 10:00:26 2015 +0100 +++ b/data/fft/FFTDataServer.cpp Wed Jun 10 13:14:32 2015 +0100 @@ -902,6 +902,7 @@ if (!cache) return 0; if (!cache->haveSetColumnAt(col)) { + if (getError() != "") return false; Profiler profiler("FFTDataServer::getMagnitudeAt: filling"); #ifdef DEBUG_FFT_SERVER std::cerr << "FFTDataServer::getMagnitudeAt: calling fillColumn(" @@ -938,6 +939,7 @@ if (!cache) return false; if (!cache->haveSetColumnAt(col)) { + if (getError() != "") return false; Profiler profiler("FFTDataServer::getMagnitudesAt: filling"); fillColumn(x); } @@ -968,6 +970,7 @@ if (!cache) return 0; if (!cache->haveSetColumnAt(col)) { + if (getError() != "") return false; Profiler profiler("FFTDataServer::getNormalizedMagnitudeAt: filling"); fillColumn(x); } @@ -1000,6 +1003,7 @@ if (!cache) return false; if (!cache->haveSetColumnAt(col)) { + if (getError() != "") return false; Profiler profiler("FFTDataServer::getNormalizedMagnitudesAt: filling"); fillColumn(x); } @@ -1032,6 +1036,7 @@ if (!cache) return 0; if (!cache->haveSetColumnAt(col)) { + if (getError() != "") return false; Profiler profiler("FFTDataServer::getMaximumMagnitudeAt: filling"); fillColumn(x); } @@ -1060,6 +1065,7 @@ if (!cache) return 0; if (!cache->haveSetColumnAt(col)) { + if (getError() != "") return false; Profiler profiler("FFTDataServer::getPhaseAt: filling"); fillColumn(x); } @@ -1092,6 +1098,7 @@ if (!cache) return false; if (!cache->haveSetColumnAt(col)) { + if (getError() != "") return false; Profiler profiler("FFTDataServer::getPhasesAt: filling"); fillColumn(x); } @@ -1131,6 +1138,11 @@ } if (!cache->haveSetColumnAt(col)) { + if (getError() != "") { + real = 0; + imaginary = 0; + return; + } Profiler profiler("FFTDataServer::getValuesAt: filling"); #ifdef DEBUG_FFT_SERVER std::cerr << "FFTDataServer::getValuesAt(" << x << ", " << y << "): filling" << std::endl; @@ -1165,6 +1177,7 @@ if (!cache) return false; if (!cache->haveSetColumnAt(col)) { + if (getError() != "") return false; Profiler profiler("FFTDataServer::getValuesAt: filling"); fillColumn(x); } @@ -1395,9 +1408,19 @@ QString FFTDataServer::getError() const { - if (m_error != "") return m_error; - else if (m_fillThread) return m_fillThread->getError(); - else return ""; + QString err; + if (m_error != "") { + err = m_error; + cerr << "FFTDataServer::getError: err (server " << this << ") = " << err << endl; + } else { + MutexLocker locker(&m_fftBuffersLock, "FFTDataServer::getError"); + if (m_fillThread) { + err = m_fillThread->getError(); + cerr << "FFTDataServer::getError: err (server " << this << ", from thread " << m_fillThread + << ") = " << err << endl; + } + } + return err; } int @@ -1475,8 +1498,10 @@ try { m_server.fillColumn(int((f - start) / m_server.m_windowIncrement)); } catch (std::exception &e) { - std::cerr << "FFTDataServer::FillThread::run: exception: " << e.what() << std::endl; - m_error = e.what(); + MutexLocker locker(&m_server.m_fftBuffersLock, + "FFTDataServer::run::m_fftBuffersLock [err]"); + m_threadError = e.what(); + std::cerr << "FFTDataServer::FillThread::run: exception: " << m_threadError << " (thread = " << this << " from server " << &m_server << ")" << std::endl; m_server.fillComplete(); m_completion = 100; m_extent = end; @@ -1524,8 +1549,10 @@ try { m_server.fillColumn(int((f - start) / m_server.m_windowIncrement)); } catch (std::exception &e) { - std::cerr << "FFTDataServer::FillThread::run: exception: " << e.what() << std::endl; - m_error = e.what(); + MutexLocker locker(&m_server.m_fftBuffersLock, + "FFTDataServer::run::m_fftBuffersLock [err]"); + m_threadError = e.what(); + std::cerr << "FFTDataServer::FillThread::run: exception: " << m_threadError << " (thread = " << this << " from server " << &m_server << ")" << std::endl; m_server.fillComplete(); m_completion = 100; m_extent = end; diff -r c1b522cfd1ff -r 7fb788f891d0 data/fft/FFTDataServer.h --- a/data/fft/FFTDataServer.h Fri Jun 05 10:00:26 2015 +0100 +++ b/data/fft/FFTDataServer.h Wed Jun 10 13:14:32 2015 +0100 @@ -70,13 +70,13 @@ const DenseTimeValueModel *getModel() const { return m_model; } int getChannel() const { return m_channel; } WindowType getWindowType() const { return m_windower.getType(); } - int getWindowSize() const { return m_windowSize; } - int getWindowIncrement() const { return m_windowIncrement; } - int getFFTSize() const { return m_fftSize; } + int getWindowSize() const { return m_windowSize; } + int getWindowIncrement() const { return m_windowIncrement; } + int getFFTSize() const { return m_fftSize; } bool getPolar() const { return m_polar; } - int getWidth() const { return m_width; } - int getHeight() const { return m_height; } + int getWidth() const { return m_width; } + int getHeight() const { return m_height; } float getMagnitudeAt(int x, int y); float getNormalizedMagnitudeAt(int x, int y); @@ -196,6 +196,7 @@ return 0; } m_cacheVectorLock.unlock(); + if (getError() != "") return 0; if (!makeCache(c)) return 0; return getCacheReader(x, col); } @@ -230,7 +231,7 @@ void getStorageAdvice(int w, int h, bool &memory, bool &compact); - QMutex m_fftBuffersLock; + mutable QMutex m_fftBuffersLock; QWaitCondition m_condition; fftsample *m_fftInput; @@ -247,7 +248,7 @@ sv_frame_t getExtent() const { return m_extent; } int getCompletion() const { return m_completion ? m_completion : 1; } - QString getError() const { return m_error; } + QString getError() const { return m_threadError; } virtual void run(); protected: @@ -255,7 +256,7 @@ sv_frame_t m_extent; int m_completion; sv_frame_t m_fillFrom; - QString m_error; + QString m_threadError; }; bool m_exiting; diff -r c1b522cfd1ff -r 7fb788f891d0 data/fft/FFTFileCacheReader.cpp --- a/data/fft/FFTFileCacheReader.cpp Fri Jun 05 10:00:26 2015 +0100 +++ b/data/fft/FFTFileCacheReader.cpp Wed Jun 10 13:14:32 2015 +0100 @@ -24,6 +24,8 @@ #include +//#define DEBUG_FFT_FILE_CACHE_READER 1 + // The underlying matrix has height (m_height * 2 + 1). In each // column we store magnitude at [0], [2] etc and phase at [1], [3] @@ -44,7 +46,9 @@ writer->getWidth(), writer->getHeight() * 2 + m_factorSize)) { -// cerr << "FFTFileCacheReader: storage type is " << (storageType == FFTCache::Compact ? "Compact" : storageType == Polar ? "Polar" : "Rectangular") << endl; +#ifdef DEBUG_FFT_FILE_CACHE_READER + cerr << "FFTFileCacheReader: storage type is " << (m_storageType == FFTCache::Compact ? "Compact" : m_storageType == FFTCache::Polar ? "Polar" : "Rectangular") << endl; +#endif } FFTFileCacheReader::~FFTFileCacheReader() diff -r c1b522cfd1ff -r 7fb788f891d0 data/fileio/MatrixFile.cpp --- a/data/fileio/MatrixFile.cpp Fri Jun 05 10:00:26 2015 +0100 +++ b/data/fileio/MatrixFile.cpp Wed Jun 10 13:14:32 2015 +0100 @@ -89,6 +89,16 @@ throw FileOperationFailed(fileName, "create"); } + // Use floating-point here to avoid integer overflow. We can be + // approximate so long as we are on the cautious side + if ((double(m_width) * m_height) * m_cellSize + m_headerSize + m_width >= + pow(2, 31) - 10.0) { // bit of slack there + cerr << "ERROR: MatrixFile::MatrixFile: width " << m_width + << " is too large for height " << m_height << " and cell size " + << m_cellSize << " (should be using multiple files)" << endl; + throw FileOperationFailed(fileName, "size"); + } + m_flags = 0; m_fmode = S_IRUSR | S_IWUSR; @@ -203,7 +213,7 @@ off_t off = m_headerSize + (m_width * m_height * m_cellSize) + m_width; #ifdef DEBUG_MATRIX_FILE - cerr << "MatrixFile[" << m_fd << "]::initialise(" << m_width << ", " << m_height << "): cell size " << m_cellSize << ", header size " << m_headerSize << ", resizing file" << endl; + cerr << "MatrixFile[" << m_fd << "]::initialise(" << m_width << ", " << m_height << "): cell size " << m_cellSize << ", header size " << m_headerSize << ", resizing fd " << m_fd << " to " << off << endl; #endif if (::lseek(m_fd, off - 1, SEEK_SET) < 0) { diff -r c1b522cfd1ff -r 7fb788f891d0 data/model/FFTModel.h --- a/data/model/FFTModel.h Fri Jun 05 10:00:26 2015 +0100 +++ b/data/model/FFTModel.h Wed Jun 10 13:14:32 2015 +0100 @@ -118,6 +118,9 @@ return const_cast(this)->getMagnitudeAt(x, y); } virtual bool isOK() const { + // Return true if the model was constructed successfully (not + // necessarily whether an error has occurred since + // construction, use getError for that) return m_server && m_server->getModel(); } virtual sv_frame_t getStartFrame() const { diff -r c1b522cfd1ff -r 7fb788f891d0 transform/FeatureExtractionModelTransformer.cpp --- a/transform/FeatureExtractionModelTransformer.cpp Fri Jun 05 10:00:26 2015 +0100 +++ b/transform/FeatureExtractionModelTransformer.cpp Wed Jun 10 13:14:32 2015 +0100 @@ -44,7 +44,7 @@ ModelTransformer(in, transform), m_plugin(0) { -// SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: plugin " << pluginId << ", outputName " << m_transform.getOutput() << endl; + SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: plugin " << m_transforms.begin()->getPluginIdentifier() << ", outputName " << m_transforms.begin()->getOutput() << endl; initialise(); } @@ -54,8 +54,12 @@ ModelTransformer(in, transforms), m_plugin(0) { -// SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: plugin " << pluginId << ", outputName " << m_transform.getOutput() << endl; - + if (m_transforms.empty()) { + SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: " << transforms.size() << " transform(s)" << endl; + } else { + SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: " << transforms.size() << " transform(s), first has plugin " << m_transforms.begin()->getPluginIdentifier() << ", outputName " << m_transforms.begin()->getOutput() << endl; + } + initialise(); } @@ -605,16 +609,18 @@ blockSize, false, StorageAdviser::PrecisionCritical); - if (!model->isOK()) { + if (!model->isOK() || model->getError() != "") { + QString err = model->getError(); delete model; for (int j = 0; j < (int)m_outputNos.size(); ++j) { setCompletion(j, 100); } //!!! need a better way to handle this -- previously we were using a QMessageBox but that isn't an appropriate thing to do here either - throw AllocationFailed("Failed to create the FFT model for this feature extraction model transformer"); + throw AllocationFailed("Failed to create the FFT model for this feature extraction model transformer: error is: " + err); } model->resume(); fftModels.push_back(model); + cerr << "created model for channel " << ch << endl; } } @@ -697,6 +703,7 @@ cerr << "FeatureExtractionModelTransformer::run: Abandoning, error is " << error << endl; m_abandoned = true; m_message = error; + break; } } } else { diff -r c1b522cfd1ff -r 7fb788f891d0 transform/ModelTransformer.h --- a/transform/ModelTransformer.h Fri Jun 05 10:00:26 2015 +0100 +++ b/transform/ModelTransformer.h Wed Jun 10 13:14:32 2015 +0100 @@ -68,6 +68,12 @@ void abandon() { m_abandoned = true; } /** + * Return true if the processing thread is being or has been + * abandoned, i.e. if abandon() has been called. + */ + bool isAbandoned() const { return m_abandoned; } + + /** * Return the input model for the transform. */ Model *getInputModel() { return m_input.getModel(); } diff -r c1b522cfd1ff -r 7fb788f891d0 transform/ModelTransformerFactory.cpp --- a/transform/ModelTransformerFactory.cpp Fri Jun 05 10:00:26 2015 +0100 +++ b/transform/ModelTransformerFactory.cpp Wed Jun 10 13:14:32 2015 +0100 @@ -285,6 +285,12 @@ m_handlers.erase(transformer); } + if (transformer->isAbandoned()) { + if (transformer->getMessage() != "") { + emit transformFailed("", transformer->getMessage()); + } + } + transformer->wait(); // unnecessary but reassuring delete transformer; } diff -r c1b522cfd1ff -r 7fb788f891d0 transform/ModelTransformerFactory.h --- a/transform/ModelTransformerFactory.h Fri Jun 05 10:00:26 2015 +0100 +++ b/transform/ModelTransformerFactory.h Wed Jun 10 13:14:32 2015 +0100 @@ -145,6 +145,9 @@ QString &message, AdditionalModelHandler *handler = 0); +signals: + void transformFailed(QString transformName, QString message); + protected slots: void transformerFinished();