Chris@148: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@148:
Chris@148: /*
Chris@148: Sonic Visualiser
Chris@148: An audio file viewer and annotation editor.
Chris@148: Centre for Digital Music, Queen Mary, University of London.
Chris@148: This file copyright 2006 Chris Cannam.
Chris@148:
Chris@148: This program is free software; you can redistribute it and/or
Chris@148: modify it under the terms of the GNU General Public License as
Chris@148: published by the Free Software Foundation; either version 2 of the
Chris@148: License, or (at your option) any later version. See the file
Chris@148: COPYING included with this distribution for more information.
Chris@148: */
Chris@148:
Chris@148: #include "CSVFileReader.h"
Chris@148:
Chris@150: #include "model/Model.h"
Chris@148: #include "base/RealTime.h"
Chris@148: #include "model/SparseOneDimensionalModel.h"
Chris@148: #include "model/SparseTimeValueModel.h"
Chris@152: #include "model/EditableDenseThreeDimensionalModel.h"
Chris@308: #include "DataFileReaderFactory.h"
Chris@148:
Chris@148: #include <QFile>
Chris@148: #include <QString>
Chris@148: #include <QRegExp>
Chris@148: #include <QStringList>
Chris@148: #include <QTextStream>
Chris@148:
Chris@148: #include <iostream>
Chris@148:
Chris@392: CSVFileReader::CSVFileReader(QString path, CSVFormat format,
Chris@392: size_t mainModelSampleRate) :
Chris@392: m_format(format),
Chris@148: m_file(0),
Chris@148: m_mainModelSampleRate(mainModelSampleRate)
Chris@148: {
Chris@148: m_file = new QFile(path);
Chris@148: bool good = false;
Chris@148:
Chris@148: if (!m_file->exists()) {
Chris@148: m_error = QFile::tr("File \"%1\" does not exist").arg(path);
Chris@148: } else if (!m_file->open(QIODevice::ReadOnly | QIODevice::Text)) {
Chris@148: m_error = QFile::tr("Failed to open file \"%1\"").arg(path);
Chris@148: } else {
Chris@148: good = true;
Chris@148: }
Chris@148:
Chris@148: if (!good) {
Chris@148: delete m_file;
Chris@148: m_file = 0;
Chris@148: }
Chris@148: }
Chris@148:
Chris@148: CSVFileReader::~CSVFileReader()
Chris@148: {
Chris@148: std::cerr << "CSVFileReader::~CSVFileReader: file is " << m_file << std::endl;
Chris@148:
Chris@148: if (m_file) {
Chris@148: std::cerr << "CSVFileReader::CSVFileReader: Closing file" << std::endl;
Chris@148: m_file->close();
Chris@148: }
Chris@148: delete m_file;
Chris@148: }
Chris@148:
Chris@148: bool
Chris@148: CSVFileReader::isOK() const
Chris@148: {
Chris@148: return (m_file != 0);
Chris@148: }
Chris@148:
Chris@148: QString
Chris@148: CSVFileReader::getError() const
Chris@148: {
Chris@148: return m_error;
Chris@148: }
Chris@148:
Chris@148: Model *
Chris@148: CSVFileReader::load() const
Chris@148: {
Chris@148: if (!m_file) return 0;
Chris@392: /*!!!
Chris@148: CSVFormatDialog *dialog = new CSVFormatDialog
Chris@148: (0, m_file, m_mainModelSampleRate);
Chris@148:
Chris@148: if (dialog->exec() == QDialog::Rejected) {
Chris@148: delete dialog;
Chris@308: throw DataFileReaderFactory::ImportCancelled;
Chris@148: }
Chris@392: */
Chris@148:
Chris@392: CSVFormat::ModelType modelType = m_format.getModelType();
Chris@392: CSVFormat::TimingType timingType = m_format.getTimingType();
Chris@392: CSVFormat::TimeUnits timeUnits = m_format.getTimeUnits();
Chris@392: QString separator = m_format.getSeparator();
Chris@392: QString::SplitBehavior behaviour = m_format.getSplitBehaviour();
Chris@392: size_t sampleRate = m_format.getSampleRate();
Chris@392: size_t windowSize = m_format.getWindowSize();
Chris@148:
Chris@392: if (timingType == CSVFormat::ExplicitTiming) {
Chris@148: windowSize = 1;
Chris@392: if (timeUnits == CSVFormat::TimeSeconds) {
Chris@148: sampleRate = m_mainModelSampleRate;
Chris@148: }
Chris@148: }
Chris@148:
Chris@148: SparseOneDimensionalModel *model1 = 0;
Chris@148: SparseTimeValueModel *model2 = 0;
Chris@152: EditableDenseThreeDimensionalModel *model3 = 0;
Chris@148: Model *model = 0;
Chris@148:
Chris@148: QTextStream in(m_file);
Chris@148: in.seek(0);
Chris@148:
Chris@148: unsigned int warnings = 0, warnLimit = 10;
Chris@148: unsigned int lineno = 0;
Chris@148:
Chris@148: float min = 0.0, max = 0.0;
Chris@148:
Chris@148: size_t frameNo = 0;
Chris@148:
Chris@148: while (!in.atEnd()) {
Chris@148:
Chris@283: // QTextStream's readLine doesn't cope with old-style Mac
Chris@283: // CR-only line endings. Why did they bother making the class
Chris@283: // cope with more than one sort of line ending, if it still
Chris@283: // can't be configured to cope with all the common sorts?
Chris@148:
Chris@283: // For the time being we'll deal with this case (which is
Chris@283: // relatively uncommon for us, but still necessary to handle)
Chris@283: // by reading the entire file using a single readLine, and
Chris@283: // splitting it. For CR and CR/LF line endings this will just
Chris@283: // read a line at a time, and that's obviously OK.
Chris@148:
Chris@283: QString chunk = in.readLine();
Chris@283: QStringList lines = chunk.split('\r', QString::SkipEmptyParts);
Chris@283:
Chris@283: for (size_t li = 0; li < lines.size(); ++li) {
Chris@148:
Chris@283: QString line = lines[li];
Chris@148:
Chris@283: if (line.startsWith("#")) continue;
Chris@283:
Chris@390: QStringList list = line.split(separator, behaviour);
Chris@283:
Chris@283: if (!model) {
Chris@283:
Chris@283: switch (modelType) {
Chris@283:
Chris@392: case CSVFormat::OneDimensionalModel:
Chris@283: model1 = new SparseOneDimensionalModel(sampleRate, windowSize);
Chris@283: model = model1;
Chris@283: break;
Chris@148:
Chris@392: case CSVFormat::TwoDimensionalModel:
Chris@283: model2 = new SparseTimeValueModel(sampleRate, windowSize, false);
Chris@283: model = model2;
Chris@283: break;
Chris@148:
Chris@392: case CSVFormat::ThreeDimensionalModel:
Chris@535: model3 = new EditableDenseThreeDimensionalModel
Chris@535: (sampleRate,
Chris@535: windowSize,
Chris@535: list.size(),
Chris@535: EditableDenseThreeDimensionalModel::NoCompression);
Chris@283: model = model3;
Chris@283: break;
Chris@283: }
Chris@283: }
Chris@148:
Chris@283: QStringList tidyList;
Chris@390: QRegExp nonNumericRx("[^0-9eE.,+-]");
Chris@148:
Chris@283: for (int i = 0; i < list.size(); ++i) {
Chris@148:
Chris@283: QString s(list[i].trimmed());
Chris@148:
Chris@283: if (s.length() >= 2 && s.startsWith("\"") && s.endsWith("\"")) {
Chris@283: s = s.mid(1, s.length() - 2);
Chris@283: } else if (s.length() >= 2 && s.startsWith("'") && s.endsWith("'")) {
Chris@283: s = s.mid(1, s.length() - 2);
Chris@283: }
Chris@148:
Chris@392: if (i == 0 && timingType == CSVFormat::ExplicitTiming) {
Chris@148:
Chris@283: bool ok = false;
Chris@283: QString numeric = s;
Chris@283: numeric.remove(nonNumericRx);
Chris@148:
Chris@392: if (timeUnits == CSVFormat::TimeSeconds) {
Chris@148:
Chris@283: double time = numeric.toDouble(&ok);
Chris@491: frameNo = int(time * sampleRate + 0.5);
Chris@148:
Chris@283: } else {
Chris@148:
Chris@283: frameNo = numeric.toInt(&ok);
Chris@148:
Chris@392: if (timeUnits == CSVFormat::TimeWindows) {
Chris@283: frameNo *= windowSize;
Chris@283: }
Chris@283: }
Chris@148:
Chris@283: if (!ok) {
Chris@283: if (warnings < warnLimit) {
Chris@283: std::cerr << "WARNING: CSVFileReader::load: "
Chris@283: << "Bad time format (\"" << s.toStdString()
Chris@283: << "\") in data line "
Chris@491: << lineno+1 << ":" << std::endl;
Chris@283: std::cerr << line.toStdString() << std::endl;
Chris@283: } else if (warnings == warnLimit) {
Chris@283: std::cerr << "WARNING: Too many warnings" << std::endl;
Chris@283: }
Chris@283: ++warnings;
Chris@283: }
Chris@283: } else {
Chris@283: tidyList.push_back(s);
Chris@283: }
Chris@283: }
Chris@148:
Chris@392: if (modelType == CSVFormat::OneDimensionalModel) {
Chris@148:
Chris@283: SparseOneDimensionalModel::Point point
Chris@283: (frameNo,
Chris@283: tidyList.size() > 0 ? tidyList[tidyList.size()-1] :
Chris@491: QString("%1").arg(lineno+1));
Chris@148:
Chris@283: model1->addPoint(point);
Chris@148:
Chris@392: } else if (modelType == CSVFormat::TwoDimensionalModel) {
Chris@148:
Chris@283: SparseTimeValueModel::Point point
Chris@283: (frameNo,
Chris@283: tidyList.size() > 0 ? tidyList[0].toFloat() : 0.0,
Chris@491: tidyList.size() > 1 ? tidyList[1] : QString("%1").arg(lineno+1));
Chris@148:
Chris@283: model2->addPoint(point);
Chris@148:
Chris@392: } else if (modelType == CSVFormat::ThreeDimensionalModel) {
Chris@148:
Chris@283: DenseThreeDimensionalModel::Column values;
Chris@148:
Chris@283: for (int i = 0; i < tidyList.size(); ++i) {
Chris@148:
Chris@283: bool ok = false;
Chris@283: float value = list[i].toFloat(&ok);
Chris@283: values.push_back(value);
Chris@148:
Chris@283: if ((lineno == 0 && i == 0) || value < min) min = value;
Chris@283: if ((lineno == 0 && i == 0) || value > max) max = value;
Chris@148:
Chris@283: if (!ok) {
Chris@283: if (warnings < warnLimit) {
Chris@283: std::cerr << "WARNING: CSVFileReader::load: "
Chris@390: << "Non-numeric value \""
Chris@390: << list[i].toStdString()
Chris@491: << "\" in data line " << lineno+1
Chris@283: << ":" << std::endl;
Chris@283: std::cerr << line.toStdString() << std::endl;
Chris@283: ++warnings;
Chris@283: } else if (warnings == warnLimit) {
Chris@390: // std::cerr << "WARNING: Too many warnings" << std::endl;
Chris@283: }
Chris@283: }
Chris@283: }
Chris@148:
Chris@390: // std::cerr << "Setting bin values for count " << lineno << ", frame "
Chris@390: // << frameNo << ", time " << RealTime::frame2RealTime(frameNo, sampleRate) << std::endl;
Chris@148:
Chris@283: model3->setColumn(frameNo / model3->getResolution(), values);
Chris@283: }
Chris@148:
Chris@283: ++lineno;
Chris@392: if (timingType == CSVFormat::ImplicitTiming ||
Chris@283: list.size() == 0) {
Chris@283: frameNo += windowSize;
Chris@283: }
Chris@283: }
Chris@148: }
Chris@148:
Chris@392: if (modelType == CSVFormat::ThreeDimensionalModel) {
Chris@148: model3->setMinimumLevel(min);
Chris@148: model3->setMaximumLevel(max);
Chris@148: }
Chris@148:
Chris@148: return model;
Chris@148: }
Chris@148: