Chris@148: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
Chris@148: 
Chris@148: /*
Chris@148:     Sonic Visualiser
Chris@148:     An audio file viewer and annotation editor.
Chris@148:     Centre for Digital Music, Queen Mary, University of London.
Chris@148:     This file copyright 2006 Chris Cannam.
Chris@148:     
Chris@148:     This program is free software; you can redistribute it and/or
Chris@148:     modify it under the terms of the GNU General Public License as
Chris@148:     published by the Free Software Foundation; either version 2 of the
Chris@148:     License, or (at your option) any later version.  See the file
Chris@148:     COPYING included with this distribution for more information.
Chris@148: */
Chris@148: 
Chris@148: #include "CSVFileReader.h"
Chris@148: 
Chris@150: #include "model/Model.h"
Chris@148: #include "base/RealTime.h"
Chris@148: #include "model/SparseOneDimensionalModel.h"
Chris@148: #include "model/SparseTimeValueModel.h"
Chris@152: #include "model/EditableDenseThreeDimensionalModel.h"
Chris@308: #include "DataFileReaderFactory.h"
Chris@148: 
Chris@148: #include <QFile>
Chris@148: #include <QString>
Chris@148: #include <QRegExp>
Chris@148: #include <QStringList>
Chris@148: #include <QTextStream>
Chris@148: 
Chris@148: #include <iostream>
Chris@148: 
Chris@392: CSVFileReader::CSVFileReader(QString path, CSVFormat format,
Chris@392:                              size_t mainModelSampleRate) :
Chris@392:     m_format(format),
Chris@148:     m_file(0),
Chris@148:     m_mainModelSampleRate(mainModelSampleRate)
Chris@148: {
Chris@148:     m_file = new QFile(path);
Chris@148:     bool good = false;
Chris@148:     
Chris@148:     if (!m_file->exists()) {
Chris@148: 	m_error = QFile::tr("File \"%1\" does not exist").arg(path);
Chris@148:     } else if (!m_file->open(QIODevice::ReadOnly | QIODevice::Text)) {
Chris@148: 	m_error = QFile::tr("Failed to open file \"%1\"").arg(path);
Chris@148:     } else {
Chris@148: 	good = true;
Chris@148:     }
Chris@148: 
Chris@148:     if (!good) {
Chris@148: 	delete m_file;
Chris@148: 	m_file = 0;
Chris@148:     }
Chris@148: }
Chris@148: 
Chris@148: CSVFileReader::~CSVFileReader()
Chris@148: {
Chris@148:     std::cerr << "CSVFileReader::~CSVFileReader: file is " << m_file << std::endl;
Chris@148: 
Chris@148:     if (m_file) {
Chris@148:         std::cerr << "CSVFileReader::CSVFileReader: Closing file" << std::endl;
Chris@148:         m_file->close();
Chris@148:     }
Chris@148:     delete m_file;
Chris@148: }
Chris@148: 
Chris@148: bool
Chris@148: CSVFileReader::isOK() const
Chris@148: {
Chris@148:     return (m_file != 0);
Chris@148: }
Chris@148: 
Chris@148: QString
Chris@148: CSVFileReader::getError() const
Chris@148: {
Chris@148:     return m_error;
Chris@148: }
Chris@148: 
Chris@148: Model *
Chris@148: CSVFileReader::load() const
Chris@148: {
Chris@148:     if (!m_file) return 0;
Chris@392: /*!!!
Chris@148:     CSVFormatDialog *dialog = new CSVFormatDialog
Chris@148: 	(0, m_file, m_mainModelSampleRate);
Chris@148: 
Chris@148:     if (dialog->exec() == QDialog::Rejected) {
Chris@148: 	delete dialog;
Chris@308:         throw DataFileReaderFactory::ImportCancelled;
Chris@148:     }
Chris@392: */
Chris@148: 
Chris@392:     CSVFormat::ModelType   modelType = m_format.getModelType();
Chris@392:     CSVFormat::TimingType timingType = m_format.getTimingType();
Chris@392:     CSVFormat::TimeUnits   timeUnits = m_format.getTimeUnits();
Chris@392:     QString separator = m_format.getSeparator();
Chris@392:     QString::SplitBehavior behaviour = m_format.getSplitBehaviour();
Chris@392:     size_t sampleRate = m_format.getSampleRate();
Chris@392:     size_t windowSize = m_format.getWindowSize();
Chris@148: 
Chris@392:     if (timingType == CSVFormat::ExplicitTiming) {
Chris@611:         if (modelType == CSVFormat::ThreeDimensionalModel) {
Chris@611:             // This will be overridden later if more than one line
Chris@611:             // appears in our file, but we want to choose a default
Chris@611:             // that's likely to be visible
Chris@611:             windowSize = 1024;
Chris@611:         } else {
Chris@611:             windowSize = 1;
Chris@611:         }
Chris@392: 	if (timeUnits == CSVFormat::TimeSeconds) {
Chris@148: 	    sampleRate = m_mainModelSampleRate;
Chris@148: 	}
Chris@148:     }
Chris@148: 
Chris@148:     SparseOneDimensionalModel *model1 = 0;
Chris@148:     SparseTimeValueModel *model2 = 0;
Chris@152:     EditableDenseThreeDimensionalModel *model3 = 0;
Chris@148:     Model *model = 0;
Chris@148: 
Chris@148:     QTextStream in(m_file);
Chris@148:     in.seek(0);
Chris@148: 
Chris@148:     unsigned int warnings = 0, warnLimit = 10;
Chris@148:     unsigned int lineno = 0;
Chris@148: 
Chris@148:     float min = 0.0, max = 0.0;
Chris@148: 
Chris@148:     size_t frameNo = 0;
Chris@611:     size_t startFrame = 0; // for calculation of dense model resolution
Chris@148: 
Chris@148:     while (!in.atEnd()) {
Chris@148: 
Chris@283:         // QTextStream's readLine doesn't cope with old-style Mac
Chris@283:         // CR-only line endings.  Why did they bother making the class
Chris@283:         // cope with more than one sort of line ending, if it still
Chris@283:         // can't be configured to cope with all the common sorts?
Chris@148: 
Chris@283:         // For the time being we'll deal with this case (which is
Chris@283:         // relatively uncommon for us, but still necessary to handle)
Chris@283:         // by reading the entire file using a single readLine, and
Chris@283:         // splitting it.  For CR and CR/LF line endings this will just
Chris@283:         // read a line at a time, and that's obviously OK.
Chris@148: 
Chris@283:         QString chunk = in.readLine();
Chris@283:         QStringList lines = chunk.split('\r', QString::SkipEmptyParts);
Chris@283:         
Chris@283:         for (size_t li = 0; li < lines.size(); ++li) {
Chris@148: 
Chris@283:             QString line = lines[li];
Chris@148: 
Chris@283:             if (line.startsWith("#")) continue;
Chris@283: 
Chris@390:             QStringList list = line.split(separator, behaviour);
Chris@283: 
Chris@283:             if (!model) {
Chris@283: 
Chris@283:                 switch (modelType) {
Chris@283: 
Chris@392:                 case CSVFormat::OneDimensionalModel:
Chris@283:                     model1 = new SparseOneDimensionalModel(sampleRate, windowSize);
Chris@283:                     model = model1;
Chris@283:                     break;
Chris@148: 		
Chris@392:                 case CSVFormat::TwoDimensionalModel:
Chris@283:                     model2 = new SparseTimeValueModel(sampleRate, windowSize, false);
Chris@283:                     model = model2;
Chris@283:                     break;
Chris@148: 		
Chris@392:                 case CSVFormat::ThreeDimensionalModel:
Chris@535:                     model3 = new EditableDenseThreeDimensionalModel
Chris@535:                         (sampleRate,
Chris@535:                          windowSize,
Chris@535:                          list.size(),
Chris@535:                          EditableDenseThreeDimensionalModel::NoCompression);
Chris@283:                     model = model3;
Chris@283:                     break;
Chris@283:                 }
Chris@283:             }
Chris@148: 
Chris@283:             QStringList tidyList;
Chris@390:             QRegExp nonNumericRx("[^0-9eE.,+-]");
Chris@148: 
Chris@283:             for (int i = 0; i < list.size(); ++i) {
Chris@148: 	    
Chris@283:                 QString s(list[i].trimmed());
Chris@148: 
Chris@283:                 if (s.length() >= 2 && s.startsWith("\"") && s.endsWith("\"")) {
Chris@283:                     s = s.mid(1, s.length() - 2);
Chris@283:                 } else if (s.length() >= 2 && s.startsWith("'") && s.endsWith("'")) {
Chris@283:                     s = s.mid(1, s.length() - 2);
Chris@283:                 }
Chris@148: 
Chris@392:                 if (i == 0 && timingType == CSVFormat::ExplicitTiming) {
Chris@148: 
Chris@283:                     bool ok = false;
Chris@283:                     QString numeric = s;
Chris@283:                     numeric.remove(nonNumericRx);
Chris@148: 
Chris@392:                     if (timeUnits == CSVFormat::TimeSeconds) {
Chris@148: 
Chris@283:                         double time = numeric.toDouble(&ok);
Chris@491:                         frameNo = int(time * sampleRate + 0.5);
Chris@148: 
Chris@283:                     } else {
Chris@148: 
Chris@283:                         frameNo = numeric.toInt(&ok);
Chris@148: 
Chris@392:                         if (timeUnits == CSVFormat::TimeWindows) {
Chris@283:                             frameNo *= windowSize;
Chris@283:                         }
Chris@283:                     }
Chris@148: 			       
Chris@283:                     if (!ok) {
Chris@283:                         if (warnings < warnLimit) {
Chris@283:                             std::cerr << "WARNING: CSVFileReader::load: "
Chris@283:                                       << "Bad time format (\"" << s.toStdString()
Chris@283:                                       << "\") in data line "
Chris@491:                                       << lineno+1 << ":" << std::endl;
Chris@283:                             std::cerr << line.toStdString() << std::endl;
Chris@283:                         } else if (warnings == warnLimit) {
Chris@283:                             std::cerr << "WARNING: Too many warnings" << std::endl;
Chris@283:                         }
Chris@283:                         ++warnings;
Chris@283:                     }
Chris@283:                 } else {
Chris@283:                     tidyList.push_back(s);
Chris@283:                 }
Chris@283:             }
Chris@148: 
Chris@392:             if (modelType == CSVFormat::OneDimensionalModel) {
Chris@148: 	    
Chris@283:                 SparseOneDimensionalModel::Point point
Chris@283:                     (frameNo,
Chris@283:                      tidyList.size() > 0 ? tidyList[tidyList.size()-1] :
Chris@491:                      QString("%1").arg(lineno+1));
Chris@148: 
Chris@283:                 model1->addPoint(point);
Chris@148: 
Chris@392:             } else if (modelType == CSVFormat::TwoDimensionalModel) {
Chris@148: 
Chris@283:                 SparseTimeValueModel::Point point
Chris@283:                     (frameNo,
Chris@283:                      tidyList.size() > 0 ? tidyList[0].toFloat() : 0.0,
Chris@491:                      tidyList.size() > 1 ? tidyList[1] : QString("%1").arg(lineno+1));
Chris@148: 
Chris@283:                 model2->addPoint(point);
Chris@148: 
Chris@392:             } else if (modelType == CSVFormat::ThreeDimensionalModel) {
Chris@148: 
Chris@283:                 DenseThreeDimensionalModel::Column values;
Chris@148: 
Chris@283:                 for (int i = 0; i < tidyList.size(); ++i) {
Chris@148: 
Chris@283:                     bool ok = false;
Chris@283:                     float value = list[i].toFloat(&ok);
Chris@611: 
Chris@611:                     if (i > 0 || timingType != CSVFormat::ExplicitTiming) {
Chris@611:                         values.push_back(value);
Chris@611:                     }
Chris@148: 	    
Chris@611:                     bool firstEver = (lineno == 0 && i == 0);
Chris@611: 
Chris@611:                     if (firstEver || value < min) min = value;
Chris@611:                     if (firstEver || value > max) max = value;
Chris@611: 
Chris@611:                     if (firstEver) {
Chris@611:                         startFrame = frameNo;
Chris@611:                         model3->setStartFrame(startFrame);
Chris@611:                     } else if (lineno == 1 &&
Chris@611:                                timingType == CSVFormat::ExplicitTiming) {
Chris@611:                         model3->setResolution(frameNo - startFrame);
Chris@611:                     }
Chris@148: 
Chris@283:                     if (!ok) {
Chris@283:                         if (warnings < warnLimit) {
Chris@283:                             std::cerr << "WARNING: CSVFileReader::load: "
Chris@390:                                       << "Non-numeric value \""
Chris@390:                                       << list[i].toStdString()
Chris@491:                                       << "\" in data line " << lineno+1
Chris@283:                                       << ":" << std::endl;
Chris@283:                             std::cerr << line.toStdString() << std::endl;
Chris@283:                             ++warnings;
Chris@283:                         } else if (warnings == warnLimit) {
Chris@390: //                            std::cerr << "WARNING: Too many warnings" << std::endl;
Chris@283:                         }
Chris@283:                     }
Chris@283:                 }
Chris@148: 	
Chris@390: //                std::cerr << "Setting bin values for count " << lineno << ", frame "
Chris@390: //                          << frameNo << ", time " << RealTime::frame2RealTime(frameNo, sampleRate) << std::endl;
Chris@148: 
Chris@611:                 model3->setColumn(lineno, values);
Chris@283:             }
Chris@148: 
Chris@283:             ++lineno;
Chris@392:             if (timingType == CSVFormat::ImplicitTiming ||
Chris@283:                 list.size() == 0) {
Chris@283:                 frameNo += windowSize;
Chris@283:             }
Chris@283:         }
Chris@148:     }
Chris@148: 
Chris@392:     if (modelType == CSVFormat::ThreeDimensionalModel) {
Chris@148: 	model3->setMinimumLevel(min);
Chris@148: 	model3->setMaximumLevel(max);
Chris@148:     }
Chris@148: 
Chris@148:     return model;
Chris@148: }
Chris@148: