annotate data/fileio/CSVFileReader.cpp @ 390:21e79997e80f

* Fix failure to handle scientific notation for time field; tidy up some
  warnings and handling of empty fields

/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */

/*
    Sonic Visualiser
    An audio file viewer and annotation editor.
    Centre for Digital Music, Queen Mary, University of London.
    This file copyright 2006 Chris Cannam.
    
    This program is free software; you can redistribute it and/or
    modify it under the terms of the GNU General Public License as
    published by the Free Software Foundation; either version 2 of the
    License, or (at your option) any later version.  See the file
    COPYING included with this distribution for more information.
*/

#include "CSVFileReader.h"

#include "model/Model.h"
#include "base/RealTime.h"
#include "model/SparseOneDimensionalModel.h"
#include "model/SparseTimeValueModel.h"
#include "model/EditableDenseThreeDimensionalModel.h"
#include "DataFileReaderFactory.h"

#include <QFile>
#include <QString>
#include <QRegExp>
#include <QStringList>
#include <QTextStream>
#include <QFrame>
#include <QGridLayout>
#include <QPushButton>
#include <QHBoxLayout>
#include <QVBoxLayout>
#include <QTableWidget>
#include <QComboBox>
#include <QLabel>

#include <iostream>

CSVFileReader::CSVFileReader(QString path, size_t mainModelSampleRate) :
    m_file(0),
    m_mainModelSampleRate(mainModelSampleRate)
{
    m_file = new QFile(path);
    bool good = false;
    
    if (!m_file->exists()) {
	m_error = QFile::tr("File \"%1\" does not exist").arg(path);
    } else if (!m_file->open(QIODevice::ReadOnly | QIODevice::Text)) {
	m_error = QFile::tr("Failed to open file \"%1\"").arg(path);
    } else {
	good = true;
    }

    if (!good) {
	delete m_file;
	m_file = 0;
    }
}

CSVFileReader::~CSVFileReader()
{
    std::cerr << "CSVFileReader::~CSVFileReader: file is " << m_file << std::endl;

    if (m_file) {
        std::cerr << "CSVFileReader::CSVFileReader: Closing file" << std::endl;
        m_file->close();
    }
    delete m_file;
}

bool
CSVFileReader::isOK() const
{
    return (m_file != 0);
}

QString
CSVFileReader::getError() const
{
    return m_error;
}

Model *
CSVFileReader::load() const
{
    if (!m_file) return 0;

    CSVFormatDialog *dialog = new CSVFormatDialog
	(0, m_file, m_mainModelSampleRate);

    if (dialog->exec() == QDialog::Rejected) {
	delete dialog;
        throw DataFileReaderFactory::ImportCancelled;
    }

    CSVFormatDialog::ModelType   modelType = dialog->getModelType();
    CSVFormatDialog::TimingType timingType = dialog->getTimingType();
    CSVFormatDialog::TimeUnits   timeUnits = dialog->getTimeUnits();
    QString separator = dialog->getSeparator();
    QString::SplitBehavior behaviour = dialog->getSplitBehaviour();
    size_t sampleRate = dialog->getSampleRate();
    size_t windowSize = dialog->getWindowSize();

    delete dialog;

    if (timingType == CSVFormatDialog::ExplicitTiming) {
	windowSize = 1;
	if (timeUnits == CSVFormatDialog::TimeSeconds) {
	    sampleRate = m_mainModelSampleRate;
	}
    }

    SparseOneDimensionalModel *model1 = 0;
    SparseTimeValueModel *model2 = 0;
    EditableDenseThreeDimensionalModel *model3 = 0;
    Model *model = 0;

    QTextStream in(m_file);
    in.seek(0);

    unsigned int warnings = 0, warnLimit = 10;
    unsigned int lineno = 0;

    float min = 0.0, max = 0.0;

    size_t frameNo = 0;

    while (!in.atEnd()) {

        // QTextStream's readLine doesn't cope with old-style Mac
        // CR-only line endings.  Why did they bother making the class
        // cope with more than one sort of line ending, if it still
        // can't be configured to cope with all the common sorts?

        // For the time being we'll deal with this case (which is
        // relatively uncommon for us, but still necessary to handle)
        // by reading the entire file using a single readLine, and
        // splitting it.  For CR and CR/LF line endings this will just
        // read a line at a time, and that's obviously OK.

        QString chunk = in.readLine();
        QStringList lines = chunk.split('\r', QString::SkipEmptyParts);
        
        for (size_t li = 0; li < lines.size(); ++li) {

            QString line = lines[li];

            if (line.startsWith("#")) continue;

            QStringList list = line.split(separator, behaviour);

            if (!model) {

                switch (modelType) {

                case CSVFormatDialog::OneDimensionalModel:
                    model1 = new SparseOneDimensionalModel(sampleRate, windowSize);
                    model = model1;
                    break;
		
                case CSVFormatDialog::TwoDimensionalModel:
                    model2 = new SparseTimeValueModel(sampleRate, windowSize, false);
                    model = model2;
                    break;
		
                case CSVFormatDialog::ThreeDimensionalModel:
                    model3 = new EditableDenseThreeDimensionalModel(sampleRate,
                                                                    windowSize,
                                                                    list.size());
                    model = model3;
                    break;
                }
            }

            QStringList tidyList;
            QRegExp nonNumericRx("[^0-9eE.,+-]");

            for (int i = 0; i < list.size(); ++i) {
	    
                QString s(list[i].trimmed());

                if (s.length() >= 2 && s.startsWith("\"") && s.endsWith("\"")) {
                    s = s.mid(1, s.length() - 2);
                } else if (s.length() >= 2 && s.startsWith("'") && s.endsWith("'")) {
                    s = s.mid(1, s.length() - 2);
                }

                if (i == 0 && timingType == CSVFormatDialog::ExplicitTiming) {

                    bool ok = false;
                    QString numeric = s;
                    numeric.remove(nonNumericRx);

                    if (timeUnits == CSVFormatDialog::TimeSeconds) {

                        double time = numeric.toDouble(&ok);
                        frameNo = int(time * sampleRate + 0.00001);

                    } else {

                        frameNo = numeric.toInt(&ok);

                        if (timeUnits == CSVFormatDialog::TimeWindows) {
                            frameNo *= windowSize;
                        }
                    }
			       
                    if (!ok) {
                        if (warnings < warnLimit) {
                            std::cerr << "WARNING: CSVFileReader::load: "
                                      << "Bad time format (\"" << s.toStdString()
                                      << "\") in data line "
                                      << lineno << ":" << std::endl;
                            std::cerr << line.toStdString() << std::endl;
                        } else if (warnings == warnLimit) {
                            std::cerr << "WARNING: Too many warnings" << std::endl;
                        }
                        ++warnings;
                    }
                } else {
                    tidyList.push_back(s);
                }
            }

            if (modelType == CSVFormatDialog::OneDimensionalModel) {
	    
                SparseOneDimensionalModel::Point point
                    (frameNo,
                     tidyList.size() > 0 ? tidyList[tidyList.size()-1] :
                     QString("%1").arg(lineno));

                model1->addPoint(point);

            } else if (modelType == CSVFormatDialog::TwoDimensionalModel) {

                SparseTimeValueModel::Point point
                    (frameNo,
                     tidyList.size() > 0 ? tidyList[0].toFloat() : 0.0,
                     tidyList.size() > 1 ? tidyList[1] : QString("%1").arg(lineno));

                model2->addPoint(point);

            } else if (modelType == CSVFormatDialog::ThreeDimensionalModel) {

                DenseThreeDimensionalModel::Column values;

                for (int i = 0; i < tidyList.size(); ++i) {

                    bool ok = false;
                    float value = list[i].toFloat(&ok);
                    values.push_back(value);
	    
                    if ((lineno == 0 && i == 0) || value < min) min = value;
                    if ((lineno == 0 && i == 0) || value > max) max = value;

                    if (!ok) {
                        if (warnings < warnLimit) {
                            std::cerr << "WARNING: CSVFileReader::load: "
                                      << "Non-numeric value \""
                                      << list[i].toStdString()
                                      << "\" in data line " << lineno
                                      << ":" << std::endl;
                            std::cerr << line.toStdString() << std::endl;
                            ++warnings;
                        } else if (warnings == warnLimit) {
//                            std::cerr << "WARNING: Too many warnings" << std::endl;
                        }
                    }
                }
	
//                std::cerr << "Setting bin values for count " << lineno << ", frame "
//                          << frameNo << ", time " << RealTime::frame2RealTime(frameNo, sampleRate) << std::endl;

                model3->setColumn(frameNo / model3->getResolution(), values);
            }

            ++lineno;
            if (timingType == CSVFormatDialog::ImplicitTiming ||
                list.size() == 0) {
                frameNo += windowSize;
            }
        }
    }

    if (modelType == CSVFormatDialog::ThreeDimensionalModel) {
	model3->setMinimumLevel(min);
	model3->setMaximumLevel(max);
    }

    return model;
}


CSVFormatDialog::CSVFormatDialog(QWidget *parent, QFile *file,
				 size_t defaultSampleRate) :
    QDialog(parent),
    m_modelType(OneDimensionalModel),
    m_timingType(ExplicitTiming),
    m_timeUnits(TimeAudioFrames),
    m_separator(""),
    m_behaviour(QString::KeepEmptyParts)
{
    setModal(true);
    setWindowTitle(tr("Select Data Format"));

    (void)guessFormat(file);

    QGridLayout *layout = new QGridLayout;

    layout->addWidget(new QLabel(tr("<b>Select Data Format</b><p>Please select the correct data format for this file.")),
		      0, 0, 1, 4);

    layout->addWidget(new QLabel(tr("Each row specifies:")), 1, 0);

    m_modelTypeCombo = new QComboBox;
    m_modelTypeCombo->addItem(tr("A point in time"));
    m_modelTypeCombo->addItem(tr("A value at a time"));
    m_modelTypeCombo->addItem(tr("A set of values"));
    layout->addWidget(m_modelTypeCombo, 1, 1, 1, 2);
    connect(m_modelTypeCombo, SIGNAL(activated(int)),
	    this, SLOT(modelTypeChanged(int)));
    m_modelTypeCombo->setCurrentIndex(int(m_modelType));

    layout->addWidget(new QLabel(tr("The first column contains:")), 2, 0);
    
    m_timingTypeCombo = new QComboBox;
    m_timingTypeCombo->addItem(tr("Time, in seconds"));
    m_timingTypeCombo->addItem(tr("Time, in audio sample frames"));
    m_timingTypeCombo->addItem(tr("Data (rows are consecutive in time)"));
    layout->addWidget(m_timingTypeCombo, 2, 1, 1, 2);
    connect(m_timingTypeCombo, SIGNAL(activated(int)),
	    this, SLOT(timingTypeChanged(int)));
    m_timingTypeCombo->setCurrentIndex(m_timingType == ExplicitTiming ?
                                       m_timeUnits == TimeSeconds ? 0 : 1 : 2);

    m_sampleRateLabel = new QLabel(tr("Audio sample rate (Hz):"));
    layout->addWidget(m_sampleRateLabel, 3, 0);
    
    size_t sampleRates[] = {
	8000, 11025, 12000, 22050, 24000, 32000,
	44100, 48000, 88200, 96000, 176400, 192000
    };

    m_sampleRateCombo = new QComboBox;
    m_sampleRate = defaultSampleRate;
    for (size_t i = 0; i < sizeof(sampleRates) / sizeof(sampleRates[0]); ++i) {
	m_sampleRateCombo->addItem(QString("%1").arg(sampleRates[i]));
	if (sampleRates[i] == m_sampleRate) m_sampleRateCombo->setCurrentIndex(i);
    }
    m_sampleRateCombo->setEditable(true);

    layout->addWidget(m_sampleRateCombo, 3, 1);
    connect(m_sampleRateCombo, SIGNAL(activated(QString)),
	    this, SLOT(sampleRateChanged(QString)));
    connect(m_sampleRateCombo, SIGNAL(editTextChanged(QString)),
	    this, SLOT(sampleRateChanged(QString)));

    m_windowSizeLabel = new QLabel(tr("Frame increment between rows:"));
    layout->addWidget(m_windowSizeLabel, 4, 0);

    m_windowSizeCombo = new QComboBox;
    m_windowSize = 1024;
    for (int i = 0; i <= 16; ++i) {
	int value = 1 << i;
	m_windowSizeCombo->addItem(QString("%1").arg(value));
	if (value == int(m_windowSize)) m_windowSizeCombo->setCurrentIndex(i);
    }
    m_windowSizeCombo->setEditable(true);

    layout->addWidget(m_windowSizeCombo, 4, 1);
    connect(m_windowSizeCombo, SIGNAL(activated(QString)),
	    this, SLOT(windowSizeChanged(QString)));
    connect(m_windowSizeCombo, SIGNAL(editTextChanged(QString)),
	    this, SLOT(windowSizeChanged(QString)));

    layout->addWidget(new QLabel(tr("\nExample data from file:")), 5, 0, 1, 4);

    m_exampleWidget = new QTableWidget
	(std::min(10, m_example.size()), m_maxExampleCols);

    layout->addWidget(m_exampleWidget, 6, 0, 1, 4);
    layout->setColumnStretch(3, 10);
    layout->setRowStretch(4, 10);

    QPushButton *ok = new QPushButton(tr("OK"));
    connect(ok, SIGNAL(clicked()), this, SLOT(accept()));
    ok->setDefault(true);

    QPushButton *cancel = new QPushButton(tr("Cancel"));
    connect(cancel, SIGNAL(clicked()), this, SLOT(reject()));

    QHBoxLayout *buttonLayout = new QHBoxLayout;
    buttonLayout->addStretch(1);
    buttonLayout->addWidget(ok);
    buttonLayout->addWidget(cancel);

    QVBoxLayout *mainLayout = new QVBoxLayout;
    mainLayout->addLayout(layout);
    mainLayout->addLayout(buttonLayout);

    setLayout(mainLayout);
    
    timingTypeChanged(m_timingTypeCombo->currentIndex());
}

CSVFormatDialog::~CSVFormatDialog()
{
}

void
CSVFormatDialog::populateExample()
{
    m_exampleWidget->setColumnCount
	(m_timingType == ExplicitTiming ?
	 m_maxExampleCols - 1 : m_maxExampleCols);

    m_exampleWidget->setHorizontalHeaderLabels(QStringList());

    for (int i = 0; i < m_example.size(); ++i) {
	for (int j = 0; j < m_example[i].size(); ++j) {

	    QTableWidgetItem *item = new QTableWidgetItem(m_example[i][j]);

	    if (j == 0) {
		if (m_timingType == ExplicitTiming) {
		    m_exampleWidget->setVerticalHeaderItem(i, item);
		    continue;
		} else {
		    QTableWidgetItem *header =
			new QTableWidgetItem(QString("%1").arg(i));
		    header->setFlags(Qt::ItemIsEnabled);
		    m_exampleWidget->setVerticalHeaderItem(i, header);
		}
	    }
	    int index = j;
	    if (m_timingType == ExplicitTiming) --index;
	    item->setFlags(Qt::ItemIsEnabled);
	    m_exampleWidget->setItem(i, index, item);
	}
    }
}

void
CSVFormatDialog::modelTypeChanged(int type)
{
    m_modelType = (ModelType)type;

    if (m_modelType == ThreeDimensionalModel) {
        // We can't load 3d models with explicit timing, because the 3d
        // model is dense so we need a fixed sample increment
        m_timingTypeCombo->setCurrentIndex(2);
        timingTypeChanged(2);
    }
}

void
CSVFormatDialog::timingTypeChanged(int type)
{
    switch (type) {

    case 0:
	m_timingType = ExplicitTiming;
	m_timeUnits = TimeSeconds;
	m_sampleRateCombo->setEnabled(false);
	m_sampleRateLabel->setEnabled(false);
	m_windowSizeCombo->setEnabled(false);
	m_windowSizeLabel->setEnabled(false);
        if (m_modelType == ThreeDimensionalModel) {
            m_modelTypeCombo->setCurrentIndex(1);
            modelTypeChanged(1);
        }
	break;

    case 1:
	m_timingType = ExplicitTiming;
	m_timeUnits = TimeAudioFrames;
	m_sampleRateCombo->setEnabled(true);
	m_sampleRateLabel->setEnabled(true);
	m_windowSizeCombo->setEnabled(false);
	m_windowSizeLabel->setEnabled(false);
        if (m_modelType == ThreeDimensionalModel) {
            m_modelTypeCombo->setCurrentIndex(1);
            modelTypeChanged(1);
        }
	break;

    case 2:
	m_timingType = ImplicitTiming;
	m_timeUnits = TimeWindows;
	m_sampleRateCombo->setEnabled(true);
	m_sampleRateLabel->setEnabled(true);
	m_windowSizeCombo->setEnabled(true);
	m_windowSizeLabel->setEnabled(true);
	break;
    }

    populateExample();
}

void
CSVFormatDialog::sampleRateChanged(QString rateString)
{
    bool ok = false;
    int sampleRate = rateString.toInt(&ok);
    if (ok) m_sampleRate = sampleRate;
}

void
CSVFormatDialog::windowSizeChanged(QString sizeString)
{
    bool ok = false;
    int size = sizeString.toInt(&ok);
    if (ok) m_windowSize = size;
}

bool
CSVFormatDialog::guessFormat(QFile *file)
{
    QTextStream in(file);
    in.seek(0);

    unsigned int lineno = 0;

    bool nonIncreasingPrimaries = false;
    bool nonNumericPrimaries = false;
    bool floatPrimaries = false;
    bool variableItemCount = false;
    int itemCount = 1;
    int earliestNonNumericItem = -1;

    float prevPrimary = 0.0;

    m_maxExampleCols = 0;

    while (!in.atEnd()) {

        // See comment about line endings in load() above

        QString chunk = in.readLine();
        QStringList lines = chunk.split('\r', QString::SkipEmptyParts);

        for (size_t li = 0; li < lines.size(); ++li) {

            QString line = lines[li];

            if (line.startsWith("#")) continue;

            m_behaviour = QString::KeepEmptyParts;

            if (m_separator == "") {
                //!!! to do: ask the user
                if (line.split(",").size() >= 2) m_separator = ",";
                else if (line.split("\t").size() >= 2) m_separator = "\t";
                else if (line.split("|").size() >= 2) m_separator = "|";
                else if (line.split("/").size() >= 2) m_separator = "/";
                else if (line.split(":").size() >= 2) m_separator = ":";
                else {
                    m_separator = " ";
                    m_behaviour = QString::SkipEmptyParts;
                }
            }

            QStringList list = line.split(m_separator, m_behaviour);
            QStringList tidyList;

            for (int i = 0; i < list.size(); ++i) {
	    
                QString s(list[i]);
                bool numeric = false;

                if (s.length() >= 2 && s.startsWith("\"") && s.endsWith("\"")) {
                    s = s.mid(1, s.length() - 2);
                } else if (s.length() >= 2 && s.startsWith("'") && s.endsWith("'")) {
                    s = s.mid(1, s.length() - 2);
                } else {
                    (void)s.toFloat(&numeric);
                }

                tidyList.push_back(s);

                if (lineno == 0 || (list.size() < itemCount)) {
                    itemCount = list.size();
                } else {
                    if (itemCount != list.size()) {
                        variableItemCount = true;
                    }
                }
	    
                if (i == 0) { // primary

                    if (numeric) {

                        float primary = s.toFloat();

                        if (lineno > 0 && primary <= prevPrimary) {
                            nonIncreasingPrimaries = true;
                        }

                        if (s.contains(".") || s.contains(",")) {
                            floatPrimaries = true;
                        }

                        prevPrimary = primary;

                    } else {
                        nonNumericPrimaries = true;
                    }
                } else { // secondary

                    if (!numeric) {
                        if (earliestNonNumericItem < 0 ||
                            i < earliestNonNumericItem) {
                            earliestNonNumericItem = i;
                        }
                    }
                }
            }

            if (lineno < 10) {
                m_example.push_back(tidyList);
                if (lineno == 0 || tidyList.size() > m_maxExampleCols) {
                    m_maxExampleCols = tidyList.size();
                }
            }

            ++lineno;

            if (lineno == 50) break;
        }
    }

    if (nonNumericPrimaries || nonIncreasingPrimaries) {
	
	// Primaries are probably not a series of times

	m_timingType = ImplicitTiming;
	m_timeUnits = TimeWindows;
	
	if (nonNumericPrimaries) {
	    m_modelType = OneDimensionalModel;
	} else if (itemCount == 1 || variableItemCount ||
		   (earliestNonNumericItem != -1)) {
	    m_modelType = TwoDimensionalModel;
	} else {
	    m_modelType = ThreeDimensionalModel;
	}

    } else {

	// Increasing numeric primaries -- likely to be time

	m_timingType = ExplicitTiming;

	if (floatPrimaries) {
	    m_timeUnits = TimeSeconds;
	} else {
	    m_timeUnits = TimeAudioFrames;
	}

	if (itemCount == 1) {
	    m_modelType = OneDimensionalModel;
	} else if (variableItemCount || (earliestNonNumericItem != -1)) {
	    if (earliestNonNumericItem != -1 && earliestNonNumericItem < 2) {
		m_modelType = OneDimensionalModel;
	    } else {
		m_modelType = TwoDimensionalModel;
	    }
	} else {
	    m_modelType = ThreeDimensionalModel;
	}
    }

    std::cerr << "Estimated model type: " << m_modelType << std::endl;
    std::cerr << "Estimated timing type: " << m_timingType << std::endl;
    std::cerr << "Estimated units: " << m_timeUnits << std::endl;

    in.seek(0);
    return true;
}