annotate data/fileio/CSVFormat.cpp @ 1346:75ad55315db4 3.0-integration

More work on getting tests (especially file encoding ones) running on Windows. Various problems here to do with interaction with test filenames in Hg repos
author Chris Cannam
date Fri, 06 Jan 2017 15:44:55 +0000
parents 1888ca033a84
children 1bf38a4b91c4
rev   line source
Chris@392 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@392 2
Chris@392 3 /*
Chris@392 4 Sonic Visualiser
Chris@392 5 An audio file viewer and annotation editor.
Chris@392 6 Centre for Digital Music, Queen Mary, University of London.
Chris@392 7 This file copyright 2006 Chris Cannam.
Chris@392 8
Chris@392 9 This program is free software; you can redistribute it and/or
Chris@392 10 modify it under the terms of the GNU General Public License as
Chris@392 11 published by the Free Software Foundation; either version 2 of the
Chris@392 12 License, or (at your option) any later version. See the file
Chris@392 13 COPYING included with this distribution for more information.
Chris@392 14 */
Chris@392 15
Chris@392 16 #include "CSVFormat.h"
Chris@392 17
Chris@629 18 #include "base/StringBits.h"
Chris@629 19
Chris@392 20 #include <QFile>
Chris@392 21 #include <QString>
Chris@392 22 #include <QRegExp>
Chris@392 23 #include <QStringList>
Chris@392 24 #include <QTextStream>
Chris@392 25
Chris@392 26 #include <iostream>
Chris@392 27
Chris@629 28 CSVFormat::CSVFormat(QString path) :
Chris@629 29 m_separator(""),
Chris@392 30 m_sampleRate(44100),
Chris@392 31 m_windowSize(1024),
Chris@629 32 m_allowQuoting(true)
Chris@392 33 {
Chris@629 34 guessFormatFor(path);
Chris@629 35 }
Chris@629 36
Chris@629 37 void
Chris@629 38 CSVFormat::guessFormatFor(QString path)
Chris@629 39 {
Chris@629 40 m_modelType = TwoDimensionalModel;
Chris@629 41 m_timingType = ExplicitTiming;
Chris@629 42 m_timeUnits = TimeSeconds;
Chris@629 43
Chris@629 44 m_maxExampleCols = 0;
Chris@629 45 m_columnCount = 0;
Chris@629 46 m_variableColumnCount = false;
Chris@629 47
Chris@629 48 m_example.clear();
Chris@629 49 m_columnQualities.clear();
Chris@629 50 m_columnPurposes.clear();
Chris@629 51 m_prevValues.clear();
Chris@629 52
Chris@629 53 QFile file(path);
Chris@392 54 if (!file.exists()) return;
Chris@392 55 if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) return;
Chris@392 56
Chris@392 57 QTextStream in(&file);
Chris@392 58 in.seek(0);
Chris@392 59
Chris@629 60 int lineno = 0;
Chris@392 61
Chris@392 62 while (!in.atEnd()) {
Chris@392 63
Chris@392 64 // See comment about line endings in CSVFileReader::load()
Chris@392 65
Chris@392 66 QString chunk = in.readLine();
Chris@392 67 QStringList lines = chunk.split('\r', QString::SkipEmptyParts);
Chris@392 68
Chris@897 69 for (int li = 0; li < lines.size(); ++li) {
Chris@392 70
Chris@392 71 QString line = lines[li];
Chris@629 72 if (line.startsWith("#") || line == "") continue;
Chris@392 73
Chris@629 74 guessQualities(line, lineno);
Chris@392 75
Chris@840 76 ++lineno;
Chris@629 77 }
Chris@840 78
Chris@840 79 if (lineno >= 50) break;
Chris@629 80 }
Chris@392 81
Chris@629 82 guessPurposes();
Chris@629 83 }
Chris@629 84
Chris@629 85 void
Chris@629 86 CSVFormat::guessSeparator(QString line)
Chris@629 87 {
Chris@629 88 char candidates[] = { ',', '\t', ' ', '|', '/', ':' };
Chris@897 89 for (int i = 0; i < int(sizeof(candidates)/sizeof(candidates[0])); ++i) {
Chris@629 90 if (StringBits::split(line, candidates[i], m_allowQuoting).size() >= 2) {
Chris@629 91 m_separator = candidates[i];
Chris@629 92 return;
Chris@629 93 }
Chris@629 94 }
Chris@629 95 m_separator = " ";
Chris@629 96 }
Chris@629 97
Chris@629 98 void
Chris@629 99 CSVFormat::guessQualities(QString line, int lineno)
Chris@629 100 {
Chris@629 101 if (m_separator == "") guessSeparator(line);
Chris@629 102
Chris@629 103 QStringList list = StringBits::split(line, m_separator[0], m_allowQuoting);
Chris@629 104
Chris@629 105 int cols = list.size();
Chris@991 106 if (lineno == 0 || (cols > m_columnCount)) m_columnCount = cols;
Chris@629 107 if (cols != m_columnCount) m_variableColumnCount = true;
Chris@629 108
Chris@629 109 // All columns are regarded as having these qualities until we see
Chris@629 110 // something that indicates otherwise:
Chris@629 111
Chris@629 112 ColumnQualities defaultQualities =
Chris@1021 113 ColumnNumeric | ColumnIntegral | ColumnIncreasing | ColumnNearEmpty;
Chris@629 114
Chris@629 115 for (int i = 0; i < cols; ++i) {
Chris@629 116
Chris@629 117 while (m_columnQualities.size() <= i) {
Chris@629 118 m_columnQualities.push_back(defaultQualities);
Chris@629 119 m_prevValues.push_back(0.f);
Chris@629 120 }
Chris@629 121
Chris@629 122 QString s(list[i]);
Chris@629 123 bool ok = false;
Chris@629 124
Chris@629 125 ColumnQualities qualities = m_columnQualities[i];
Chris@629 126
Chris@629 127 bool numeric = (qualities & ColumnNumeric);
Chris@629 128 bool integral = (qualities & ColumnIntegral);
Chris@629 129 bool increasing = (qualities & ColumnIncreasing);
Chris@629 130 bool large = (qualities & ColumnLarge); // this one defaults to off
Chris@1021 131 bool emptyish = (qualities & ColumnNearEmpty);
Chris@629 132
Chris@1021 133 if (lineno > 1 && s.trimmed() != "") {
Chris@1021 134 emptyish = false;
Chris@1021 135 }
Chris@1021 136
Chris@629 137 float value = 0.f;
Chris@629 138
Chris@629 139 //!!! how to take into account headers?
Chris@629 140
Chris@629 141 if (numeric) {
Chris@629 142 value = s.toFloat(&ok);
Chris@629 143 if (!ok) {
Chris@629 144 value = (float)StringBits::stringToDoubleLocaleFree(s, &ok);
Chris@629 145 }
Chris@629 146 if (ok) {
Chris@629 147 if (lineno < 2 && value > 1000.f) large = true;
Chris@629 148 } else {
Chris@629 149 numeric = false;
Chris@629 150 }
Chris@629 151 }
Chris@629 152
Chris@629 153 if (numeric) {
Chris@629 154
Chris@629 155 if (integral) {
Chris@629 156 if (s.contains('.') || s.contains(',')) {
Chris@629 157 integral = false;
Chris@392 158 }
Chris@392 159 }
Chris@392 160
Chris@629 161 if (increasing) {
Chris@629 162 if (lineno > 0 && value <= m_prevValues[i]) {
Chris@629 163 increasing = false;
Chris@392 164 }
Chris@392 165 }
Chris@392 166
Chris@629 167 m_prevValues[i] = value;
Chris@629 168 }
Chris@392 169
Chris@629 170 m_columnQualities[i] =
Chris@629 171 (numeric ? ColumnNumeric : 0) |
Chris@629 172 (integral ? ColumnIntegral : 0) |
Chris@629 173 (increasing ? ColumnIncreasing : 0) |
Chris@1021 174 (large ? ColumnLarge : 0) |
Chris@1021 175 (emptyish ? ColumnNearEmpty : 0);
Chris@629 176 }
Chris@392 177
Chris@629 178 if (lineno < 10) {
Chris@629 179 m_example.push_back(list);
Chris@629 180 if (lineno == 0 || cols > m_maxExampleCols) {
Chris@629 181 m_maxExampleCols = cols;
Chris@392 182 }
Chris@392 183 }
Chris@392 184
Chris@843 185 // cerr << "Estimated column qualities: ";
Chris@676 186 // for (int i = 0; i < m_columnCount; ++i) {
Chris@843 187 // cerr << int(m_columnQualities[i]) << " ";
Chris@676 188 // }
Chris@843 189 // cerr << endl;
Chris@629 190 }
Chris@629 191
Chris@629 192 void
Chris@629 193 CSVFormat::guessPurposes()
Chris@629 194 {
Chris@629 195 m_timingType = CSVFormat::ImplicitTiming;
Chris@629 196 m_timeUnits = CSVFormat::TimeWindows;
Chris@392 197
Chris@629 198 int timingColumnCount = 0;
Chris@1021 199
Chris@1021 200 // if our first column has zero or one entries in it and the rest
Chris@1021 201 // have more, then we'll default to ignoring the first column and
Chris@1021 202 // counting the next one as primary. (e.g. Sonic Annotator output
Chris@1021 203 // with filename at start of first column.)
Chris@1021 204
Chris@1021 205 int primaryColumnNo = 0;
Chris@1021 206
Chris@1021 207 if (m_columnCount >= 2) {
Chris@1021 208 if ( (m_columnQualities[0] & ColumnNearEmpty) &&
Chris@1021 209 !(m_columnQualities[1] & ColumnNearEmpty)) {
Chris@1021 210 primaryColumnNo = 1;
Chris@1021 211 }
Chris@1021 212 }
Chris@629 213
Chris@629 214 for (int i = 0; i < m_columnCount; ++i) {
Chris@629 215
Chris@629 216 ColumnPurpose purpose = ColumnUnknown;
Chris@1021 217
Chris@1021 218 if (i < primaryColumnNo) {
Chris@1021 219 setColumnPurpose(i, purpose);
Chris@1021 220 continue;
Chris@1021 221 }
Chris@1021 222
Chris@1021 223 bool primary = (i == primaryColumnNo);
Chris@392 224
Chris@629 225 ColumnQualities qualities = m_columnQualities[i];
Chris@392 226
Chris@629 227 bool numeric = (qualities & ColumnNumeric);
Chris@629 228 bool integral = (qualities & ColumnIntegral);
Chris@629 229 bool increasing = (qualities & ColumnIncreasing);
Chris@629 230 bool large = (qualities & ColumnLarge);
Chris@629 231
Chris@629 232 bool timingColumn = (numeric && increasing);
Chris@629 233
Chris@629 234 if (timingColumn) {
Chris@629 235
Chris@629 236 ++timingColumnCount;
Chris@629 237
Chris@629 238 if (primary) {
Chris@629 239
Chris@629 240 purpose = ColumnStartTime;
Chris@629 241
Chris@629 242 m_timingType = ExplicitTiming;
Chris@629 243
Chris@629 244 if (integral && large) {
Chris@629 245 m_timeUnits = TimeAudioFrames;
Chris@629 246 } else {
Chris@629 247 m_timeUnits = TimeSeconds;
Chris@629 248 }
Chris@629 249
Chris@629 250 } else {
Chris@629 251
Chris@629 252 if (timingColumnCount == 2 && m_timingType == ExplicitTiming) {
Chris@629 253 purpose = ColumnEndTime;
Chris@629 254 }
Chris@629 255 }
Chris@629 256 }
Chris@629 257
Chris@629 258 if (purpose == ColumnUnknown) {
Chris@629 259 if (numeric) {
Chris@629 260 purpose = ColumnValue;
Chris@629 261 } else {
Chris@629 262 purpose = ColumnLabel;
Chris@629 263 }
Chris@629 264 }
Chris@629 265
Chris@631 266 setColumnPurpose(i, purpose);
Chris@629 267 }
Chris@629 268
Chris@629 269 int valueCount = 0;
Chris@629 270 for (int i = 0; i < m_columnCount; ++i) {
Chris@629 271 if (m_columnPurposes[i] == ColumnValue) ++valueCount;
Chris@629 272 }
Chris@629 273
Chris@630 274 if (valueCount == 2 && timingColumnCount == 1) {
Chris@630 275 // If we have exactly two apparent value columns and only one
Chris@630 276 // timing column, but one value column is integral and the
Chris@630 277 // other is not, guess that whichever one matches the integral
Chris@630 278 // status of the time column is either duration or end time
Chris@630 279 if (m_timingType == ExplicitTiming) {
Chris@630 280 int a = -1, b = -1;
Chris@630 281 for (int i = 0; i < m_columnCount; ++i) {
Chris@630 282 if (m_columnPurposes[i] == ColumnValue) {
Chris@630 283 if (a == -1) a = i;
Chris@630 284 else b = i;
Chris@630 285 }
Chris@630 286 }
Chris@630 287 if ((m_columnQualities[a] & ColumnIntegral) !=
Chris@630 288 (m_columnQualities[b] & ColumnIntegral)) {
Chris@630 289 int timecol = a;
Chris@630 290 if ((m_columnQualities[a] & ColumnIntegral) !=
Chris@630 291 (m_columnQualities[0] & ColumnIntegral)) {
Chris@630 292 timecol = b;
Chris@630 293 }
Chris@630 294 if (m_columnQualities[timecol] & ColumnIncreasing) {
Chris@630 295 // This shouldn't happen; should have been settled above
Chris@630 296 m_columnPurposes[timecol] = ColumnEndTime;
Chris@630 297 } else {
Chris@630 298 m_columnPurposes[timecol] = ColumnDuration;
Chris@630 299 }
Chris@630 300 --valueCount;
Chris@630 301 }
Chris@630 302 }
Chris@630 303 }
Chris@630 304
Chris@631 305 if (timingColumnCount > 1) {
Chris@631 306 m_modelType = TwoDimensionalModelWithDuration;
Chris@392 307 } else {
Chris@631 308 if (valueCount == 0) {
Chris@631 309 m_modelType = OneDimensionalModel;
Chris@631 310 } else if (valueCount == 1) {
Chris@631 311 m_modelType = TwoDimensionalModel;
Chris@631 312 } else {
Chris@631 313 m_modelType = ThreeDimensionalModel;
Chris@631 314 }
Chris@629 315 }
Chris@392 316
Chris@843 317 // cerr << "Estimated column purposes: ";
Chris@676 318 // for (int i = 0; i < m_columnCount; ++i) {
Chris@843 319 // cerr << int(m_columnPurposes[i]) << " ";
Chris@676 320 // }
Chris@843 321 // cerr << endl;
Chris@392 322
Chris@843 323 // cerr << "Estimated model type: " << m_modelType << endl;
Chris@843 324 // cerr << "Estimated timing type: " << m_timingType << endl;
Chris@843 325 // cerr << "Estimated units: " << m_timeUnits << endl;
Chris@392 326 }
Chris@392 327
Chris@631 328 CSVFormat::ColumnPurpose
Chris@631 329 CSVFormat::getColumnPurpose(int i)
Chris@631 330 {
Chris@631 331 while (m_columnPurposes.size() <= i) {
Chris@631 332 m_columnPurposes.push_back(ColumnUnknown);
Chris@631 333 }
Chris@631 334 return m_columnPurposes[i];
Chris@631 335 }
Chris@629 336
Chris@631 337 CSVFormat::ColumnPurpose
Chris@631 338 CSVFormat::getColumnPurpose(int i) const
Chris@631 339 {
Chris@668 340 if (m_columnPurposes.size() <= i) {
Chris@668 341 return ColumnUnknown;
Chris@668 342 }
Chris@631 343 return m_columnPurposes[i];
Chris@631 344 }
Chris@631 345
Chris@631 346 void
Chris@631 347 CSVFormat::setColumnPurpose(int i, ColumnPurpose p)
Chris@631 348 {
Chris@631 349 while (m_columnPurposes.size() <= i) {
Chris@631 350 m_columnPurposes.push_back(ColumnUnknown);
Chris@631 351 }
Chris@631 352 m_columnPurposes[i] = p;
Chris@631 353 }
Chris@631 354
Chris@631 355
Chris@631 356
Chris@631 357