Chris@148
|
1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
|
Chris@148
|
2
|
Chris@148
|
3 /*
|
Chris@148
|
4 Sonic Visualiser
|
Chris@148
|
5 An audio file viewer and annotation editor.
|
Chris@148
|
6 Centre for Digital Music, Queen Mary, University of London.
|
Chris@148
|
7 This file copyright 2006 Chris Cannam.
|
Chris@148
|
8
|
Chris@148
|
9 This program is free software; you can redistribute it and/or
|
Chris@148
|
10 modify it under the terms of the GNU General Public License as
|
Chris@148
|
11 published by the Free Software Foundation; either version 2 of the
|
Chris@148
|
12 License, or (at your option) any later version. See the file
|
Chris@148
|
13 COPYING included with this distribution for more information.
|
Chris@148
|
14 */
|
Chris@148
|
15
|
Chris@148
|
16 #include "CSVFileReader.h"
|
Chris@148
|
17
|
Chris@150
|
18 #include "model/Model.h"
|
Chris@148
|
19 #include "base/RealTime.h"
|
Chris@631
|
20 #include "base/StringBits.h"
|
Chris@1491
|
21 #include "base/ProgressReporter.h"
|
Chris@1519
|
22 #include "base/RecordDirectory.h"
|
Chris@148
|
23 #include "model/SparseOneDimensionalModel.h"
|
Chris@148
|
24 #include "model/SparseTimeValueModel.h"
|
Chris@152
|
25 #include "model/EditableDenseThreeDimensionalModel.h"
|
Chris@628
|
26 #include "model/RegionModel.h"
|
Chris@897
|
27 #include "model/NoteModel.h"
|
Chris@1793
|
28 #include "model/BoxModel.h"
|
Chris@1488
|
29 #include "model/WritableWaveFileModel.h"
|
Chris@308
|
30 #include "DataFileReaderFactory.h"
|
Chris@148
|
31
|
Chris@148
|
32 #include <QFile>
|
Chris@1519
|
33 #include <QDir>
|
Chris@1030
|
34 #include <QFileInfo>
|
Chris@148
|
35 #include <QString>
|
Chris@148
|
36 #include <QRegExp>
|
Chris@148
|
37 #include <QStringList>
|
Chris@148
|
38 #include <QTextStream>
|
Chris@1519
|
39 #include <QDateTime>
|
Chris@148
|
40
|
Chris@148
|
41 #include <iostream>
|
Chris@628
|
42 #include <map>
|
Chris@1428
|
43 #include <string>
|
Chris@148
|
44
|
Chris@1113
|
45 using namespace std;
|
Chris@1113
|
46
|
Chris@392
|
47 CSVFileReader::CSVFileReader(QString path, CSVFormat format,
|
Chris@1491
|
48 sv_samplerate_t mainModelSampleRate,
|
Chris@1491
|
49 ProgressReporter *reporter) :
|
Chris@392
|
50 m_format(format),
|
Chris@1582
|
51 m_device(nullptr),
|
Chris@1009
|
52 m_ownDevice(true),
|
Chris@631
|
53 m_warnings(0),
|
Chris@1491
|
54 m_mainModelSampleRate(mainModelSampleRate),
|
Chris@1491
|
55 m_fileSize(0),
|
Chris@1491
|
56 m_readCount(0),
|
Chris@1492
|
57 m_progress(-1),
|
Chris@1491
|
58 m_reporter(reporter)
|
Chris@148
|
59 {
|
Chris@1009
|
60 QFile *file = new QFile(path);
|
Chris@148
|
61 bool good = false;
|
Chris@148
|
62
|
Chris@1009
|
63 if (!file->exists()) {
|
Chris@1429
|
64 m_error = QFile::tr("File \"%1\" does not exist").arg(path);
|
Chris@1009
|
65 } else if (!file->open(QIODevice::ReadOnly | QIODevice::Text)) {
|
Chris@1429
|
66 m_error = QFile::tr("Failed to open file \"%1\"").arg(path);
|
Chris@148
|
67 } else {
|
Chris@1429
|
68 good = true;
|
Chris@148
|
69 }
|
Chris@148
|
70
|
Chris@1009
|
71 if (good) {
|
Chris@1009
|
72 m_device = file;
|
Chris@1030
|
73 m_filename = QFileInfo(path).fileName();
|
Chris@1491
|
74 m_fileSize = file->size();
|
Chris@1491
|
75 if (m_reporter) m_reporter->setDefinite(true);
|
Chris@1009
|
76 } else {
|
Chris@1429
|
77 delete file;
|
Chris@148
|
78 }
|
Chris@148
|
79 }
|
Chris@148
|
80
|
Chris@1009
|
81 CSVFileReader::CSVFileReader(QIODevice *device, CSVFormat format,
|
Chris@1491
|
82 sv_samplerate_t mainModelSampleRate,
|
Chris@1491
|
83 ProgressReporter *reporter) :
|
Chris@1009
|
84 m_format(format),
|
Chris@1009
|
85 m_device(device),
|
Chris@1009
|
86 m_ownDevice(false),
|
Chris@1009
|
87 m_warnings(0),
|
Chris@1491
|
88 m_mainModelSampleRate(mainModelSampleRate),
|
Chris@1491
|
89 m_fileSize(0),
|
Chris@1491
|
90 m_readCount(0),
|
Chris@1492
|
91 m_progress(-1),
|
Chris@1491
|
92 m_reporter(reporter)
|
Chris@1009
|
93 {
|
Chris@1491
|
94 if (m_reporter) m_reporter->setDefinite(false);
|
Chris@1009
|
95 }
|
Chris@1009
|
96
|
Chris@148
|
97 CSVFileReader::~CSVFileReader()
|
Chris@148
|
98 {
|
Chris@1009
|
99 SVDEBUG << "CSVFileReader::~CSVFileReader: device is " << m_device << endl;
|
Chris@148
|
100
|
Chris@1009
|
101 if (m_device && m_ownDevice) {
|
Chris@1009
|
102 SVDEBUG << "CSVFileReader::CSVFileReader: Closing device" << endl;
|
Chris@1009
|
103 m_device->close();
|
Chris@1009
|
104 delete m_device;
|
Chris@148
|
105 }
|
Chris@148
|
106 }
|
Chris@148
|
107
|
Chris@148
|
108 bool
|
Chris@148
|
109 CSVFileReader::isOK() const
|
Chris@148
|
110 {
|
Chris@1582
|
111 return (m_device != nullptr);
|
Chris@148
|
112 }
|
Chris@148
|
113
|
Chris@148
|
114 QString
|
Chris@148
|
115 CSVFileReader::getError() const
|
Chris@148
|
116 {
|
Chris@148
|
117 return m_error;
|
Chris@148
|
118 }
|
Chris@148
|
119
|
Chris@1867
|
120 bool
|
Chris@1047
|
121 CSVFileReader::convertTimeValue(QString s, int lineno,
|
Chris@1047
|
122 sv_samplerate_t sampleRate,
|
Chris@1867
|
123 int windowSize,
|
Chris@1867
|
124 sv_frame_t &calculatedFrame) const
|
Chris@631
|
125 {
|
Chris@631
|
126 QRegExp nonNumericRx("[^0-9eE.,+-]");
|
Chris@897
|
127 int warnLimit = 10;
|
Chris@631
|
128
|
Chris@631
|
129 CSVFormat::TimeUnits timeUnits = m_format.getTimeUnits();
|
Chris@631
|
130
|
Chris@1867
|
131 calculatedFrame = 0;
|
Chris@631
|
132
|
Chris@631
|
133 bool ok = false;
|
Chris@631
|
134 QString numeric = s;
|
Chris@631
|
135 numeric.remove(nonNumericRx);
|
Chris@631
|
136
|
Chris@631
|
137 if (timeUnits == CSVFormat::TimeSeconds) {
|
Chris@631
|
138
|
Chris@631
|
139 double time = numeric.toDouble(&ok);
|
Chris@631
|
140 if (!ok) time = StringBits::stringToDoubleLocaleFree(numeric, &ok);
|
Chris@1038
|
141 calculatedFrame = sv_frame_t(time * sampleRate + 0.5);
|
Chris@990
|
142
|
Chris@990
|
143 } else if (timeUnits == CSVFormat::TimeMilliseconds) {
|
Chris@990
|
144
|
Chris@990
|
145 double time = numeric.toDouble(&ok);
|
Chris@990
|
146 if (!ok) time = StringBits::stringToDoubleLocaleFree(numeric, &ok);
|
Chris@1038
|
147 calculatedFrame = sv_frame_t((time / 1000.0) * sampleRate + 0.5);
|
Chris@631
|
148
|
Chris@631
|
149 } else {
|
Chris@631
|
150
|
Chris@631
|
151 long n = numeric.toLong(&ok);
|
Chris@631
|
152 if (n >= 0) calculatedFrame = n;
|
Chris@631
|
153
|
Chris@631
|
154 if (timeUnits == CSVFormat::TimeWindows) {
|
Chris@631
|
155 calculatedFrame *= windowSize;
|
Chris@631
|
156 }
|
Chris@631
|
157 }
|
Chris@631
|
158
|
Chris@631
|
159 if (!ok) {
|
Chris@631
|
160 if (m_warnings < warnLimit) {
|
Chris@1428
|
161 SVCERR << "WARNING: CSVFileReader::load: "
|
Chris@844
|
162 << "Bad time format (\"" << s
|
Chris@631
|
163 << "\") in data line "
|
Chris@843
|
164 << lineno+1 << endl;
|
Chris@631
|
165 } else if (m_warnings == warnLimit) {
|
Chris@1428
|
166 SVCERR << "WARNING: Too many warnings" << endl;
|
Chris@631
|
167 }
|
Chris@631
|
168 ++m_warnings;
|
Chris@631
|
169 }
|
Chris@631
|
170
|
Chris@631
|
171 return calculatedFrame;
|
Chris@631
|
172 }
|
Chris@631
|
173
|
Chris@148
|
174 Model *
|
Chris@148
|
175 CSVFileReader::load() const
|
Chris@148
|
176 {
|
Chris@1582
|
177 if (!m_device) return nullptr;
|
Chris@148
|
178
|
Chris@628
|
179 CSVFormat::ModelType modelType = m_format.getModelType();
|
Chris@392
|
180 CSVFormat::TimingType timingType = m_format.getTimingType();
|
Chris@628
|
181 CSVFormat::TimeUnits timeUnits = m_format.getTimeUnits();
|
Chris@1047
|
182 sv_samplerate_t sampleRate = m_format.getSampleRate();
|
Chris@929
|
183 int windowSize = m_format.getWindowSize();
|
Chris@631
|
184 QChar separator = m_format.getSeparator();
|
Chris@631
|
185 bool allowQuoting = m_format.getAllowQuoting();
|
Chris@148
|
186
|
Chris@392
|
187 if (timingType == CSVFormat::ExplicitTiming) {
|
Chris@611
|
188 if (modelType == CSVFormat::ThreeDimensionalModel) {
|
Chris@611
|
189 // This will be overridden later if more than one line
|
Chris@611
|
190 // appears in our file, but we want to choose a default
|
Chris@611
|
191 // that's likely to be visible
|
Chris@611
|
192 windowSize = 1024;
|
Chris@611
|
193 } else {
|
Chris@611
|
194 windowSize = 1;
|
Chris@611
|
195 }
|
Chris@1429
|
196 if (timeUnits == CSVFormat::TimeSeconds ||
|
Chris@990
|
197 timeUnits == CSVFormat::TimeMilliseconds) {
|
Chris@1429
|
198 sampleRate = m_mainModelSampleRate;
|
Chris@1429
|
199 }
|
Chris@148
|
200 }
|
Chris@148
|
201
|
Chris@1582
|
202 SparseOneDimensionalModel *model1 = nullptr;
|
Chris@1582
|
203 SparseTimeValueModel *model2 = nullptr;
|
Chris@1582
|
204 RegionModel *model2a = nullptr;
|
Chris@1582
|
205 NoteModel *model2b = nullptr;
|
Chris@1793
|
206 BoxModel *model2c = nullptr;
|
Chris@1582
|
207 EditableDenseThreeDimensionalModel *model3 = nullptr;
|
Chris@1582
|
208 WritableWaveFileModel *modelW = nullptr;
|
Chris@1582
|
209 Model *model = nullptr;
|
Chris@148
|
210
|
Chris@1009
|
211 QTextStream in(m_device);
|
Chris@148
|
212
|
Chris@148
|
213 unsigned int warnings = 0, warnLimit = 10;
|
Chris@148
|
214 unsigned int lineno = 0;
|
Chris@148
|
215
|
Chris@148
|
216 float min = 0.0, max = 0.0;
|
Chris@148
|
217
|
Chris@1038
|
218 sv_frame_t frameNo = 0;
|
Chris@1038
|
219 sv_frame_t duration = 0;
|
Chris@1038
|
220 sv_frame_t endFrame = 0;
|
Chris@631
|
221
|
Chris@631
|
222 bool haveAnyValue = false;
|
Chris@631
|
223 bool haveEndTime = false;
|
Chris@897
|
224 bool pitchLooksLikeMIDI = true;
|
Chris@631
|
225
|
Chris@1038
|
226 sv_frame_t startFrame = 0; // for calculation of dense model resolution
|
Chris@631
|
227 bool firstEverValue = true;
|
Chris@631
|
228
|
Chris@676
|
229 int valueColumns = 0;
|
Chris@676
|
230 for (int i = 0; i < m_format.getColumnCount(); ++i) {
|
Chris@676
|
231 if (m_format.getColumnPurpose(i) == CSVFormat::ColumnValue) {
|
Chris@676
|
232 ++valueColumns;
|
Chris@676
|
233 }
|
Chris@676
|
234 }
|
Chris@676
|
235
|
Chris@1518
|
236 int audioChannels = 0;
|
Chris@1582
|
237 float **audioSamples = nullptr;
|
Chris@1518
|
238 float sampleShift = 0.f;
|
Chris@1518
|
239 float sampleScale = 1.f;
|
Chris@1518
|
240
|
Chris@1518
|
241 if (modelType == CSVFormat::WaveFileModel) {
|
Chris@1518
|
242
|
Chris@1518
|
243 audioChannels = valueColumns;
|
Chris@1518
|
244
|
Chris@1518
|
245 audioSamples =
|
Chris@1518
|
246 breakfastquay::allocate_and_zero_channels<float>
|
Chris@1518
|
247 (audioChannels, 1);
|
Chris@1518
|
248
|
Chris@1518
|
249 switch (m_format.getAudioSampleRange()) {
|
Chris@1518
|
250 case CSVFormat::SampleRangeSigned1:
|
Chris@1518
|
251 case CSVFormat::SampleRangeOther:
|
Chris@1518
|
252 sampleShift = 0.f;
|
Chris@1518
|
253 sampleScale = 1.f;
|
Chris@1518
|
254 break;
|
Chris@1518
|
255 case CSVFormat::SampleRangeUnsigned255:
|
Chris@1518
|
256 sampleShift = -128.f;
|
Chris@1518
|
257 sampleScale = 1.f / 128.f;
|
Chris@1518
|
258 break;
|
Chris@1518
|
259 case CSVFormat::SampleRangeSigned32767:
|
Chris@1518
|
260 sampleShift = 0.f;
|
Chris@1518
|
261 sampleScale = 1.f / 32768.f;
|
Chris@1518
|
262 break;
|
Chris@1518
|
263 }
|
Chris@1518
|
264 }
|
Chris@1518
|
265
|
Chris@1518
|
266 map<QString, int> labelCountMap;
|
Chris@1518
|
267
|
Chris@1509
|
268 bool abandoned = false;
|
Chris@1509
|
269
|
Chris@1509
|
270 while (!in.atEnd() && !abandoned) {
|
Chris@148
|
271
|
Chris@283
|
272 // QTextStream's readLine doesn't cope with old-style Mac
|
Chris@283
|
273 // CR-only line endings. Why did they bother making the class
|
Chris@283
|
274 // cope with more than one sort of line ending, if it still
|
Chris@283
|
275 // can't be configured to cope with all the common sorts?
|
Chris@148
|
276
|
Chris@283
|
277 // For the time being we'll deal with this case (which is
|
Chris@283
|
278 // relatively uncommon for us, but still necessary to handle)
|
Chris@283
|
279 // by reading the entire file using a single readLine, and
|
Chris@283
|
280 // splitting it. For CR and CR/LF line endings this will just
|
Chris@283
|
281 // read a line at a time, and that's obviously OK.
|
Chris@148
|
282
|
Chris@283
|
283 QString chunk = in.readLine();
|
Chris@283
|
284 QStringList lines = chunk.split('\r', QString::SkipEmptyParts);
|
Chris@1491
|
285
|
Chris@1491
|
286 m_readCount += chunk.size() + 1;
|
Chris@1491
|
287
|
Chris@1491
|
288 if (m_reporter) {
|
Chris@1491
|
289 if (m_reporter->wasCancelled()) {
|
Chris@1509
|
290 abandoned = true;
|
Chris@1491
|
291 break;
|
Chris@1491
|
292 }
|
Chris@1491
|
293 int progress;
|
Chris@1491
|
294 if (m_fileSize > 0) {
|
Chris@1491
|
295 progress = int((double(m_readCount) / double(m_fileSize))
|
Chris@1491
|
296 * 100.0);
|
Chris@1491
|
297 } else {
|
Chris@1491
|
298 progress = int(m_readCount / 10000);
|
Chris@1491
|
299 }
|
Chris@1491
|
300 if (progress != m_progress) {
|
Chris@1491
|
301 m_reporter->setProgress(progress);
|
Chris@1491
|
302 m_progress = progress;
|
Chris@1491
|
303 }
|
Chris@1491
|
304 }
|
Chris@283
|
305
|
Chris@897
|
306 for (int li = 0; li < lines.size(); ++li) {
|
Chris@148
|
307
|
Chris@283
|
308 QString line = lines[li];
|
Chris@1009
|
309
|
Chris@283
|
310 if (line.startsWith("#")) continue;
|
Chris@283
|
311
|
Chris@631
|
312 QStringList list = StringBits::split(line, separator, allowQuoting);
|
Chris@283
|
313 if (!model) {
|
Chris@283
|
314
|
Chris@1519
|
315 QString modelName = m_filename;
|
Chris@1519
|
316
|
Chris@283
|
317 switch (modelType) {
|
Chris@283
|
318
|
Chris@392
|
319 case CSVFormat::OneDimensionalModel:
|
Chris@1867
|
320 SVDEBUG << "CSVFileReader: Creating sparse one-dimensional model" << endl;
|
Chris@283
|
321 model1 = new SparseOneDimensionalModel(sampleRate, windowSize);
|
Chris@283
|
322 model = model1;
|
Chris@283
|
323 break;
|
Chris@1429
|
324
|
Chris@392
|
325 case CSVFormat::TwoDimensionalModel:
|
Chris@1867
|
326 SVDEBUG << "CSVFileReader: Creating sparse time-value model" << endl;
|
Chris@283
|
327 model2 = new SparseTimeValueModel(sampleRate, windowSize, false);
|
Chris@283
|
328 model = model2;
|
Chris@283
|
329 break;
|
Chris@1429
|
330
|
Chris@628
|
331 case CSVFormat::TwoDimensionalModelWithDuration:
|
Chris@1867
|
332 SVDEBUG << "CSVFileReader: Creating region model" << endl;
|
Chris@628
|
333 model2a = new RegionModel(sampleRate, windowSize, false);
|
Chris@628
|
334 model = model2a;
|
Chris@628
|
335 break;
|
Chris@1429
|
336
|
Chris@897
|
337 case CSVFormat::TwoDimensionalModelWithDurationAndPitch:
|
Chris@1867
|
338 SVDEBUG << "CSVFileReader: Creating note model" << endl;
|
Chris@897
|
339 model2b = new NoteModel(sampleRate, windowSize, false);
|
Chris@897
|
340 model = model2b;
|
Chris@897
|
341 break;
|
Chris@1429
|
342
|
Chris@1793
|
343 case CSVFormat::TwoDimensionalModelWithDurationAndExtent:
|
Chris@1867
|
344 SVDEBUG << "CSVFileReader: Creating box model" << endl;
|
Chris@1793
|
345 model2c = new BoxModel(sampleRate, windowSize, false);
|
Chris@1793
|
346 model = model2c;
|
Chris@1793
|
347 break;
|
Chris@1793
|
348
|
Chris@392
|
349 case CSVFormat::ThreeDimensionalModel:
|
Chris@1867
|
350 SVDEBUG << "CSVFileReader: Creating editable dense three-dimensional model" << endl;
|
Chris@535
|
351 model3 = new EditableDenseThreeDimensionalModel
|
Chris@1777
|
352 (sampleRate, windowSize, valueColumns);
|
Chris@283
|
353 model = model3;
|
Chris@283
|
354 break;
|
Chris@1488
|
355
|
Chris@1488
|
356 case CSVFormat::WaveFileModel:
|
Chris@1517
|
357 {
|
Chris@1867
|
358 SVDEBUG << "CSVFileReader: Creating writable wave-file model" << endl;
|
Chris@1517
|
359 bool normalise = (m_format.getAudioSampleRange()
|
Chris@1517
|
360 == CSVFormat::SampleRangeOther);
|
Chris@1519
|
361 QString path = getConvertedAudioFilePath();
|
Chris@1488
|
362 modelW = new WritableWaveFileModel
|
Chris@1520
|
363 (path, sampleRate, valueColumns,
|
Chris@1520
|
364 normalise ?
|
Chris@1520
|
365 WritableWaveFileModel::Normalisation::Peak :
|
Chris@1520
|
366 WritableWaveFileModel::Normalisation::None);
|
Chris@1519
|
367 modelName = QFileInfo(path).fileName();
|
Chris@1488
|
368 model = modelW;
|
Chris@1488
|
369 break;
|
Chris@283
|
370 }
|
Chris@1517
|
371 }
|
Chris@1030
|
372
|
Chris@1508
|
373 if (model && model->isOK()) {
|
Chris@1519
|
374 if (modelName != "") {
|
Chris@1519
|
375 model->setObjectName(modelName);
|
Chris@1030
|
376 }
|
Chris@1030
|
377 }
|
Chris@283
|
378 }
|
Chris@148
|
379
|
Chris@1508
|
380 if (!model || !model->isOK()) {
|
Chris@1508
|
381 SVCERR << "Failed to create model to load CSV file into"
|
Chris@1508
|
382 << endl;
|
Chris@1508
|
383 if (model) {
|
Chris@1508
|
384 delete model;
|
Chris@1582
|
385 model = nullptr;
|
Chris@1793
|
386 model1 = nullptr; model2 = nullptr;
|
Chris@1793
|
387 model2a = nullptr; model2b = nullptr; model2c = nullptr;
|
Chris@1582
|
388 model3 = nullptr; modelW = nullptr;
|
Chris@1508
|
389 }
|
Chris@1509
|
390 abandoned = true;
|
Chris@1508
|
391 break;
|
Chris@1508
|
392 }
|
Chris@1508
|
393
|
Chris@631
|
394 float value = 0.f;
|
Chris@1793
|
395 float otherValue = 0.f;
|
Chris@897
|
396 float pitch = 0.f;
|
Chris@631
|
397 QString label = "";
|
Chris@1867
|
398 bool ok = true;
|
Chris@148
|
399
|
Chris@631
|
400 duration = 0.f;
|
Chris@631
|
401 haveEndTime = false;
|
Chris@1518
|
402
|
Chris@283
|
403 for (int i = 0; i < list.size(); ++i) {
|
Chris@148
|
404
|
Chris@631
|
405 QString s = list[i];
|
Chris@631
|
406
|
Chris@631
|
407 CSVFormat::ColumnPurpose purpose = m_format.getColumnPurpose(i);
|
Chris@631
|
408
|
Chris@631
|
409 switch (purpose) {
|
Chris@631
|
410
|
Chris@631
|
411 case CSVFormat::ColumnUnknown:
|
Chris@631
|
412 break;
|
Chris@631
|
413
|
Chris@631
|
414 case CSVFormat::ColumnStartTime:
|
Chris@1867
|
415 if (!convertTimeValue(s, lineno, sampleRate, windowSize, frameNo)) {
|
Chris@1867
|
416 ok = false;
|
Chris@1867
|
417 }
|
Chris@631
|
418 break;
|
Chris@631
|
419
|
Chris@631
|
420 case CSVFormat::ColumnEndTime:
|
Chris@1867
|
421 if (convertTimeValue(s, lineno, sampleRate, windowSize, endFrame)) {
|
Chris@1867
|
422 haveEndTime = true;
|
Chris@1867
|
423 }
|
Chris@631
|
424 break;
|
Chris@631
|
425
|
Chris@631
|
426 case CSVFormat::ColumnDuration:
|
Chris@1867
|
427 if (!convertTimeValue(s, lineno, sampleRate, windowSize, duration)) {
|
Chris@1867
|
428 ok = false;
|
Chris@1867
|
429 }
|
Chris@631
|
430 break;
|
Chris@631
|
431
|
Chris@631
|
432 case CSVFormat::ColumnValue:
|
Chris@1793
|
433 if (haveAnyValue) {
|
Chris@1793
|
434 otherValue = value;
|
Chris@1793
|
435 }
|
Chris@631
|
436 value = s.toFloat();
|
Chris@631
|
437 haveAnyValue = true;
|
Chris@631
|
438 break;
|
Chris@631
|
439
|
Chris@897
|
440 case CSVFormat::ColumnPitch:
|
Chris@897
|
441 pitch = s.toFloat();
|
Chris@897
|
442 if (pitch < 0.f || pitch > 127.f) {
|
Chris@897
|
443 pitchLooksLikeMIDI = false;
|
Chris@897
|
444 }
|
Chris@897
|
445 break;
|
Chris@897
|
446
|
Chris@631
|
447 case CSVFormat::ColumnLabel:
|
Chris@631
|
448 label = s;
|
Chris@631
|
449 break;
|
Chris@283
|
450 }
|
Chris@631
|
451 }
|
Chris@148
|
452
|
Chris@1867
|
453 if (!ok) {
|
Chris@1867
|
454 continue;
|
Chris@1867
|
455 }
|
Chris@1867
|
456
|
Chris@1113
|
457 ++labelCountMap[label];
|
Chris@1113
|
458
|
Chris@631
|
459 if (haveEndTime) { // ... calculate duration now all cols read
|
Chris@631
|
460 if (endFrame > frameNo) {
|
Chris@631
|
461 duration = endFrame - frameNo;
|
Chris@628
|
462 }
|
Chris@283
|
463 }
|
Chris@148
|
464
|
Chris@392
|
465 if (modelType == CSVFormat::OneDimensionalModel) {
|
Chris@1429
|
466
|
Chris@1658
|
467 Event point(frameNo, label);
|
Chris@1658
|
468 model1->add(point);
|
Chris@148
|
469
|
Chris@392
|
470 } else if (modelType == CSVFormat::TwoDimensionalModel) {
|
Chris@148
|
471
|
Chris@1651
|
472 Event point(frameNo, value, label);
|
Chris@1651
|
473 model2->add(point);
|
Chris@148
|
474
|
Chris@628
|
475 } else if (modelType == CSVFormat::TwoDimensionalModelWithDuration) {
|
Chris@628
|
476
|
Chris@1649
|
477 Event region(frameNo, value, duration, label);
|
Chris@1649
|
478 model2a->add(region);
|
Chris@628
|
479
|
Chris@897
|
480 } else if (modelType == CSVFormat::TwoDimensionalModelWithDurationAndPitch) {
|
Chris@897
|
481
|
Chris@897
|
482 float level = ((value >= 0.f && value <= 1.f) ? value : 1.f);
|
Chris@1643
|
483 Event note(frameNo, pitch, duration, level, label);
|
Chris@1644
|
484 model2b->add(note);
|
Chris@897
|
485
|
Chris@1793
|
486 } else if (modelType == CSVFormat::TwoDimensionalModelWithDurationAndExtent) {
|
Chris@1793
|
487
|
Chris@1793
|
488 float level = 0.f;
|
Chris@1793
|
489 if (value > otherValue) {
|
Chris@1793
|
490 level = value - otherValue;
|
Chris@1793
|
491 value = otherValue;
|
Chris@1793
|
492 } else {
|
Chris@1793
|
493 level = otherValue - value;
|
Chris@1793
|
494 }
|
Chris@1793
|
495 Event box(frameNo, value, duration, level, label);
|
Chris@1793
|
496 model2c->add(box);
|
Chris@1793
|
497
|
Chris@392
|
498 } else if (modelType == CSVFormat::ThreeDimensionalModel) {
|
Chris@148
|
499
|
Chris@283
|
500 DenseThreeDimensionalModel::Column values;
|
Chris@148
|
501
|
Chris@631
|
502 for (int i = 0; i < list.size(); ++i) {
|
Chris@148
|
503
|
Chris@676
|
504 if (m_format.getColumnPurpose(i) != CSVFormat::ColumnValue) {
|
Chris@676
|
505 continue;
|
Chris@676
|
506 }
|
Chris@676
|
507
|
Chris@283
|
508 bool ok = false;
|
Chris@283
|
509 float value = list[i].toFloat(&ok);
|
Chris@611
|
510
|
Chris@676
|
511 values.push_back(value);
|
Chris@1429
|
512
|
Chris@631
|
513 if (firstEverValue || value < min) min = value;
|
Chris@631
|
514 if (firstEverValue || value > max) max = value;
|
Chris@676
|
515
|
Chris@631
|
516 if (firstEverValue) {
|
Chris@611
|
517 startFrame = frameNo;
|
Chris@611
|
518 model3->setStartFrame(startFrame);
|
Chris@611
|
519 } else if (lineno == 1 &&
|
Chris@611
|
520 timingType == CSVFormat::ExplicitTiming) {
|
Chris@1038
|
521 model3->setResolution(int(frameNo - startFrame));
|
Chris@611
|
522 }
|
Chris@631
|
523
|
Chris@631
|
524 firstEverValue = false;
|
Chris@148
|
525
|
Chris@283
|
526 if (!ok) {
|
Chris@283
|
527 if (warnings < warnLimit) {
|
Chris@1428
|
528 SVCERR << "WARNING: CSVFileReader::load: "
|
Chris@390
|
529 << "Non-numeric value \""
|
Chris@844
|
530 << list[i]
|
Chris@491
|
531 << "\" in data line " << lineno+1
|
Chris@843
|
532 << ":" << endl;
|
Chris@1428
|
533 SVCERR << line << endl;
|
Chris@283
|
534 ++warnings;
|
Chris@283
|
535 } else if (warnings == warnLimit) {
|
Chris@1428
|
536 // SVCERR << "WARNING: Too many warnings" << endl;
|
Chris@283
|
537 }
|
Chris@283
|
538 }
|
Chris@283
|
539 }
|
Chris@1429
|
540
|
Chris@690
|
541 // SVDEBUG << "Setting bin values for count " << lineno << ", frame "
|
Chris@687
|
542 // << frameNo << ", time " << RealTime::frame2RealTime(frameNo, sampleRate) << endl;
|
Chris@148
|
543
|
Chris@611
|
544 model3->setColumn(lineno, values);
|
Chris@1488
|
545
|
Chris@1488
|
546 } else if (modelType == CSVFormat::WaveFileModel) {
|
Chris@1488
|
547
|
Chris@1518
|
548 int channel = 0;
|
Chris@1490
|
549
|
Chris@1518
|
550 for (int i = 0;
|
Chris@1518
|
551 i < list.size() && channel < audioChannels;
|
Chris@1518
|
552 ++i) {
|
Chris@1488
|
553
|
Chris@1490
|
554 if (m_format.getColumnPurpose(i) !=
|
Chris@1490
|
555 CSVFormat::ColumnValue) {
|
Chris@1488
|
556 continue;
|
Chris@1488
|
557 }
|
Chris@1488
|
558
|
Chris@1488
|
559 bool ok = false;
|
Chris@1488
|
560 float value = list[i].toFloat(&ok);
|
Chris@1518
|
561 if (!ok) {
|
Chris@1518
|
562 value = 0.f;
|
Chris@1518
|
563 }
|
Chris@1517
|
564
|
Chris@1518
|
565 value += sampleShift;
|
Chris@1518
|
566 value *= sampleScale;
|
Chris@1488
|
567
|
Chris@1518
|
568 audioSamples[channel][0] = value;
|
Chris@1490
|
569
|
Chris@1490
|
570 ++channel;
|
Chris@1488
|
571 }
|
Chris@1488
|
572
|
Chris@1518
|
573 while (channel < audioChannels) {
|
Chris@1518
|
574 audioSamples[channel][0] = 0.f;
|
Chris@1518
|
575 ++channel;
|
Chris@1518
|
576 }
|
Chris@1488
|
577
|
Chris@1518
|
578 bool ok = modelW->addSamples(audioSamples, 1);
|
Chris@1488
|
579
|
Chris@1488
|
580 if (!ok) {
|
Chris@1488
|
581 if (warnings < warnLimit) {
|
Chris@1488
|
582 SVCERR << "WARNING: CSVFileReader::load: "
|
Chris@1488
|
583 << "Unable to add sample to wave-file model"
|
Chris@1488
|
584 << endl;
|
Chris@1488
|
585 SVCERR << line << endl;
|
Chris@1488
|
586 ++warnings;
|
Chris@1488
|
587 }
|
Chris@1488
|
588 }
|
Chris@283
|
589 }
|
Chris@1488
|
590
|
Chris@283
|
591 ++lineno;
|
Chris@392
|
592 if (timingType == CSVFormat::ImplicitTiming ||
|
Chris@283
|
593 list.size() == 0) {
|
Chris@283
|
594 frameNo += windowSize;
|
Chris@283
|
595 }
|
Chris@283
|
596 }
|
Chris@148
|
597 }
|
Chris@148
|
598
|
Chris@631
|
599 if (!haveAnyValue) {
|
Chris@631
|
600 if (model2a) {
|
Chris@631
|
601 // assign values for regions based on label frequency; we
|
Chris@631
|
602 // have this in our labelCountMap, sort of
|
Chris@631
|
603
|
Chris@1113
|
604 map<int, map<QString, float> > countLabelValueMap;
|
Chris@1113
|
605 for (map<QString, int>::iterator i = labelCountMap.begin();
|
Chris@631
|
606 i != labelCountMap.end(); ++i) {
|
Chris@1113
|
607 countLabelValueMap[i->second][i->first] = -1.f;
|
Chris@631
|
608 }
|
Chris@631
|
609
|
Chris@631
|
610 float v = 0.f;
|
Chris@1113
|
611 for (map<int, map<QString, float> >::iterator i =
|
Chris@631
|
612 countLabelValueMap.end(); i != countLabelValueMap.begin(); ) {
|
Chris@631
|
613 --i;
|
Chris@1428
|
614 SVCERR << "count -> " << i->first << endl;
|
Chris@1113
|
615 for (map<QString, float>::iterator j = i->second.begin();
|
Chris@631
|
616 j != i->second.end(); ++j) {
|
Chris@631
|
617 j->second = v;
|
Chris@1428
|
618 SVCERR << "label -> " << j->first << ", value " << v << endl;
|
Chris@631
|
619 v = v + 1.f;
|
Chris@631
|
620 }
|
Chris@631
|
621 }
|
Chris@631
|
622
|
Chris@1649
|
623 map<Event, Event> eventMap;
|
Chris@1649
|
624
|
Chris@1649
|
625 EventVector allEvents = model2a->getAllEvents();
|
Chris@1649
|
626 for (const Event &e: allEvents) {
|
Chris@1649
|
627 int count = labelCountMap[e.getLabel()];
|
Chris@1649
|
628 v = countLabelValueMap[count][e.getLabel()];
|
Chris@1649
|
629 // SVCERR << "mapping from label \"" << p.label
|
Chris@1649
|
630 // << "\" (count " << count
|
Chris@1649
|
631 // << ") to value " << v << endl;
|
Chris@1649
|
632 eventMap[e] = Event(e.getFrame(), v,
|
Chris@1649
|
633 e.getDuration(), e.getLabel());
|
Chris@631
|
634 }
|
Chris@631
|
635
|
Chris@1649
|
636 for (const auto &i: eventMap) {
|
Chris@1113
|
637 // There could be duplicate regions; if so replace
|
Chris@1113
|
638 // them all -- but we need to check we're not
|
Chris@1113
|
639 // replacing a region by itself (or else this will
|
Chris@1113
|
640 // never terminate)
|
Chris@1649
|
641 if (i.first.getValue() == i.second.getValue()) {
|
Chris@1113
|
642 continue;
|
Chris@1113
|
643 }
|
Chris@1649
|
644 while (model2a->containsEvent(i.first)) {
|
Chris@1649
|
645 model2a->remove(i.first);
|
Chris@1649
|
646 model2a->add(i.second);
|
Chris@1113
|
647 }
|
Chris@631
|
648 }
|
Chris@631
|
649 }
|
Chris@631
|
650 }
|
Chris@631
|
651
|
Chris@897
|
652 if (model2b) {
|
Chris@897
|
653 if (pitchLooksLikeMIDI) {
|
Chris@897
|
654 model2b->setScaleUnits("MIDI Pitch");
|
Chris@897
|
655 } else {
|
Chris@897
|
656 model2b->setScaleUnits("Hz");
|
Chris@897
|
657 }
|
Chris@897
|
658 }
|
Chris@897
|
659
|
Chris@961
|
660 if (model3) {
|
Chris@1429
|
661 model3->setMinimumLevel(min);
|
Chris@1429
|
662 model3->setMaximumLevel(max);
|
Chris@148
|
663 }
|
Chris@148
|
664
|
Chris@1489
|
665 if (modelW) {
|
Chris@1518
|
666 breakfastquay::deallocate_channels(audioSamples, audioChannels);
|
Chris@1493
|
667 modelW->updateModel();
|
Chris@1489
|
668 modelW->writeComplete();
|
Chris@1489
|
669 }
|
Chris@1489
|
670
|
Chris@148
|
671 return model;
|
Chris@148
|
672 }
|
Chris@148
|
673
|
Chris@1519
|
674 QString
|
Chris@1519
|
675 CSVFileReader::getConvertedAudioFilePath() const
|
Chris@1519
|
676 {
|
Chris@1519
|
677 QString base = m_filename;
|
Chris@1519
|
678 base.replace(QRegExp("[/\\,.:;~<>\"'|?%*]+"), "_");
|
Chris@1519
|
679
|
Chris@1519
|
680 QString convertedFileDir = RecordDirectory::getConvertedAudioDirectory();
|
Chris@1519
|
681 if (convertedFileDir == "") {
|
Chris@1519
|
682 SVCERR << "WARNING: CSVFileReader::getConvertedAudioFilePath: Failed to retrieve converted audio directory" << endl;
|
Chris@1519
|
683 return "";
|
Chris@1519
|
684 }
|
Chris@1519
|
685
|
Chris@1519
|
686 auto ms = QDateTime::currentDateTime().toMSecsSinceEpoch();
|
Chris@1519
|
687 auto s = ms / 1000; // there is a toSecsSinceEpoch in Qt 5.8 but
|
Chris@1519
|
688 // we currently want to support older versions
|
Chris@1519
|
689
|
Chris@1519
|
690 return QDir(convertedFileDir).filePath
|
Chris@1519
|
691 (QString("%1-%2.wav").arg(base).arg(s));
|
Chris@1519
|
692 }
|
Chris@1519
|
693
|