comparison rdf/RDFImporter.cpp @ 440:5746c559af15

* Merge revisions 1131 to 1201 from sv-rdf-import branch
author Chris Cannam
date Thu, 18 Sep 2008 12:33:30 +0000
parents beb2948baa77
children 2fb0061c5d23
comparison
equal deleted inserted replaced
439:beb2948baa77 440:5746c559af15
52 typedef std::vector<float> ValueList; 52 typedef std::vector<float> ValueList;
53 typedef std::map<RealTime, ValueList> TimeValueMap; 53 typedef std::map<RealTime, ValueList> TimeValueMap;
54 typedef std::map<QString, TimeValueMap> TypeTimeValueMap; 54 typedef std::map<QString, TimeValueMap> TypeTimeValueMap;
55 typedef std::map<QString, TypeTimeValueMap> SourceTypeTimeValueMap; 55 typedef std::map<QString, TypeTimeValueMap> SourceTypeTimeValueMap;
56 56
57 void getDataModelsSparse(std::vector<Model *> &, ProgressReporter *);
58 void getDataModelsDense(std::vector<Model *> &, ProgressReporter *);
59
60 void getDenseFeatureProperties(QString featureUri,
61 int &sampleRate, int &windowLength,
62 int &hopSize, int &width, int &height);
63
57 void extractStructure(const TimeValueMap &map, bool &sparse, 64 void extractStructure(const TimeValueMap &map, bool &sparse,
58 int &minValueCount, int &maxValueCount); 65 int &minValueCount, int &maxValueCount);
59 66
60 void fillModel(SparseOneDimensionalModel *, const TimeValueMap &); 67 void fillModel(SparseOneDimensionalModel *, const TimeValueMap &);
61 void fillModel(SparseTimeValueModel *, const TimeValueMap &); 68 void fillModel(SparseTimeValueModel *, const TimeValueMap &);
122 std::vector<Model *> 129 std::vector<Model *>
123 RDFImporterImpl::getDataModels(ProgressReporter *reporter) 130 RDFImporterImpl::getDataModels(ProgressReporter *reporter)
124 { 131 {
125 std::vector<Model *> models; 132 std::vector<Model *> models;
126 133
134 getDataModelsDense(models, reporter);
135
136 QString error;
137 if (!isOK()) error = m_errorString;
138 m_errorString = "";
139
140 getDataModelsSparse(models, reporter);
141
142 if (isOK()) m_errorString = error;
143
144 return models;
145 }
146
147 void
148 RDFImporterImpl::getDataModelsDense(std::vector<Model *> &models,
149 ProgressReporter *reporter)
150 {
151 SimpleSPARQLQuery query = SimpleSPARQLQuery
152 (QString
153 (
154 " PREFIX mo: <http://purl.org/ontology/mo/>"
155 " PREFIX af: <http://purl.org/ontology/af/>"
156
157 " SELECT ?feature ?signal_source ?feature_signal_type ?value "
158 " FROM <%1> "
159
160 " WHERE { "
161
162 " ?signal a mo:Signal ; "
163 " mo:available_as ?signal_source ; "
164 " af:signal_feature ?feature . "
165
166 " ?feature a ?feature_signal_type ; "
167 " af:value ?value . "
168
169 " } "
170 )
171 .arg(m_uristring));
172
173 SimpleSPARQLQuery::ResultList results = query.execute();
174
175 if (!query.isOK()) {
176 m_errorString = query.getErrorString();
177 return;
178 }
179
180 if (query.wasCancelled()) {
181 m_errorString = "Query cancelled";
182 return;
183 }
184
185 for (int i = 0; i < results.size(); ++i) {
186
187 QString feature = results[i]["feature"].value;
188 QString source = results[i]["signal_source"].value;
189 QString type = results[i]["feature_signal_type"].value;
190 QString value = results[i]["value"].value;
191
192 int sampleRate = 0;
193 int windowLength = 0;
194 int hopSize = 0;
195 int width = 0;
196 int height = 0;
197 getDenseFeatureProperties
198 (feature, sampleRate, windowLength, hopSize, width, height);
199
200 if (sampleRate != 0 && sampleRate != m_sampleRate) {
201 cerr << "WARNING: Sample rate in dense feature description does not match our underlying rate -- using rate from feature description" << endl;
202 }
203 if (sampleRate == 0) sampleRate = m_sampleRate;
204
205 if (hopSize == 0) {
206 cerr << "WARNING: Dense feature description does not specify a hop size -- assuming 1" << endl;
207 hopSize = 1;
208 }
209
210 if (height == 0) {
211 cerr << "WARNING: Dense feature description does not specify feature signal dimensions -- assuming one-dimensional (height = 1)" << endl;
212 height = 1;
213 }
214
215 QStringList values = value.split(' ', QString::SkipEmptyParts);
216
217 if (values.empty()) {
218 cerr << "WARNING: Dense feature description does not specify any values!" << endl;
219 continue;
220 }
221
222 if (height == 1) {
223
224 SparseTimeValueModel *m = new SparseTimeValueModel
225 (sampleRate, hopSize, false);
226
227 for (int j = 0; j < values.size(); ++j) {
228 float f = values[j].toFloat();
229 SparseTimeValueModel::Point point(j * hopSize, f, "");
230 m->addPoint(point);
231 }
232
233 models.push_back(m);
234
235 } else {
236
237 EditableDenseThreeDimensionalModel *m =
238 new EditableDenseThreeDimensionalModel(sampleRate, hopSize,
239 height, false);
240
241 EditableDenseThreeDimensionalModel::Column column;
242
243 int x = 0;
244
245 for (int j = 0; j < values.size(); ++j) {
246 if (j % height == 0 && !column.empty()) {
247 m->setColumn(x++, column);
248 column.clear();
249 }
250 column.push_back(values[j].toFloat());
251 }
252
253 if (!column.empty()) {
254 m->setColumn(x++, column);
255 }
256
257 models.push_back(m);
258 }
259 }
260 }
261
262 void
263 RDFImporterImpl::getDenseFeatureProperties(QString featureUri,
264 int &sampleRate, int &windowLength,
265 int &hopSize, int &width, int &height)
266 {
267 QString dimensionsQuery
268 (
269 " PREFIX mo: <http://purl.org/ontology/mo/>"
270 " PREFIX af: <http://purl.org/ontology/af/>"
271
272 " SELECT ?dimensions "
273 " FROM <%1> "
274
275 " WHERE { "
276
277 " <%2> af:dimensions ?dimensions . "
278
279 " } "
280 );
281
282 SimpleSPARQLQuery::Value dimensionsValue =
283 SimpleSPARQLQuery::singleResultQuery(dimensionsQuery
284 .arg(m_uristring).arg(featureUri),
285 "dimensions");
286
287 cerr << "Dimensions = \"" << dimensionsValue.value.toStdString() << "\""
288 << endl;
289
290 if (dimensionsValue.value != "") {
291 QStringList dl = dimensionsValue.value.split(" ");
292 if (dl.empty()) dl.push_back(dimensionsValue.value);
293 if (dl.size() > 0) height = dl[0].toInt();
294 if (dl.size() > 1) width = dl[1].toInt();
295 }
296
297 QString queryTemplate
298 (
299 " PREFIX mo: <http://purl.org/ontology/mo/>"
300 " PREFIX af: <http://purl.org/ontology/af/>"
301 " PREFIX tl: <http://purl.org/NET/c4dm/timeline.owl#>"
302
303 " SELECT ?%3 "
304 " FROM <%1> "
305
306 " WHERE { "
307
308 " <%2> mo:time ?time . "
309
310 " ?time a tl:Interval ; "
311 " tl:onTimeLine ?timeline . "
312
313 " ?map tl:rangeTimeLine ?timeline . "
314
315 " ?map tl:%3 ?%3 . "
316
317 " } "
318 );
319
320 // Another laborious workaround for rasqal's failure to handle
321 // multiple optionals properly
322
323 SimpleSPARQLQuery::Value srValue =
324 SimpleSPARQLQuery::singleResultQuery(queryTemplate
325 .arg(m_uristring).arg(featureUri)
326 .arg("sampleRate"),
327 "sampleRate");
328 if (srValue.value != "") {
329 sampleRate = srValue.value.toInt();
330 }
331
332 SimpleSPARQLQuery::Value hopValue =
333 SimpleSPARQLQuery::singleResultQuery(queryTemplate
334 .arg(m_uristring).arg(featureUri)
335 .arg("hopSize"),
336 "hopSize");
337 if (srValue.value != "") {
338 hopSize = hopValue.value.toInt();
339 }
340
341 SimpleSPARQLQuery::Value winValue =
342 SimpleSPARQLQuery::singleResultQuery(queryTemplate
343 .arg(m_uristring).arg(featureUri)
344 .arg("windowLength"),
345 "windowLength");
346 if (winValue.value != "") {
347 windowLength = winValue.value.toInt();
348 }
349
350 cerr << "sr = " << sampleRate << ", hop = " << hopSize << ", win = " << windowLength << endl;
351 }
352
353 void
354 RDFImporterImpl::getDataModelsSparse(std::vector<Model *> &models,
355 ProgressReporter *reporter)
356 {
127 // Our query is intended to retrieve every thing that has a time, 357 // Our query is intended to retrieve every thing that has a time,
128 // and every feature type and value associated with a thing that 358 // and every feature type and value associated with a thing that
129 // has a time. 359 // has a time.
130 360
131 // We will then need to refine this big bag of results into a set 361 // We will then need to refine this big bag of results into a set
161 " PREFIX event: <http://purl.org/NET/c4dm/event.owl#>" 391 " PREFIX event: <http://purl.org/NET/c4dm/event.owl#>"
162 " PREFIX time: <http://purl.org/NET/c4dm/timeline.owl#>" 392 " PREFIX time: <http://purl.org/NET/c4dm/timeline.owl#>"
163 " PREFIX mo: <http://purl.org/ontology/mo/>" 393 " PREFIX mo: <http://purl.org/ontology/mo/>"
164 " PREFIX af: <http://purl.org/ontology/af/>" 394 " PREFIX af: <http://purl.org/ontology/af/>"
165 395
166 " SELECT ?signalSource ?time ?eventType ?value" 396 " SELECT ?signal_source ?time ?event_type ?value"
167 " FROM <%1>" 397 " FROM <%1>"
168 398
169 " WHERE {" 399 " WHERE {"
170 " ?signal mo:available_as ?signalSource ." 400
401 " ?signal mo:available_as ?signal_source ."
402 " ?signal a mo:Signal ."
403
171 " ?signal mo:time ?interval ." 404 " ?signal mo:time ?interval ."
172 " ?interval time:onTimeLine ?tl ." 405 " ?interval time:onTimeLine ?tl ."
173 " ?t time:onTimeLine ?tl ." 406 " ?t time:onTimeLine ?tl ."
174 " ?t time:at ?time ." 407 " ?t time:at ?time ."
175 " ?timedThing event:time ?t ." 408 " ?timed_thing event:time ?t ."
176 " ?timedThing a ?eventType ." 409 " ?timed_thing a ?event_type ."
410
177 " OPTIONAL {" 411 " OPTIONAL {"
178 " ?timedThing af:hasFeature ?feature ." 412 " ?timed_thing af:feature ?value"
179 " ?feature af:value ?value"
180 " }" 413 " }"
181 " }" 414 " }"
182 415
183 ).arg(m_uristring); 416 ).arg(m_uristring);
184 417
189 422
190 SimpleSPARQLQuery::ResultList results = query.execute(); 423 SimpleSPARQLQuery::ResultList results = query.execute();
191 424
192 if (!query.isOK()) { 425 if (!query.isOK()) {
193 m_errorString = query.getErrorString(); 426 m_errorString = query.getErrorString();
194 return models; 427 return;
195 } 428 }
196 429
197 if (query.wasCancelled()) { 430 if (query.wasCancelled()) {
198 m_errorString = "Query cancelled"; 431 m_errorString = "Query cancelled";
199 return models; 432 return;
200 } 433 }
201 434
202 for (int i = 0; i < results.size(); ++i) { 435 for (int i = 0; i < results.size(); ++i) {
203 436
204 QString source = results[i]["signalSource"].value; 437 QString source = results[i]["signal_source"].value;
205 438
206 QString timestring = results[i]["time"].value; 439 QString timestring = results[i]["time"].value;
207 RealTime time; 440 RealTime time;
208 time = RealTime::fromXsdDuration(timestring.toStdString()); 441 time = RealTime::fromXsdDuration(timestring.toStdString());
209 cerr << "time = " << time.toString() << " (from xsd:duration \"" 442 cerr << "time = " << time.toString() << " (from xsd:duration \""
210 << timestring.toStdString() << "\")" << endl; 443 << timestring.toStdString() << "\")" << endl;
211 444
212 QString type = results[i]["eventType"].value; 445 QString type = results[i]["event_type"].value;
213 446
214 QString valuestring = results[i]["value"].value; 447 QString valuestring = results[i]["value"].value;
215 float value = 0.f; 448 float value = 0.f;
216 bool haveValue = false; 449 bool haveValue = false;
217 if (valuestring != "") { 450 if (valuestring != "") {
451 //!!! no -- runner actually writes a "CSV literal"
218 value = valuestring.toFloat(&haveValue); 452 value = valuestring.toFloat(&haveValue);
219 cerr << "value = " << value << endl; 453 cerr << "value = " << value << endl;
220 } 454 }
221 455
222 if (haveValue) { 456 if (haveValue) {
320 fillModel(model, ttvi->second); 554 fillModel(model, ttvi->second);
321 models.push_back(model); 555 models.push_back(model);
322 } 556 }
323 } 557 }
324 } 558 }
325
326
327 return models;
328 } 559 }
329 560
330 void 561 void
331 RDFImporterImpl::extractStructure(const TimeValueMap &tvm, 562 RDFImporterImpl::extractStructure(const TimeValueMap &tvm,
332 bool &sparse, 563 bool &sparse,