mas01cr@0
|
1 #include "audioDB.h"
|
mas01cr@0
|
2
|
mas01mc@308
|
3 LSH* SERVER_LSH_INDEX_SINGLETON;
|
mas01mc@324
|
4 char* SERVER_ADB_ROOT;
|
mas01mc@324
|
5 char* SERVER_ADB_FEATURE_ROOT;
|
mas01mc@308
|
6
|
mas01mc@292
|
7 PointPair::PointPair(Uns32T a, Uns32T b, Uns32T c):trackID(a),qpos(b),spos(c){};
|
mas01mc@292
|
8
|
mas01mc@292
|
9 bool operator<(const PointPair& a, const PointPair& b){
|
mas01mc@324
|
10 return ( (a.trackID<b.trackID) ||
|
mas01mc@324
|
11 ( (a.trackID==b.trackID) &&
|
mas01mc@324
|
12 ( (a.spos<b.spos) || ( (a.spos==b.spos) && (a.qpos < b.qpos) )) ) );
|
mas01mc@292
|
13 }
|
mas01mc@292
|
14
|
mas01mc@292
|
15 bool operator>(const PointPair& a, const PointPair& b){
|
mas01mc@324
|
16 return ( (a.trackID>b.trackID) ||
|
mas01mc@324
|
17 ( (a.trackID==b.trackID) &&
|
mas01mc@324
|
18 ( (a.spos>b.spos) || ( (a.spos==b.spos) && (a.qpos > b.qpos) )) ) );
|
mas01mc@292
|
19 }
|
mas01mc@292
|
20
|
mas01mc@292
|
21 bool operator==(const PointPair& a, const PointPair& b){
|
mas01mc@292
|
22 return ( (a.trackID==b.trackID) && (a.qpos==b.qpos) && (a.spos==b.spos) );
|
mas01mc@292
|
23 }
|
mas01mc@292
|
24
|
mas01cr@76
|
25 audioDB::audioDB(const unsigned argc, char* const argv[]): O2_AUDIODB_INITIALIZERS
|
mas01cr@76
|
26 {
|
mas01cr@0
|
27 if(processArgs(argc, argv)<0){
|
mas01cr@0
|
28 printf("No command found.\n");
|
mas01cr@0
|
29 cmdline_parser_print_version ();
|
mas01cr@0
|
30 if (strlen(gengetopt_args_info_purpose) > 0)
|
mas01cr@0
|
31 printf("%s\n", gengetopt_args_info_purpose);
|
mas01cr@0
|
32 printf("%s\n", gengetopt_args_info_usage);
|
mas01cr@0
|
33 printf("%s\n", gengetopt_args_info_help[1]);
|
mas01cr@0
|
34 printf("%s\n", gengetopt_args_info_help[2]);
|
mas01cr@0
|
35 printf("%s\n", gengetopt_args_info_help[0]);
|
mas01cr@151
|
36 error("No command found");
|
mas01cr@0
|
37 }
|
mas01cr@77
|
38
|
mas01mc@324
|
39 // Perform database prefix substitution
|
mas01mc@328
|
40 if(dbName && adb_root)
|
mas01mc@324
|
41 prefix_name((char** const)&dbName, adb_root);
|
mas01mc@324
|
42
|
mas01mc@328
|
43 if(inFile && adb_feature_root)
|
mas01mc@328
|
44 prefix_name((char** const)&inFile, adb_feature_root);
|
mas01mc@328
|
45
|
mas01cr@0
|
46 if(O2_ACTION(COM_SERVER))
|
mas01cr@0
|
47 startServer();
|
mas01cr@0
|
48
|
mas01cr@0
|
49 else if(O2_ACTION(COM_CREATE))
|
mas01cr@0
|
50 create(dbName);
|
mas01cr@0
|
51
|
mas01cr@0
|
52 else if(O2_ACTION(COM_INSERT))
|
mas01cr@0
|
53 insert(dbName, inFile);
|
mas01cr@0
|
54
|
mas01cr@0
|
55 else if(O2_ACTION(COM_BATCHINSERT))
|
mas01cr@0
|
56 batchinsert(dbName, inFile);
|
mas01cr@0
|
57
|
mas01cr@0
|
58 else if(O2_ACTION(COM_QUERY))
|
mas01mc@307
|
59 if(isClient){
|
mas01mc@329
|
60 if(query_from_key){
|
mas01mc@329
|
61 VERB_LOG(1, "Calling web services query %s on database %s, query=%s\n", radius>0?"(Radius)":"(NN)", dbName, strlen(key)?key:inFile);
|
mas01mc@328
|
62 ws_query_by_key(dbName, key, inFile, (char*)hostport);
|
mas01mc@329
|
63 }
|
mas01mc@329
|
64 else{
|
mas01mc@329
|
65 VERB_LOG(1, "Calling web services query on database %s, query=%s\n", dbName, strlen(key)?key:inFile);
|
mas01mc@307
|
66 ws_query(dbName, inFile, (char*)hostport);
|
mas01mc@329
|
67 }
|
mas01mc@307
|
68 }
|
mas01cr@0
|
69 else
|
mas01cr@76
|
70 query(dbName, inFile);
|
mas01cr@0
|
71
|
mas01cr@0
|
72 else if(O2_ACTION(COM_STATUS))
|
mas01cr@0
|
73 if(isClient)
|
mas01cr@0
|
74 ws_status(dbName,(char*)hostport);
|
mas01cr@0
|
75 else
|
mas01cr@0
|
76 status(dbName);
|
mas01cr@280
|
77
|
mas01cr@280
|
78 else if(O2_ACTION(COM_SAMPLE))
|
mas01cr@280
|
79 sample(dbName);
|
mas01cr@0
|
80
|
mas01cr@0
|
81 else if(O2_ACTION(COM_L2NORM))
|
mas01cr@0
|
82 l2norm(dbName);
|
mas01cr@0
|
83
|
mas01cr@193
|
84 else if(O2_ACTION(COM_POWER))
|
mas01cr@193
|
85 power_flag(dbName);
|
mas01cr@193
|
86
|
mas01cr@0
|
87 else if(O2_ACTION(COM_DUMP))
|
mas01cr@0
|
88 dump(dbName);
|
mas01mc@292
|
89
|
mas01mc@292
|
90 else if(O2_ACTION(COM_INDEX))
|
mas01mc@292
|
91 index_index_db(dbName);
|
mas01cr@0
|
92
|
mas01cr@0
|
93 else
|
mas01cr@0
|
94 error("Unrecognized command",command);
|
mas01cr@0
|
95 }
|
mas01cr@0
|
96
|
mas01cr@133
|
97 audioDB::audioDB(const unsigned argc, char* const argv[], adb__queryResponse *adbQueryResponse): O2_AUDIODB_INITIALIZERS
|
mas01cr@76
|
98 {
|
mas01cr@97
|
99 try {
|
mas01cr@151
|
100 isServer = 1; // FIXME: Hack
|
mas01cr@97
|
101 processArgs(argc, argv);
|
mas01mc@324
|
102 // Perform database prefix substitution
|
mas01mc@328
|
103 if(dbName && adb_root)
|
mas01mc@324
|
104 prefix_name((char** const)&dbName, adb_root);
|
mas01mc@328
|
105 if(inFile && adb_feature_root)
|
mas01mc@328
|
106 prefix_name((char** const)&inFile, adb_feature_root);
|
mas01cr@97
|
107 assert(O2_ACTION(COM_QUERY));
|
mas01cr@133
|
108 query(dbName, inFile, adbQueryResponse);
|
mas01cr@97
|
109 } catch(char *err) {
|
mas01cr@97
|
110 cleanup();
|
mas01cr@97
|
111 throw(err);
|
mas01cr@97
|
112 }
|
mas01cr@76
|
113 }
|
mas01cr@76
|
114
|
mas01cr@133
|
115 audioDB::audioDB(const unsigned argc, char* const argv[], adb__statusResponse *adbStatusResponse): O2_AUDIODB_INITIALIZERS
|
mas01cr@76
|
116 {
|
mas01cr@97
|
117 try {
|
mas01cr@151
|
118 isServer = 1; // FIXME: Hack
|
mas01cr@97
|
119 processArgs(argc, argv);
|
mas01mc@324
|
120 // Perform database prefix substitution
|
mas01mc@328
|
121 if(dbName && adb_root)
|
mas01mc@324
|
122 prefix_name((char** const)&dbName, adb_root);
|
mas01mc@328
|
123 if(inFile && adb_feature_root)
|
mas01mc@328
|
124 prefix_name((char** const)&inFile, adb_feature_root);
|
mas01cr@97
|
125 assert(O2_ACTION(COM_STATUS));
|
mas01cr@133
|
126 status(dbName, adbStatusResponse);
|
mas01cr@97
|
127 } catch(char *err) {
|
mas01cr@97
|
128 cleanup();
|
mas01cr@97
|
129 throw(err);
|
mas01cr@97
|
130 }
|
mas01cr@76
|
131 }
|
mas01cr@76
|
132
|
mas01cr@97
|
133 void audioDB::cleanup() {
|
mas01cr@122
|
134 cmdline_parser_free(&args_info);
|
mas01cr@0
|
135 if(indata)
|
mas01cr@0
|
136 munmap(indata,statbuf.st_size);
|
mas01cr@0
|
137 if(db)
|
mas01cr@196
|
138 munmap(db,getpagesize());
|
mas01cr@196
|
139 if(fileTable)
|
mas01cr@196
|
140 munmap(fileTable, fileTableLength);
|
mas01cr@196
|
141 if(trackTable)
|
mas01cr@196
|
142 munmap(trackTable, trackTableLength);
|
mas01cr@196
|
143 if(dataBuf)
|
mas01cr@196
|
144 munmap(dataBuf, dataBufLength);
|
mas01cr@196
|
145 if(timesTable)
|
mas01cr@196
|
146 munmap(timesTable, timesTableLength);
|
mas01mc@314
|
147 if(powerTable)
|
mas01mc@314
|
148 munmap(powerTable, powerTableLength);
|
mas01cr@196
|
149 if(l2normTable)
|
mas01cr@196
|
150 munmap(l2normTable, l2normTableLength);
|
mas01mc@324
|
151 if(featureFileNameTable)
|
mas01mc@324
|
152 munmap(featureFileNameTable, fileTableLength);
|
mas01mc@324
|
153 if(timesFileNameTable)
|
mas01mc@324
|
154 munmap(timesFileNameTable, fileTableLength);
|
mas01mc@324
|
155 if(powerFileNameTable)
|
mas01mc@324
|
156 munmap(powerFileNameTable, fileTableLength);
|
mas01mc@292
|
157 if(trackOffsetTable)
|
mas01mc@292
|
158 delete trackOffsetTable;
|
mas01mc@292
|
159 if(reporter)
|
mas01mc@292
|
160 delete reporter;
|
mas01mc@292
|
161 if(exact_evaluation_queue)
|
mas01mc@292
|
162 delete exact_evaluation_queue;
|
mas01cr@284
|
163 if(rng)
|
mas01cr@284
|
164 gsl_rng_free(rng);
|
mas01mc@292
|
165 if(vv)
|
mas01mc@292
|
166 delete vv;
|
mas01cr@0
|
167 if(dbfid>0)
|
mas01cr@0
|
168 close(dbfid);
|
mas01cr@0
|
169 if(infid>0)
|
mas01cr@0
|
170 close(infid);
|
mas01cr@0
|
171 if(dbH)
|
mas01cr@0
|
172 delete dbH;
|
mas01mc@308
|
173 if(lsh!=SERVER_LSH_INDEX_SINGLETON)
|
mas01mc@308
|
174 delete lsh;
|
mas01cr@0
|
175 }
|
mas01cr@0
|
176
|
mas01cr@97
|
177 audioDB::~audioDB(){
|
mas01cr@97
|
178 cleanup();
|
mas01cr@97
|
179 }
|
mas01cr@97
|
180
|
mas01cr@0
|
181 int audioDB::processArgs(const unsigned argc, char* const argv[]){
|
mas01cr@0
|
182
|
mas01cr@0
|
183 if(argc<2){
|
mas01cr@0
|
184 cmdline_parser_print_version ();
|
mas01cr@0
|
185 if (strlen(gengetopt_args_info_purpose) > 0)
|
mas01cr@0
|
186 printf("%s\n", gengetopt_args_info_purpose);
|
mas01cr@0
|
187 printf("%s\n", gengetopt_args_info_usage);
|
mas01cr@0
|
188 printf("%s\n", gengetopt_args_info_help[1]);
|
mas01cr@0
|
189 printf("%s\n", gengetopt_args_info_help[2]);
|
mas01cr@0
|
190 printf("%s\n", gengetopt_args_info_help[0]);
|
mas01cr@0
|
191 exit(0);
|
mas01cr@0
|
192 }
|
mas01cr@0
|
193
|
mas01cr@0
|
194 if (cmdline_parser (argc, argv, &args_info) != 0)
|
mas01cr@151
|
195 error("Error parsing command line");
|
mas01cr@0
|
196
|
mas01cr@0
|
197 if(args_info.help_given){
|
mas01cr@0
|
198 cmdline_parser_print_help();
|
mas01cr@0
|
199 exit(0);
|
mas01cr@0
|
200 }
|
mas01cr@0
|
201
|
mas01cr@0
|
202 if(args_info.verbosity_given){
|
mas01cr@239
|
203 verbosity = args_info.verbosity_arg;
|
mas01cr@239
|
204 if(verbosity < 0 || verbosity > 10){
|
mas01cr@239
|
205 std::cerr << "Warning: verbosity out of range, setting to 1" << std::endl;
|
mas01cr@239
|
206 verbosity = 1;
|
mas01cr@0
|
207 }
|
mas01cr@0
|
208 }
|
mas01cr@0
|
209
|
mas01cr@129
|
210 if(args_info.size_given) {
|
mas01cr@256
|
211 if(args_info.datasize_given) {
|
mas01cr@256
|
212 error("both --size and --datasize given", "");
|
mas01cr@256
|
213 }
|
mas01cr@256
|
214 if(args_info.ntracks_given) {
|
mas01cr@256
|
215 error("both --size and --ntracks given", "");
|
mas01cr@256
|
216 }
|
mas01cr@256
|
217 if(args_info.datadim_given) {
|
mas01cr@256
|
218 error("both --size and --datadim given", "");
|
mas01cr@256
|
219 }
|
mas01cr@196
|
220 if (args_info.size_arg < 50 || args_info.size_arg > 32000) {
|
mas01cr@129
|
221 error("Size out of range", "");
|
mas01cr@129
|
222 }
|
mas01cr@256
|
223 double ratio = (double) args_info.size_arg * 1000000 / ((double) O2_DEFAULTDBSIZE);
|
mas01cr@256
|
224 /* FIXME: what's the safe way of doing this? */
|
mas01cr@256
|
225 datasize = (unsigned int) ceil(datasize * ratio);
|
mas01cr@256
|
226 ntracks = (unsigned int) ceil(ntracks * ratio);
|
mas01cr@256
|
227 } else {
|
mas01cr@256
|
228 if(args_info.datasize_given) {
|
mas01cr@256
|
229 datasize = args_info.datasize_arg;
|
mas01cr@256
|
230 }
|
mas01cr@256
|
231 if(args_info.ntracks_given) {
|
mas01cr@256
|
232 ntracks = args_info.ntracks_arg;
|
mas01cr@256
|
233 }
|
mas01cr@256
|
234 if(args_info.datadim_given) {
|
mas01cr@256
|
235 datadim = args_info.datadim_arg;
|
mas01cr@256
|
236 }
|
mas01cr@129
|
237 }
|
mas01cr@129
|
238
|
mas01cr@239
|
239 if(args_info.radius_given) {
|
mas01cr@239
|
240 radius = args_info.radius_arg;
|
mas01mc@307
|
241 if(radius < 0 || radius > 1000000000) {
|
mas01cr@77
|
242 error("radius out of range");
|
mas01cr@239
|
243 } else {
|
mas01cr@239
|
244 VERB_LOG(3, "Setting radius to %f\n", radius);
|
mas01mc@17
|
245 }
|
mas01mc@17
|
246 }
|
mas01mc@17
|
247
|
mas01mc@292
|
248 sequenceLength = args_info.sequencelength_arg;
|
mas01mc@292
|
249 if(sequenceLength < 1 || sequenceLength > 1000) {
|
mas01mc@292
|
250 error("seqlen out of range: 1 <= seqlen <= 1000");
|
mas01mc@292
|
251 }
|
mas01mc@292
|
252 sequenceHop = args_info.sequencehop_arg;
|
mas01mc@292
|
253 if(sequenceHop < 1 || sequenceHop > 1000) {
|
mas01mc@292
|
254 error("seqhop out of range: 1 <= seqhop <= 1000");
|
mas01mc@292
|
255 }
|
mas01mc@292
|
256
|
mas01mc@292
|
257 if (args_info.absolute_threshold_given) {
|
mas01mc@292
|
258 if (args_info.absolute_threshold_arg >= 0) {
|
mas01mc@292
|
259 error("absolute threshold out of range: should be negative");
|
mas01mc@292
|
260 }
|
mas01mc@292
|
261 use_absolute_threshold = true;
|
mas01mc@292
|
262 absolute_threshold = args_info.absolute_threshold_arg;
|
mas01mc@292
|
263 }
|
mas01mc@292
|
264 if (args_info.relative_threshold_given) {
|
mas01mc@292
|
265 use_relative_threshold = true;
|
mas01mc@292
|
266 relative_threshold = args_info.relative_threshold_arg;
|
mas01mc@292
|
267 }
|
mas01mc@292
|
268
|
mas01mc@324
|
269 if (args_info.adb_root_given){
|
mas01mc@324
|
270 adb_root = args_info.adb_root_arg;
|
mas01mc@324
|
271 }
|
mas01mc@324
|
272
|
mas01mc@324
|
273 if (args_info.adb_feature_root_given){
|
mas01mc@324
|
274 adb_feature_root = args_info.adb_feature_root_arg;
|
mas01mc@324
|
275 }
|
mas01mc@324
|
276
|
mas01mc@324
|
277 // perform dbName path prefix SERVER-side subsitution
|
mas01mc@324
|
278 if(SERVER_ADB_ROOT && !adb_root)
|
mas01mc@324
|
279 adb_root = SERVER_ADB_ROOT;
|
mas01mc@324
|
280 if(SERVER_ADB_FEATURE_ROOT && !adb_feature_root)
|
mas01mc@324
|
281 adb_feature_root = SERVER_ADB_FEATURE_ROOT;
|
mas01mc@324
|
282
|
mas01cr@0
|
283 if(args_info.SERVER_given){
|
mas01cr@0
|
284 command=COM_SERVER;
|
mas01cr@0
|
285 port=args_info.SERVER_arg;
|
mas01cr@0
|
286 if(port<100 || port > 100000)
|
mas01cr@0
|
287 error("port out of range");
|
mas01cr@151
|
288 isServer = 1;
|
mas01cr@105
|
289 #if defined(O2_DEBUG)
|
mas01cr@104
|
290 struct sigaction sa;
|
mas01cr@104
|
291 sa.sa_sigaction = sigterm_action;
|
mas01cr@104
|
292 sa.sa_flags = SA_SIGINFO | SA_RESTART | SA_NODEFER;
|
mas01cr@104
|
293 sigaction(SIGTERM, &sa, NULL);
|
mas01cr@104
|
294 sa.sa_sigaction = sighup_action;
|
mas01cr@104
|
295 sa.sa_flags = SA_SIGINFO | SA_RESTART | SA_NODEFER;
|
mas01cr@104
|
296 sigaction(SIGHUP, &sa, NULL);
|
mas01cr@105
|
297 #endif
|
mas01mc@308
|
298 if(args_info.load_index_given){
|
mas01mc@308
|
299 if(!args_info.database_given)
|
mas01mc@308
|
300 error("load_index requires a --database argument");
|
mas01mc@308
|
301 else
|
mas01mc@308
|
302 dbName=args_info.database_arg;
|
mas01mc@308
|
303 if(!args_info.radius_given)
|
mas01mc@308
|
304 error("load_index requires a --radius argument");
|
mas01mc@308
|
305 if(!args_info.sequencelength_given)
|
mas01mc@308
|
306 error("load_index requires a --sequenceLength argument");
|
mas01mc@308
|
307 WS_load_index = true;
|
mas01mc@308
|
308 }
|
mas01cr@0
|
309 return 0;
|
mas01cr@0
|
310 }
|
mas01cr@0
|
311
|
mas01cr@0
|
312 // No return on client command, find database command
|
mas01cr@105
|
313 if(args_info.client_given){
|
mas01cr@105
|
314 command=COM_CLIENT;
|
mas01cr@105
|
315 hostport=args_info.client_arg;
|
mas01cr@105
|
316 isClient=1;
|
mas01cr@105
|
317 }
|
mas01cr@0
|
318
|
mas01cr@105
|
319 if(args_info.NEW_given){
|
mas01cr@105
|
320 command=COM_CREATE;
|
mas01cr@105
|
321 dbName=args_info.database_arg;
|
mas01cr@105
|
322 return 0;
|
mas01cr@105
|
323 }
|
mas01cr@0
|
324
|
mas01cr@105
|
325 if(args_info.STATUS_given){
|
mas01cr@105
|
326 command=COM_STATUS;
|
mas01cr@105
|
327 dbName=args_info.database_arg;
|
mas01cr@105
|
328 return 0;
|
mas01cr@105
|
329 }
|
mas01cr@0
|
330
|
mas01cr@280
|
331 if(args_info.SAMPLE_given) {
|
mas01cr@280
|
332 command = COM_SAMPLE;
|
mas01cr@280
|
333 dbName = args_info.database_arg;
|
mas01cr@280
|
334 sequenceLength = args_info.sequencelength_arg;
|
mas01cr@280
|
335 if(sequenceLength < 1 || sequenceLength > 1000) {
|
mas01cr@280
|
336 error("seqlen out of range: 1 <= seqlen <= 1000");
|
mas01cr@280
|
337 }
|
mas01cr@280
|
338 nsamples = args_info.nsamples_arg;
|
mas01cr@280
|
339 return 0;
|
mas01cr@280
|
340 }
|
mas01cr@280
|
341
|
mas01cr@105
|
342 if(args_info.DUMP_given){
|
mas01cr@105
|
343 command=COM_DUMP;
|
mas01cr@105
|
344 dbName=args_info.database_arg;
|
mas01cr@131
|
345 output = args_info.output_arg;
|
mas01cr@105
|
346 return 0;
|
mas01cr@105
|
347 }
|
mas01cr@0
|
348
|
mas01cr@105
|
349 if(args_info.L2NORM_given){
|
mas01cr@105
|
350 command=COM_L2NORM;
|
mas01cr@105
|
351 dbName=args_info.database_arg;
|
mas01cr@105
|
352 return 0;
|
mas01cr@105
|
353 }
|
mas01cr@0
|
354
|
mas01cr@193
|
355 if(args_info.POWER_given){
|
mas01cr@193
|
356 command=COM_POWER;
|
mas01cr@193
|
357 dbName=args_info.database_arg;
|
mas01cr@193
|
358 return 0;
|
mas01cr@193
|
359 }
|
mas01cr@193
|
360
|
mas01cr@105
|
361 if(args_info.INSERT_given){
|
mas01cr@105
|
362 command=COM_INSERT;
|
mas01cr@105
|
363 dbName=args_info.database_arg;
|
mas01cr@105
|
364 inFile=args_info.features_arg;
|
mas01cr@105
|
365 if(args_info.key_given)
|
mas01mc@292
|
366 if(!args_info.features_given)
|
mas01mc@292
|
367 error("INSERT: '-k key' argument depends on '-f features'");
|
mas01mc@292
|
368 else
|
mas01mc@292
|
369 key=args_info.key_arg;
|
mas01cr@105
|
370 if(args_info.times_given){
|
mas01cr@105
|
371 timesFileName=args_info.times_arg;
|
mas01cr@105
|
372 if(strlen(timesFileName)>0){
|
mas01cr@239
|
373 if(!(timesFile = new std::ifstream(timesFileName,std::ios::in)))
|
mas01cr@105
|
374 error("Could not open times file for reading", timesFileName);
|
mas01cr@105
|
375 usingTimes=1;
|
mas01cr@105
|
376 }
|
mas01cr@105
|
377 }
|
mas01cr@193
|
378 if (args_info.power_given) {
|
mas01cr@193
|
379 powerFileName = args_info.power_arg;
|
mas01cr@193
|
380 if (strlen(powerFileName) > 0) {
|
mas01cr@193
|
381 if (!(powerfd = open(powerFileName, O_RDONLY))) {
|
mas01cr@193
|
382 error("Could not open power file for reading", powerFileName, "open");
|
mas01cr@193
|
383 }
|
mas01cr@193
|
384 usingPower = 1;
|
mas01cr@193
|
385 }
|
mas01cr@193
|
386 }
|
mas01cr@105
|
387 return 0;
|
mas01cr@105
|
388 }
|
mas01cr@105
|
389
|
mas01cr@105
|
390 if(args_info.BATCHINSERT_given){
|
mas01cr@105
|
391 command=COM_BATCHINSERT;
|
mas01cr@105
|
392 dbName=args_info.database_arg;
|
mas01cr@105
|
393 inFile=args_info.featureList_arg;
|
mas01cr@105
|
394 if(args_info.keyList_given)
|
mas01tc@298
|
395 if(!args_info.featureList_given)
|
mas01tc@300
|
396 error("BATCHINSERT: '-K keyList' argument depends on '-F featureList'");
|
mas01mc@292
|
397 else
|
mas01cr@304
|
398 key=args_info.keyList_arg; // INCONSISTENT NO CHECK
|
mas01cr@0
|
399
|
mas01cr@105
|
400 /* TO DO: REPLACE WITH
|
mas01cr@0
|
401 if(args_info.keyList_given){
|
mas01mc@18
|
402 trackFileName=args_info.keyList_arg;
|
mas01cr@239
|
403 if(strlen(trackFileName)>0 && !(trackFile = new std::ifstream(trackFileName,std::ios::in)))
|
mas01mc@18
|
404 error("Could not open keyList file for reading",trackFileName);
|
mas01cr@0
|
405 }
|
mas01cr@0
|
406 AND UPDATE BATCHINSERT()
|
mas01cr@105
|
407 */
|
mas01cr@105
|
408
|
mas01cr@105
|
409 if(args_info.timesList_given){
|
mas01cr@105
|
410 timesFileName=args_info.timesList_arg;
|
mas01cr@105
|
411 if(strlen(timesFileName)>0){
|
mas01cr@239
|
412 if(!(timesFile = new std::ifstream(timesFileName,std::ios::in)))
|
mas01cr@105
|
413 error("Could not open timesList file for reading", timesFileName);
|
mas01cr@105
|
414 usingTimes=1;
|
mas01cr@105
|
415 }
|
mas01cr@105
|
416 }
|
mas01cr@193
|
417 if(args_info.powerList_given){
|
mas01cr@193
|
418 powerFileName=args_info.powerList_arg;
|
mas01cr@193
|
419 if(strlen(powerFileName)>0){
|
mas01cr@239
|
420 if(!(powerFile = new std::ifstream(powerFileName,std::ios::in)))
|
mas01cr@193
|
421 error("Could not open powerList file for reading", powerFileName);
|
mas01cr@193
|
422 usingPower=1;
|
mas01cr@193
|
423 }
|
mas01cr@193
|
424 }
|
mas01cr@105
|
425 return 0;
|
mas01cr@105
|
426 }
|
mas01mc@292
|
427
|
mas01mc@292
|
428 // Set no_unit_norm flag
|
mas01mc@292
|
429 no_unit_norming = args_info.no_unit_norming_flag;
|
mas01mc@292
|
430 lsh_use_u_functions = args_info.lsh_use_u_functions_flag;
|
mas01mc@292
|
431
|
mas01mc@292
|
432 // LSH Index Command
|
mas01mc@292
|
433 if(args_info.INDEX_given){
|
mas01mc@292
|
434 if(radius <= 0 )
|
mas01mc@292
|
435 error("INDEXing requires a Radius argument");
|
mas01mc@292
|
436 if(!(sequenceLength>0 && sequenceLength <= O2_MAXSEQLEN))
|
mas01mc@292
|
437 error("INDEXing requires 1 <= sequenceLength <= 1000");
|
mas01mc@292
|
438 command=COM_INDEX;
|
mas01mc@292
|
439 dbName=args_info.database_arg;
|
mas01mc@292
|
440
|
mas01mc@292
|
441 // Whether to store LSH hash tables for query in core (FORMAT2)
|
mas01mc@297
|
442 lsh_in_core = !args_info.lsh_on_disk_flag; // This flag is set to 0 if on_disk requested
|
mas01mc@292
|
443
|
mas01mc@292
|
444 lsh_param_w = args_info.lsh_w_arg;
|
mas01mc@292
|
445 if(!(lsh_param_w>0 && lsh_param_w<=O2_SERIAL_MAX_BINWIDTH))
|
mas01mc@292
|
446 error("Indexing parameter w out of range (0.0 < w <= 100.0)");
|
mas01mc@292
|
447
|
mas01mc@292
|
448 lsh_param_k = args_info.lsh_k_arg;
|
mas01mc@292
|
449 if(!(lsh_param_k>0 && lsh_param_k<=O2_SERIAL_MAX_FUNS))
|
mas01mc@292
|
450 error("Indexing parameter k out of range (1 <= k <= 100)");
|
mas01mc@292
|
451
|
mas01mc@292
|
452 lsh_param_m = args_info.lsh_m_arg;
|
mas01mc@292
|
453 if(!(lsh_param_m>0 && lsh_param_m<= (1 + (sqrt(1 + O2_SERIAL_MAX_TABLES*8.0)))/2.0))
|
mas01mc@292
|
454 error("Indexing parameter m out of range (1 <= m <= 20)");
|
mas01mc@292
|
455
|
mas01mc@292
|
456 lsh_param_N = args_info.lsh_N_arg;
|
mas01mc@292
|
457 if(!(lsh_param_N>0 && lsh_param_N<=O2_SERIAL_MAX_ROWS))
|
mas01mc@292
|
458 error("Indexing parameter N out of range (1 <= N <= 1000000)");
|
mas01mc@292
|
459
|
mas01mc@292
|
460 lsh_param_b = args_info.lsh_b_arg;
|
mas01mc@292
|
461 if(!(lsh_param_b>0 && lsh_param_b<=O2_SERIAL_MAX_TRACKBATCH))
|
mas01mc@292
|
462 error("Indexing parameter b out of range (1 <= b <= 10000)");
|
mas01mc@292
|
463
|
mas01mc@296
|
464 lsh_param_ncols = args_info.lsh_ncols_arg;
|
mas01mc@296
|
465 if(lsh_in_core) // We don't want to block rows with FORMAT2 indexing
|
mas01mc@296
|
466 lsh_param_ncols = O2_SERIAL_MAX_COLS;
|
mas01mc@292
|
467 if( !(lsh_param_ncols>0 && lsh_param_ncols<=O2_SERIAL_MAX_COLS))
|
mas01mc@292
|
468 error("Indexing parameter ncols out of range (1 <= ncols <= 1000");
|
mas01mc@292
|
469
|
mas01mc@292
|
470 return 0;
|
mas01mc@292
|
471 }
|
mas01mc@292
|
472
|
mas01cr@105
|
473 // Query command and arguments
|
mas01cr@105
|
474 if(args_info.QUERY_given){
|
mas01cr@105
|
475 command=COM_QUERY;
|
mas01cr@105
|
476 dbName=args_info.database_arg;
|
mas01mc@292
|
477 // XOR features and key search
|
mas01mc@292
|
478 if(!args_info.features_given && !args_info.key_given || (args_info.features_given && args_info.key_given))
|
mas01mc@292
|
479 error("QUERY requires exactly one of either -f features or -k key");
|
mas01mc@292
|
480 if(args_info.features_given)
|
mas01mc@292
|
481 inFile=args_info.features_arg; // query from file
|
mas01mc@292
|
482 else{
|
mas01mc@292
|
483 query_from_key = true;
|
mas01mc@292
|
484 key=args_info.key_arg; // query from key
|
mas01mc@292
|
485 }
|
mas01mc@292
|
486
|
mas01cr@105
|
487 if(args_info.keyList_given){
|
mas01cr@105
|
488 trackFileName=args_info.keyList_arg;
|
mas01cr@239
|
489 if(strlen(trackFileName)>0 && !(trackFile = new std::ifstream(trackFileName,std::ios::in)))
|
mas01cr@105
|
490 error("Could not open keyList file for reading",trackFileName);
|
mas01cr@105
|
491 }
|
mas01cr@105
|
492
|
mas01cr@105
|
493 if(args_info.times_given){
|
mas01cr@105
|
494 timesFileName=args_info.times_arg;
|
mas01cr@105
|
495 if(strlen(timesFileName)>0){
|
mas01cr@239
|
496 if(!(timesFile = new std::ifstream(timesFileName,std::ios::in)))
|
mas01cr@105
|
497 error("Could not open times file for reading", timesFileName);
|
mas01cr@105
|
498 usingTimes=1;
|
mas01cr@105
|
499 }
|
mas01cr@105
|
500 }
|
mas01cr@193
|
501
|
mas01cr@193
|
502 if(args_info.power_given){
|
mas01cr@193
|
503 powerFileName=args_info.power_arg;
|
mas01cr@193
|
504 if(strlen(powerFileName)>0){
|
mas01cr@193
|
505 if (!(powerfd = open(powerFileName, O_RDONLY))) {
|
mas01cr@193
|
506 error("Could not open power file for reading", powerFileName, "open");
|
mas01cr@193
|
507 }
|
mas01cr@193
|
508 usingPower = 1;
|
mas01cr@193
|
509 }
|
mas01cr@193
|
510 }
|
mas01cr@105
|
511
|
mas01cr@105
|
512 // query type
|
mas01cr@105
|
513 if(strncmp(args_info.QUERY_arg, "track", MAXSTR)==0)
|
mas01cr@105
|
514 queryType=O2_TRACK_QUERY;
|
mas01cr@105
|
515 else if(strncmp(args_info.QUERY_arg, "point", MAXSTR)==0)
|
mas01cr@105
|
516 queryType=O2_POINT_QUERY;
|
mas01cr@105
|
517 else if(strncmp(args_info.QUERY_arg, "sequence", MAXSTR)==0)
|
mas01cr@105
|
518 queryType=O2_SEQUENCE_QUERY;
|
mas01mc@248
|
519 else if(strncmp(args_info.QUERY_arg, "nsequence", MAXSTR)==0)
|
mas01mc@248
|
520 queryType=O2_N_SEQUENCE_QUERY;
|
mas01mc@263
|
521 else if(strncmp(args_info.QUERY_arg, "onetoonensequence", MAXSTR)==0)
|
mas01mc@263
|
522 queryType=O2_ONE_TO_ONE_N_SEQUENCE_QUERY;
|
mas01cr@105
|
523 else
|
mas01cr@105
|
524 error("unsupported query type",args_info.QUERY_arg);
|
mas01cr@105
|
525
|
mas01cr@105
|
526 if(!args_info.exhaustive_flag){
|
mas01cr@105
|
527 queryPoint = args_info.qpoint_arg;
|
mas01cr@105
|
528 usingQueryPoint=1;
|
mas01cr@105
|
529 if(queryPoint<0 || queryPoint >10000)
|
mas01cr@105
|
530 error("queryPoint out of range: 0 <= queryPoint <= 10000");
|
mas01cr@105
|
531 }
|
mas01mc@292
|
532
|
mas01mc@296
|
533 // Whether to pre-load LSH hash tables for query (default on, if flag set then off)
|
mas01mc@297
|
534 lsh_in_core = !args_info.lsh_on_disk_flag;
|
mas01mc@292
|
535
|
mas01mc@292
|
536 // Whether to perform exact evaluation of points returned by LSH
|
mas01mc@292
|
537 lsh_exact = args_info.lsh_exact_flag;
|
mas01mc@292
|
538
|
mas01cr@105
|
539 pointNN = args_info.pointnn_arg;
|
mas01mc@263
|
540 if(pointNN < 1 || pointNN > O2_MAXNN) {
|
mas01mc@263
|
541 error("pointNN out of range: 1 <= pointNN <= 1000000");
|
mas01cr@105
|
542 }
|
mas01cr@105
|
543 trackNN = args_info.resultlength_arg;
|
mas01mc@263
|
544 if(trackNN < 1 || trackNN > O2_MAXNN) {
|
mas01mc@263
|
545 error("resultlength out of range: 1 <= resultlength <= 1000000");
|
mas01cr@105
|
546 }
|
mas01cr@105
|
547 return 0;
|
mas01cr@105
|
548 }
|
mas01cr@105
|
549 return -1; // no command found
|
mas01cr@0
|
550 }
|
mas01cr@0
|
551
|
mas01cr@133
|
552 void audioDB::status(const char* dbName, adb__statusResponse *adbStatusResponse){
|
mas01cr@0
|
553 if(!dbH)
|
mas01cr@196
|
554 initTables(dbName, 0);
|
mas01cr@0
|
555
|
mas01cr@0
|
556 unsigned dudCount=0;
|
mas01cr@0
|
557 unsigned nullCount=0;
|
mas01cr@0
|
558 for(unsigned k=0; k<dbH->numFiles; k++){
|
mas01mc@18
|
559 if(trackTable[k]<sequenceLength){
|
mas01cr@0
|
560 dudCount++;
|
mas01mc@18
|
561 if(!trackTable[k])
|
mas01cr@76
|
562 nullCount++;
|
mas01cr@0
|
563 }
|
mas01cr@0
|
564 }
|
mas01cr@76
|
565
|
mas01cr@133
|
566 if(adbStatusResponse == 0) {
|
mas01cr@76
|
567
|
mas01cr@76
|
568 // Update Header information
|
mas01cr@239
|
569 std::cout << "num files:" << dbH->numFiles << std::endl;
|
mas01cr@239
|
570 std::cout << "data dim:" << dbH->dim <<std::endl;
|
mas01cr@76
|
571 if(dbH->dim>0){
|
mas01cr@239
|
572 std::cout << "total vectors:" << dbH->length/(sizeof(double)*dbH->dim)<<std::endl;
|
mas01mc@324
|
573 if(dbH->flags & O2_FLAG_LARGE_ADB)
|
mas01mc@324
|
574 std::cout << "vectors available:" << O2_MAX_VECTORS - (dbH->length / (sizeof(double)*dbH->dim)) << std::endl;
|
mas01mc@324
|
575 else
|
mas01mc@324
|
576 std::cout << "vectors available:" << (dbH->timesTableOffset-(dbH->dataOffset+dbH->length))/(sizeof(double)*dbH->dim) << std::endl;
|
mas01cr@76
|
577 }
|
mas01mc@324
|
578 if( ! (dbH->flags & O2_FLAG_LARGE_ADB) ){
|
mas01mc@324
|
579 std::cout << "total bytes:" << dbH->length << " (" << (100.0*dbH->length)/(dbH->timesTableOffset-dbH->dataOffset) << "%)" << std::endl;
|
mas01mc@324
|
580 std::cout << "bytes available:" << dbH->timesTableOffset-(dbH->dataOffset+dbH->length) << " (" <<
|
mas01mc@324
|
581 (100.0*(dbH->timesTableOffset-(dbH->dataOffset+dbH->length)))/(dbH->timesTableOffset-dbH->dataOffset) << "%)" << std::endl;
|
mas01mc@324
|
582 }
|
mas01mc@301
|
583 std::cout << "flags:" << " l2norm[" << DISPLAY_FLAG(dbH->flags&O2_FLAG_L2NORM)
|
mas01mc@301
|
584 << "] minmax[" << DISPLAY_FLAG(dbH->flags&O2_FLAG_MINMAX)
|
mas01mc@301
|
585 << "] power[" << DISPLAY_FLAG(dbH->flags&O2_FLAG_POWER)
|
mas01mc@324
|
586 << "] times[" << DISPLAY_FLAG(dbH->flags&O2_FLAG_TIMES)
|
mas01mc@324
|
587 << "] largeADB[" << DISPLAY_FLAG(dbH->flags&O2_FLAG_LARGE_ADB)
|
mas01mc@324
|
588 << "]" << endl;
|
mas01mc@324
|
589
|
mas01cr@239
|
590 std::cout << "null count: " << nullCount << " small sequence count " << dudCount-nullCount << std::endl;
|
mas01cr@76
|
591 } else {
|
mas01cr@133
|
592 adbStatusResponse->result.numFiles = dbH->numFiles;
|
mas01cr@133
|
593 adbStatusResponse->result.dim = dbH->dim;
|
mas01cr@133
|
594 adbStatusResponse->result.length = dbH->length;
|
mas01cr@133
|
595 adbStatusResponse->result.dudCount = dudCount;
|
mas01cr@133
|
596 adbStatusResponse->result.nullCount = nullCount;
|
mas01cr@133
|
597 adbStatusResponse->result.flags = dbH->flags;
|
mas01cr@76
|
598 }
|
mas01cr@0
|
599 }
|
mas01cr@0
|
600
|
mas01cr@196
|
601 void audioDB::l2norm(const char* dbName) {
|
mas01cr@196
|
602 forWrite = true;
|
mas01cr@196
|
603 initTables(dbName, 0);
|
mas01mc@324
|
604 if( !(dbH->flags & O2_FLAG_LARGE_ADB ) && (dbH->length>0) ){
|
mas01cr@196
|
605 /* FIXME: should probably be uint64_t */
|
mas01cr@0
|
606 unsigned numVectors = dbH->length/(sizeof(double)*dbH->dim);
|
mas01cr@196
|
607 CHECKED_MMAP(double *, dataBuf, dbH->dataOffset, dataBufLength);
|
mas01cr@0
|
608 unitNormAndInsertL2(dataBuf, dbH->dim, numVectors, 0); // No append
|
mas01cr@0
|
609 }
|
mas01cr@0
|
610 // Update database flags
|
mas01cr@0
|
611 dbH->flags = dbH->flags|O2_FLAG_L2NORM;
|
mas01cr@0
|
612 memcpy (db, dbH, O2_HEADERSIZE);
|
mas01cr@0
|
613 }
|
mas01cr@193
|
614
|
mas01cr@193
|
615 void audioDB::power_flag(const char *dbName) {
|
mas01cr@196
|
616 forWrite = true;
|
mas01mc@324
|
617 initTables(dbName, 0);
|
mas01mc@324
|
618 if( !(dbH->flags & O2_FLAG_LARGE_ADB ) && (dbH->length>0) ){
|
mas01cr@193
|
619 error("cannot turn on power storage for non-empty database", dbName);
|
mas01cr@193
|
620 }
|
mas01cr@193
|
621 dbH->flags |= O2_FLAG_POWER;
|
mas01cr@193
|
622 memcpy(db, dbH, O2_HEADERSIZE);
|
mas01cr@193
|
623 }
|
mas01cr@193
|
624
|
mas01cr@239
|
625 // Unit norm block of features
|
mas01cr@0
|
626
|
mas01cr@239
|
627 /* FIXME: in fact this does not unit norm a block of features, it just
|
mas01cr@239
|
628 records the L2 norms somewhere. unitNorm() does in fact unit norm
|
mas01cr@239
|
629 a block of features. */
|
mas01cr@0
|
630 void audioDB::unitNormAndInsertL2(double* X, unsigned dim, unsigned n, unsigned append=0){
|
mas01cr@0
|
631 unsigned d;
|
mas01cr@59
|
632 double *p;
|
mas01cr@0
|
633 unsigned nn = n;
|
mas01cr@0
|
634
|
mas01cr@0
|
635 assert(l2normTable);
|
mas01cr@0
|
636
|
mas01mc@324
|
637 if( !(dbH->flags & O2_FLAG_LARGE_ADB) && !append && (dbH->flags & O2_FLAG_L2NORM) )
|
mas01cr@0
|
638 error("Database is already L2 normed", "automatic norm on insert is enabled");
|
mas01cr@0
|
639
|
mas01cr@239
|
640 VERB_LOG(2, "norming %u vectors...", n);
|
mas01cr@0
|
641
|
mas01cr@0
|
642 double* l2buf = new double[n];
|
mas01cr@0
|
643 double* l2ptr = l2buf;
|
mas01cr@0
|
644 assert(l2buf);
|
mas01cr@0
|
645 assert(X);
|
mas01cr@0
|
646
|
mas01cr@0
|
647 while(nn--){
|
mas01cr@0
|
648 p=X;
|
mas01cr@0
|
649 *l2ptr=0.0;
|
mas01cr@0
|
650 d=dim;
|
mas01cr@0
|
651 while(d--){
|
mas01cr@0
|
652 *l2ptr+=*p**p;
|
mas01cr@0
|
653 p++;
|
mas01cr@0
|
654 }
|
mas01mc@17
|
655 l2ptr++;
|
mas01mc@17
|
656 X+=dim;
|
mas01cr@0
|
657 }
|
mas01cr@0
|
658 unsigned offset;
|
mas01cr@84
|
659 if(append) {
|
mas01cr@84
|
660 // FIXME: a hack, a very palpable hack: the vectors have already
|
mas01cr@84
|
661 // been inserted, and dbH->length has already been updated. We
|
mas01cr@84
|
662 // need to subtract off again the number of vectors that we've
|
mas01cr@84
|
663 // inserted this time...
|
mas01cr@84
|
664 offset=(dbH->length/(dbH->dim*sizeof(double)))-n; // number of vectors
|
mas01cr@84
|
665 } else {
|
mas01cr@0
|
666 offset=0;
|
mas01cr@84
|
667 }
|
mas01cr@0
|
668 memcpy(l2normTable+offset, l2buf, n*sizeof(double));
|
mas01cr@0
|
669 if(l2buf)
|
mas01mc@17
|
670 delete[] l2buf;
|
mas01cr@239
|
671 VERB_LOG(2, " done.");
|
mas01cr@193
|
672 }
|
mas01cr@193
|
673
|
mas01mc@308
|
674 // This entry point is visited once per instance
|
mas01mc@308
|
675 // so it is a good place to set any global state variables
|
mas01cr@0
|
676 int main(const unsigned argc, char* const argv[]){
|
mas01mc@308
|
677 SERVER_LSH_INDEX_SINGLETON = 0; // Initialize global variables
|
mas01mc@324
|
678 SERVER_ADB_ROOT = 0; // Server-side database root prefix
|
mas01mc@324
|
679 SERVER_ADB_FEATURE_ROOT = 0; // Server-side features root prefix
|
mas01cr@0
|
680 audioDB(argc, argv);
|
mas01cr@0
|
681 }
|