annotate audioDB.cpp @ 332:a0493f861531

Added new tests for indexed query over WS. Fixed potential NULL string dereference in audioDB.cpp
author mas01mc
date Sun, 31 Aug 2008 08:45:22 +0000
parents 8f11ea4d9cd2
children 100cf66a5825
rev   line source
mas01cr@0 1 #include "audioDB.h"
mas01cr@0 2
mas01mc@308 3 LSH* SERVER_LSH_INDEX_SINGLETON;
mas01mc@324 4 char* SERVER_ADB_ROOT;
mas01mc@324 5 char* SERVER_ADB_FEATURE_ROOT;
mas01mc@308 6
mas01mc@292 7 PointPair::PointPair(Uns32T a, Uns32T b, Uns32T c):trackID(a),qpos(b),spos(c){};
mas01mc@292 8
mas01mc@292 9 bool operator<(const PointPair& a, const PointPair& b){
mas01mc@324 10 return ( (a.trackID<b.trackID) ||
mas01mc@324 11 ( (a.trackID==b.trackID) &&
mas01mc@324 12 ( (a.spos<b.spos) || ( (a.spos==b.spos) && (a.qpos < b.qpos) )) ) );
mas01mc@292 13 }
mas01mc@292 14
mas01mc@292 15 bool operator>(const PointPair& a, const PointPair& b){
mas01mc@324 16 return ( (a.trackID>b.trackID) ||
mas01mc@324 17 ( (a.trackID==b.trackID) &&
mas01mc@324 18 ( (a.spos>b.spos) || ( (a.spos==b.spos) && (a.qpos > b.qpos) )) ) );
mas01mc@292 19 }
mas01mc@292 20
mas01mc@292 21 bool operator==(const PointPair& a, const PointPair& b){
mas01mc@292 22 return ( (a.trackID==b.trackID) && (a.qpos==b.qpos) && (a.spos==b.spos) );
mas01mc@292 23 }
mas01mc@292 24
mas01cr@76 25 audioDB::audioDB(const unsigned argc, char* const argv[]): O2_AUDIODB_INITIALIZERS
mas01cr@76 26 {
mas01cr@0 27 if(processArgs(argc, argv)<0){
mas01cr@0 28 printf("No command found.\n");
mas01cr@0 29 cmdline_parser_print_version ();
mas01cr@0 30 if (strlen(gengetopt_args_info_purpose) > 0)
mas01cr@0 31 printf("%s\n", gengetopt_args_info_purpose);
mas01cr@0 32 printf("%s\n", gengetopt_args_info_usage);
mas01cr@0 33 printf("%s\n", gengetopt_args_info_help[1]);
mas01cr@0 34 printf("%s\n", gengetopt_args_info_help[2]);
mas01cr@0 35 printf("%s\n", gengetopt_args_info_help[0]);
mas01cr@151 36 error("No command found");
mas01cr@0 37 }
mas01cr@77 38
mas01mc@324 39 // Perform database prefix substitution
mas01mc@328 40 if(dbName && adb_root)
mas01mc@324 41 prefix_name((char** const)&dbName, adb_root);
mas01mc@324 42
mas01mc@328 43 if(inFile && adb_feature_root)
mas01mc@328 44 prefix_name((char** const)&inFile, adb_feature_root);
mas01mc@328 45
mas01cr@0 46 if(O2_ACTION(COM_SERVER))
mas01cr@0 47 startServer();
mas01cr@0 48
mas01cr@0 49 else if(O2_ACTION(COM_CREATE))
mas01cr@0 50 create(dbName);
mas01cr@0 51
mas01cr@0 52 else if(O2_ACTION(COM_INSERT))
mas01cr@0 53 insert(dbName, inFile);
mas01cr@0 54
mas01cr@0 55 else if(O2_ACTION(COM_BATCHINSERT))
mas01cr@0 56 batchinsert(dbName, inFile);
mas01cr@0 57
mas01cr@0 58 else if(O2_ACTION(COM_QUERY))
mas01mc@307 59 if(isClient){
mas01mc@329 60 if(query_from_key){
mas01mc@332 61 VERB_LOG(1, "Calling web services query %s on database %s, query=%s\n", radius>0?"(Radius)":"(NN)", dbName, (key&&strlen(key))?key:inFile);
mas01mc@328 62 ws_query_by_key(dbName, key, inFile, (char*)hostport);
mas01mc@329 63 }
mas01mc@329 64 else{
mas01mc@332 65 VERB_LOG(1, "Calling web services query on database %s, query=%s\n", dbName, (key&&strlen(key))?key:inFile);
mas01mc@307 66 ws_query(dbName, inFile, (char*)hostport);
mas01mc@329 67 }
mas01mc@307 68 }
mas01cr@0 69 else
mas01cr@76 70 query(dbName, inFile);
mas01cr@0 71
mas01cr@0 72 else if(O2_ACTION(COM_STATUS))
mas01cr@0 73 if(isClient)
mas01cr@0 74 ws_status(dbName,(char*)hostport);
mas01cr@0 75 else
mas01cr@0 76 status(dbName);
mas01cr@280 77
mas01cr@280 78 else if(O2_ACTION(COM_SAMPLE))
mas01cr@280 79 sample(dbName);
mas01cr@0 80
mas01cr@0 81 else if(O2_ACTION(COM_L2NORM))
mas01cr@0 82 l2norm(dbName);
mas01cr@0 83
mas01cr@193 84 else if(O2_ACTION(COM_POWER))
mas01cr@193 85 power_flag(dbName);
mas01cr@193 86
mas01cr@0 87 else if(O2_ACTION(COM_DUMP))
mas01cr@0 88 dump(dbName);
mas01mc@292 89
mas01mc@292 90 else if(O2_ACTION(COM_INDEX))
mas01mc@292 91 index_index_db(dbName);
mas01cr@0 92
mas01cr@0 93 else
mas01cr@0 94 error("Unrecognized command",command);
mas01cr@0 95 }
mas01cr@0 96
mas01cr@133 97 audioDB::audioDB(const unsigned argc, char* const argv[], adb__queryResponse *adbQueryResponse): O2_AUDIODB_INITIALIZERS
mas01cr@76 98 {
mas01cr@97 99 try {
mas01cr@151 100 isServer = 1; // FIXME: Hack
mas01cr@97 101 processArgs(argc, argv);
mas01mc@324 102 // Perform database prefix substitution
mas01mc@328 103 if(dbName && adb_root)
mas01mc@324 104 prefix_name((char** const)&dbName, adb_root);
mas01mc@328 105 if(inFile && adb_feature_root)
mas01mc@328 106 prefix_name((char** const)&inFile, adb_feature_root);
mas01cr@97 107 assert(O2_ACTION(COM_QUERY));
mas01cr@133 108 query(dbName, inFile, adbQueryResponse);
mas01cr@97 109 } catch(char *err) {
mas01cr@97 110 cleanup();
mas01cr@97 111 throw(err);
mas01cr@97 112 }
mas01cr@76 113 }
mas01cr@76 114
mas01cr@133 115 audioDB::audioDB(const unsigned argc, char* const argv[], adb__statusResponse *adbStatusResponse): O2_AUDIODB_INITIALIZERS
mas01cr@76 116 {
mas01cr@97 117 try {
mas01cr@151 118 isServer = 1; // FIXME: Hack
mas01cr@97 119 processArgs(argc, argv);
mas01mc@324 120 // Perform database prefix substitution
mas01mc@328 121 if(dbName && adb_root)
mas01mc@324 122 prefix_name((char** const)&dbName, adb_root);
mas01mc@328 123 if(inFile && adb_feature_root)
mas01mc@328 124 prefix_name((char** const)&inFile, adb_feature_root);
mas01cr@97 125 assert(O2_ACTION(COM_STATUS));
mas01cr@133 126 status(dbName, adbStatusResponse);
mas01cr@97 127 } catch(char *err) {
mas01cr@97 128 cleanup();
mas01cr@97 129 throw(err);
mas01cr@97 130 }
mas01cr@76 131 }
mas01cr@76 132
mas01cr@97 133 void audioDB::cleanup() {
mas01cr@122 134 cmdline_parser_free(&args_info);
mas01cr@0 135 if(indata)
mas01cr@0 136 munmap(indata,statbuf.st_size);
mas01cr@0 137 if(db)
mas01cr@196 138 munmap(db,getpagesize());
mas01cr@196 139 if(fileTable)
mas01cr@196 140 munmap(fileTable, fileTableLength);
mas01cr@196 141 if(trackTable)
mas01cr@196 142 munmap(trackTable, trackTableLength);
mas01cr@196 143 if(dataBuf)
mas01cr@196 144 munmap(dataBuf, dataBufLength);
mas01cr@196 145 if(timesTable)
mas01cr@196 146 munmap(timesTable, timesTableLength);
mas01mc@314 147 if(powerTable)
mas01mc@314 148 munmap(powerTable, powerTableLength);
mas01cr@196 149 if(l2normTable)
mas01cr@196 150 munmap(l2normTable, l2normTableLength);
mas01mc@324 151 if(featureFileNameTable)
mas01mc@324 152 munmap(featureFileNameTable, fileTableLength);
mas01mc@324 153 if(timesFileNameTable)
mas01mc@324 154 munmap(timesFileNameTable, fileTableLength);
mas01mc@324 155 if(powerFileNameTable)
mas01mc@324 156 munmap(powerFileNameTable, fileTableLength);
mas01mc@292 157 if(trackOffsetTable)
mas01mc@292 158 delete trackOffsetTable;
mas01mc@292 159 if(reporter)
mas01mc@292 160 delete reporter;
mas01mc@292 161 if(exact_evaluation_queue)
mas01mc@292 162 delete exact_evaluation_queue;
mas01cr@284 163 if(rng)
mas01cr@284 164 gsl_rng_free(rng);
mas01mc@292 165 if(vv)
mas01mc@292 166 delete vv;
mas01cr@0 167 if(dbfid>0)
mas01cr@0 168 close(dbfid);
mas01cr@0 169 if(infid>0)
mas01cr@0 170 close(infid);
mas01cr@0 171 if(dbH)
mas01cr@0 172 delete dbH;
mas01mc@308 173 if(lsh!=SERVER_LSH_INDEX_SINGLETON)
mas01mc@308 174 delete lsh;
mas01cr@0 175 }
mas01cr@0 176
mas01cr@97 177 audioDB::~audioDB(){
mas01cr@97 178 cleanup();
mas01cr@97 179 }
mas01cr@97 180
mas01cr@0 181 int audioDB::processArgs(const unsigned argc, char* const argv[]){
mas01cr@0 182
mas01cr@0 183 if(argc<2){
mas01cr@0 184 cmdline_parser_print_version ();
mas01cr@0 185 if (strlen(gengetopt_args_info_purpose) > 0)
mas01cr@0 186 printf("%s\n", gengetopt_args_info_purpose);
mas01cr@0 187 printf("%s\n", gengetopt_args_info_usage);
mas01cr@0 188 printf("%s\n", gengetopt_args_info_help[1]);
mas01cr@0 189 printf("%s\n", gengetopt_args_info_help[2]);
mas01cr@0 190 printf("%s\n", gengetopt_args_info_help[0]);
mas01cr@0 191 exit(0);
mas01cr@0 192 }
mas01cr@0 193
mas01cr@0 194 if (cmdline_parser (argc, argv, &args_info) != 0)
mas01cr@151 195 error("Error parsing command line");
mas01cr@0 196
mas01cr@0 197 if(args_info.help_given){
mas01cr@0 198 cmdline_parser_print_help();
mas01cr@0 199 exit(0);
mas01cr@0 200 }
mas01cr@0 201
mas01cr@0 202 if(args_info.verbosity_given){
mas01cr@239 203 verbosity = args_info.verbosity_arg;
mas01cr@239 204 if(verbosity < 0 || verbosity > 10){
mas01cr@239 205 std::cerr << "Warning: verbosity out of range, setting to 1" << std::endl;
mas01cr@239 206 verbosity = 1;
mas01cr@0 207 }
mas01cr@0 208 }
mas01cr@0 209
mas01cr@129 210 if(args_info.size_given) {
mas01cr@256 211 if(args_info.datasize_given) {
mas01cr@256 212 error("both --size and --datasize given", "");
mas01cr@256 213 }
mas01cr@256 214 if(args_info.ntracks_given) {
mas01cr@256 215 error("both --size and --ntracks given", "");
mas01cr@256 216 }
mas01cr@256 217 if(args_info.datadim_given) {
mas01cr@256 218 error("both --size and --datadim given", "");
mas01cr@256 219 }
mas01cr@196 220 if (args_info.size_arg < 50 || args_info.size_arg > 32000) {
mas01cr@129 221 error("Size out of range", "");
mas01cr@129 222 }
mas01cr@256 223 double ratio = (double) args_info.size_arg * 1000000 / ((double) O2_DEFAULTDBSIZE);
mas01cr@256 224 /* FIXME: what's the safe way of doing this? */
mas01cr@256 225 datasize = (unsigned int) ceil(datasize * ratio);
mas01cr@256 226 ntracks = (unsigned int) ceil(ntracks * ratio);
mas01cr@256 227 } else {
mas01cr@256 228 if(args_info.datasize_given) {
mas01cr@256 229 datasize = args_info.datasize_arg;
mas01cr@256 230 }
mas01cr@256 231 if(args_info.ntracks_given) {
mas01cr@256 232 ntracks = args_info.ntracks_arg;
mas01cr@256 233 }
mas01cr@256 234 if(args_info.datadim_given) {
mas01cr@256 235 datadim = args_info.datadim_arg;
mas01cr@256 236 }
mas01cr@129 237 }
mas01cr@129 238
mas01cr@239 239 if(args_info.radius_given) {
mas01cr@239 240 radius = args_info.radius_arg;
mas01mc@307 241 if(radius < 0 || radius > 1000000000) {
mas01cr@77 242 error("radius out of range");
mas01cr@239 243 } else {
mas01cr@239 244 VERB_LOG(3, "Setting radius to %f\n", radius);
mas01mc@17 245 }
mas01mc@17 246 }
mas01mc@17 247
mas01mc@292 248 sequenceLength = args_info.sequencelength_arg;
mas01mc@292 249 if(sequenceLength < 1 || sequenceLength > 1000) {
mas01mc@292 250 error("seqlen out of range: 1 <= seqlen <= 1000");
mas01mc@292 251 }
mas01mc@292 252 sequenceHop = args_info.sequencehop_arg;
mas01mc@292 253 if(sequenceHop < 1 || sequenceHop > 1000) {
mas01mc@292 254 error("seqhop out of range: 1 <= seqhop <= 1000");
mas01mc@292 255 }
mas01mc@292 256
mas01mc@292 257 if (args_info.absolute_threshold_given) {
mas01mc@292 258 if (args_info.absolute_threshold_arg >= 0) {
mas01mc@292 259 error("absolute threshold out of range: should be negative");
mas01mc@292 260 }
mas01mc@292 261 use_absolute_threshold = true;
mas01mc@292 262 absolute_threshold = args_info.absolute_threshold_arg;
mas01mc@292 263 }
mas01mc@292 264 if (args_info.relative_threshold_given) {
mas01mc@292 265 use_relative_threshold = true;
mas01mc@292 266 relative_threshold = args_info.relative_threshold_arg;
mas01mc@292 267 }
mas01mc@292 268
mas01mc@324 269 if (args_info.adb_root_given){
mas01mc@324 270 adb_root = args_info.adb_root_arg;
mas01mc@324 271 }
mas01mc@324 272
mas01mc@324 273 if (args_info.adb_feature_root_given){
mas01mc@324 274 adb_feature_root = args_info.adb_feature_root_arg;
mas01mc@324 275 }
mas01mc@324 276
mas01mc@324 277 // perform dbName path prefix SERVER-side subsitution
mas01mc@324 278 if(SERVER_ADB_ROOT && !adb_root)
mas01mc@324 279 adb_root = SERVER_ADB_ROOT;
mas01mc@324 280 if(SERVER_ADB_FEATURE_ROOT && !adb_feature_root)
mas01mc@324 281 adb_feature_root = SERVER_ADB_FEATURE_ROOT;
mas01mc@324 282
mas01cr@0 283 if(args_info.SERVER_given){
mas01cr@0 284 command=COM_SERVER;
mas01cr@0 285 port=args_info.SERVER_arg;
mas01cr@0 286 if(port<100 || port > 100000)
mas01cr@0 287 error("port out of range");
mas01cr@151 288 isServer = 1;
mas01cr@105 289 #if defined(O2_DEBUG)
mas01cr@104 290 struct sigaction sa;
mas01cr@104 291 sa.sa_sigaction = sigterm_action;
mas01cr@104 292 sa.sa_flags = SA_SIGINFO | SA_RESTART | SA_NODEFER;
mas01cr@104 293 sigaction(SIGTERM, &sa, NULL);
mas01cr@104 294 sa.sa_sigaction = sighup_action;
mas01cr@104 295 sa.sa_flags = SA_SIGINFO | SA_RESTART | SA_NODEFER;
mas01cr@104 296 sigaction(SIGHUP, &sa, NULL);
mas01cr@105 297 #endif
mas01mc@308 298 if(args_info.load_index_given){
mas01mc@308 299 if(!args_info.database_given)
mas01mc@308 300 error("load_index requires a --database argument");
mas01mc@308 301 else
mas01mc@308 302 dbName=args_info.database_arg;
mas01mc@308 303 if(!args_info.radius_given)
mas01mc@308 304 error("load_index requires a --radius argument");
mas01mc@308 305 if(!args_info.sequencelength_given)
mas01mc@308 306 error("load_index requires a --sequenceLength argument");
mas01mc@308 307 WS_load_index = true;
mas01mc@308 308 }
mas01cr@0 309 return 0;
mas01cr@0 310 }
mas01cr@0 311
mas01cr@0 312 // No return on client command, find database command
mas01cr@105 313 if(args_info.client_given){
mas01cr@105 314 command=COM_CLIENT;
mas01cr@105 315 hostport=args_info.client_arg;
mas01cr@105 316 isClient=1;
mas01cr@105 317 }
mas01cr@0 318
mas01cr@105 319 if(args_info.NEW_given){
mas01cr@105 320 command=COM_CREATE;
mas01cr@105 321 dbName=args_info.database_arg;
mas01cr@105 322 return 0;
mas01cr@105 323 }
mas01cr@0 324
mas01cr@105 325 if(args_info.STATUS_given){
mas01cr@105 326 command=COM_STATUS;
mas01cr@105 327 dbName=args_info.database_arg;
mas01cr@105 328 return 0;
mas01cr@105 329 }
mas01cr@0 330
mas01cr@280 331 if(args_info.SAMPLE_given) {
mas01cr@280 332 command = COM_SAMPLE;
mas01cr@280 333 dbName = args_info.database_arg;
mas01cr@280 334 sequenceLength = args_info.sequencelength_arg;
mas01cr@280 335 if(sequenceLength < 1 || sequenceLength > 1000) {
mas01cr@280 336 error("seqlen out of range: 1 <= seqlen <= 1000");
mas01cr@280 337 }
mas01cr@280 338 nsamples = args_info.nsamples_arg;
mas01cr@280 339 return 0;
mas01cr@280 340 }
mas01cr@280 341
mas01cr@105 342 if(args_info.DUMP_given){
mas01cr@105 343 command=COM_DUMP;
mas01cr@105 344 dbName=args_info.database_arg;
mas01cr@131 345 output = args_info.output_arg;
mas01cr@105 346 return 0;
mas01cr@105 347 }
mas01cr@0 348
mas01cr@105 349 if(args_info.L2NORM_given){
mas01cr@105 350 command=COM_L2NORM;
mas01cr@105 351 dbName=args_info.database_arg;
mas01cr@105 352 return 0;
mas01cr@105 353 }
mas01cr@0 354
mas01cr@193 355 if(args_info.POWER_given){
mas01cr@193 356 command=COM_POWER;
mas01cr@193 357 dbName=args_info.database_arg;
mas01cr@193 358 return 0;
mas01cr@193 359 }
mas01cr@193 360
mas01cr@105 361 if(args_info.INSERT_given){
mas01cr@105 362 command=COM_INSERT;
mas01cr@105 363 dbName=args_info.database_arg;
mas01cr@105 364 inFile=args_info.features_arg;
mas01cr@105 365 if(args_info.key_given)
mas01mc@292 366 if(!args_info.features_given)
mas01mc@292 367 error("INSERT: '-k key' argument depends on '-f features'");
mas01mc@292 368 else
mas01mc@292 369 key=args_info.key_arg;
mas01cr@105 370 if(args_info.times_given){
mas01cr@105 371 timesFileName=args_info.times_arg;
mas01cr@105 372 if(strlen(timesFileName)>0){
mas01cr@239 373 if(!(timesFile = new std::ifstream(timesFileName,std::ios::in)))
mas01cr@105 374 error("Could not open times file for reading", timesFileName);
mas01cr@105 375 usingTimes=1;
mas01cr@105 376 }
mas01cr@105 377 }
mas01cr@193 378 if (args_info.power_given) {
mas01cr@193 379 powerFileName = args_info.power_arg;
mas01cr@193 380 if (strlen(powerFileName) > 0) {
mas01cr@193 381 if (!(powerfd = open(powerFileName, O_RDONLY))) {
mas01cr@193 382 error("Could not open power file for reading", powerFileName, "open");
mas01cr@193 383 }
mas01cr@193 384 usingPower = 1;
mas01cr@193 385 }
mas01cr@193 386 }
mas01cr@105 387 return 0;
mas01cr@105 388 }
mas01cr@105 389
mas01cr@105 390 if(args_info.BATCHINSERT_given){
mas01cr@105 391 command=COM_BATCHINSERT;
mas01cr@105 392 dbName=args_info.database_arg;
mas01cr@105 393 inFile=args_info.featureList_arg;
mas01cr@105 394 if(args_info.keyList_given)
mas01tc@298 395 if(!args_info.featureList_given)
mas01tc@300 396 error("BATCHINSERT: '-K keyList' argument depends on '-F featureList'");
mas01mc@292 397 else
mas01cr@304 398 key=args_info.keyList_arg; // INCONSISTENT NO CHECK
mas01cr@0 399
mas01cr@105 400 /* TO DO: REPLACE WITH
mas01cr@0 401 if(args_info.keyList_given){
mas01mc@18 402 trackFileName=args_info.keyList_arg;
mas01cr@239 403 if(strlen(trackFileName)>0 && !(trackFile = new std::ifstream(trackFileName,std::ios::in)))
mas01mc@18 404 error("Could not open keyList file for reading",trackFileName);
mas01cr@0 405 }
mas01cr@0 406 AND UPDATE BATCHINSERT()
mas01cr@105 407 */
mas01cr@105 408
mas01cr@105 409 if(args_info.timesList_given){
mas01cr@105 410 timesFileName=args_info.timesList_arg;
mas01cr@105 411 if(strlen(timesFileName)>0){
mas01cr@239 412 if(!(timesFile = new std::ifstream(timesFileName,std::ios::in)))
mas01cr@105 413 error("Could not open timesList file for reading", timesFileName);
mas01cr@105 414 usingTimes=1;
mas01cr@105 415 }
mas01cr@105 416 }
mas01cr@193 417 if(args_info.powerList_given){
mas01cr@193 418 powerFileName=args_info.powerList_arg;
mas01cr@193 419 if(strlen(powerFileName)>0){
mas01cr@239 420 if(!(powerFile = new std::ifstream(powerFileName,std::ios::in)))
mas01cr@193 421 error("Could not open powerList file for reading", powerFileName);
mas01cr@193 422 usingPower=1;
mas01cr@193 423 }
mas01cr@193 424 }
mas01cr@105 425 return 0;
mas01cr@105 426 }
mas01mc@292 427
mas01mc@292 428 // Set no_unit_norm flag
mas01mc@292 429 no_unit_norming = args_info.no_unit_norming_flag;
mas01mc@292 430 lsh_use_u_functions = args_info.lsh_use_u_functions_flag;
mas01mc@292 431
mas01mc@292 432 // LSH Index Command
mas01mc@292 433 if(args_info.INDEX_given){
mas01mc@292 434 if(radius <= 0 )
mas01mc@292 435 error("INDEXing requires a Radius argument");
mas01mc@292 436 if(!(sequenceLength>0 && sequenceLength <= O2_MAXSEQLEN))
mas01mc@292 437 error("INDEXing requires 1 <= sequenceLength <= 1000");
mas01mc@292 438 command=COM_INDEX;
mas01mc@292 439 dbName=args_info.database_arg;
mas01mc@292 440
mas01mc@292 441 // Whether to store LSH hash tables for query in core (FORMAT2)
mas01mc@297 442 lsh_in_core = !args_info.lsh_on_disk_flag; // This flag is set to 0 if on_disk requested
mas01mc@292 443
mas01mc@292 444 lsh_param_w = args_info.lsh_w_arg;
mas01mc@292 445 if(!(lsh_param_w>0 && lsh_param_w<=O2_SERIAL_MAX_BINWIDTH))
mas01mc@292 446 error("Indexing parameter w out of range (0.0 < w <= 100.0)");
mas01mc@292 447
mas01mc@292 448 lsh_param_k = args_info.lsh_k_arg;
mas01mc@292 449 if(!(lsh_param_k>0 && lsh_param_k<=O2_SERIAL_MAX_FUNS))
mas01mc@292 450 error("Indexing parameter k out of range (1 <= k <= 100)");
mas01mc@292 451
mas01mc@292 452 lsh_param_m = args_info.lsh_m_arg;
mas01mc@292 453 if(!(lsh_param_m>0 && lsh_param_m<= (1 + (sqrt(1 + O2_SERIAL_MAX_TABLES*8.0)))/2.0))
mas01mc@292 454 error("Indexing parameter m out of range (1 <= m <= 20)");
mas01mc@292 455
mas01mc@292 456 lsh_param_N = args_info.lsh_N_arg;
mas01mc@292 457 if(!(lsh_param_N>0 && lsh_param_N<=O2_SERIAL_MAX_ROWS))
mas01mc@292 458 error("Indexing parameter N out of range (1 <= N <= 1000000)");
mas01mc@292 459
mas01mc@292 460 lsh_param_b = args_info.lsh_b_arg;
mas01mc@292 461 if(!(lsh_param_b>0 && lsh_param_b<=O2_SERIAL_MAX_TRACKBATCH))
mas01mc@292 462 error("Indexing parameter b out of range (1 <= b <= 10000)");
mas01mc@292 463
mas01mc@296 464 lsh_param_ncols = args_info.lsh_ncols_arg;
mas01mc@296 465 if(lsh_in_core) // We don't want to block rows with FORMAT2 indexing
mas01mc@296 466 lsh_param_ncols = O2_SERIAL_MAX_COLS;
mas01mc@292 467 if( !(lsh_param_ncols>0 && lsh_param_ncols<=O2_SERIAL_MAX_COLS))
mas01mc@292 468 error("Indexing parameter ncols out of range (1 <= ncols <= 1000");
mas01mc@292 469
mas01mc@292 470 return 0;
mas01mc@292 471 }
mas01mc@292 472
mas01cr@105 473 // Query command and arguments
mas01cr@105 474 if(args_info.QUERY_given){
mas01cr@105 475 command=COM_QUERY;
mas01cr@105 476 dbName=args_info.database_arg;
mas01mc@292 477 // XOR features and key search
mas01mc@292 478 if(!args_info.features_given && !args_info.key_given || (args_info.features_given && args_info.key_given))
mas01mc@292 479 error("QUERY requires exactly one of either -f features or -k key");
mas01mc@292 480 if(args_info.features_given)
mas01mc@292 481 inFile=args_info.features_arg; // query from file
mas01mc@292 482 else{
mas01mc@292 483 query_from_key = true;
mas01mc@292 484 key=args_info.key_arg; // query from key
mas01mc@292 485 }
mas01mc@292 486
mas01cr@105 487 if(args_info.keyList_given){
mas01cr@105 488 trackFileName=args_info.keyList_arg;
mas01cr@239 489 if(strlen(trackFileName)>0 && !(trackFile = new std::ifstream(trackFileName,std::ios::in)))
mas01cr@105 490 error("Could not open keyList file for reading",trackFileName);
mas01cr@105 491 }
mas01cr@105 492
mas01cr@105 493 if(args_info.times_given){
mas01cr@105 494 timesFileName=args_info.times_arg;
mas01cr@105 495 if(strlen(timesFileName)>0){
mas01cr@239 496 if(!(timesFile = new std::ifstream(timesFileName,std::ios::in)))
mas01cr@105 497 error("Could not open times file for reading", timesFileName);
mas01cr@105 498 usingTimes=1;
mas01cr@105 499 }
mas01cr@105 500 }
mas01cr@193 501
mas01cr@193 502 if(args_info.power_given){
mas01cr@193 503 powerFileName=args_info.power_arg;
mas01cr@193 504 if(strlen(powerFileName)>0){
mas01cr@193 505 if (!(powerfd = open(powerFileName, O_RDONLY))) {
mas01cr@193 506 error("Could not open power file for reading", powerFileName, "open");
mas01cr@193 507 }
mas01cr@193 508 usingPower = 1;
mas01cr@193 509 }
mas01cr@193 510 }
mas01cr@105 511
mas01cr@105 512 // query type
mas01cr@105 513 if(strncmp(args_info.QUERY_arg, "track", MAXSTR)==0)
mas01cr@105 514 queryType=O2_TRACK_QUERY;
mas01cr@105 515 else if(strncmp(args_info.QUERY_arg, "point", MAXSTR)==0)
mas01cr@105 516 queryType=O2_POINT_QUERY;
mas01cr@105 517 else if(strncmp(args_info.QUERY_arg, "sequence", MAXSTR)==0)
mas01cr@105 518 queryType=O2_SEQUENCE_QUERY;
mas01mc@248 519 else if(strncmp(args_info.QUERY_arg, "nsequence", MAXSTR)==0)
mas01mc@248 520 queryType=O2_N_SEQUENCE_QUERY;
mas01mc@263 521 else if(strncmp(args_info.QUERY_arg, "onetoonensequence", MAXSTR)==0)
mas01mc@263 522 queryType=O2_ONE_TO_ONE_N_SEQUENCE_QUERY;
mas01cr@105 523 else
mas01cr@105 524 error("unsupported query type",args_info.QUERY_arg);
mas01cr@105 525
mas01cr@105 526 if(!args_info.exhaustive_flag){
mas01cr@105 527 queryPoint = args_info.qpoint_arg;
mas01cr@105 528 usingQueryPoint=1;
mas01cr@105 529 if(queryPoint<0 || queryPoint >10000)
mas01cr@105 530 error("queryPoint out of range: 0 <= queryPoint <= 10000");
mas01cr@105 531 }
mas01mc@292 532
mas01mc@296 533 // Whether to pre-load LSH hash tables for query (default on, if flag set then off)
mas01mc@297 534 lsh_in_core = !args_info.lsh_on_disk_flag;
mas01mc@292 535
mas01mc@292 536 // Whether to perform exact evaluation of points returned by LSH
mas01mc@292 537 lsh_exact = args_info.lsh_exact_flag;
mas01mc@292 538
mas01cr@105 539 pointNN = args_info.pointnn_arg;
mas01mc@263 540 if(pointNN < 1 || pointNN > O2_MAXNN) {
mas01mc@263 541 error("pointNN out of range: 1 <= pointNN <= 1000000");
mas01cr@105 542 }
mas01cr@105 543 trackNN = args_info.resultlength_arg;
mas01mc@263 544 if(trackNN < 1 || trackNN > O2_MAXNN) {
mas01mc@263 545 error("resultlength out of range: 1 <= resultlength <= 1000000");
mas01cr@105 546 }
mas01cr@105 547 return 0;
mas01cr@105 548 }
mas01cr@105 549 return -1; // no command found
mas01cr@0 550 }
mas01cr@0 551
mas01cr@133 552 void audioDB::status(const char* dbName, adb__statusResponse *adbStatusResponse){
mas01cr@0 553 if(!dbH)
mas01cr@196 554 initTables(dbName, 0);
mas01cr@0 555
mas01cr@0 556 unsigned dudCount=0;
mas01cr@0 557 unsigned nullCount=0;
mas01cr@0 558 for(unsigned k=0; k<dbH->numFiles; k++){
mas01mc@18 559 if(trackTable[k]<sequenceLength){
mas01cr@0 560 dudCount++;
mas01mc@18 561 if(!trackTable[k])
mas01cr@76 562 nullCount++;
mas01cr@0 563 }
mas01cr@0 564 }
mas01cr@76 565
mas01cr@133 566 if(adbStatusResponse == 0) {
mas01cr@76 567
mas01cr@76 568 // Update Header information
mas01cr@239 569 std::cout << "num files:" << dbH->numFiles << std::endl;
mas01cr@239 570 std::cout << "data dim:" << dbH->dim <<std::endl;
mas01cr@76 571 if(dbH->dim>0){
mas01cr@239 572 std::cout << "total vectors:" << dbH->length/(sizeof(double)*dbH->dim)<<std::endl;
mas01mc@324 573 if(dbH->flags & O2_FLAG_LARGE_ADB)
mas01mc@324 574 std::cout << "vectors available:" << O2_MAX_VECTORS - (dbH->length / (sizeof(double)*dbH->dim)) << std::endl;
mas01mc@324 575 else
mas01mc@324 576 std::cout << "vectors available:" << (dbH->timesTableOffset-(dbH->dataOffset+dbH->length))/(sizeof(double)*dbH->dim) << std::endl;
mas01cr@76 577 }
mas01mc@324 578 if( ! (dbH->flags & O2_FLAG_LARGE_ADB) ){
mas01mc@324 579 std::cout << "total bytes:" << dbH->length << " (" << (100.0*dbH->length)/(dbH->timesTableOffset-dbH->dataOffset) << "%)" << std::endl;
mas01mc@324 580 std::cout << "bytes available:" << dbH->timesTableOffset-(dbH->dataOffset+dbH->length) << " (" <<
mas01mc@324 581 (100.0*(dbH->timesTableOffset-(dbH->dataOffset+dbH->length)))/(dbH->timesTableOffset-dbH->dataOffset) << "%)" << std::endl;
mas01mc@324 582 }
mas01mc@301 583 std::cout << "flags:" << " l2norm[" << DISPLAY_FLAG(dbH->flags&O2_FLAG_L2NORM)
mas01mc@301 584 << "] minmax[" << DISPLAY_FLAG(dbH->flags&O2_FLAG_MINMAX)
mas01mc@301 585 << "] power[" << DISPLAY_FLAG(dbH->flags&O2_FLAG_POWER)
mas01mc@324 586 << "] times[" << DISPLAY_FLAG(dbH->flags&O2_FLAG_TIMES)
mas01mc@324 587 << "] largeADB[" << DISPLAY_FLAG(dbH->flags&O2_FLAG_LARGE_ADB)
mas01mc@324 588 << "]" << endl;
mas01mc@324 589
mas01cr@239 590 std::cout << "null count: " << nullCount << " small sequence count " << dudCount-nullCount << std::endl;
mas01cr@76 591 } else {
mas01cr@133 592 adbStatusResponse->result.numFiles = dbH->numFiles;
mas01cr@133 593 adbStatusResponse->result.dim = dbH->dim;
mas01cr@133 594 adbStatusResponse->result.length = dbH->length;
mas01cr@133 595 adbStatusResponse->result.dudCount = dudCount;
mas01cr@133 596 adbStatusResponse->result.nullCount = nullCount;
mas01cr@133 597 adbStatusResponse->result.flags = dbH->flags;
mas01cr@76 598 }
mas01cr@0 599 }
mas01cr@0 600
mas01cr@196 601 void audioDB::l2norm(const char* dbName) {
mas01cr@196 602 forWrite = true;
mas01cr@196 603 initTables(dbName, 0);
mas01mc@324 604 if( !(dbH->flags & O2_FLAG_LARGE_ADB ) && (dbH->length>0) ){
mas01cr@196 605 /* FIXME: should probably be uint64_t */
mas01cr@0 606 unsigned numVectors = dbH->length/(sizeof(double)*dbH->dim);
mas01cr@196 607 CHECKED_MMAP(double *, dataBuf, dbH->dataOffset, dataBufLength);
mas01cr@0 608 unitNormAndInsertL2(dataBuf, dbH->dim, numVectors, 0); // No append
mas01cr@0 609 }
mas01cr@0 610 // Update database flags
mas01cr@0 611 dbH->flags = dbH->flags|O2_FLAG_L2NORM;
mas01cr@0 612 memcpy (db, dbH, O2_HEADERSIZE);
mas01cr@0 613 }
mas01cr@193 614
mas01cr@193 615 void audioDB::power_flag(const char *dbName) {
mas01cr@196 616 forWrite = true;
mas01mc@324 617 initTables(dbName, 0);
mas01mc@324 618 if( !(dbH->flags & O2_FLAG_LARGE_ADB ) && (dbH->length>0) ){
mas01cr@193 619 error("cannot turn on power storage for non-empty database", dbName);
mas01cr@193 620 }
mas01cr@193 621 dbH->flags |= O2_FLAG_POWER;
mas01cr@193 622 memcpy(db, dbH, O2_HEADERSIZE);
mas01cr@193 623 }
mas01cr@193 624
mas01cr@239 625 // Unit norm block of features
mas01cr@0 626
mas01cr@239 627 /* FIXME: in fact this does not unit norm a block of features, it just
mas01cr@239 628 records the L2 norms somewhere. unitNorm() does in fact unit norm
mas01cr@239 629 a block of features. */
mas01cr@0 630 void audioDB::unitNormAndInsertL2(double* X, unsigned dim, unsigned n, unsigned append=0){
mas01cr@0 631 unsigned d;
mas01cr@59 632 double *p;
mas01cr@0 633 unsigned nn = n;
mas01cr@0 634
mas01cr@0 635 assert(l2normTable);
mas01cr@0 636
mas01mc@324 637 if( !(dbH->flags & O2_FLAG_LARGE_ADB) && !append && (dbH->flags & O2_FLAG_L2NORM) )
mas01cr@0 638 error("Database is already L2 normed", "automatic norm on insert is enabled");
mas01cr@0 639
mas01cr@239 640 VERB_LOG(2, "norming %u vectors...", n);
mas01cr@0 641
mas01cr@0 642 double* l2buf = new double[n];
mas01cr@0 643 double* l2ptr = l2buf;
mas01cr@0 644 assert(l2buf);
mas01cr@0 645 assert(X);
mas01cr@0 646
mas01cr@0 647 while(nn--){
mas01cr@0 648 p=X;
mas01cr@0 649 *l2ptr=0.0;
mas01cr@0 650 d=dim;
mas01cr@0 651 while(d--){
mas01cr@0 652 *l2ptr+=*p**p;
mas01cr@0 653 p++;
mas01cr@0 654 }
mas01mc@17 655 l2ptr++;
mas01mc@17 656 X+=dim;
mas01cr@0 657 }
mas01cr@0 658 unsigned offset;
mas01cr@84 659 if(append) {
mas01cr@84 660 // FIXME: a hack, a very palpable hack: the vectors have already
mas01cr@84 661 // been inserted, and dbH->length has already been updated. We
mas01cr@84 662 // need to subtract off again the number of vectors that we've
mas01cr@84 663 // inserted this time...
mas01cr@84 664 offset=(dbH->length/(dbH->dim*sizeof(double)))-n; // number of vectors
mas01cr@84 665 } else {
mas01cr@0 666 offset=0;
mas01cr@84 667 }
mas01cr@0 668 memcpy(l2normTable+offset, l2buf, n*sizeof(double));
mas01cr@0 669 if(l2buf)
mas01mc@17 670 delete[] l2buf;
mas01cr@239 671 VERB_LOG(2, " done.");
mas01cr@193 672 }
mas01cr@193 673
mas01mc@308 674 // This entry point is visited once per instance
mas01mc@308 675 // so it is a good place to set any global state variables
mas01cr@0 676 int main(const unsigned argc, char* const argv[]){
mas01mc@308 677 SERVER_LSH_INDEX_SINGLETON = 0; // Initialize global variables
mas01mc@324 678 SERVER_ADB_ROOT = 0; // Server-side database root prefix
mas01mc@324 679 SERVER_ADB_FEATURE_ROOT = 0; // Server-side features root prefix
mas01cr@0 680 audioDB(argc, argv);
mas01cr@0 681 }