mas01cr@0: #include "audioDB.h" mas01cr@385: extern "C" { mas01cr@385: #include "audioDB_API.h" mas01cr@402: #include "audioDB-internals.h" mas01cr@385: } mas01cr@0: mas01mc@308: LSH* SERVER_LSH_INDEX_SINGLETON; mas01mc@324: char* SERVER_ADB_ROOT; mas01mc@324: char* SERVER_ADB_FEATURE_ROOT; mas01mc@308: mas01mc@292: PointPair::PointPair(Uns32T a, Uns32T b, Uns32T c):trackID(a),qpos(b),spos(c){}; mas01mc@292: mas01mc@292: bool operator<(const PointPair& a, const PointPair& b){ mas01mc@324: return ( (a.trackID(const PointPair& a, const PointPair& b){ mas01mc@324: return ( (a.trackID>b.trackID) || mas01mc@324: ( (a.trackID==b.trackID) && mas01mc@324: ( (a.spos>b.spos) || ( (a.spos==b.spos) && (a.qpos > b.qpos) )) ) ); mas01mc@292: } mas01mc@292: mas01mc@292: bool operator==(const PointPair& a, const PointPair& b){ mas01mc@292: return ( (a.trackID==b.trackID) && (a.qpos==b.qpos) && (a.spos==b.spos) ); mas01mc@292: } mas01mc@292: mas01cr@370: audioDB::audioDB(const unsigned argc, const char *argv[]): O2_AUDIODB_INITIALIZERS mas01cr@76: { mas01cr@0: if(processArgs(argc, argv)<0){ mas01cr@0: printf("No command found.\n"); mas01cr@0: cmdline_parser_print_version (); mas01cr@0: if (strlen(gengetopt_args_info_purpose) > 0) mas01cr@0: printf("%s\n", gengetopt_args_info_purpose); mas01cr@0: printf("%s\n", gengetopt_args_info_usage); mas01cr@0: printf("%s\n", gengetopt_args_info_help[1]); mas01cr@0: printf("%s\n", gengetopt_args_info_help[2]); mas01cr@0: printf("%s\n", gengetopt_args_info_help[0]); mas01cr@151: error("No command found"); mas01cr@0: } mas01cr@77: mas01mc@324: // Perform database prefix substitution mas01mc@328: if(dbName && adb_root) mas01mc@324: prefix_name((char** const)&dbName, adb_root); mas01mc@324: mas01cr@0: if(O2_ACTION(COM_SERVER)) mas01cr@371: #ifdef LIBRARY mas01cr@371: ; mas01cr@371: #else mas01cr@0: startServer(); mas01cr@371: #endif mas01cr@0: mas01cr@0: else if(O2_ACTION(COM_CREATE)) mas01cr@0: create(dbName); mas01cr@0: mas01cr@0: else if(O2_ACTION(COM_INSERT)) mas01cr@0: insert(dbName, inFile); mas01cr@0: mas01cr@0: else if(O2_ACTION(COM_BATCHINSERT)) mas01cr@0: batchinsert(dbName, inFile); mas01cr@0: mas01cr@0: else if(O2_ACTION(COM_QUERY)) mas01mc@307: if(isClient){ mas01cr@371: #ifdef LIBRARY mas01cr@371: ; mas01cr@371: #else mas01mc@329: if(query_from_key){ mas01mc@332: VERB_LOG(1, "Calling web services query %s on database %s, query=%s\n", radius>0?"(Radius)":"(NN)", dbName, (key&&strlen(key))?key:inFile); mas01mc@328: ws_query_by_key(dbName, key, inFile, (char*)hostport); mas01mc@329: } mas01mc@329: else{ mas01mc@332: VERB_LOG(1, "Calling web services query on database %s, query=%s\n", dbName, (key&&strlen(key))?key:inFile); mas01mc@307: ws_query(dbName, inFile, (char*)hostport); mas01mc@329: } mas01cr@371: #endif mas01mc@307: } mas01cr@0: else mas01cr@76: query(dbName, inFile); mas01cr@0: mas01cr@0: else if(O2_ACTION(COM_STATUS)) mas01cr@0: if(isClient) mas01cr@371: #ifdef LIBRARY mas01cr@371: ; mas01cr@371: #else mas01cr@0: ws_status(dbName,(char*)hostport); mas01cr@371: #endif mas01cr@0: else mas01cr@0: status(dbName); mas01cr@280: mas01cr@280: else if(O2_ACTION(COM_SAMPLE)) mas01cr@280: sample(dbName); mas01cr@0: mas01cr@0: else if(O2_ACTION(COM_L2NORM)) mas01cr@0: l2norm(dbName); mas01cr@0: mas01cr@193: else if(O2_ACTION(COM_POWER)) mas01cr@193: power_flag(dbName); mas01cr@193: mas01cr@0: else if(O2_ACTION(COM_DUMP)) mas01cr@0: dump(dbName); mas01mc@292: mas01mc@334: else if(O2_ACTION(COM_LISZT)) mas01mc@334: if(isClient) mas01cr@371: #ifdef LIBRARY mas01cr@371: ; mas01cr@371: #else mas01mc@334: ws_liszt(dbName, (char*) hostport); mas01cr@371: #endif mas01mc@334: else mas01mc@334: liszt(dbName, lisztOffset, lisztLength); mas01mc@334: mas01mc@292: else if(O2_ACTION(COM_INDEX)) mas01mc@292: index_index_db(dbName); mas01cr@0: mas01cr@0: else mas01cr@0: error("Unrecognized command",command); mas01cr@0: } mas01cr@0: mas01cr@370: audioDB::audioDB(const unsigned argc, const char *argv[], adb__queryResponse *adbQueryResponse): O2_AUDIODB_INITIALIZERS mas01cr@76: { mas01cr@97: try { mas01mc@338: isServer = 1; // Set to make errors report over SOAP mas01cr@97: processArgs(argc, argv); mas01mc@324: // Perform database prefix substitution mas01mc@328: if(dbName && adb_root) mas01mc@324: prefix_name((char** const)&dbName, adb_root); mas01cr@97: assert(O2_ACTION(COM_QUERY)); mas01cr@133: query(dbName, inFile, adbQueryResponse); mas01cr@97: } catch(char *err) { mas01cr@97: cleanup(); mas01cr@97: throw(err); mas01cr@97: } mas01cr@76: } mas01cr@76: mas01cr@370: audioDB::audioDB(const unsigned argc, const char *argv[], adb__statusResponse *adbStatusResponse): O2_AUDIODB_INITIALIZERS mas01cr@76: { mas01cr@97: try { mas01mc@338: isServer = 1; // Set to make errors report over SOAP mas01cr@97: processArgs(argc, argv); mas01mc@324: // Perform database prefix substitution mas01mc@328: if(dbName && adb_root) mas01mc@324: prefix_name((char** const)&dbName, adb_root); mas01cr@97: assert(O2_ACTION(COM_STATUS)); mas01cr@133: status(dbName, adbStatusResponse); mas01cr@97: } catch(char *err) { mas01cr@97: cleanup(); mas01cr@97: throw(err); mas01cr@97: } mas01cr@76: } mas01cr@76: mas01cr@370: audioDB::audioDB(const unsigned argc, const char *argv[], adb__lisztResponse *adbLisztResponse): O2_AUDIODB_INITIALIZERS mas01mc@334: { mas01mc@334: try { mas01mc@338: isServer = 1; // Set to make errors report over SOAP mas01mc@338: processArgs(argc, argv); mas01mc@334: // Perform database prefix substitution mas01mc@334: if(dbName && adb_root) mas01mc@334: prefix_name((char** const)&dbName, adb_root); mas01mc@334: assert(O2_ACTION(COM_LISZT)); mas01mc@334: liszt(dbName, lisztOffset, lisztLength, adbLisztResponse); mas01mc@334: } catch(char *err) { mas01mc@334: cleanup(); mas01mc@334: throw(err); mas01mc@334: } mas01mc@334: } mas01mc@334: mas01cr@395: //for API query mas01cr@395: audioDB::audioDB(const unsigned argc, const char *argv[],adb__queryResponse *adbQueryResponse, int * apierror, adb_t *a): O2_AUDIODB_INITIALIZERS mas01ik@355: { mas01ik@355: mas01ik@355: try { mas01ik@355: UseApiError=1; mas01cr@395: adb = a; mas01ik@355: mas01ik@355: if(processArgs(argc, argv)<0){ mas01ik@355: printf("No command found.\n"); mas01ik@355: cmdline_parser_print_version (); mas01ik@355: if (strlen(gengetopt_args_info_purpose) > 0) mas01ik@355: printf("%s\n", gengetopt_args_info_purpose); mas01ik@355: printf("%s\n", gengetopt_args_info_usage); mas01ik@355: printf("%s\n", gengetopt_args_info_help[1]); mas01ik@355: printf("%s\n", gengetopt_args_info_help[2]); mas01ik@355: printf("%s\n", gengetopt_args_info_help[0]); mas01ik@355: error("No command found"); mas01ik@355: } mas01ik@355: mas01ik@355: query(dbName, inFile, adbQueryResponse); mas01ik@355: mas01ik@355: } catch(int a) { mas01ik@355: *apierror=a; mas01ik@355: return; mas01ik@355: mas01ik@355: } mas01ik@355: *apierror=apierrortemp; mas01ik@355: return; mas01ik@355: mas01ik@355: } mas01ik@355: mas01cr@97: void audioDB::cleanup() { mas01cr@122: cmdline_parser_free(&args_info); mas01cr@0: if(indata) mas01cr@0: munmap(indata,statbuf.st_size); mas01cr@0: if(db) mas01cr@196: munmap(db,getpagesize()); mas01cr@196: if(fileTable) mas01cr@196: munmap(fileTable, fileTableLength); mas01cr@196: if(trackTable) mas01cr@196: munmap(trackTable, trackTableLength); mas01cr@196: if(dataBuf) mas01cr@196: munmap(dataBuf, dataBufLength); mas01cr@196: if(timesTable) mas01cr@196: munmap(timesTable, timesTableLength); mas01mc@314: if(powerTable) mas01mc@314: munmap(powerTable, powerTableLength); mas01cr@196: if(l2normTable) mas01cr@196: munmap(l2normTable, l2normTableLength); mas01mc@324: if(featureFileNameTable) mas01mc@324: munmap(featureFileNameTable, fileTableLength); mas01mc@324: if(timesFileNameTable) mas01mc@324: munmap(timesFileNameTable, fileTableLength); mas01mc@324: if(powerFileNameTable) mas01mc@324: munmap(powerFileNameTable, fileTableLength); mas01mc@292: if(trackOffsetTable) mas01ik@355: delete [] trackOffsetTable; mas01mc@292: if(reporter) mas01mc@292: delete reporter; mas01mc@292: if(exact_evaluation_queue) mas01mc@292: delete exact_evaluation_queue; mas01cr@284: if(rng) mas01cr@284: gsl_rng_free(rng); mas01mc@292: if(vv) mas01mc@292: delete vv; mas01cr@0: if(infid>0) mas01cr@0: close(infid); mas01cr@395: if(adb && !UseApiError) { mas01cr@392: audiodb_close(adb); mas01cr@392: adb = NULL; mas01cr@392: } mas01mc@308: if(lsh!=SERVER_LSH_INDEX_SINGLETON) mas01mc@308: delete lsh; mas01cr@0: } mas01cr@0: mas01cr@97: audioDB::~audioDB(){ mas01cr@97: cleanup(); mas01cr@97: } mas01cr@97: mas01cr@370: int audioDB::processArgs(const unsigned argc, const char *argv[]){ mas01cr@0: mas01cr@0: if(argc<2){ mas01cr@0: cmdline_parser_print_version (); mas01cr@0: if (strlen(gengetopt_args_info_purpose) > 0) mas01cr@0: printf("%s\n", gengetopt_args_info_purpose); mas01cr@0: printf("%s\n", gengetopt_args_info_usage); mas01cr@0: printf("%s\n", gengetopt_args_info_help[1]); mas01cr@0: printf("%s\n", gengetopt_args_info_help[2]); mas01cr@0: printf("%s\n", gengetopt_args_info_help[0]); mas01cr@0: exit(0); mas01cr@0: } mas01cr@0: mas01cr@345: /* KLUDGE: gengetopt generates a function which is not completely mas01cr@345: const-clean in its declaration. We cast argv here to keep the mas01cr@345: compiler happy. -- CSR, 2008-10-08 */ mas01cr@345: if (cmdline_parser (argc, (char *const *) argv, &args_info) != 0) mas01cr@151: error("Error parsing command line"); mas01cr@0: mas01cr@0: if(args_info.help_given){ mas01cr@0: cmdline_parser_print_help(); mas01cr@0: exit(0); mas01cr@0: } mas01cr@0: mas01cr@0: if(args_info.verbosity_given){ mas01cr@239: verbosity = args_info.verbosity_arg; mas01cr@239: if(verbosity < 0 || verbosity > 10){ mas01cr@239: std::cerr << "Warning: verbosity out of range, setting to 1" << std::endl; mas01cr@239: verbosity = 1; mas01cr@0: } mas01cr@0: } mas01cr@0: mas01cr@129: if(args_info.size_given) { mas01cr@256: if(args_info.datasize_given) { mas01cr@256: error("both --size and --datasize given", ""); mas01cr@256: } mas01cr@256: if(args_info.ntracks_given) { mas01cr@256: error("both --size and --ntracks given", ""); mas01cr@256: } mas01cr@256: if(args_info.datadim_given) { mas01cr@256: error("both --size and --datadim given", ""); mas01cr@256: } mas01cr@196: if (args_info.size_arg < 50 || args_info.size_arg > 32000) { mas01cr@129: error("Size out of range", ""); mas01cr@129: } mas01cr@256: double ratio = (double) args_info.size_arg * 1000000 / ((double) O2_DEFAULTDBSIZE); mas01cr@256: /* FIXME: what's the safe way of doing this? */ mas01cr@256: datasize = (unsigned int) ceil(datasize * ratio); mas01cr@256: ntracks = (unsigned int) ceil(ntracks * ratio); mas01cr@256: } else { mas01cr@256: if(args_info.datasize_given) { mas01cr@256: datasize = args_info.datasize_arg; mas01cr@256: } mas01cr@256: if(args_info.ntracks_given) { mas01cr@256: ntracks = args_info.ntracks_arg; mas01cr@256: } mas01cr@256: if(args_info.datadim_given) { mas01cr@256: datadim = args_info.datadim_arg; mas01cr@256: } mas01cr@129: } mas01cr@129: mas01cr@239: if(args_info.radius_given) { mas01cr@239: radius = args_info.radius_arg; mas01mc@307: if(radius < 0 || radius > 1000000000) { mas01cr@77: error("radius out of range"); mas01cr@239: } else { mas01cr@239: VERB_LOG(3, "Setting radius to %f\n", radius); mas01mc@17: } mas01mc@17: } mas01mc@17: mas01mc@292: sequenceLength = args_info.sequencelength_arg; mas01mc@292: if(sequenceLength < 1 || sequenceLength > 1000) { mas01mc@292: error("seqlen out of range: 1 <= seqlen <= 1000"); mas01mc@292: } mas01mc@292: sequenceHop = args_info.sequencehop_arg; mas01mc@292: if(sequenceHop < 1 || sequenceHop > 1000) { mas01mc@292: error("seqhop out of range: 1 <= seqhop <= 1000"); mas01mc@292: } mas01mc@292: mas01mc@292: if (args_info.absolute_threshold_given) { mas01mc@292: if (args_info.absolute_threshold_arg >= 0) { mas01mc@292: error("absolute threshold out of range: should be negative"); mas01mc@292: } mas01mc@292: use_absolute_threshold = true; mas01mc@292: absolute_threshold = args_info.absolute_threshold_arg; mas01mc@292: } mas01mc@292: if (args_info.relative_threshold_given) { mas01mc@292: use_relative_threshold = true; mas01mc@292: relative_threshold = args_info.relative_threshold_arg; mas01mc@292: } mas01mc@292: mas01mc@324: if (args_info.adb_root_given){ mas01mc@324: adb_root = args_info.adb_root_arg; mas01mc@324: } mas01mc@324: mas01mc@324: if (args_info.adb_feature_root_given){ mas01mc@324: adb_feature_root = args_info.adb_feature_root_arg; mas01mc@324: } mas01mc@324: mas01mc@324: // perform dbName path prefix SERVER-side subsitution mas01mc@324: if(SERVER_ADB_ROOT && !adb_root) mas01mc@324: adb_root = SERVER_ADB_ROOT; mas01mc@324: if(SERVER_ADB_FEATURE_ROOT && !adb_feature_root) mas01mc@324: adb_feature_root = SERVER_ADB_FEATURE_ROOT; mas01mc@339: mas01cr@0: if(args_info.SERVER_given){ mas01cr@0: command=COM_SERVER; mas01cr@0: port=args_info.SERVER_arg; mas01cr@0: if(port<100 || port > 100000) mas01cr@0: error("port out of range"); mas01cr@105: #if defined(O2_DEBUG) mas01cr@104: struct sigaction sa; mas01cr@104: sa.sa_sigaction = sigterm_action; mas01cr@104: sa.sa_flags = SA_SIGINFO | SA_RESTART | SA_NODEFER; mas01cr@104: sigaction(SIGTERM, &sa, NULL); mas01cr@104: sa.sa_sigaction = sighup_action; mas01cr@104: sa.sa_flags = SA_SIGINFO | SA_RESTART | SA_NODEFER; mas01cr@104: sigaction(SIGHUP, &sa, NULL); mas01cr@105: #endif mas01mc@308: if(args_info.load_index_given){ mas01mc@308: if(!args_info.database_given) mas01mc@308: error("load_index requires a --database argument"); mas01mc@308: else mas01mc@308: dbName=args_info.database_arg; mas01mc@308: if(!args_info.radius_given) mas01mc@308: error("load_index requires a --radius argument"); mas01mc@308: if(!args_info.sequencelength_given) mas01mc@308: error("load_index requires a --sequenceLength argument"); mas01mc@308: WS_load_index = true; mas01mc@308: } mas01cr@0: return 0; mas01cr@0: } mas01cr@0: mas01cr@0: // No return on client command, find database command mas01cr@105: if(args_info.client_given){ mas01cr@105: command=COM_CLIENT; mas01cr@105: hostport=args_info.client_arg; mas01cr@105: isClient=1; mas01cr@105: } mas01cr@0: mas01cr@105: if(args_info.NEW_given){ mas01cr@105: command=COM_CREATE; mas01cr@105: dbName=args_info.database_arg; mas01cr@105: return 0; mas01cr@105: } mas01cr@0: mas01cr@105: if(args_info.STATUS_given){ mas01cr@105: command=COM_STATUS; mas01cr@105: dbName=args_info.database_arg; mas01cr@105: return 0; mas01cr@105: } mas01cr@0: mas01cr@280: if(args_info.SAMPLE_given) { mas01cr@280: command = COM_SAMPLE; mas01cr@280: dbName = args_info.database_arg; mas01cr@280: sequenceLength = args_info.sequencelength_arg; mas01cr@280: if(sequenceLength < 1 || sequenceLength > 1000) { mas01cr@280: error("seqlen out of range: 1 <= seqlen <= 1000"); mas01cr@280: } mas01cr@280: nsamples = args_info.nsamples_arg; mas01cr@280: return 0; mas01cr@280: } mas01cr@280: mas01cr@105: if(args_info.DUMP_given){ mas01cr@105: command=COM_DUMP; mas01cr@105: dbName=args_info.database_arg; mas01cr@131: output = args_info.output_arg; mas01cr@105: return 0; mas01cr@105: } mas01cr@0: mas01cr@105: if(args_info.L2NORM_given){ mas01cr@105: command=COM_L2NORM; mas01cr@105: dbName=args_info.database_arg; mas01cr@105: return 0; mas01cr@105: } mas01cr@0: mas01cr@193: if(args_info.POWER_given){ mas01cr@193: command=COM_POWER; mas01cr@193: dbName=args_info.database_arg; mas01cr@193: return 0; mas01cr@193: } mas01cr@193: mas01cr@370: if(args_info.INSERT_given) { mas01cr@105: command=COM_INSERT; mas01cr@105: dbName=args_info.database_arg; mas01cr@105: inFile=args_info.features_arg; mas01cr@370: if(args_info.key_given) { mas01cr@370: if(!args_info.features_given) { mas01mc@292: error("INSERT: '-k key' argument depends on '-f features'"); mas01cr@370: } else { mas01mc@292: key=args_info.key_arg; mas01cr@370: } mas01cr@370: } mas01cr@370: if(args_info.times_given) { mas01cr@105: timesFileName=args_info.times_arg; mas01cr@370: if(strlen(timesFileName)>0) { mas01cr@370: if(!(timesFile = new std::ifstream(timesFileName,std::ios::in))) { mas01cr@105: error("Could not open times file for reading", timesFileName); mas01cr@370: } mas01cr@105: usingTimes=1; mas01cr@105: } mas01cr@105: } mas01cr@193: if (args_info.power_given) { mas01cr@193: powerFileName = args_info.power_arg; mas01cr@193: if (strlen(powerFileName) > 0) { mas01cr@193: if (!(powerfd = open(powerFileName, O_RDONLY))) { mas01cr@193: error("Could not open power file for reading", powerFileName, "open"); mas01cr@193: } mas01cr@193: usingPower = 1; mas01cr@193: } mas01cr@193: } mas01cr@105: return 0; mas01cr@105: } mas01cr@105: mas01cr@370: if(args_info.BATCHINSERT_given) { mas01cr@105: command=COM_BATCHINSERT; mas01cr@105: dbName=args_info.database_arg; mas01cr@105: inFile=args_info.featureList_arg; mas01cr@370: if(args_info.keyList_given) { mas01cr@370: if(!args_info.featureList_given) { mas01tc@300: error("BATCHINSERT: '-K keyList' argument depends on '-F featureList'"); mas01cr@370: } else { mas01cr@304: key=args_info.keyList_arg; // INCONSISTENT NO CHECK mas01cr@370: } mas01cr@370: } mas01cr@105: /* TO DO: REPLACE WITH mas01cr@0: if(args_info.keyList_given){ mas01mc@18: trackFileName=args_info.keyList_arg; mas01cr@239: if(strlen(trackFileName)>0 && !(trackFile = new std::ifstream(trackFileName,std::ios::in))) mas01mc@18: error("Could not open keyList file for reading",trackFileName); mas01cr@0: } mas01cr@0: AND UPDATE BATCHINSERT() mas01cr@105: */ mas01cr@105: mas01cr@370: if(args_info.timesList_given) { mas01cr@105: timesFileName=args_info.timesList_arg; mas01cr@370: if(strlen(timesFileName)>0) { mas01cr@239: if(!(timesFile = new std::ifstream(timesFileName,std::ios::in))) mas01cr@105: error("Could not open timesList file for reading", timesFileName); mas01cr@105: usingTimes=1; mas01cr@105: } mas01cr@105: } mas01cr@370: if(args_info.powerList_given) { mas01cr@193: powerFileName=args_info.powerList_arg; mas01cr@370: if(strlen(powerFileName)>0) { mas01cr@239: if(!(powerFile = new std::ifstream(powerFileName,std::ios::in))) mas01cr@193: error("Could not open powerList file for reading", powerFileName); mas01cr@193: usingPower=1; mas01cr@193: } mas01cr@193: } mas01cr@105: return 0; mas01cr@105: } mas01mc@292: mas01mc@292: // Set no_unit_norm flag mas01mc@292: no_unit_norming = args_info.no_unit_norming_flag; mas01mc@292: lsh_use_u_functions = args_info.lsh_use_u_functions_flag; mas01mc@292: mas01mc@292: // LSH Index Command mas01mc@292: if(args_info.INDEX_given){ mas01mc@292: if(radius <= 0 ) mas01mc@292: error("INDEXing requires a Radius argument"); mas01mc@292: if(!(sequenceLength>0 && sequenceLength <= O2_MAXSEQLEN)) mas01mc@292: error("INDEXing requires 1 <= sequenceLength <= 1000"); mas01mc@292: command=COM_INDEX; mas01mc@337: if(!args_info.database_given) mas01mc@337: error("INDEXing requires a database"); mas01mc@292: dbName=args_info.database_arg; mas01mc@292: mas01mc@292: // Whether to store LSH hash tables for query in core (FORMAT2) mas01mc@297: lsh_in_core = !args_info.lsh_on_disk_flag; // This flag is set to 0 if on_disk requested mas01mc@292: mas01mc@292: lsh_param_w = args_info.lsh_w_arg; mas01mc@292: if(!(lsh_param_w>0 && lsh_param_w<=O2_SERIAL_MAX_BINWIDTH)) mas01mc@292: error("Indexing parameter w out of range (0.0 < w <= 100.0)"); mas01mc@292: mas01mc@292: lsh_param_k = args_info.lsh_k_arg; mas01mc@292: if(!(lsh_param_k>0 && lsh_param_k<=O2_SERIAL_MAX_FUNS)) mas01mc@292: error("Indexing parameter k out of range (1 <= k <= 100)"); mas01mc@292: mas01mc@292: lsh_param_m = args_info.lsh_m_arg; mas01mc@292: if(!(lsh_param_m>0 && lsh_param_m<= (1 + (sqrt(1 + O2_SERIAL_MAX_TABLES*8.0)))/2.0)) mas01mc@292: error("Indexing parameter m out of range (1 <= m <= 20)"); mas01mc@292: mas01mc@292: lsh_param_N = args_info.lsh_N_arg; mas01mc@292: if(!(lsh_param_N>0 && lsh_param_N<=O2_SERIAL_MAX_ROWS)) mas01mc@292: error("Indexing parameter N out of range (1 <= N <= 1000000)"); mas01mc@292: mas01mc@292: lsh_param_b = args_info.lsh_b_arg; mas01mc@292: if(!(lsh_param_b>0 && lsh_param_b<=O2_SERIAL_MAX_TRACKBATCH)) mas01mc@292: error("Indexing parameter b out of range (1 <= b <= 10000)"); mas01mc@292: mas01mc@296: lsh_param_ncols = args_info.lsh_ncols_arg; mas01mc@296: if(lsh_in_core) // We don't want to block rows with FORMAT2 indexing mas01mc@296: lsh_param_ncols = O2_SERIAL_MAX_COLS; mas01mc@292: if( !(lsh_param_ncols>0 && lsh_param_ncols<=O2_SERIAL_MAX_COLS)) mas01mc@292: error("Indexing parameter ncols out of range (1 <= ncols <= 1000"); mas01mc@292: mas01mc@292: return 0; mas01mc@292: } mas01mc@292: mas01cr@105: // Query command and arguments mas01cr@105: if(args_info.QUERY_given){ mas01cr@105: command=COM_QUERY; mas01cr@105: dbName=args_info.database_arg; mas01mc@292: // XOR features and key search mas01cr@370: if((!args_info.features_given && !args_info.key_given) || (args_info.features_given && args_info.key_given)) mas01mc@292: error("QUERY requires exactly one of either -f features or -k key"); mas01mc@292: if(args_info.features_given) mas01mc@292: inFile=args_info.features_arg; // query from file mas01mc@292: else{ mas01mc@292: query_from_key = true; mas01mc@292: key=args_info.key_arg; // query from key mas01mc@292: } mas01mc@292: mas01cr@105: if(args_info.keyList_given){ mas01cr@105: trackFileName=args_info.keyList_arg; mas01cr@239: if(strlen(trackFileName)>0 && !(trackFile = new std::ifstream(trackFileName,std::ios::in))) mas01cr@105: error("Could not open keyList file for reading",trackFileName); mas01cr@105: } mas01cr@105: mas01cr@105: if(args_info.times_given){ mas01cr@105: timesFileName=args_info.times_arg; mas01cr@105: if(strlen(timesFileName)>0){ mas01cr@239: if(!(timesFile = new std::ifstream(timesFileName,std::ios::in))) mas01cr@105: error("Could not open times file for reading", timesFileName); mas01cr@105: usingTimes=1; mas01cr@105: } mas01cr@105: } mas01cr@193: mas01cr@193: if(args_info.power_given){ mas01cr@193: powerFileName=args_info.power_arg; mas01cr@193: if(strlen(powerFileName)>0){ mas01cr@193: if (!(powerfd = open(powerFileName, O_RDONLY))) { mas01cr@193: error("Could not open power file for reading", powerFileName, "open"); mas01cr@193: } mas01cr@193: usingPower = 1; mas01cr@193: } mas01cr@193: } mas01cr@105: mas01cr@105: // query type mas01cr@105: if(strncmp(args_info.QUERY_arg, "track", MAXSTR)==0) mas01cr@105: queryType=O2_TRACK_QUERY; mas01cr@105: else if(strncmp(args_info.QUERY_arg, "point", MAXSTR)==0) mas01cr@105: queryType=O2_POINT_QUERY; mas01cr@105: else if(strncmp(args_info.QUERY_arg, "sequence", MAXSTR)==0) mas01cr@105: queryType=O2_SEQUENCE_QUERY; mas01mc@248: else if(strncmp(args_info.QUERY_arg, "nsequence", MAXSTR)==0) mas01mc@248: queryType=O2_N_SEQUENCE_QUERY; mas01mc@263: else if(strncmp(args_info.QUERY_arg, "onetoonensequence", MAXSTR)==0) mas01mc@263: queryType=O2_ONE_TO_ONE_N_SEQUENCE_QUERY; mas01cr@105: else mas01cr@105: error("unsupported query type",args_info.QUERY_arg); mas01cr@105: mas01cr@105: if(!args_info.exhaustive_flag){ mas01cr@105: queryPoint = args_info.qpoint_arg; mas01cr@105: usingQueryPoint=1; mas01cr@105: if(queryPoint<0 || queryPoint >10000) mas01cr@105: error("queryPoint out of range: 0 <= queryPoint <= 10000"); mas01cr@105: } mas01mc@292: mas01mc@296: // Whether to pre-load LSH hash tables for query (default on, if flag set then off) mas01mc@297: lsh_in_core = !args_info.lsh_on_disk_flag; mas01mc@292: mas01mc@292: // Whether to perform exact evaluation of points returned by LSH mas01mc@292: lsh_exact = args_info.lsh_exact_flag; mas01mc@292: mas01cr@105: pointNN = args_info.pointnn_arg; mas01mc@263: if(pointNN < 1 || pointNN > O2_MAXNN) { mas01mc@263: error("pointNN out of range: 1 <= pointNN <= 1000000"); mas01cr@105: } mas01cr@105: trackNN = args_info.resultlength_arg; mas01mc@263: if(trackNN < 1 || trackNN > O2_MAXNN) { mas01mc@263: error("resultlength out of range: 1 <= resultlength <= 1000000"); mas01cr@105: } mas01cr@105: return 0; mas01cr@105: } mas01mc@334: mas01mc@334: if(args_info.LISZT_given){ mas01mc@334: command = COM_LISZT; mas01mc@334: dbName=args_info.database_arg; mas01mc@334: lisztOffset = args_info.lisztOffset_arg; mas01mc@334: lisztLength = args_info.lisztLength_arg; mas01mc@334: if(args_info.lisztOffset_arg<0) // check upper bound later when database is opened mas01mc@334: error("lisztOffset cannot be negative"); mas01mc@334: if(args_info.lisztLength_arg<0) mas01mc@334: error("lisztLength cannot be negative"); mas01mc@334: if(lisztLength >1000000) mas01mc@334: error("lisztLength too large (>1000000)"); mas01mc@334: return 0; mas01mc@334: } mas01mc@334: mas01cr@105: return -1; // no command found mas01cr@0: } mas01cr@0: mas01cr@133: void audioDB::status(const char* dbName, adb__statusResponse *adbStatusResponse){ mas01cr@395: adb_status_t status; mas01cr@399: if(!adb) { mas01cr@399: if(!(adb = audiodb_open(dbName, O_RDONLY))) { mas01cr@399: error("Failed to open database file", dbName); mas01cr@399: } mas01cr@0: } mas01cr@395: if(audiodb_status(adb, &status)) { mas01cr@395: error("Failed to retrieve database status", dbName); mas01cr@395: } mas01cr@76: mas01cr@133: if(adbStatusResponse == 0) { mas01cr@395: std::cout << "num files:" << status.numFiles << std::endl; mas01cr@395: std::cout << "data dim:" << status.dim < 0) { mas01cr@397: size_t bytes_per_vector = sizeof(double) * status.dim; mas01cr@397: off_t nvectors = status.length / bytes_per_vector; mas01cr@397: off_t data_region_vectors = status.data_region_size / bytes_per_vector; mas01cr@395: std::cout << "total vectors:" << nvectors << std::endl; mas01cr@395: std::cout << "vectors available:"; mas01cr@395: if(status.flags & O2_FLAG_LARGE_ADB) { mas01cr@395: std::cout << O2_MAX_VECTORS - nvectors << std::endl; mas01cr@395: } else { mas01cr@395: std::cout << data_region_vectors - nvectors << std::endl; mas01cr@395: } mas01cr@76: } mas01cr@395: if(!(status.flags & O2_FLAG_LARGE_ADB)) { mas01cr@395: double used_frac = ((double) status.length) / status.data_region_size; mas01cr@395: std::cout << "total bytes:" << status.length << mas01cr@395: " (" << (100.0*used_frac) << "%)" << std::endl; mas01cr@395: std::cout << "bytes available:" << status.data_region_size - status.length << mas01cr@395: " (" << (100.0*(1-used_frac)) << "%)" << std::endl; mas01mc@324: } mas01cr@395: std::cout << "flags:" << " l2norm[" << DISPLAY_FLAG(status.flags&O2_FLAG_L2NORM) mas01cr@395: << "] minmax[" << DISPLAY_FLAG(status.flags&O2_FLAG_MINMAX) mas01cr@395: << "] power[" << DISPLAY_FLAG(status.flags&O2_FLAG_POWER) mas01cr@395: << "] times[" << DISPLAY_FLAG(status.flags&O2_FLAG_TIMES) mas01cr@395: << "] largeADB[" << DISPLAY_FLAG(status.flags&O2_FLAG_LARGE_ADB) mas01mc@324: << "]" << endl; mas01mc@324: mas01cr@395: std::cout << "null count: " << status.nullCount << " small sequence count " << status.dudCount-status.nullCount << std::endl; mas01cr@76: } else { mas01cr@395: adbStatusResponse->result.numFiles = status.numFiles; mas01cr@395: adbStatusResponse->result.dim = status.dim; mas01cr@395: adbStatusResponse->result.length = status.length; mas01cr@395: adbStatusResponse->result.dudCount = status.dudCount; mas01cr@395: adbStatusResponse->result.nullCount = status.nullCount; mas01cr@395: adbStatusResponse->result.flags = status.flags; mas01cr@76: } mas01cr@0: } mas01cr@0: mas01cr@196: void audioDB::l2norm(const char* dbName) { mas01cr@401: if(!adb) { mas01cr@401: if(!(adb = audiodb_open(dbName, O_RDWR))) { mas01cr@401: error("Failed to open database file", dbName); mas01cr@401: } mas01cr@0: } mas01cr@401: if(audiodb_l2norm(adb)) { mas01cr@401: error("failed to turn on l2norm flag for database", dbName); mas01cr@401: } mas01cr@0: } mas01cr@193: mas01cr@193: void audioDB::power_flag(const char *dbName) { mas01cr@400: if(!adb) { mas01cr@400: if(!(adb = audiodb_open(dbName, O_RDWR))) { mas01cr@400: error("Failed to open database file", dbName); mas01cr@400: } mas01cr@193: } mas01cr@400: if(audiodb_power(adb)) { mas01cr@400: error("can't turn on power flag for database", dbName); mas01cr@400: } mas01cr@193: } mas01cr@193: mas01cr@385: void audioDB::create(const char *dbName) { mas01cr@399: if(adb) { mas01cr@399: error("Already have an adb in this object", ""); mas01cr@399: } mas01cr@391: if(!(adb = audiodb_create(dbName, datasize, ntracks, datadim))) { mas01cr@392: error("Failed to create database file", dbName); mas01cr@385: } mas01cr@385: } mas01cr@385: mas01cr@399: void audioDB::dump(const char *dbName) { mas01cr@399: if(!adb) { mas01cr@399: if(!(adb = audiodb_open(dbName, O_RDONLY))) { mas01cr@399: error("Failed to open database file", dbName); mas01cr@399: } mas01cr@399: } mas01cr@399: if(audiodb_dump(adb, output)) { mas01cr@399: error("Failed to dump database to ", output); mas01cr@399: } mas01cr@399: status(dbName); mas01cr@399: } mas01cr@193: mas01cr@409: void audioDB::insert(const char* dbName, const char* inFile) { mas01cr@409: if(!adb) { mas01cr@409: if(!(adb = audiodb_open(dbName, O_RDWR))) { mas01cr@409: error("failed to open database", dbName); mas01cr@409: } mas01cr@409: } mas01cr@409: mas01cr@409: /* at this point, we have powerfd (an fd), timesFile (a mas01cr@409: * std::ifstream *) and inFile (a char *). Wacky, huh? Ignore mas01cr@409: * the wackiness and just use the names. */ mas01cr@409: adb_insert_t insert; mas01cr@409: insert.features = inFile; mas01cr@409: insert.times = timesFileName; mas01cr@409: insert.power = powerFileName; mas01cr@409: insert.key = key; mas01cr@409: mas01cr@409: if(audiodb_insert(adb, &insert)) { mas01cr@409: error("insertion failure", inFile); mas01cr@409: } mas01cr@409: status(dbName); mas01cr@409: } mas01cr@409: mas01cr@409: void audioDB::batchinsert(const char* dbName, const char* inFile) { mas01cr@409: if(!adb) { mas01cr@409: if(!(adb = audiodb_open(dbName, O_RDWR))) { mas01cr@409: error("failed to open database", dbName); mas01cr@409: } mas01cr@409: } mas01cr@409: mas01cr@409: if(!key) mas01cr@409: key=inFile; mas01cr@409: std::ifstream *filesIn = 0; mas01cr@409: std::ifstream *keysIn = 0; mas01cr@409: mas01cr@409: if(!(filesIn = new std::ifstream(inFile))) mas01cr@409: error("Could not open batch in file", inFile); mas01cr@409: if(key && key!=inFile) mas01cr@409: if(!(keysIn = new std::ifstream(key))) mas01cr@409: error("Could not open batch key file",key); mas01cr@409: mas01cr@409: unsigned totalVectors=0; mas01cr@409: char *thisFile = new char[MAXSTR]; mas01cr@409: char *thisKey = 0; mas01cr@409: if (key && (key != inFile)) { mas01cr@409: thisKey = new char[MAXSTR]; mas01cr@409: } mas01cr@409: char *thisTimesFileName = new char[MAXSTR]; mas01cr@409: char *thisPowerFileName = new char[MAXSTR]; mas01cr@409: mas01cr@409: do { mas01cr@409: filesIn->getline(thisFile,MAXSTR); mas01cr@409: if(key && key!=inFile) { mas01cr@409: keysIn->getline(thisKey,MAXSTR); mas01cr@409: } else { mas01cr@409: thisKey = thisFile; mas01cr@409: } mas01cr@409: if(usingTimes) { mas01cr@409: timesFile->getline(thisTimesFileName,MAXSTR); mas01cr@409: } mas01cr@409: if(usingPower) { mas01cr@409: powerFile->getline(thisPowerFileName, MAXSTR); mas01cr@409: } mas01cr@409: mas01cr@409: if(filesIn->eof()) { mas01cr@409: break; mas01cr@409: } mas01cr@409: if(usingTimes){ mas01cr@409: if(timesFile->eof()) { mas01cr@409: error("not enough timestamp files in timesList", timesFileName); mas01cr@409: } mas01cr@409: } mas01cr@409: if (usingPower) { mas01cr@409: if(powerFile->eof()) { mas01cr@409: error("not enough power files in powerList", powerFileName); mas01cr@409: } mas01cr@409: } mas01cr@409: adb_insert_t insert; mas01cr@409: insert.features = thisFile; mas01cr@409: insert.times = usingTimes ? thisTimesFileName : NULL; mas01cr@409: insert.power = usingPower ? thisPowerFileName : NULL; mas01cr@409: insert.key = thisKey; mas01cr@409: if(audiodb_insert(adb, &insert)) { mas01cr@409: error("insertion failure", thisFile); mas01cr@409: } mas01cr@409: } while(!filesIn->eof()); mas01cr@409: mas01cr@409: VERB_LOG(0, "%s %s %u vectors %ju bytes.\n", COM_BATCHINSERT, dbName, totalVectors, (intmax_t) (totalVectors * dbH->dim * sizeof(double))); mas01cr@409: mas01cr@409: delete [] thisPowerFileName; mas01cr@409: if(key && (key != inFile)) { mas01cr@409: delete [] thisKey; mas01cr@409: } mas01cr@409: delete [] thisFile; mas01cr@409: delete [] thisTimesFileName; mas01cr@409: mas01cr@409: delete filesIn; mas01cr@409: delete keysIn; mas01cr@409: mas01cr@409: // Report status mas01cr@409: status(dbName); mas01cr@409: } mas01cr@409: mas01mc@308: // This entry point is visited once per instance mas01mc@308: // so it is a good place to set any global state variables mas01cr@370: int main(const int argc, const char* argv[]){ mas01mc@308: SERVER_LSH_INDEX_SINGLETON = 0; // Initialize global variables mas01mc@324: SERVER_ADB_ROOT = 0; // Server-side database root prefix mas01mc@324: SERVER_ADB_FEATURE_ROOT = 0; // Server-side features root prefix mas01cr@0: audioDB(argc, argv); mas01cr@0: } mas01ik@355: mas01ik@355: mas01ik@355: extern "C" { mas01ik@355: mas01ik@355: /* for API questions contact mas01ik@355: * Christophe Rhodes c.rhodes@gold.ac.uk mas01ik@355: * Ian Knopke mas01ik@gold.ac.uk, ian.knopke@gmail.com */ mas01ik@355: mas01ik@355: int audiodb_query(adb_ptr mydb, adb_query_ptr adbq, adb_queryresult_ptr adbqr){ mas01ik@355: mas01cr@370: const char *argv[32]; mas01ik@355: int argvctr=0; mas01ik@355: char tempstr1[200]; mas01ik@355: char tempstr2[200]; mas01ik@355: char tempstr3[200]; mas01ik@355: int apierror=0; mas01ik@355: mas01ik@355: adb__queryResponse adbQueryResponse; mas01ik@355: mas01ik@355: /* TODO: may need error checking here */ mas01ik@355: /* currently counting on audioDB binary to fail for me */ mas01ik@355: argv[argvctr++]="audioDB"; mas01ik@355: mas01ik@355: if(adbq->querytype){ mas01ik@355: argv[argvctr++]="-Q"; mas01ik@355: argv[argvctr++]=adbq->querytype; mas01ik@355: } mas01ik@355: mas01cr@388: if(mydb->path){ mas01ik@355: argv[argvctr++]="-d"; mas01cr@388: argv[argvctr++]=mydb->path; mas01ik@355: } mas01ik@355: mas01ik@355: if (adbq->feature){ mas01ik@355: argv[argvctr++]="-f"; mas01ik@355: argv[argvctr++]=adbq->feature; mas01ik@355: } mas01ik@355: mas01ik@355: if (adbq->power){ mas01ik@355: argv[argvctr++]="-w"; mas01ik@355: argv[argvctr++]=adbq->power; mas01ik@355: } mas01ik@355: mas01ik@355: if (adbq->qpoint){ mas01ik@355: argv[argvctr++]="-p"; mas01ik@355: argv[argvctr++]=adbq->qpoint; mas01ik@355: } mas01ik@355: if (adbq->numpoints){ mas01ik@355: argv[argvctr++]="-n"; mas01ik@355: argv[argvctr++]=adbq->numpoints; mas01ik@355: } mas01ik@355: if (adbq->radius){ mas01ik@355: argv[argvctr++]="-R"; mas01ik@355: argv[argvctr++]=adbq->radius; mas01ik@355: } mas01ik@355: if(adbq->resultlength){ mas01ik@355: argv[argvctr++]="-r"; mas01ik@355: argv[argvctr++]=adbq->resultlength; mas01ik@355: } mas01ik@355: if(adbq->sequencelength){ mas01ik@355: argv[argvctr++]="-l"; mas01ik@355: argv[argvctr++]=adbq->sequencelength; mas01ik@355: } mas01ik@355: if(adbq->sequencehop){ mas01ik@355: argv[argvctr++]="-h"; mas01ik@355: argv[argvctr++]=adbq->sequencehop; mas01ik@355: } mas01ik@355: mas01ik@355: if (adbq->absolute_threshold){ mas01ik@355: argv[argvctr++]="--absolute-threshold"; mas01ik@355: snprintf(tempstr1,sizeof(tempstr1),"%f",adbq->absolute_threshold); mas01ik@355: argv[argvctr++]=tempstr1; mas01ik@355: } mas01ik@355: mas01ik@355: if (adbq->relative_threshold){ mas01ik@355: argv[argvctr++]="--relative-threshold"; mas01ik@355: snprintf(tempstr2,sizeof(tempstr2),"%f",adbq->relative_threshold); mas01ik@355: argv[argvctr++]=tempstr2; mas01ik@355: } mas01ik@355: mas01ik@355: if (adbq->exhaustive){ mas01ik@355: argv[argvctr++]="--exhaustive"; mas01ik@355: } mas01ik@355: mas01ik@355: if (adbq->expandfactor){ mas01ik@355: argv[argvctr++]="--expandfactor"; mas01ik@355: snprintf(tempstr3,sizeof(tempstr3),"%f",adbq->expandfactor); mas01ik@355: argv[argvctr++]=tempstr3; mas01ik@355: } mas01ik@355: mas01ik@355: if (adbq->rotate){ mas01ik@355: argv[argvctr++]="--rotate"; mas01ik@355: } mas01ik@355: mas01ik@355: if (adbq->keylist){ mas01ik@355: argv[argvctr++]="-K"; mas01ik@355: argv[argvctr++]=adbq->keylist; mas01ik@355: } mas01cr@382: argv[argvctr]='\0'; mas01ik@355: mas01ik@355: /* debugging */ mas01ik@355: mas01cr@395: audioDB::audioDB(argvctr,argv, &adbQueryResponse, &apierror,mydb); mas01ik@355: mas01ik@355: //copy data over here from adbQueryResponse to adbqr mas01ik@355: adbqr->sizeRlist=adbQueryResponse.result.__sizeRlist; mas01ik@355: adbqr->sizeDist=adbQueryResponse.result.__sizeDist; mas01ik@355: adbqr->sizeQpos=adbQueryResponse.result.__sizeQpos; mas01ik@355: adbqr->sizeSpos=adbQueryResponse.result.__sizeSpos; mas01ik@355: adbqr->Rlist=adbQueryResponse.result.Rlist; mas01ik@355: adbqr->Dist=adbQueryResponse.result.Dist; mas01ik@355: adbqr->Qpos=adbQueryResponse.result.Qpos; mas01ik@355: adbqr->Spos=adbQueryResponse.result.Spos; mas01ik@355: mas01ik@355: return apierror; mas01ik@355: } mas01ik@355: } mas01ik@355: