Mercurial > hg > audiodb
changeset 498:342822c2d49a
Merge api-inversion branch (-r656:771, but I don't expect to return to
that branch) into the trunk.
I expect there to be minor performance regressions (e.g. in the SOAP
server index cacheing, which I have forcibly removed) and minor
unplugged memory leaks (e.g. in audioDB::query(), where I don't free up
the datum). I hope that these leaks and performance regressions can be
plugged in short order. I also expect that some (but maybe not all) of
the issues currently addressed in the memory-leaks branch are superseded
or fixed by this merge.
There remains much work to be done; go forth and do it.
line wrap: on
line diff
--- a/Makefile Sat Jan 10 11:11:27 2009 +0000 +++ b/Makefile Sat Jan 10 16:47:57 2009 +0000 @@ -8,10 +8,8 @@ SHARED_LIB_FLAGS=-shared -Wl,-soname, - - -LIBOBJS=insert.o create.o common.o dump.o query.o sample.o index.o lshlib.o cmdline.o -OBJS=$(LIBOBJS) soap.o audioDB.o +LIBOBJS=query.o index.o insert.o create.o common.o open.o close.o status.o dump.o power.o l2norm.o lshlib.o lock.o pointpair.o +OBJS=$(LIBOBJS) soap.o liszt.o sample.o cmdline.o audioDB.o common.o EXECUTABLE=audioDB @@ -50,20 +48,16 @@ soapServer.cpp soapClient.cpp soapC.cpp adb.nsmap: audioDBws.h $(SOAPCPP2) audioDBws.h -%.o: %.cpp audioDB.h adb.nsmap cmdline.h reporter.h ReporterBase.h lshlib.h +%.o: %.cpp audioDB.h audioDB_API.h adb.nsmap cmdline.h reporter.h ReporterBase.h lshlib.h g++ -c $(CFLAGS) $(GSOAP_INCLUDE) $(GSL_INCLUDE) -Wall $< cmdline.o: cmdline.c cmdline.h gcc -c $(CFLAGS) $< -audioDB_library.o: audioDB.cpp - g++ -c -o $@ $(CFLAGS) $(GSOAP_INCLUDE) -Wall -DLIBRARY $< - $(EXECUTABLE): $(OBJS) soapServer.cpp soapClient.cpp soapC.cpp g++ -o $(EXECUTABLE) $(CFLAGS) $^ $(LIBGSL) $(GSOAP_INCLUDE) $(GSOAP_CPP) - -$(LIBRARY): $(LIBOBJS) audioDB_library.o +$(LIBRARY): $(LIBOBJS) g++ $(SHARED_LIB_FLAGS)$(LIBRARY) -o $(LIBRARY) $(CFLAGS) $(LIBGSL) $^ tags: @@ -79,7 +73,7 @@ -rm xthresh -sh -c "cd tests && sh ./clean.sh" -sh -c "cd libtests && sh ./clean.sh" - -rm $(LIBRARY) audioDB_library.o + -rm $(LIBRARY) -rm tags distclean: clean @@ -99,4 +93,3 @@ ln -sf /usr/local/lib/lib$(EXECUTABLE).so.$(SOVERSION) /usr/local/lib/lib$(EXECUTABLE).so ldconfig cp audioDB_API.h /usr/local/include/ -
--- a/ReporterBase.h Sat Jan 10 11:11:27 2009 +0000 +++ b/ReporterBase.h Sat Jan 10 16:47:57 2009 +0000 @@ -6,7 +6,7 @@ public: virtual ~ReporterBase(){}; virtual void add_point(unsigned int trackID, unsigned int qpos, unsigned int spos, double dist) = 0; - virtual void report(char*,void*) = 0; + virtual void report(adb_t *,void *) = 0; }; #endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/accumulator.h Sat Jan 10 16:47:57 2009 +0000 @@ -0,0 +1,11 @@ +#ifndef ACCUMULATOR_H +#define ACCUMULATOR_H + +class Accumulator { +public: + virtual ~Accumulator() {}; + virtual void add_point(adb_result_t *r) = 0; + virtual adb_query_results_t *get_points() = 0; +}; + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/accumulator_test.cpp Sat Jan 10 16:47:57 2009 +0000 @@ -0,0 +1,34 @@ +#include "audioDB.h" +extern "C" { +#include "audioDB_API.h" +} +#include "audioDB-internals.h" + +#include "accumulators.h" + +static NearestAccumulator<adb_result_dist_lt> *foo = new NearestAccumulator<adb_result_dist_lt>(); + +int main() { + adb_result_t r; + r.key = "hello"; + r.ipos = 0; + r.qpos = 0; + r.dist = 3; + foo->add_point(&r); + r.ipos = 1; + r.dist = 2; + foo->add_point(&r); + r.qpos = 1; + foo->add_point(&r); + + adb_query_results_t *rs; + rs = foo->get_points(); + for (unsigned int k = 0; k < rs->nresults; k++) { + r = rs->results[k]; + printf("%s: %f %u %u\n", r.key, r.dist, r.qpos, r.ipos); + } + free(rs->results); + free(rs); + delete foo; + return 1; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/accumulators.h Sat Jan 10 16:47:57 2009 +0000 @@ -0,0 +1,15 @@ +#ifndef ACCUMULATORS_H +#define ACCUMULATORS_H + +#include <string> +#include <set> +#include <queue> +#include <map> +#include <functional> + +#include "accumulator.h" +#include "dbaccumulator.h" +#include "pertrackaccumulator.h" +#include "nearestaccumulator.h" + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/audioDB-internals.h Sat Jan 10 16:47:57 2009 +0000 @@ -0,0 +1,260 @@ +#include "accumulator.h" + +/* this struct is for writing polymorphic routines as puns. When + * inserting, we might have a "datum" (with actual numerical data) or + * a "reference" (with strings denoting pathnames containing numerical + * data), but most of the operations are the same. This struct, used + * only internally, allows us to write the main body of the insert + * code only once. + */ +typedef struct adb_datum_internal { + uint32_t nvectors; + uint32_t dim; + const char *key; + void *data; + void *times; + void *power; +} adb_datum_internal_t; + +/* this struct is to collect together a bunch of information about a + * query (or, in fact, a single database entry, or even a whole + * database). The _data pointers are immutable (hey, FIXME: should + * they be constified in some way?) so that free() can work on them + * later, while the ones without the suffix are mutable to maintain + * the "current" position in some way. mean_duration points to a + * (possibly single-element) array of mean durations for each track. + */ +typedef struct adb_qpointers_internal { + uint32_t nvectors; + double *l2norm_data; + double *l2norm; + double *power_data; + double *power; + double *mean_duration; +} adb_qpointers_internal_t; + +/* this struct is for maintaining per-query state. We don't want to + * store this stuff in the adb struct itself, because (a) it doesn't + * belong there and (b) in principle people might do two queries in + * parallel using the same adb handle. (b) is in practice a little + * bit academic because at the moment we're seeking all over the disk + * using adb->fd, but changing to use pread() might win us + * threadsafety eventually. + */ +typedef struct adb_qstate_internal { + Accumulator *accumulator; + std::set<std::string> *allowed_keys; + std::priority_queue<PointPair> *exact_evaluation_queue; + LSH *lsh; +} adb_qstate_internal_t; + +/* the transparent version of the opaque (forward-declared) adb_t. */ +struct adb { + char *path; + int fd; + int flags; + adb_header_t *header; + std::vector<std::string> *keys; + std::map<std::string,uint32_t> *keymap; + std::vector<uint32_t> *track_lengths; + std::vector<off_t> *track_offsets; + LSH *cached_lsh; +}; + +typedef struct { + bool operator() (const adb_result_t &r1, const adb_result_t &r2) { + return strcmp(r1.key, r2.key) < 0; + } +} adb_result_key_lt; + +typedef struct { + bool operator() (const adb_result_t &r1, const adb_result_t &r2) { + return r1.qpos < r2.qpos; + } +} adb_result_qpos_lt; + +typedef struct { + bool operator() (const adb_result_t &r1, const adb_result_t &r2) { + return r1.dist < r2.dist; + } +} adb_result_dist_lt; + +typedef struct { + bool operator() (const adb_result_t &r1, const adb_result_t &r2) { + return r1.dist > r2.dist; + } +} adb_result_dist_gt; + +typedef struct { + bool operator() (const adb_result_t &r1, const adb_result_t &r2) { + return ((r1.ipos < r2.ipos) || + ((r1.ipos == r2.ipos) && + ((r1.qpos < r2.qpos) || + ((r1.qpos == r2.qpos) && (strcmp(r1.key, r2.key) < 0))))); + } +} adb_result_triple_lt; + +/* We could go gcc-specific here and use typeof() instead of passing + * in an explicit type. Answers on a postcard as to whether that's a + * good plan or not. */ +#define mmap_or_goto_error(type, var, start, length) \ + { void *tmp = mmap(0, length, PROT_READ, MAP_SHARED, adb->fd, (start)); \ + if(tmp == (void *) -1) { \ + goto error; \ + } \ + var = (type) tmp; \ + } + +#define maybe_munmap(table, length) \ + { if(table) { \ + munmap(table, length); \ + } \ + } + +#define write_or_goto_error(fd, buffer, size) \ + { ssize_t tmp = size; \ + if(write(fd, buffer, size) != tmp) { \ + goto error; \ + } \ + } + +#define read_or_goto_error(fd, buffer, size) \ + { ssize_t tmp = size; \ + if(read(fd, buffer, size) != tmp) { \ + goto error; \ + } \ + } + +static inline int audiodb_sync_header(adb_t *adb) { + off_t pos; + pos = lseek(adb->fd, (off_t) 0, SEEK_CUR); + if(pos == (off_t) -1) { + goto error; + } + if(lseek(adb->fd, (off_t) 0, SEEK_SET) == (off_t) -1) { + goto error; + } + if(write(adb->fd, adb->header, O2_HEADERSIZE) != O2_HEADERSIZE) { + goto error; + } + + /* can be fsync() if fdatasync() is racily exciting and new */ + fdatasync(adb->fd); + if(lseek(adb->fd, pos, SEEK_SET) == (off_t) -1) { + goto error; + } + return 0; + + error: + return 1; +} + +static inline double audiodb_dot_product(double *p, double *q, size_t count) { + double result = 0; + while(count--) { + result += *p++ * *q++; + } + return result; +} + +static inline void audiodb_l2norm_buffer(double *d, size_t dim, size_t nvectors, double *l) { + while(nvectors--) { + double *d1 = d; + double *d2 = d; + *l++ = audiodb_dot_product(d1, d2, dim); + d += dim; + } +} + +// This is a common pattern in sequence queries: what we are doing is +// taking a window of length seqlen over a buffer of length length, +// and placing the sum of the elements in that window in the first +// element of the window: thus replacing all but the last seqlen +// elements in the buffer with the corresponding windowed sum. +static inline void audiodb_sequence_sum(double *buffer, int length, int seqlen) { + double tmp1, tmp2, *ps; + int j, w; + + tmp1 = *buffer; + j = 1; + w = seqlen - 1; + while(w--) { + *buffer += buffer[j++]; + } + ps = buffer + 1; + w = length - seqlen; // +1 - 1 + while(w--) { + tmp2 = *ps; + if(isfinite(tmp1)) { + *ps = *(ps - 1) - tmp1 + *(ps + seqlen - 1); + } else { + for(int i = 1; i < seqlen; i++) { + *ps += *(ps + i); + } + } + tmp1 = tmp2; + ps++; + } +} + +// In contrast to audiodb_sequence_sum() above, +// audiodb_sequence_sqrt() and audiodb_sequence_average() below are +// simple mappers across the sequence. +static inline void audiodb_sequence_sqrt(double *buffer, int length, int seqlen) { + int w = length - seqlen + 1; + while(w--) { + *buffer = sqrt(*buffer); + buffer++; + } +} + +static inline void audiodb_sequence_average(double *buffer, int length, int seqlen) { + int w = length - seqlen + 1; + while(w--) { + *buffer /= seqlen; + buffer++; + } +} + +static inline uint32_t audiodb_key_index(adb_t *adb, const char *key) { + std::map<std::string,uint32_t>::iterator it; + it = adb->keymap->find(key); + if(it == adb->keymap->end()) { + return (uint32_t) -1; + } else { + return (*it).second; + } +} + +static inline const char *audiodb_index_key(adb_t *adb, uint32_t index) { + return (*adb->keys)[index].c_str(); +} + +static inline uint32_t audiodb_index_to_track_id(uint32_t lshid, uint32_t n_point_bits) { + return (lshid >> n_point_bits); +} + +static inline uint32_t audiodb_index_to_track_pos(uint32_t lshid, uint32_t n_point_bits) { + return (lshid & ((1 << n_point_bits) - 1)); +} + +static inline uint32_t audiodb_index_from_trackinfo(uint32_t track_id, uint32_t track_pos, uint32_t n_point_bits) { + return ((track_id << n_point_bits) | track_pos); +} + +static inline uint32_t audiodb_lsh_n_point_bits(adb_t *adb) { + uint32_t nbits = adb->header->flags >> 28; + return (nbits ? nbits : O2_DEFAULT_LSH_N_POINT_BITS); +} + +int audiodb_read_data(adb_t *, int, int, double **, size_t *); +int audiodb_insert_create_datum(adb_insert_t *, adb_datum_t *); +int audiodb_track_id_datum(adb_t *, uint32_t, adb_datum_t *); +int audiodb_free_datum(adb_datum_t *); +int audiodb_datum_qpointers(adb_datum_t *, uint32_t, double **, double **, adb_qpointers_internal_t *); +int audiodb_query_spec_qpointers(adb_t *, const adb_query_spec_t *, double **, double **, adb_qpointers_internal_t *); +int audiodb_query_queue_loop(adb_t *, const adb_query_spec_t *, adb_qstate_internal_t *, double *, adb_qpointers_internal_t *); +int audiodb_query_loop(adb_t *, const adb_query_spec_t *, adb_qstate_internal_t *); +char *audiodb_index_get_name(const char *, double, uint32_t); +bool audiodb_index_exists(const char *, double, uint32_t); +int audiodb_index_query_loop(adb_t *, const adb_query_spec_t *, adb_qstate_internal_t *);
--- a/audioDB.cpp Sat Jan 10 11:11:27 2009 +0000 +++ b/audioDB.cpp Sat Jan 10 16:47:57 2009 +0000 @@ -1,27 +1,13 @@ #include "audioDB.h" +extern "C" { +#include "audioDB_API.h" +#include "audioDB-internals.h" +} +#include "reporter.h" -LSH* SERVER_LSH_INDEX_SINGLETON; char* SERVER_ADB_ROOT; char* SERVER_ADB_FEATURE_ROOT; -PointPair::PointPair(Uns32T a, Uns32T b, Uns32T c):trackID(a),qpos(b),spos(c){}; - -bool operator<(const PointPair& a, const PointPair& b){ - return ( (a.trackID<b.trackID) || - ( (a.trackID==b.trackID) && - ( (a.spos<b.spos) || ( (a.spos==b.spos) && (a.qpos < b.qpos) )) ) ); -} - -bool operator>(const PointPair& a, const PointPair& b){ - return ( (a.trackID>b.trackID) || - ( (a.trackID==b.trackID) && - ( (a.spos>b.spos) || ( (a.spos==b.spos) && (a.qpos > b.qpos) )) ) ); -} - -bool operator==(const PointPair& a, const PointPair& b){ - return ( (a.trackID==b.trackID) && (a.qpos==b.qpos) && (a.spos==b.spos) ); -} - audioDB::audioDB(const unsigned argc, const char *argv[]): O2_AUDIODB_INITIALIZERS { if(processArgs(argc, argv)<0){ @@ -41,13 +27,7 @@ prefix_name((char** const)&dbName, adb_root); if(O2_ACTION(COM_SERVER)){ -#ifdef LIBRARY - ; -#else startServer(); - if(SERVER_LSH_INDEX_SINGLETON) - delete lsh; -#endif } else if(O2_ACTION(COM_CREATE)) create(dbName); @@ -60,9 +40,6 @@ else if(O2_ACTION(COM_QUERY)) if(isClient){ -#ifdef LIBRARY - ; -#else if(query_from_key){ VERB_LOG(1, "Calling web services query %s on database %s, query=%s\n", radius>0?"(Radius)":"(NN)", dbName, (key&&strlen(key))?key:inFile); ws_query_by_key(dbName, key, inFile, (char*)hostport); @@ -71,18 +48,13 @@ VERB_LOG(1, "Calling web services query on database %s, query=%s\n", dbName, (key&&strlen(key))?key:inFile); ws_query(dbName, inFile, (char*)hostport); } -#endif } else query(dbName, inFile); else if(O2_ACTION(COM_STATUS)) if(isClient) -#ifdef LIBRARY - ; -#else ws_status(dbName,(char*)hostport); -#endif else status(dbName); @@ -100,11 +72,7 @@ else if(O2_ACTION(COM_LISZT)) if(isClient) -#ifdef LIBRARY - ; -#else ws_liszt(dbName, (char*) hostport); -#endif else liszt(dbName, lisztOffset, lisztLength); @@ -163,153 +131,12 @@ } } - -//for the lib / API -audioDB::audioDB(const unsigned argc, const char *argv[], int * apierror): O2_AUDIODB_INITIALIZERS -{ - - try { - UseApiError=1; - - if(processArgs(argc, argv)<0){ - printf("No command found.\n"); - cmdline_parser_print_version (); - if (strlen(gengetopt_args_info_purpose) > 0) - printf("%s\n", gengetopt_args_info_purpose); - printf("%s\n", gengetopt_args_info_usage); - printf("%s\n", gengetopt_args_info_help[1]); - printf("%s\n", gengetopt_args_info_help[2]); - printf("%s\n", gengetopt_args_info_help[0]); - error("No command found"); - } - - adb__queryResponse adbq; - - if(O2_ACTION(COM_CREATE)) - create(dbName); - - else if(O2_ACTION(COM_INSERT)) - insert(dbName, inFile); - - else if(O2_ACTION(COM_BATCHINSERT)) - batchinsert(dbName, inFile); - - else if(O2_ACTION(COM_QUERY)) - if(isClient) - ;//ws_query(dbName, inFile, (char*)hostport); - else - query(dbName, inFile, &adbq); - //query(dbName, inFile); - - else if(O2_ACTION(COM_STATUS)) - if(isClient) - ;//ws_status(dbName,(char*)hostport); - else - status(dbName); - - else if(O2_ACTION(COM_L2NORM)) - l2norm(dbName); - - else if(O2_ACTION(COM_POWER)) - power_flag(dbName); - - else if(O2_ACTION(COM_DUMP)) - dump(dbName); - - else - error("Unrecognized command",command); - - } catch(int a) { - *apierror=a; - return; - - } - *apierror=apierrortemp; - return; - -} - -//for API status -audioDB::audioDB(const unsigned argc, const char *argv[], cppstatusptr stat, int * apierror): O2_AUDIODB_INITIALIZERS -{ - - try { - UseApiError=1; - - - if(processArgs(argc, argv)<0){ - printf("No command found.\n"); - cmdline_parser_print_version (); - if (strlen(gengetopt_args_info_purpose) > 0) - printf("%s\n", gengetopt_args_info_purpose); - printf("%s\n", gengetopt_args_info_usage); - printf("%s\n", gengetopt_args_info_help[1]); - printf("%s\n", gengetopt_args_info_help[2]); - printf("%s\n", gengetopt_args_info_help[0]); - error("No command found"); - } - - status(dbName, stat); - - - } catch(int a) { - *apierror=a; - return; - - } - *apierror=apierrortemp; - return; - -} - - -//for API query -audioDB::audioDB(const unsigned argc, const char *argv[],adb__queryResponse *adbQueryResponse, int * apierror): O2_AUDIODB_INITIALIZERS -{ - - try { - UseApiError=1; - - if(processArgs(argc, argv)<0){ - printf("No command found.\n"); - cmdline_parser_print_version (); - if (strlen(gengetopt_args_info_purpose) > 0) - printf("%s\n", gengetopt_args_info_purpose); - printf("%s\n", gengetopt_args_info_usage); - printf("%s\n", gengetopt_args_info_help[1]); - printf("%s\n", gengetopt_args_info_help[2]); - printf("%s\n", gengetopt_args_info_help[0]); - error("No command found"); - } - - query(dbName, inFile, adbQueryResponse); - - } catch(int a) { - *apierror=a; - return; - - } - *apierror=apierrortemp; - return; - -} - - - - - void audioDB::cleanup() { cmdline_parser_free(&args_info); - if(indata) - munmap(indata,statbuf.st_size); - if(db) - munmap(db,getpagesize()); if(fileTable) munmap(fileTable, fileTableLength); if(trackTable) munmap(trackTable, trackTableLength); - if(dataBuf) - munmap(dataBuf, dataBufLength); if(timesTable) munmap(timesTable, timesTableLength); if(powerTable) @@ -322,25 +149,19 @@ munmap(timesFileNameTable, fileTableLength); if(powerFileNameTable) munmap(powerFileNameTable, fileTableLength); - if(trackOffsetTable) - delete [] trackOffsetTable; if(reporter) delete reporter; - if(exact_evaluation_queue) - delete exact_evaluation_queue; if(allowed_keys) delete allowed_keys; if(rng) gsl_rng_free(rng); - if(vv) - delete vv; - if(dbfid>0) - close(dbfid); if(infid>0) close(infid); - if(dbH) - delete dbH; - if(lsh!=SERVER_LSH_INDEX_SINGLETON) + if(adb) { + audiodb_close(adb); + adb = NULL; + } + if(lsh) delete lsh; } @@ -743,613 +564,388 @@ } void audioDB::status(const char* dbName, adb__statusResponse *adbStatusResponse){ - if(!dbH) - initTables(dbName, 0); - - unsigned dudCount=0; - unsigned nullCount=0; - for(unsigned k=0; k<dbH->numFiles; k++){ - if(trackTable[k]<sequenceLength){ - dudCount++; - if(!trackTable[k]) - nullCount++; + adb_status_t status; + if(!adb) { + if(!(adb = audiodb_open(dbName, O_RDONLY))) { + error("Failed to open database file", dbName); } } + if(audiodb_status(adb, &status)) { + error("Failed to retrieve database status", dbName); + } if(adbStatusResponse == 0) { - - // Update Header information - std::cout << "num files:" << dbH->numFiles << std::endl; - std::cout << "data dim:" << dbH->dim <<std::endl; - if(dbH->dim>0){ - std::cout << "total vectors:" << dbH->length/(sizeof(double)*dbH->dim)<<std::endl; - if(dbH->flags & O2_FLAG_LARGE_ADB) - std::cout << "vectors available:" << O2_MAX_VECTORS - (dbH->length / (sizeof(double)*dbH->dim)) << std::endl; - else - std::cout << "vectors available:" << (dbH->timesTableOffset-(dbH->dataOffset+dbH->length))/(sizeof(double)*dbH->dim) << std::endl; + std::cout << "num files:" << status.numFiles << std::endl; + std::cout << "data dim:" << status.dim <<std::endl; + if(status.dim > 0) { + size_t bytes_per_vector = sizeof(double) * status.dim; + off_t nvectors = status.length / bytes_per_vector; + off_t data_region_vectors = status.data_region_size / bytes_per_vector; + std::cout << "total vectors:" << nvectors << std::endl; + std::cout << "vectors available:"; + if(status.flags & O2_FLAG_LARGE_ADB) { + std::cout << O2_MAX_VECTORS - nvectors << std::endl; + } else { + std::cout << data_region_vectors - nvectors << std::endl; + } } - if( ! (dbH->flags & O2_FLAG_LARGE_ADB) ){ - std::cout << "total bytes:" << dbH->length << " (" << (100.0*dbH->length)/(dbH->timesTableOffset-dbH->dataOffset) << "%)" << std::endl; - std::cout << "bytes available:" << dbH->timesTableOffset-(dbH->dataOffset+dbH->length) << " (" << - (100.0*(dbH->timesTableOffset-(dbH->dataOffset+dbH->length)))/(dbH->timesTableOffset-dbH->dataOffset) << "%)" << std::endl; + if(!(status.flags & O2_FLAG_LARGE_ADB)) { + double used_frac = ((double) status.length) / status.data_region_size; + std::cout << "total bytes:" << status.length << + " (" << (100.0*used_frac) << "%)" << std::endl; + std::cout << "bytes available:" << status.data_region_size - status.length << + " (" << (100.0*(1-used_frac)) << "%)" << std::endl; } - std::cout << "flags:" << " l2norm[" << DISPLAY_FLAG(dbH->flags&O2_FLAG_L2NORM) - << "] minmax[" << DISPLAY_FLAG(dbH->flags&O2_FLAG_MINMAX) - << "] power[" << DISPLAY_FLAG(dbH->flags&O2_FLAG_POWER) - << "] times[" << DISPLAY_FLAG(dbH->flags&O2_FLAG_TIMES) - << "] largeADB[" << DISPLAY_FLAG(dbH->flags&O2_FLAG_LARGE_ADB) + std::cout << "flags:" << " l2norm[" << DISPLAY_FLAG(status.flags&O2_FLAG_L2NORM) + << "] minmax[" << DISPLAY_FLAG(status.flags&O2_FLAG_MINMAX) + << "] power[" << DISPLAY_FLAG(status.flags&O2_FLAG_POWER) + << "] times[" << DISPLAY_FLAG(status.flags&O2_FLAG_TIMES) + << "] largeADB[" << DISPLAY_FLAG(status.flags&O2_FLAG_LARGE_ADB) << "]" << endl; - std::cout << "null count: " << nullCount << " small sequence count " << dudCount-nullCount << std::endl; + std::cout << "null count: " << status.nullCount << " small sequence count " << status.dudCount-status.nullCount << std::endl; } else { - adbStatusResponse->result.numFiles = dbH->numFiles; - adbStatusResponse->result.dim = dbH->dim; - adbStatusResponse->result.length = dbH->length; - adbStatusResponse->result.dudCount = dudCount; - adbStatusResponse->result.nullCount = nullCount; - adbStatusResponse->result.flags = dbH->flags; + adbStatusResponse->result.numFiles = status.numFiles; + adbStatusResponse->result.dim = status.dim; + adbStatusResponse->result.length = status.length; + adbStatusResponse->result.dudCount = status.dudCount; + adbStatusResponse->result.nullCount = status.nullCount; + adbStatusResponse->result.flags = status.flags; } } -///used by lib/API -void audioDB::status(const char* dbName, cppstatusptr status){ - if(!dbH) { - initTables(dbName, 0); - } - - unsigned dudCount=0; - unsigned nullCount=0; - for(unsigned k=0; k<dbH->numFiles; k++){ - if(trackTable[k]<sequenceLength){ - dudCount++; - if(!trackTable[k]) - nullCount++; +void audioDB::l2norm(const char* dbName) { + if(!adb) { + if(!(adb = audiodb_open(dbName, O_RDWR))) { + error("Failed to open database file", dbName); } } - - status->numFiles = dbH->numFiles; - status->dim = dbH->dim; - status->length = dbH->length; - status->dudCount = dudCount; - status->nullCount = nullCount; - status->flags = dbH->flags; - - return; -} - - - - -void audioDB::l2norm(const char* dbName) { - forWrite = true; - initTables(dbName, 0); - if( !(dbH->flags & O2_FLAG_LARGE_ADB ) && (dbH->length>0) ){ - /* FIXME: should probably be uint64_t */ - unsigned numVectors = dbH->length/(sizeof(double)*dbH->dim); - CHECKED_MMAP(double *, dataBuf, dbH->dataOffset, dataBufLength); - unitNormAndInsertL2(dataBuf, dbH->dim, numVectors, 0); // No append + if(audiodb_l2norm(adb)) { + error("failed to turn on l2norm flag for database", dbName); } - // Update database flags - dbH->flags = dbH->flags|O2_FLAG_L2NORM; - memcpy (db, dbH, O2_HEADERSIZE); } void audioDB::power_flag(const char *dbName) { - forWrite = true; - initTables(dbName, 0); - if( !(dbH->flags & O2_FLAG_LARGE_ADB ) && (dbH->length>0) ){ - error("cannot turn on power storage for non-empty database", dbName); + if(!adb) { + if(!(adb = audiodb_open(dbName, O_RDWR))) { + error("Failed to open database file", dbName); + } } - dbH->flags |= O2_FLAG_POWER; - memcpy(db, dbH, O2_HEADERSIZE); + if(audiodb_power(adb)) { + error("can't turn on power flag for database", dbName); + } } -// Unit norm block of features +void audioDB::create(const char *dbName) { + if(adb) { + error("Already have an adb in this object", ""); + } + if(!(adb = audiodb_create(dbName, datasize, ntracks, datadim))) { + error("Failed to create database file", dbName); + } +} -/* FIXME: in fact this does not unit norm a block of features, it just - records the L2 norms somewhere. unitNorm() does in fact unit norm - a block of features. */ -void audioDB::unitNormAndInsertL2(double* X, unsigned dim, unsigned n, unsigned append=0){ - unsigned d; - double *p; - unsigned nn = n; +void audioDB::dump(const char *dbName) { + if(!adb) { + if(!(adb = audiodb_open(dbName, O_RDONLY))) { + error("Failed to open database file", dbName); + } + } + if(audiodb_dump(adb, output)) { + error("Failed to dump database to ", output); + } + status(dbName); +} - assert(l2normTable); +void audioDB::insert(const char* dbName, const char* inFile) { + if(!adb) { + if(!(adb = audiodb_open(dbName, O_RDWR))) { + error("failed to open database", dbName); + } + } - if( !(dbH->flags & O2_FLAG_LARGE_ADB) && !append && (dbH->flags & O2_FLAG_L2NORM) ) - error("Database is already L2 normed", "automatic norm on insert is enabled"); + /* at this point, we have powerfd (an fd), timesFile (a + * std::ifstream *) and inFile (a char *). Wacky, huh? Ignore + * the wackiness and just use the names. */ + adb_insert_t insert; + insert.features = inFile; + insert.times = timesFileName; + insert.power = powerFileName; + insert.key = key; - VERB_LOG(2, "norming %u vectors...", n); + if(audiodb_insert(adb, &insert)) { + error("insertion failure", inFile); + } + status(dbName); +} - double* l2buf = new double[n]; - double* l2ptr = l2buf; - assert(l2buf); - assert(X); +void audioDB::batchinsert(const char* dbName, const char* inFile) { + if(!adb) { + if(!(adb = audiodb_open(dbName, O_RDWR))) { + error("failed to open database", dbName); + } + } - while(nn--){ - p=X; - *l2ptr=0.0; - d=dim; - while(d--){ - *l2ptr+=*p**p; - p++; + if(!key) + key=inFile; + std::ifstream *filesIn = 0; + std::ifstream *keysIn = 0; + + if(!(filesIn = new std::ifstream(inFile))) + error("Could not open batch in file", inFile); + if(key && key!=inFile) + if(!(keysIn = new std::ifstream(key))) + error("Could not open batch key file",key); + + unsigned totalVectors=0; + char *thisFile = new char[MAXSTR]; + char *thisKey = 0; + if (key && (key != inFile)) { + thisKey = new char[MAXSTR]; + } + char *thisTimesFileName = new char[MAXSTR]; + char *thisPowerFileName = new char[MAXSTR]; + + do { + filesIn->getline(thisFile,MAXSTR); + if(key && key!=inFile) { + keysIn->getline(thisKey,MAXSTR); + } else { + thisKey = thisFile; } - l2ptr++; - X+=dim; + if(usingTimes) { + timesFile->getline(thisTimesFileName,MAXSTR); + } + if(usingPower) { + powerFile->getline(thisPowerFileName, MAXSTR); + } + + if(filesIn->eof()) { + break; + } + if(usingTimes){ + if(timesFile->eof()) { + error("not enough timestamp files in timesList", timesFileName); + } + } + if (usingPower) { + if(powerFile->eof()) { + error("not enough power files in powerList", powerFileName); + } + } + adb_insert_t insert; + insert.features = thisFile; + insert.times = usingTimes ? thisTimesFileName : NULL; + insert.power = usingPower ? thisPowerFileName : NULL; + insert.key = thisKey; + if(audiodb_insert(adb, &insert)) { + error("insertion failure", thisFile); + } + } while(!filesIn->eof()); + + VERB_LOG(0, "%s %s %u vectors %ju bytes.\n", COM_BATCHINSERT, dbName, totalVectors, (intmax_t) (totalVectors * dbH->dim * sizeof(double))); + + delete [] thisPowerFileName; + if(key && (key != inFile)) { + delete [] thisKey; } - unsigned offset; - if(append) { - // FIXME: a hack, a very palpable hack: the vectors have already - // been inserted, and dbH->length has already been updated. We - // need to subtract off again the number of vectors that we've - // inserted this time... - offset=(dbH->length/(dbH->dim*sizeof(double)))-n; // number of vectors + delete [] thisFile; + delete [] thisTimesFileName; + + delete filesIn; + delete keysIn; + + // Report status + status(dbName); +} + +void audioDB::query(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse) { + + if(!adb) { + if(!(adb = audiodb_open(dbName, O_RDWR))) { + error("failed to open database", dbName); + } + } + + /* FIXME: we only need this for getting nfiles, which we only need + * because the reporters aren't desperately well implemented, + * relying on statically-sized vectors rather than adjustable data + * structures. Rework reporter.h to be less lame. */ + adb_status_t status; + audiodb_status(adb, &status); + uint32_t nfiles = status.numFiles; + + adb_query_spec_t qspec; + adb_datum_t datum = {0}; + + qspec.refine.flags = 0; + if(trackFile) { + qspec.refine.flags |= ADB_REFINE_INCLUDE_KEYLIST; + std::vector<const char *> v; + char *k = new char[MAXSTR]; + trackFile->getline(k, MAXSTR); + while(!trackFile->eof()) { + v.push_back(k); + k = new char[MAXSTR]; + trackFile->getline(k, MAXSTR); + } + delete [] k; + qspec.refine.include.nkeys = v.size(); + qspec.refine.include.keys = new const char *[qspec.refine.include.nkeys]; + for(unsigned int k = 0; k < qspec.refine.include.nkeys; k++) { + qspec.refine.include.keys[k] = v[k]; + } + } + if(query_from_key) { + qspec.refine.flags |= ADB_REFINE_EXCLUDE_KEYLIST; + qspec.refine.exclude.nkeys = 1; + qspec.refine.exclude.keys = &key; + } + if(radius) { + qspec.refine.flags |= ADB_REFINE_RADIUS; + qspec.refine.radius = radius; + } + if(use_absolute_threshold) { + qspec.refine.flags |= ADB_REFINE_ABSOLUTE_THRESHOLD; + qspec.refine.absolute_threshold = absolute_threshold; + } + if(use_relative_threshold) { + qspec.refine.flags |= ADB_REFINE_RELATIVE_THRESHOLD; + qspec.refine.relative_threshold = relative_threshold; + } + if(usingTimes) { + qspec.refine.flags |= ADB_REFINE_DURATION_RATIO; + qspec.refine.duration_ratio = timesTol; + } + /* FIXME: not sure about this any more; maybe it belongs in + query_id? Or maybe we just don't need a flag for it? */ + qspec.refine.hopsize = sequenceHop; + if(sequenceHop != 1) { + qspec.refine.flags |= ADB_REFINE_HOP_SIZE; + } + + if(query_from_key) { + datum.key = key; } else { - offset=0; + int fd; + struct stat st; + + /* FIXME: around here there are all sorts of hideous leaks. */ + fd = open(inFile, O_RDONLY); + if(fd < 0) { + error("failed to open feature file", inFile); + } + fstat(fd, &st); + read(fd, &datum.dim, sizeof(uint32_t)); + datum.nvectors = (st.st_size - sizeof(uint32_t)) / (datum.dim * sizeof(double)); + datum.data = (double *) malloc(st.st_size - sizeof(uint32_t)); + read(fd, datum.data, st.st_size - sizeof(uint32_t)); + close(fd); + if(usingPower) { + uint32_t one; + fd = open(powerFileName, O_RDONLY); + if(fd < 0) { + error("failed to open power file", powerFileName); + } + read(fd, &one, sizeof(uint32_t)); + if(one != 1) { + error("malformed power file dimensionality", powerFileName); + } + datum.power = (double *) malloc(datum.nvectors * sizeof(double)); + if(read(fd, datum.power, datum.nvectors * sizeof(double)) != (ssize_t) (datum.nvectors * sizeof(double))) { + error("malformed power file", powerFileName); + } + close(fd); + } + if(usingTimes) { + datum.times = (double *) malloc(2 * datum.nvectors * sizeof(double)); + insertTimeStamps(datum.nvectors, timesFile, datum.times); + } } - memcpy(l2normTable+offset, l2buf, n*sizeof(double)); - if(l2buf) - delete[] l2buf; - VERB_LOG(2, " done."); + + qspec.qid.datum = &datum; + qspec.qid.sequence_length = sequenceLength; + qspec.qid.flags = 0; + qspec.qid.flags |= usingQueryPoint ? 0 : ADB_QID_FLAG_EXHAUSTIVE; + qspec.qid.flags |= lsh_exact ? 0 : ADB_QID_FLAG_ALLOW_FALSE_POSITIVES; + qspec.qid.sequence_start = queryPoint; + + switch(queryType) { + case O2_POINT_QUERY: + qspec.qid.sequence_length = 1; + qspec.params.accumulation = ADB_ACCUMULATION_DB; + qspec.params.distance = ADB_DISTANCE_DOT_PRODUCT; + qspec.params.npoints = pointNN; + qspec.params.ntracks = 0; + reporter = new pointQueryReporter< std::greater < NNresult > >(pointNN); + break; + case O2_TRACK_QUERY: + qspec.qid.sequence_length = 1; + qspec.params.accumulation = ADB_ACCUMULATION_PER_TRACK; + qspec.params.distance = ADB_DISTANCE_DOT_PRODUCT; + qspec.params.npoints = pointNN; + qspec.params.ntracks = trackNN; + reporter = new trackAveragingReporter< std::greater< NNresult > >(pointNN, trackNN, nfiles); + break; + case O2_SEQUENCE_QUERY: + case O2_N_SEQUENCE_QUERY: + qspec.params.accumulation = ADB_ACCUMULATION_PER_TRACK; + qspec.params.distance = no_unit_norming ? ADB_DISTANCE_EUCLIDEAN : ADB_DISTANCE_EUCLIDEAN_NORMED; + qspec.params.npoints = pointNN; + qspec.params.ntracks = trackNN; + switch(queryType) { + case O2_SEQUENCE_QUERY: + if(!(qspec.refine.flags & ADB_REFINE_RADIUS)) { + reporter = new trackAveragingReporter< std::less< NNresult > >(pointNN, trackNN, nfiles); + } else { + reporter = new trackSequenceQueryRadReporter(trackNN, nfiles); + } + break; + case O2_N_SEQUENCE_QUERY: + if(!(qspec.refine.flags & ADB_REFINE_RADIUS)) { + reporter = new trackSequenceQueryNNReporter< std::less < NNresult > >(pointNN, trackNN, nfiles); + } else { + reporter = new trackSequenceQueryRadNNReporter(pointNN, trackNN, nfiles); + } + break; + } + break; + case O2_ONE_TO_ONE_N_SEQUENCE_QUERY: + qspec.params.accumulation = ADB_ACCUMULATION_ONE_TO_ONE; + qspec.params.distance = no_unit_norming ? ADB_DISTANCE_EUCLIDEAN : ADB_DISTANCE_EUCLIDEAN_NORMED; + qspec.params.npoints = 0; + qspec.params.ntracks = 0; + if(!(qspec.refine.flags & ADB_REFINE_RADIUS)) { + error("query-type not yet supported"); + } else { + reporter = new trackSequenceQueryRadNNReporterOneToOne(pointNN,trackNN, dbH->numFiles); + } + break; + default: + error("unrecognized queryType"); + } + + adb_query_results_t *rs = audiodb_query_spec(adb, &qspec); + + // FIXME: free bits of datum if !query_from_key + + if(rs == NULL) { + error("audiodb_query_spec failed"); + } + + for(unsigned int k = 0; k < rs->nresults; k++) { + adb_result_t r = rs->results[k]; + reporter->add_point(audiodb_key_index(adb, r.key), r.qpos, r.ipos, r.dist); + } + audiodb_query_free_results(adb, &qspec, rs); + + reporter->report(adb, adbQueryResponse); } // This entry point is visited once per instance // so it is a good place to set any global state variables int main(const int argc, const char* argv[]){ - SERVER_LSH_INDEX_SINGLETON = 0; // Initialize global variables SERVER_ADB_ROOT = 0; // Server-side database root prefix SERVER_ADB_FEATURE_ROOT = 0; // Server-side features root prefix audioDB(argc, argv); } - - -extern "C" { - -/* for API questions contact - * Christophe Rhodes c.rhodes@gold.ac.uk - * Ian Knopke mas01ik@gold.ac.uk, ian.knopke@gmail.com */ - -#include "audioDB_API.h" - - adb_ptr audiodb_create(const char *path, unsigned datasize, unsigned ntracks, unsigned datadim) { - const char *argv[12]; - int argvctr=0; - char tempstr1[200]; - char tempstr2[200]; - char tempstr3[200]; - int apierror=0; - - - argv[argvctr++] = "audioDB"; - argv[argvctr++] = "--NEW"; - argv[argvctr++] = "-d"; - argv[argvctr++] = path; - - if (datasize >0){ - argv[argvctr++]="--datasize"; - snprintf(tempstr1,sizeof(tempstr1),"%u",datasize); - argv[argvctr++]=tempstr1; - } - - if (ntracks >0){ - argv[argvctr++]="--ntracks"; - snprintf(tempstr2,sizeof(tempstr2),"%u",ntracks); - argv[argvctr++]=tempstr2; - } - - if (datadim > 0){ - argv[argvctr++]="--datadim"; - snprintf(tempstr3,sizeof(tempstr3),"%u",datadim); - argv[argvctr++]=tempstr3; - } - - argv[argvctr]='\0'; - - audioDB::audioDB(argvctr, argv, &apierror); - - if (!apierror){ - return audiodb_open(path); - } - - /* database exists, so fail and pass NULL */ - return NULL; - } - - - - int audiodb_insert(adb_ptr mydb, adb_insert_ptr ins) { - const char *argv[15]; - int argvctr=0; - int apierror=0; - - argv[argvctr++]="audioDB"; - argv[argvctr++]="-I"; - argv[argvctr++]="-d"; - argv[argvctr++]=mydb->dbname; - argv[argvctr++]="-f"; - argv[argvctr++]=ins->features; - - if (ins->times){ - argv[argvctr++]="--times"; - argv[argvctr++]=ins->times; - } - - if (ins->power){ - argv[argvctr++]="-w"; - argv[argvctr++]=ins->power; - } - - if (ins->key){ - argv[argvctr++]="--key"; - argv[argvctr++]=ins->key; - } - argv[argvctr]='\0'; - - audioDB::audioDB(argvctr,argv,&apierror); - return apierror; - } - - - int audiodb_batchinsert(adb_ptr mydb, adb_insert_ptr ins, unsigned int size) { - - const char *argv[22]; - int argvctr=0; - unsigned int i=0; - char tempfeaturename[]="tempfeatureXXXXXX"; - char temppowername[]="temppowerXXXXXX"; - char tempkeyname[]="tempkeyXXXXXX"; - char temptimesname[]="temptimesXXXXXX"; - int tempfeaturefd = -1; - int temppowerfd = -1; - int tempkeyfd = -1; - int temptimesfd = -1; - - int flags[4]={0}; - int apierror=0; - - /* So the final API should take an array of structs. However, the current - * version requires four separate text files. So temporarily, we need to - * unpack the struct array, make four separate text files, and then reinsert - * them into the command-line call. This should change as soon as possible */ - - - argv[argvctr++]="audioDB"; - argv[argvctr++]="-B"; - argv[argvctr++]="-d"; - argv[argvctr++]=mydb->dbname; - - - /* assume struct is well formed for all entries */ - if (ins[0].features){ flags[0]++;} else { - /* short circuit the case where there are no features in the structs */ - return -1; - } ; - if (ins[0].power){ flags[1]++;}; - if (ins[0].key){ flags[2]++;}; - if (ins[0].times){ flags[3]++;}; - - - /* make four temp files */ - if ((tempfeaturefd = mkstemp(tempfeaturename)) == -1) - goto error; - if ((temppowerfd = mkstemp(temppowername)) == -1) - goto error; - if ((tempkeyfd=mkstemp(tempkeyname)) == -1) - goto error; - if ((temptimesfd=mkstemp(temptimesname)) == -1) - goto error; - - /* Ok, so we should have a working set of files to write to */ - /* I'm going to assume that the same format is kept for all structs in the array */ - /* That is, each struct should be correctly formed, and contain at least a features file, because I'm just going to pass the terms along to the text files */ - for (i = 0; i < size; i++) { - if (write(tempfeaturefd,ins[i].features,strlen(ins[i].features)) != (ssize_t) strlen(ins[i].features)) - goto error; - if (write(tempfeaturefd,"\n",1) != 1) - goto error; - - if (flags[1]) { - if (write(temppowerfd,ins[i].power,strlen(ins[i].power)) != (ssize_t) strlen(ins[i].power)) - goto error; - if (write(temppowerfd,"\n",1) != 1) - goto error; - } - if (flags[2]) { - if (write(tempkeyfd,ins[i].key,strlen(ins[i].key)) != (ssize_t) strlen(ins[i].key)) - goto error; - if (write(tempkeyfd,"\n",1) != 1) - goto error; - } - if (flags[3]) { - if (write(temptimesfd,ins[i].times,strlen(ins[i].times)) != (ssize_t) strlen(ins[i].times)) - goto error; - if (write(temptimesfd,"\n",1) != 1) - goto error; - } - } - - argv[argvctr++]="-F"; - argv[argvctr++]=tempfeaturename; - close(tempfeaturefd); - close(temppowerfd); - close(tempkeyfd); - close(temptimesfd); - - if (flags[1]){ - argv[argvctr++]="--powerList"; - argv[argvctr++]=temppowername; - } - - if (flags[2]){ - argv[argvctr++]="--keyList"; - argv[argvctr++]=tempkeyname; - } - - if (flags[3]){ - argv[argvctr++]="--timesList"; - argv[argvctr++]=temptimesname; - } - - argv[argvctr]='\0'; - - audioDB::audioDB(argvctr,argv,&apierror); - - remove(tempfeaturename); - remove(temppowername); - remove(tempkeyname); - remove(temptimesname); - - - return apierror; - - error: - if(tempfeaturefd != -1) { - close(tempfeaturefd); - remove(tempfeaturename); - } - if(temppowerfd != -1) { - close(temppowerfd); - remove(temppowername); - } - if(tempkeyfd != -1) { - close(tempkeyfd); - remove(tempkeyname); - } - if(temptimesfd != -1) { - close(temptimesfd); - remove(temptimesname); - } - return -1; - } - - - int audiodb_query(adb_ptr mydb, adb_query_ptr adbq, adb_queryresult_ptr adbqr){ - - const char *argv[32]; - int argvctr=0; - char tempstr1[200]; - char tempstr2[200]; - char tempstr3[200]; - int apierror=0; - - adb__queryResponse adbQueryResponse; - - /* TODO: may need error checking here */ - /* currently counting on audioDB binary to fail for me */ - argv[argvctr++]="audioDB"; - - if(adbq->querytype){ - argv[argvctr++]="-Q"; - argv[argvctr++]=adbq->querytype; - } - - if(mydb->dbname){ - argv[argvctr++]="-d"; - argv[argvctr++]=mydb->dbname; - } - - if (adbq->feature){ - argv[argvctr++]="-f"; - argv[argvctr++]=adbq->feature; - } - - if (adbq->key){ - argv[argvctr++]="-k"; - argv[argvctr++]=adbq->key; - } - - if (adbq->power){ - argv[argvctr++]="-w"; - argv[argvctr++]=adbq->power; - } - - if (adbq->qpoint){ - argv[argvctr++]="-p"; - argv[argvctr++]=adbq->qpoint; - } - if (adbq->numpoints){ - argv[argvctr++]="-n"; - argv[argvctr++]=adbq->numpoints; - } - if (adbq->radius){ - argv[argvctr++]="-R"; - argv[argvctr++]=adbq->radius; - } - if(adbq->resultlength){ - argv[argvctr++]="-r"; - argv[argvctr++]=adbq->resultlength; - } - if(adbq->sequencelength){ - argv[argvctr++]="-l"; - argv[argvctr++]=adbq->sequencelength; - } - if(adbq->sequencehop){ - argv[argvctr++]="-h"; - argv[argvctr++]=adbq->sequencehop; - } - - if (adbq->absolute_threshold){ - argv[argvctr++]="--absolute-threshold"; - snprintf(tempstr1,sizeof(tempstr1),"%f",adbq->absolute_threshold); - argv[argvctr++]=tempstr1; - } - - if (adbq->relative_threshold){ - argv[argvctr++]="--relative-threshold"; - snprintf(tempstr2,sizeof(tempstr2),"%f",adbq->relative_threshold); - argv[argvctr++]=tempstr2; - } - - if (adbq->exhaustive){ - argv[argvctr++]="--exhaustive"; - } - - if (adbq->expandfactor){ - argv[argvctr++]="--expandfactor"; - snprintf(tempstr3,sizeof(tempstr3),"%f",adbq->expandfactor); - argv[argvctr++]=tempstr3; - } - - if (adbq->rotate){ - argv[argvctr++]="--rotate"; - } - - if (adbq->keylist){ - argv[argvctr++]="-K"; - argv[argvctr++]=adbq->keylist; - } - argv[argvctr]='\0'; - - /* debugging */ - - audioDB::audioDB(argvctr,argv, &adbQueryResponse, &apierror); - - //copy data over here from adbQueryResponse to adbqr - adbqr->sizeRlist=adbQueryResponse.result.__sizeRlist; - adbqr->sizeDist=adbQueryResponse.result.__sizeDist; - adbqr->sizeQpos=adbQueryResponse.result.__sizeQpos; - adbqr->sizeSpos=adbQueryResponse.result.__sizeSpos; - adbqr->Rlist=adbQueryResponse.result.Rlist; - adbqr->Dist=adbQueryResponse.result.Dist; - adbqr->Qpos=adbQueryResponse.result.Qpos; - adbqr->Spos=adbQueryResponse.result.Spos; - - return apierror; - } - - ///* status command */ - int audiodb_status(adb_ptr mydb, adb_status_ptr status){ - - cppstatus sss; - int apierror=0; - - const char *argv[5]; - - apierror=0; - argv[0]="audioDB"; - argv[1]="--STATUS"; - argv[2]="-d"; - argv[3]=mydb->dbname; - argv[4]='\0'; - - audioDB::audioDB(4,argv,&sss ,&apierror); - - status->numFiles=sss.numFiles; - status->dim=sss.dim; - status->length=sss.length; - status->dudCount=sss.dudCount; - status->nullCount=sss.nullCount; - status->flags=sss.flags; - - return apierror; - } - - int audiodb_dump(adb_ptr mydb){ - return audiodb_dump_withdir(mydb,"audioDB.dump"); - } - - int audiodb_dump_withdir(adb_ptr mydb, const char *outputdir){ - - const char *argv[7]; - int argvctr=0; - int apierror=0; - - argv[argvctr++]="audioDB"; - argv[argvctr++]="--DUMP"; - argv[argvctr++]="-d"; - argv[argvctr++]=mydb->dbname; - argv[argvctr++]="--output"; - argv[argvctr++]=(char *)outputdir; - argv[argvctr]='\0'; - - audioDB::audioDB(6,argv,&apierror); - - return apierror; - } - - int audiodb_l2norm(adb_ptr mydb){ - - const char *argv[5]; - int apierror=0; - - argv[0]="audioDB"; - argv[1]="--L2NORM"; - argv[2]="-d"; - argv[3]=mydb->dbname; - argv[4]='\0'; - - audioDB::audioDB(4,argv,&apierror); - return apierror; - } - - int audiodb_power(adb_ptr mydb){ - - const char *argv[5]; - int apierror=0; - - argv[0]="audioDB"; - argv[1]="--POWER"; - argv[2]="-d"; - argv[3]=mydb->dbname; - argv[4]='\0'; - - audioDB::audioDB(4,argv,&apierror); - return apierror; - } - - adb_ptr audiodb_open(const char *path){ - - adb_ptr mydbp; - adbstatus mystatus; - - /* if db exists */ - - if (open(path, O_EXCL) != -1){ - - mydbp=(adb_ptr)malloc(sizeof(adb)); - mydbp->dbname=(char *)malloc(1+strlen(path)); - - strcpy(mydbp->dbname,path); - - audiodb_status(mydbp, &mystatus); - mydbp->ntracks=mystatus.numFiles; - mydbp->datadim=mystatus.dim; - - return mydbp; - } - - return NULL; - }; - - - - void audiodb_close(adb_ptr db){ - - free(db->dbname); - free(db); - - } - - -} -
--- a/audioDB.h Sat Jan 10 11:11:27 2009 +0000 +++ b/audioDB.h Sat Jan 10 16:47:57 2009 +0000 @@ -11,6 +11,7 @@ #include <iostream> #include <fstream> #include <set> +#include <map> #include <string> #include <math.h> #include <sys/time.h> @@ -20,13 +21,27 @@ #include <gsl/gsl_rng.h> // includes for LSH indexing +extern "C" { +#include "audioDB_API.h" +} #include "ReporterBase.h" +#include "accumulator.h" #include "lshlib.h" // includes for web services #include "soapH.h" #include "cmdline.h" +// should probably be rewritten +class PointPair{ + public: + Uns32T trackID; + Uns32T qpos; + Uns32T spos; + PointPair(Uns32T a, Uns32T b, Uns32T c); +}; +bool operator<(const PointPair& a, const PointPair& b); + #define MAXSTR 512 // Databse PRIMARY commands @@ -64,11 +79,6 @@ #define COM_LSH_EXACT "--lsh_exact" #define COM_NO_UNIT_NORMING "--no_unit_norming" -// Because LSH returns NN with P(1)<1 we want to return exact -// points above this boundary. -// Because we work in Radius^2 units, -// The sqrt of this number is the multiplier on the radius - #define O2_OLD_MAGIC ('O'|'2'<<8|'D'<<16|'B'<<24) #define O2_MAGIC ('o'|'2'<<8|'d'<<16|'b'<<24) #define O2_FORMAT_VERSION (4U) @@ -182,7 +192,6 @@ #define SAFE_DELETE(PTR) delete PTR; PTR=0; #define SAFE_DELETE_ARRAY(PTR) delete[] PTR; PTR=0; -extern LSH* SERVER_LSH_INDEX_SINGLETON; extern char* SERVER_ADB_ROOT; extern char* SERVER_ADB_FEATURE_ROOT; @@ -203,28 +212,6 @@ off_t dbSize; } dbTableHeaderT, *dbTableHeaderPtr; -typedef struct { - - unsigned numFiles; - unsigned dim; - unsigned length; - unsigned dudCount; - unsigned nullCount; - unsigned flags; - - -} cppstatus, *cppstatusptr; - -class PointPair{ - public: - Uns32T trackID; - Uns32T qpos; - Uns32T spos; - PointPair(Uns32T a, Uns32T b, Uns32T c); -}; - -bool operator<(const PointPair& a, const PointPair& b); - class audioDB{ private: gengetopt_args_info args_info; @@ -249,18 +236,14 @@ int lshfid; bool forWrite; int infid; - char* db; - char* indata; struct stat statbuf; dbTableHeaderPtr dbH; + struct adb *adb; gsl_rng *rng; char* fileTable; unsigned* trackTable; - off_t* trackOffsetTable; - double* dataBuf; - double* inBuf; double* l2normTable; double* timesTable; double* powerTable; @@ -271,7 +254,6 @@ size_t fileTableLength; size_t trackTableLength; - off_t dataBufLength; size_t timesTableLength; size_t powerTableLength; size_t l2normTableLength; @@ -310,47 +292,21 @@ double relative_threshold; ReporterBase* reporter; // track/point reporter - priority_queue<PointPair, std::vector<PointPair>, std::less<PointPair> >* exact_evaluation_queue; set<Uns32T> * allowed_keys; // search restrict list by key - // Timers - struct timeval tv1; - struct timeval tv2; - // LISZT parameters unsigned lisztOffset; unsigned lisztLength; - //for lib / API - int apierrortemp; - unsigned UseApiError; - // private methods void error(const char* a, const char* b = "", const char *sysFunc = 0); - void sequence_sum(double *buffer, int length, int seqlen); - void sequence_sqrt(double *buffer, int length, int seqlen); - void sequence_average(double *buffer, int length, int seqlen); - - void initialize_arrays(int track, unsigned int numVectors, double *query, double *data_buffer, double **D, double **DD); - void delete_arrays(int track, unsigned int numVectors, double **D, double **DD); - void read_data(int trkfid, int track, double **data_buffer_p, size_t *data_buffer_size_p); - void set_up_query(double **qp, double **vqp, double **qnp, double **vqnp, double **qpp, double **vqpp, double *mqdp, unsigned int *nvp); - void set_up_query_from_key(double **qp, double **vqp, double **qnp, double **vqnp, double **qpp, double **vqpp, double *mqdp, unsigned *nvp, Uns32T queryIndex); - void set_up_db(double **snp, double **vsnp, double **spp, double **vspp, double **mddp, unsigned int *dvp); - void query_loop(const char* dbName, Uns32T queryIndex); - void query_loop_points(double* query, double* qnPtr, double* qpPtr, double meanQdur, Uns32T numVectors); - double dot_product_points(double* q, double* p, Uns32T L); + void insertTimeStamps(unsigned n, std::ifstream* timesFile, double* timesdata); void initRNG(); void initDBHeader(const char *dbName); - void initInputFile(const char *inFile, bool loadData = true); + void initInputFile(const char *inFile); void initTables(const char* dbName, const char* inFile = 0); void initTablesFromKey(const char* dbName, const Uns32T queryIndex); - void unitNorm(double* X, unsigned d, unsigned n, double* qNorm); - void unitNormAndInsertL2(double* X, unsigned dim, unsigned n, unsigned append); - void insertTimeStamps(unsigned n, std::ifstream* timesFile, double* timesdata); - void insertPowerData(unsigned n, int powerfd, double *powerdata); - unsigned getKeyPos(char* key); void prefix_name(char** const name, const char* prefix); public: @@ -358,10 +314,6 @@ audioDB(const unsigned argc, const char *argv[], adb__queryResponse *adbQueryResponse); audioDB(const unsigned argc, const char *argv[], adb__statusResponse *adbStatusResponse); audioDB(const unsigned argc, const char *argv[], adb__lisztResponse *adbLisztResponse); - audioDB(const unsigned argc, const char *argv[], int * apierror); - audioDB(const unsigned argc, const char *argv[], cppstatusptr stat, int * apierror); - audioDB(const unsigned argc, const char *argv[],adb__queryResponse *adbQueryResponse, int * apierror); - void cleanup(); ~audioDB(); @@ -369,21 +321,15 @@ void get_lock(int fd, bool exclusive); void release_lock(int fd); void create(const char* dbName); - bool enough_per_file_space_free(); - bool enough_data_space_free(off_t size); - void insert_data_vectors(off_t offset, void *buffer, size_t size); void insert(const char* dbName, const char* inFile); void batchinsert(const char* dbName, const char* inFile); - void batchinsert_large_adb(const char* dbName, const char* inFile); void query(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse=0); void status(const char* dbName, adb__statusResponse *adbStatusResponse=0); - void status(const char* dbName, cppstatusptr status); unsigned random_track(unsigned *propTable, unsigned total); void sample(const char *dbName); void l2norm(const char* dbName); void power_flag(const char *dbName); - bool powers_acceptable(double p1, double p2); void dump(const char* dbName); void liszt(const char* dbName, unsigned offset, unsigned numLines, adb__lisztResponse* adbLisztResponse=0); @@ -399,12 +345,6 @@ Uns32T lsh_param_N; // Number of rows per hash table Uns32T lsh_param_b; // Batch size, in number of tracks, per indexing iteration Uns32T lsh_param_ncols; // Maximum number of collision in a hash-table row - Uns32T lsh_n_point_bits; // How many bits to use to encode point ID within a track - - - // LSH vector<> containers for one in-core copy of a set of feature vectors - vector<float>::iterator vi; // feature vector iterator - vector<vector<float> > *vv; // one-track's worth data // LSH indexing and retrieval methods void index_index_db(const char* dbName); @@ -412,20 +352,7 @@ void index_insert_tracks(Uns32T start_track, Uns32T end_track, double** fvpp, double** sNormpp,double** snPtrp, double** sPowerp, double** spPtrp); int index_insert_track(Uns32T trackID, double** fvpp, double** snpp, double** sppp); Uns32T index_insert_shingles(vector<vector<float> >*, Uns32T trackID, double* spp); - void index_make_shingle(vector<vector<float> >*, Uns32T idx, double* fvp, Uns32T dim, Uns32T seqLen); - int index_norm_shingles(vector<vector<float> >*, double* snp, double* spp); - int index_query_loop(const char* dbName, Uns32T queryIndex); - vector<vector<float> >* index_initialize_shingles(Uns32T sz); - int index_init_query(const char* dbName); - int index_exists(const char* dbName, double radius, Uns32T sequenceLength); - char* index_get_name(const char*dbName, double radius, Uns32T sequenceLength); - static void index_add_point(void* instance, Uns32T pointID, Uns32T qpos, float dist); // static point reporter callback method - static Uns32T index_to_trackID(Uns32T lshID, Uns32T nPntBits); // Convert lsh point index to audioDB trackID - static Uns32T index_to_trackPos(Uns32T lshID, Uns32T nPntBits); // Convert lsh point index to audioDB trackPos (spos) - static Uns32T index_from_trackInfo(Uns32T trackID, Uns32T pntID, Uns32T nPntBits); // Convert audioDB trackID and trackPos to an lsh point index - void initialize_exact_evalutation_queue(); - void index_insert_exact_evaluation_queue(Uns32T trackID, Uns32T qpos, Uns32T spos); - LSH* index_allocate(char* indexName, bool load_hashTables); + void insertPowerData(unsigned n, int powerfd, double *powerdata); void init_track_aux_data(Uns32T trackID, double* fvp, double** sNormpp,double** snPtrp, double** sPowerp, double** spPtrp); void initialize_allowed_keys(std::ifstream*); // implementation of restrict lists using STL "set" class int is_in_allowed_keys(Uns32T trackID); // test method for allowed_keys used during search @@ -460,14 +387,11 @@ lshfid(0), \ forWrite(false), \ infid(0), \ - db(0), \ - indata(0), \ dbH(0), \ + adb(0), \ rng(0), \ fileTable(0), \ trackTable(0), \ - trackOffsetTable(0), \ - dataBuf(0), \ l2normTable(0), \ timesTable(0), \ powerTable(0), \ @@ -476,7 +400,6 @@ powerFileNameTable(0), \ fileTableLength(0), \ trackTableLength(0), \ - dataBufLength(0), \ timesTableLength(0), \ powerTableLength(0), \ l2normTableLength(0), \ @@ -502,18 +425,15 @@ timesTol(0.1), \ radius(0), \ query_from_key(false), \ - query_from_key_index(O2_ERR_KEYNOTFOUND), \ + query_from_key_index((uint32_t) -1), \ use_absolute_threshold(false), \ absolute_threshold(0.0), \ use_relative_threshold(false), \ relative_threshold(0.0), \ reporter(0), \ - exact_evaluation_queue(0), \ allowed_keys(0), \ lisztOffset(0), \ lisztLength(0), \ - apierrortemp(0), \ - UseApiError(0), \ lsh(0), \ lsh_in_core(false), \ lsh_use_u_functions(false), \ @@ -523,7 +443,5 @@ lsh_param_m(0), \ lsh_param_N(0), \ lsh_param_b(0), \ - lsh_param_ncols(0), \ - lsh_n_point_bits(0), \ - vv(0) + lsh_param_ncols(0) #endif
--- a/audioDB_API.h Sat Jan 10 11:11:27 2009 +0000 +++ b/audioDB_API.h Sat Jan 10 16:47:57 2009 +0000 @@ -1,7 +1,18 @@ +#ifndef AUDIODB_API_H +#define AUDIODB_API_H + +#include <stdbool.h> +#include <stdint.h> + /* for API questions contact * Christophe Rhodes c.rhodes@gold.ac.uk * Ian Knopke mas01ik@gold.ac.uk, ian.knopke@gmail.com */ +/* Temporary workarounds */ +typedef struct dbTableHeader adb_header_t; +int acquire_lock(int, bool); +int divest_lock(int); + /*******************************************************************/ /* Data types for API */ @@ -9,36 +20,40 @@ /* The main struct that stores the name of the database, and in future will hold all * kinds of other interesting information */ /* This basically gets passed around to all of the other functions */ -struct adb { - char * dbname; - unsigned int ntracks; /* number of tracks */ - unsigned int datadim; /* dimensionality of stored data */ +/* FIXME: it might be that "adb_" isn't such a good prefix to use, and + that we should prefer "audiodb_". Or else maybe we should be + calling ourselves libadb? */ +typedef struct adb adb_t, *adb_ptr; +struct adb_datum { + uint32_t nvectors; + uint32_t dim; + const char *key; + double *data; + double *power; + double *times; }; -typedef struct adb adb_t, *adb_ptr; +typedef struct adb_datum adb_datum_t; //used for both insert and batchinsert struct adbinsert { - - char * features; - char * power; - char * key; - char * times; - + const char *features; + const char *power; + const char *key; + const char *times; }; -typedef struct adbinsert adb_insert_t, *adb_insert_ptr; +typedef struct adbinsert adb_insert_t, adb_reference_t, *adb_insert_ptr; /* struct for returning status results */ struct adbstatus { - unsigned int numFiles; unsigned int dim; - unsigned int length; unsigned int dudCount; unsigned int nullCount; unsigned int flags; - + uint64_t length; + uint64_t data_region_size; }; typedef struct adbstatus adb_status_t, *adb_status_ptr; @@ -80,14 +95,81 @@ }; typedef struct adbqueryresult adb_queryresult_t, *adb_queryresult_ptr; +/* New ("new" == December 2008) query API */ + +typedef struct adbresult { + const char *key; + double dist; + uint32_t qpos; + uint32_t ipos; +} adb_result_t; + +#define ADB_REFINE_INCLUDE_KEYLIST 1 +#define ADB_REFINE_EXCLUDE_KEYLIST 2 +#define ADB_REFINE_RADIUS 4 +#define ADB_REFINE_ABSOLUTE_THRESHOLD 8 +#define ADB_REFINE_RELATIVE_THRESHOLD 16 +#define ADB_REFINE_DURATION_RATIO 32 +#define ADB_REFINE_HOP_SIZE 64 + +typedef struct adbkeylist { + uint32_t nkeys; + const char **keys; +} adb_keylist_t; + +typedef struct adbqueryrefine { + uint32_t flags; + adb_keylist_t include; + adb_keylist_t exclude; + double radius; + double absolute_threshold; + double relative_threshold; + double duration_ratio; /* expandfactor */ + uint32_t hopsize; +} adb_query_refine_t; + +#define ADB_ACCUMULATION_DB 1 +#define ADB_ACCUMULATION_PER_TRACK 2 +#define ADB_ACCUMULATION_ONE_TO_ONE 3 + +#define ADB_DISTANCE_DOT_PRODUCT 1 +#define ADB_DISTANCE_EUCLIDEAN_NORMED 2 +#define ADB_DISTANCE_EUCLIDEAN 3 + +typedef struct adbqueryparameters { + uint32_t accumulation; + uint32_t distance; + uint32_t npoints; + uint32_t ntracks; +} adb_query_parameters_t; + +typedef struct adbqueryresults { + uint32_t nresults; + adb_result_t *results; +} adb_query_results_t; + +#define ADB_QID_FLAG_EXHAUSTIVE 1 +#define ADB_QID_FLAG_ALLOW_FALSE_POSITIVES 2 + +typedef struct adbqueryid { + adb_datum_t *datum; + uint32_t sequence_length; + uint32_t flags; + uint32_t sequence_start; +} adb_query_id_t; + +typedef struct adbqueryspec { + adb_query_id_t qid; + adb_query_parameters_t params; + adb_query_refine_t refine; +} adb_query_spec_t; /*******************************************************************/ /* Function prototypes for API */ - /* open an existing database */ /* returns a struct or NULL on failure */ -adb_ptr audiodb_open(const char *path); +adb_ptr audiodb_open(const char *path, int flags); /* create a new database */ /* returns a struct or NULL on failure */ @@ -101,17 +183,20 @@ int audiodb_power(adb_ptr mydb); /* insert functions */ +int audiodb_insert_datum(adb_t *, const adb_datum_t *); +int audiodb_insert_reference(adb_t *, const adb_reference_t *); int audiodb_insert(adb_ptr mydb, adb_insert_ptr ins); int audiodb_batchinsert(adb_ptr mydb, adb_insert_ptr ins, unsigned int size); /* query function */ int audiodb_query(adb_ptr mydb, adb_query_ptr adbq, adb_queryresult_ptr adbqres); - +adb_query_results_t *audiodb_query_spec(adb_t *, const adb_query_spec_t *); +int audiodb_query_free_results(adb_t *, const adb_query_spec_t *, adb_query_results_t *); + /* database status */ int audiodb_status(adb_ptr mydb, adb_status_ptr status); /* varoius dump formats */ -int audiodb_dump(adb_ptr mydb); -int audiodb_dump_withdir(adb_ptr mydb, const char *outputdir); +int audiodb_dump(adb_ptr mydb, const char *outputdir); - +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/close.cpp Sat Jan 10 16:47:57 2009 +0000 @@ -0,0 +1,19 @@ +#include "audioDB.h" +extern "C" { +#include "audioDB_API.h" +#include "audioDB-internals.h" +} + +void audiodb_close(adb_t *adb) { + free(adb->path); + free(adb->header); + delete adb->keys; + delete adb->keymap; + delete adb->track_lengths; + delete adb->track_offsets; + if(adb->cached_lsh) { + delete adb->cached_lsh; + } + close(adb->fd); + free(adb); +}
--- a/common.cpp Sat Jan 10 11:11:27 2009 +0000 +++ b/common.cpp Sat Jan 10 16:47:57 2009 +0000 @@ -1,4 +1,8 @@ #include "audioDB.h" +extern "C" { +#include "audioDB_API.h" +#include "audioDB-internals.h" +} #if defined(O2_DEBUG) void sigterm_action(int signal, siginfo_t *info, void *context) { @@ -11,42 +15,15 @@ #endif void audioDB::get_lock(int fd, bool exclusive) { - struct flock lock; - int status; - - lock.l_type = exclusive ? F_WRLCK : F_RDLCK; - lock.l_whence = SEEK_SET; - lock.l_start = 0; - lock.l_len = 0; /* "the whole file" */ - - retry: - do { - status = fcntl(fd, F_SETLKW, &lock); - } while (status != 0 && errno == EINTR); - - if (status) { - if (errno == EAGAIN) { - sleep(1); - goto retry; - } else { - error("fcntl lock error", "", "fcntl"); - } + if(acquire_lock(fd, exclusive)) { + error("fcntl lock error", "", "fcntl"); } } void audioDB::release_lock(int fd) { - struct flock lock; - int status; - - lock.l_type = F_UNLCK; - lock.l_whence = SEEK_SET; - lock.l_start = 0; - lock.l_len = 0; - - status = fcntl(fd, F_SETLKW, &lock); - - if (status) + if (divest_lock(fd)) { error("fcntl unlock error", "", "fcntl"); + } } void audioDB::error(const char* a, const char* b, const char *sysFunc) { @@ -63,9 +40,6 @@ structured type, so that we can throw separate faultstring and details. -- CSR, 2007-10-01 */ throw(err); - } else if (UseApiError){ - apierrortemp=-1; - throw(apierrortemp); } else { std::cerr << a << ": " << b << std::endl; if (sysFunc) { @@ -73,7 +47,6 @@ } exit(1); } - } void audioDB::initRNG() { @@ -86,46 +59,20 @@ } void audioDB::initDBHeader(const char* dbName) { - if ((dbfid = open(dbName, forWrite ? O_RDWR : O_RDONLY)) < 0) { - error("Can't open database file", dbName, "open"); + if(!adb) { + adb = audiodb_open(dbName, forWrite ? O_RDWR : O_RDONLY); + if(!adb) { + error("Failed to open database", dbName); + } } - - get_lock(dbfid, forWrite); - // Get the database header info - dbH = new dbTableHeaderT(); - assert(dbH); - - if(read(dbfid, (char *) dbH, O2_HEADERSIZE) != O2_HEADERSIZE) { - error("error reading db header", dbName, "read"); - } - - if(dbH->magic == O2_OLD_MAGIC) { - // FIXME: if anyone ever complains, write the program to convert - // from the old audioDB format to the new... - error("database file has old O2 header", dbName); - } - - if(dbH->magic != O2_MAGIC) { - std::cerr << "expected: " << O2_MAGIC << ", got: " << dbH->magic << std::endl; - error("database file has incorrect header", dbName); - } - - if(dbH->version != O2_FORMAT_VERSION) { - error("database file has incorrect version", dbName); - } - - if(dbH->headerSize != O2_HEADERSIZE) { - error("sizeof(dbTableHeader) unexpected: platform ABI mismatch?", dbName); - } - - CHECKED_MMAP(char *, db, 0, getpagesize()); + dbfid = adb->fd; + dbH = adb->header; // Make some handy tables with correct types if(forWrite || (dbH->length > 0)) { if(forWrite) { fileTableLength = dbH->trackTableOffset - dbH->fileTableOffset; trackTableLength = dbH->dataOffset - dbH->trackTableOffset; - dataBufLength = dbH->timesTableOffset - dbH->dataOffset; timesTableLength = dbH->powerTableOffset - dbH->timesTableOffset; powerTableLength = dbH->l2normTableOffset - dbH->powerTableOffset; l2normTableLength = dbH->dbSize - dbH->l2normTableOffset; @@ -133,13 +80,11 @@ fileTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLE_ENTRY_SIZE); trackTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_TRACKTABLE_ENTRY_SIZE); if( dbH->flags & O2_FLAG_LARGE_ADB ){ - dataBufLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLE_ENTRY_SIZE); timesTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLE_ENTRY_SIZE); powerTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLE_ENTRY_SIZE); l2normTableLength = 0; } else{ - dataBufLength = ALIGN_PAGE_UP(dbH->length); timesTableLength = ALIGN_PAGE_UP(2*(dbH->length / dbH->dim)); powerTableLength = ALIGN_PAGE_UP(dbH->length / dbH->dim); l2normTableLength = ALIGN_PAGE_UP(dbH->length / dbH->dim); @@ -147,16 +92,6 @@ } CHECKED_MMAP(char *, fileTable, dbH->fileTableOffset, fileTableLength); CHECKED_MMAP(unsigned *, trackTable, dbH->trackTableOffset, trackTableLength); - /* - * No more mmap() for dataBuf - * - * FIXME: Actually we do do the mmap() in the two cases where it's - * still "needed": in pointQuery and in l2norm if dbH->length is - * non-zero. Removing those cases too (and deleting the dataBuf - * variable completely) would be cool. -- CSR, 2007-11-19 - * - * CHECKED_MMAP(double *, dataBuf, dbH->dataOffset, dataBufLength); - */ if( dbH->flags & O2_FLAG_LARGE_ADB ){ CHECKED_MMAP(char *, featureFileNameTable, dbH->dataOffset, fileTableLength); if( dbH->flags & O2_FLAG_TIMES ) @@ -170,22 +105,9 @@ CHECKED_MMAP(double *, l2normTable, dbH->l2normTableOffset, l2normTableLength); } } - - // build track offset table - trackOffsetTable = new off_t[dbH->numFiles]; - Uns32T cumTrack=0; - for(Uns32T k = 0; k < dbH->numFiles; k++){ - trackOffsetTable[k] = cumTrack; - cumTrack += trackTable[k] * dbH->dim; - } - - // Assign correct number of point bits per track in LSH indexing / retrieval - lsh_n_point_bits = dbH->flags >> 28; - if( !lsh_n_point_bits ) - lsh_n_point_bits = O2_DEFAULT_LSH_N_POINT_BITS; } -void audioDB::initInputFile (const char *inFile, bool loadData) { +void audioDB::initInputFile (const char *inFile) { if (inFile) { if ((infid = open(inFile, O_RDONLY)) < 0) { error("can't open input file for reading", inFile, "open"); @@ -216,10 +138,6 @@ error("feature dimensions do not match database table dimensions", inFile); } } - - if (loadData && ((indata = (char *) mmap(0, statbuf.st_size, PROT_READ, MAP_SHARED, infid, 0)) == (caddr_t) -1)) { - error("mmap error for input", inFile, "mmap"); - } } } @@ -254,3 +172,39 @@ sprintf(prefixedName, "%s/%s", prefix, *name); *name = prefixedName; // side effect new name to old name } + +void audioDB::insertTimeStamps(unsigned numVectors, std::ifstream *timesFile, double *timesdata) { + assert(usingTimes); + + unsigned numtimes = 0; + + if(!timesFile->is_open()) { + error("problem opening times file on timestamped database", timesFileName); + } + + double timepoint, next; + *timesFile >> timepoint; + if (timesFile->eof()) { + error("no entries in times file", timesFileName); + } + numtimes++; + do { + *timesFile >> next; + if (timesFile->eof()) { + break; + } + numtimes++; + timesdata[0] = timepoint; + timepoint = (timesdata[1] = next); + timesdata += 2; + } while (numtimes < numVectors + 1); + + if (numtimes < numVectors + 1) { + error("too few timepoints in times file", timesFileName); + } + + *timesFile >> next; + if (!timesFile->eof()) { + error("too many timepoints in times file", timesFileName); + } +}
--- a/create.cpp Sat Jan 10 11:11:27 2009 +0000 +++ b/create.cpp Sat Jan 10 16:47:57 2009 +0000 @@ -1,8 +1,12 @@ #include "audioDB.h" +extern "C" { +#include "audioDB_API.h" +} +#include "audioDB-internals.h" /* Make a new database. -IF size(featuredata) < O2_LARGE_ADB_SIZE +IF size(featuredata) < O2_LARGE_ADB_SIZE The database consists of: * a header (see dbTableHeader struct definition); @@ -13,9 +17,9 @@ * timesTable: (start,end) time points for each feature vector; * powerTable: associated power for each feature vector; * l2normTable: squared l2norms for each feature vector. - + ELSE the database consists of: - + * a header (see dbTableHeader struct definition); * keyTable: list of keys of tracks * trackTable: sizes of tracks @@ -25,69 +29,97 @@ */ -void audioDB::create(const char* dbName){ - if ((dbfid = open (dbName, O_RDWR|O_CREAT|O_EXCL, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)) < 0) - error("Can't create database file", dbName, "open"); - get_lock(dbfid, 1); +adb_t *audiodb_create(const char *path, unsigned datasize, unsigned ntracks, unsigned datadim) { + int fd; + adb_header_t *header = 0; + off_t databytes, auxbytes; + if(datasize == 0) { + datasize = O2_DEFAULT_DATASIZE; + } + if(ntracks == 0) { + ntracks = O2_DEFAULT_NTRACKS; + } + if(datadim == 0) { + datadim = O2_DEFAULT_DATADIM; + } - VERB_LOG(0, "header size: %ju\n", (intmax_t) O2_HEADERSIZE); - - dbH = new dbTableHeaderT(); - assert(dbH); + if ((fd = open(path, O_RDWR|O_CREAT|O_EXCL, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)) < 0) { + goto error; + } + if (acquire_lock(fd, true)) { + goto error; + } - //unsigned int maxfiles = (unsigned int) rint((double) O2_MAXFILES * (double) size / (double) O2_DEFAULTDBSIZE); + header = (adb_header_t *) malloc(sizeof(adb_header_t)); + if(!header) { + goto error; + } // Initialize header - dbH->magic = O2_MAGIC; - dbH->version = O2_FORMAT_VERSION; - dbH->numFiles = 0; - dbH->dim = 0; - dbH->flags = 0; - dbH->headerSize = O2_HEADERSIZE; - dbH->length = 0; - dbH->fileTableOffset = ALIGN_PAGE_UP(O2_HEADERSIZE); - dbH->trackTableOffset = ALIGN_PAGE_UP(dbH->fileTableOffset + O2_FILETABLE_ENTRY_SIZE*ntracks); - dbH->dataOffset = ALIGN_PAGE_UP(dbH->trackTableOffset + O2_TRACKTABLE_ENTRY_SIZE*ntracks); + header->magic = O2_MAGIC; + header->version = O2_FORMAT_VERSION; + header->numFiles = 0; + header->dim = 0; + header->flags = 0; + header->headerSize = O2_HEADERSIZE; + header->length = 0; + header->fileTableOffset = ALIGN_PAGE_UP(O2_HEADERSIZE); + header->trackTableOffset = ALIGN_PAGE_UP(header->fileTableOffset + O2_FILETABLE_ENTRY_SIZE*ntracks); + header->dataOffset = ALIGN_PAGE_UP(header->trackTableOffset + O2_TRACKTABLE_ENTRY_SIZE*ntracks); - off_t databytes = ((off_t) datasize) * 1024 * 1024; - off_t auxbytes = databytes / datadim; + databytes = ((off_t) datasize) * 1024 * 1024; + auxbytes = databytes / datadim; + + /* FIXME: what's going on here? There are two distinct preprocessor + constants (O2_LSH_N_POINT_BITS, LSH_N_POINT_BITS); a third is + presumably some default (O2_DEFAULT_LSH_N_POINT_BITS), and then + there's this magic 28 bits [which needs to be the same as the 28 + in audiodb_lsh_n_point_bits()]. Should this really be part of + the flags structure at all? Putting it elsewhere will of course + break backwards compatibility, unless 14 is the only value that's + been used anywhere... */ // For backward-compatibility, Record the point-encoding parameter for LSH indexing in the adb header // If this value is 0 then it will be set to 14 - #if LSH_N_POINT_BITS > 31 + #if LSH_N_POINT_BITS > 15 #error "AudioDB Compile ERROR: consistency check of O2_LSH_POINT_BITS failed (>31)" #endif - - dbH->flags |= LSH_N_POINT_BITS << 27; + + header->flags |= LSH_N_POINT_BITS << 28; // If database will fit in a single file the vectors are copied into the AudioDB instance // Else all the vectors are left on the FileSystem and we use the dataOffset as storage // for the location of the features, powers and times files (assuming that arbitrary keys are used for the fileTable) if(ntracks<O2_LARGE_ADB_NTRACKS && datasize<O2_LARGE_ADB_SIZE){ - dbH->timesTableOffset = ALIGN_PAGE_UP(dbH->dataOffset + databytes); - dbH->powerTableOffset = ALIGN_PAGE_UP(dbH->timesTableOffset + 2*auxbytes); - dbH->l2normTableOffset = ALIGN_PAGE_UP(dbH->powerTableOffset + auxbytes); - dbH->dbSize = ALIGN_PAGE_UP(dbH->l2normTableOffset + auxbytes); + header->timesTableOffset = ALIGN_PAGE_UP(header->dataOffset + databytes); + header->powerTableOffset = ALIGN_PAGE_UP(header->timesTableOffset + 2*auxbytes); + header->l2normTableOffset = ALIGN_PAGE_UP(header->powerTableOffset + auxbytes); + header->dbSize = ALIGN_PAGE_UP(header->l2normTableOffset + auxbytes); + } else { // Create LARGE_ADB, features and powers kept on filesystem + header->flags |= O2_FLAG_LARGE_ADB; + header->timesTableOffset = ALIGN_PAGE_UP(header->dataOffset + O2_FILETABLE_ENTRY_SIZE*ntracks); + header->powerTableOffset = ALIGN_PAGE_UP(header->timesTableOffset + O2_FILETABLE_ENTRY_SIZE*ntracks); + header->l2normTableOffset = ALIGN_PAGE_UP(header->powerTableOffset + O2_FILETABLE_ENTRY_SIZE*ntracks); + header->dbSize = header->l2normTableOffset; } - else{ // Create LARGE_ADB, features and powers kept on filesystem - dbH->flags |= O2_FLAG_LARGE_ADB; - dbH->timesTableOffset = ALIGN_PAGE_UP(dbH->dataOffset + O2_FILETABLE_ENTRY_SIZE*ntracks); - dbH->powerTableOffset = ALIGN_PAGE_UP(dbH->timesTableOffset + O2_FILETABLE_ENTRY_SIZE*ntracks); - dbH->l2normTableOffset = ALIGN_PAGE_UP(dbH->powerTableOffset + O2_FILETABLE_ENTRY_SIZE*ntracks); - dbH->dbSize = dbH->l2normTableOffset; - } - CHECKED_WRITE(dbfid, dbH, O2_HEADERSIZE); + write_or_goto_error(fd, header, O2_HEADERSIZE); // go to the location corresponding to the last byte - if (lseek (dbfid, dbH->dbSize - 1, SEEK_SET) == -1) - error("lseek error in db file", "", "lseek"); + if (lseek (fd, header->dbSize - 1, SEEK_SET) == -1) { + goto error; + } // write a dummy byte at the last location - if (write (dbfid, "", 1) != 1) - error("write error", "", "write"); + write_or_goto_error(fd, "", 1); - VERB_LOG(0, "%s %s\n", COM_CREATE, dbName); + free(header); + return audiodb_open(path, O_RDWR); + + error: + if(header) { + free(header); + } + return NULL; } -
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/dbaccumulator.h Sat Jan 10 16:47:57 2009 +0000 @@ -0,0 +1,52 @@ +template <class T> class DBAccumulator : public Accumulator { +public: + DBAccumulator(unsigned int pointNN); + ~DBAccumulator(); + void add_point(adb_result_t *r); + adb_query_results_t *get_points(); +private: + unsigned int pointNN; + std::priority_queue< adb_result_t, std::vector<adb_result_t>, T > *queue; + std::set< adb_result_t, adb_result_triple_lt > *set; +}; + +template <class T> DBAccumulator<T>::DBAccumulator(unsigned int pointNN) + : pointNN(pointNN), queue(0), set(0) { + queue = new std::priority_queue< adb_result_t, std::vector<adb_result_t>, T>; + set = new std::set<adb_result_t, adb_result_triple_lt>; +} + +template <class T> DBAccumulator<T>::~DBAccumulator() { + if(queue) { + delete queue; + } + if(set) { + delete set; + } +} + +template <class T> void DBAccumulator<T>::add_point(adb_result_t *r) { + if(!isnan(r->dist)) { + if(set->find(*r) == set->end()) { + set->insert(*r); + queue->push(*r); + if(queue->size() > pointNN) { + queue->pop(); + } + } + } +} + +template <class T> adb_query_results_t *DBAccumulator<T>::get_points() { + unsigned int size = queue->size(); + adb_query_results_t *r = (adb_query_results_t *) malloc(sizeof(adb_query_results_t)); + adb_result_t *rs = (adb_result_t *) calloc(size, sizeof(adb_result_t)); + r->nresults = size; + r->results = rs; + + for(unsigned int k = 0; k < size; k++) { + rs[k] = queue->top(); + queue->pop(); + } + return r; +}
--- a/dump.cpp Sat Jan 10 11:11:27 2009 +0000 +++ b/dump.cpp Sat Jan 10 16:47:57 2009 +0000 @@ -1,47 +1,95 @@ #include "audioDB.h" +extern "C" { +#include "audioDB_API.h" +#include "audioDB-internals.h" +} -void audioDB::dump(const char* dbName){ - if(!dbH) { - initTables(dbName, 0); +int audiodb_dump(adb_t *adb, const char *output) { + char *fileTable = 0; /* key_table */ + double *timesTable = 0; /* timestamps_table */ + double *powerTable = 0; /* power_table */ + + size_t fileTableLength = 0; + size_t timesTableLength = 0; + size_t powerTableLength = 0; + + char *featureFileNameTable = 0; + char *powerFileNameTable = 0; + char *timesFileNameTable = 0; + + char cwd[PATH_MAX]; + int directory_changed = 0; + + int fLfd = 0, tLfd = 0, pLfd = 0, kLfd = 0; + FILE *fLFile = 0, *tLFile = 0, *pLFile = 0, *kLFile = 0; + + int times, power; + + char fName[256]; + int ffd, pfd; + FILE *tFile; + unsigned pos = 0; + double *data_buffer; + size_t data_buffer_size; + FILE *scriptFile = 0; + + unsigned nfiles = adb->header->numFiles; + + if(adb->header->length > 0) { + fileTableLength = ALIGN_PAGE_UP(nfiles * O2_FILETABLE_ENTRY_SIZE); + if(!(adb->header->flags & O2_FLAG_LARGE_ADB)) { + off_t length = adb->header->length; + unsigned dim = adb->header->dim; + timesTableLength = ALIGN_PAGE_UP(2*length/dim); + powerTableLength = ALIGN_PAGE_UP(length/dim); + } + + mmap_or_goto_error(char *, fileTable, adb->header->fileTableOffset, fileTableLength); + if (adb->header->flags & O2_FLAG_LARGE_ADB) { + mmap_or_goto_error(char *, featureFileNameTable, adb->header->dataOffset, fileTableLength); + mmap_or_goto_error(char *, powerFileNameTable, adb->header->powerTableOffset, fileTableLength); + mmap_or_goto_error(char *, timesFileNameTable, adb->header->timesTableOffset, fileTableLength); + } else { + mmap_or_goto_error(double *, powerTable, adb->header->powerTableOffset, powerTableLength); + mmap_or_goto_error(double *, timesTable, adb->header->timesTableOffset, timesTableLength); + } } if((mkdir(output, S_IRWXU|S_IRWXG|S_IRWXO)) < 0) { - error("error making output directory", output, "mkdir"); + goto error; } - char *cwd = new char[PATH_MAX]; - if ((getcwd(cwd, PATH_MAX)) == 0) { - error("error getting working directory", "", "getcwd"); + goto error; } + /* FIXME: Hrm. How does chdir(2) interact with threads? Does each + * thread have its own working directory? */ if((chdir(output)) < 0) { - error("error changing working directory", output, "chdir"); + goto error; + } + directory_changed = 1; + + if ((fLfd = open("featureList.txt", O_CREAT|O_RDWR|O_EXCL, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)) < 0) { + goto error; } - int fLfd, tLfd = 0, pLfd = 0, kLfd; - FILE *fLFile, *tLFile = 0, *pLFile = 0, *kLFile; - - if ((fLfd = open("featureList.txt", O_CREAT|O_RDWR|O_EXCL, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)) < 0) { - error("error creating featureList file", "featureList.txt", "open"); - } - - int times = dbH->flags & O2_FLAG_TIMES; + times = adb->header->flags & O2_FLAG_TIMES; if (times) { if ((tLfd = open("timesList.txt", O_CREAT|O_RDWR|O_EXCL, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)) < 0) { - error("error creating timesList file", "timesList.txt", "open"); + goto error; } } - int power = dbH->flags & O2_FLAG_POWER; + power = adb->header->flags & O2_FLAG_POWER; if (power) { if ((pLfd = open("powerList.txt", O_CREAT|O_RDWR|O_EXCL, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)) < 0) { - error("error creating powerList file", "powerList.txt", "open"); + goto error; } } if ((kLfd = open("keyList.txt", O_CREAT|O_RDWR|O_EXCL, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)) < 0) { - error("error creating keyList file", "keyList.txt", "open"); + goto error; } /* can these fail? I sincerely hope not. */ @@ -54,73 +102,65 @@ } kLFile = fdopen(kLfd, "w"); - char *fName = new char[256]; - int ffd, pfd; - FILE *tFile; - unsigned pos = 0; - lseek(dbfid, dbH->dataOffset, SEEK_SET); - double *data_buffer; - size_t data_buffer_size; - for(unsigned k = 0; k < dbH->numFiles; k++) { + lseek(adb->fd, adb->header->dataOffset, SEEK_SET); + + for(unsigned k = 0; k < nfiles; k++) { fprintf(kLFile, "%s\n", fileTable + k*O2_FILETABLE_ENTRY_SIZE); - if(dbH->flags & O2_FLAG_LARGE_ADB) { + if(adb->header->flags & O2_FLAG_LARGE_ADB) { char *featureFileName = featureFileNameTable+k*O2_FILETABLE_ENTRY_SIZE; + if(*featureFileName != '/') { + goto error; + } fprintf(fLFile, "%s\n", featureFileName); - if(*featureFileName != '/') { - error("relative path in LARGE_ADB", featureFileName); - } if(times) { char *timesFileName = timesFileNameTable + k*O2_FILETABLE_ENTRY_SIZE; + if(*timesFileName != '/') { + goto error; + } fprintf(tLFile, "%s\n", timesFileName); - if(*timesFileName != '/') { - error("relative path in LARGE_ADB", timesFileName); - } } if(power) { char *powerFileName = powerFileNameTable + k*O2_FILETABLE_ENTRY_SIZE; + if(*powerFileName != '/') { + goto error; + } fprintf(pLFile, "%s\n", powerFileName); - if(*powerFileName != '/') { - error("relative path in LARGE_ADB", powerFileName); - } } } else { snprintf(fName, 256, "%05d.features", k); if ((ffd = open(fName, O_CREAT|O_RDWR|O_EXCL, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)) < 0) { - error("error creating feature file", fName, "open"); + goto error; } - if ((write(ffd, &dbH->dim, sizeof(uint32_t))) < 0) { - error("error writing dimensions", fName, "write"); - } + write_or_goto_error(ffd, &(adb->header->dim), sizeof(uint32_t)); /* FIXME: this repeated malloc()/free() of data buffers is inefficient. */ - data_buffer_size = trackTable[k] * dbH->dim * sizeof(double); + data_buffer_size = (*adb->track_lengths)[k] * adb->header->dim * sizeof(double); { void *tmp = malloc(data_buffer_size); if (tmp == NULL) { - error("error allocating data buffer"); + goto error; } data_buffer = (double *) tmp; } - if ((read(dbfid, data_buffer, data_buffer_size)) != (ssize_t) data_buffer_size) { - error("error reading data", fName, "read"); + if ((read(adb->fd, data_buffer, data_buffer_size)) != (ssize_t) data_buffer_size) { + goto error; } - if ((write(ffd, data_buffer, data_buffer_size)) < 0) { - error("error writing data", fName, "write"); - } + write_or_goto_error(ffd, data_buffer, data_buffer_size); free(data_buffer); fprintf(fLFile, "%s\n", fName); close(ffd); - + ffd = 0; + if (times) { snprintf(fName, 256, "%05d.times", k); tFile = fopen(fName, "w"); - for(unsigned i = 0; i < trackTable[k]; i++) { + for(unsigned i = 0; i < (*adb->track_lengths)[k]; i++) { // KLUDGE: specifying 16 digits of precision after the decimal // point is (but check this!) sufficient to uniquely identify // doubles; however, that will cause ugliness, as that's @@ -129,7 +169,8 @@ // -- CSR, 2007-10-19 fprintf(tFile, "%.16e\n", *(timesTable + 2*pos + 2*i)); } - fprintf(tFile, "%.16e\n", *(timesTable + 2*pos + 2*trackTable[k]-1)); + fprintf(tFile, "%.16e\n", *(timesTable + 2*pos + 2*(*adb->track_lengths)[k]-1)); + fclose(tFile); fprintf(tLFile, "%s\n", fName); } @@ -138,24 +179,20 @@ uint32_t one = 1; snprintf(fName, 256, "%05d.power", k); if ((pfd = open(fName, O_CREAT|O_RDWR|O_EXCL, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)) < 0) { - error("error creating power file", fName, "open"); + goto error; } - if ((write(pfd, &one, sizeof(uint32_t))) < 0) { - error("error writing one", fName, "write"); - } - if ((write(pfd, powerTable + pos, trackTable[k] * sizeof(double))) < 0) { - error("error writing data", fName, "write"); - } + write_or_goto_error(pfd, &one, sizeof(uint32_t)); + write_or_goto_error(pfd, powerTable + pos, (*adb->track_lengths)[k] * sizeof(double)); fprintf(pLFile, "%s\n", fName); close(pfd); + pfd = 0; } - pos += trackTable[k]; - std::cout << fileTable+k*O2_FILETABLE_ENTRY_SIZE << " " << trackTable[k] << std::endl; + pos += (*adb->track_lengths)[k]; + std::cout << fileTable+k*O2_FILETABLE_ENTRY_SIZE << " " << (*adb->track_lengths)[k] << std::endl; } } - FILE *scriptFile; scriptFile = fopen("restore.sh", "w"); fprintf(scriptFile, "\ #! /bin/sh\n\ @@ -165,12 +202,12 @@ if [ -z \"${AUDIODB}\" ]; then echo set AUDIODB variable; exit 1; fi\n\ if [ -z \"$1\" ]; then echo usage: $0 newdb; exit 1; fi\n\n\ \"${AUDIODB}\" -d \"$1\" -N --datasize=%d --ntracks=%d --datadim=%d\n", - (int) ((dbH->timesTableOffset - dbH->dataOffset) / (1024*1024)), + (int) ((adb->header->timesTableOffset - adb->header->dataOffset) / (1024*1024)), // fileTable entries (char[256]) are bigger than trackTable // (int), so the granularity of page aligning is finer. - (int) ((dbH->trackTableOffset - dbH->fileTableOffset) / O2_FILETABLE_ENTRY_SIZE), - (int) ceil(((double) (dbH->timesTableOffset - dbH->dataOffset)) / ((double) (dbH->dbSize - dbH->l2normTableOffset)))); - if(dbH->flags & O2_FLAG_L2NORM) { + (int) ((adb->header->trackTableOffset - adb->header->fileTableOffset) / O2_FILETABLE_ENTRY_SIZE), + (int) ceil(((double) (adb->header->timesTableOffset - adb->header->dataOffset)) / ((double) (adb->header->dbSize - adb->header->l2normTableOffset)))); + if(adb->header->flags & O2_FLAG_L2NORM) { fprintf(scriptFile, "\"${AUDIODB}\" -d \"$1\" -L\n"); } if(power) { @@ -186,10 +223,6 @@ fprintf(scriptFile, "\n"); fclose(scriptFile); - if((chdir(cwd)) < 0) { - error("error changing working directory", cwd, "chdir"); - } - fclose(fLFile); if(times) { fclose(tLFile); @@ -198,40 +231,57 @@ fclose(pLFile); } fclose(kLFile); - delete[] fName; - status(dbName); -} + maybe_munmap(fileTable, fileTableLength); + maybe_munmap(timesTable, timesTableLength); + maybe_munmap(powerTable, powerTableLength); + maybe_munmap(featureFileNameTable, fileTableLength); + maybe_munmap(timesFileNameTable, fileTableLength); + maybe_munmap(powerFileNameTable, fileTableLength); -void audioDB::liszt(const char* dbName, unsigned offset, unsigned numLines, adb__lisztResponse* adbLisztResponse){ - if(!dbH) { - initTables(dbName, 0); + if((chdir(cwd)) < 0) { + /* don't goto error because the error handling will try to + * chdir() */ + return 1; } - assert(trackTable && fileTable); + return 0; - if(offset>dbH->numFiles){ - char tmpStr[MAXSTR]; - sprintf(tmpStr, "numFiles=%u, lisztOffset=%u", dbH->numFiles, offset); - error("listKeys offset out of range", tmpStr); + error: + if(fLFile) { + fclose(fLFile); + } else if(fLfd) { + close(fLfd); + } + if(tLFile) { + fclose(tLFile); + } else if(tLfd) { + close(fLfd); + } + if(pLFile) { + fclose(pLFile); + } else if(pLfd) { + close(pLfd); + } + if(kLFile) { + fclose(kLFile); + } else if(kLfd) { + close(kLfd); + } + if(scriptFile) { + fclose(scriptFile); } - if(!adbLisztResponse){ - for(Uns32T k=0; k<numLines && offset+k<dbH->numFiles; k++){ - fprintf(stdout, "[%d] %s (%d)\n", offset+k, fileTable+(offset+k)*O2_FILETABLE_ENTRY_SIZE, trackTable[offset+k]); - } + maybe_munmap(fileTable, fileTableLength); + maybe_munmap(timesTable, timesTableLength); + maybe_munmap(powerTable, powerTableLength); + maybe_munmap(featureFileNameTable, fileTableLength); + maybe_munmap(timesFileNameTable, fileTableLength); + maybe_munmap(powerFileNameTable, fileTableLength); + + if(directory_changed) { + int gcc_warning_workaround = chdir(cwd); + directory_changed = gcc_warning_workaround; } - else{ - adbLisztResponse->result.Rkey = new char*[numLines]; - adbLisztResponse->result.Rlen = new unsigned int[numLines]; - Uns32T k = 0; - for( ; k<numLines && offset+k<dbH->numFiles; k++){ - adbLisztResponse->result.Rkey[k] = new char[MAXSTR]; - snprintf(adbLisztResponse->result.Rkey[k], O2_MAXFILESTR, "%s", fileTable+(offset+k)*O2_FILETABLE_ENTRY_SIZE); - adbLisztResponse->result.Rlen[k] = trackTable[offset+k]; - } - adbLisztResponse->result.__sizeRkey = k; - adbLisztResponse->result.__sizeRlen = k; - } - + return 1; }
--- a/index.cpp Sat Jan 10 11:11:27 2009 +0000 +++ b/index.cpp Sat Jan 10 16:47:57 2009 +0000 @@ -12,77 +12,87 @@ // 19th August 2008 - added O2_FLAG_LARGE_ADB support #include "audioDB.h" -#include "ReporterBase.h" +#include "audioDB-internals.h" - -/************************* LSH point index to audioDB conversion *****************/ -Uns32T audioDB::index_to_trackID(Uns32T lshID, Uns32T nPntBits){ - assert(nPntBits); - return lshID>>nPntBits; -} - -Uns32T audioDB::index_to_trackPos(Uns32T lshID, Uns32T nPntBits){ - assert(nPntBits); - return lshID&((1<<nPntBits)-1); -} - -Uns32T audioDB::index_from_trackInfo(Uns32T trackID, Uns32T spos, Uns32T nPntBits){ - assert(nPntBits); - return (trackID << nPntBits) | spos; -} +typedef struct adb_qcallback { + adb_t *adb; + adb_qstate_internal_t *qstate; +} adb_qcallback_t; /************************* LSH indexing and query initialization *****************/ -char* audioDB::index_get_name(const char*dbName, double radius, Uns32T sequenceLength){ - char* indexName = new char[MAXSTR]; - // Attempt to make new file - if(strlen(dbName) > (MAXSTR - 32)) - error("dbName is too long for LSH index filename appendages"); +/* FIXME: there are several things wrong with this: the memory + * discipline isn't ideal, the radius printing is a bit lame, the name + * getting will succeed or fail depending on whether the path was + * relative or absolute -- but most importantly encoding all that + * information in a filename is going to lose: it's impossible to + * maintain backwards-compatibility. Instead we should probably store + * the index metadata inside the audiodb instance. */ +char *audiodb_index_get_name(const char *dbName, double radius, Uns32T sequenceLength) { + char *indexName; + if(strlen(dbName) > (MAXSTR - 32)) { + return NULL; + } + indexName = new char[MAXSTR]; strncpy(indexName, dbName, MAXSTR); sprintf(indexName+strlen(dbName), ".lsh.%019.9f.%d", radius, sequenceLength); return indexName; } -// return true if index exists else return false -int audioDB::index_exists(const char* dbName, double radius, Uns32T sequenceLength){ - // Test to see if file exists - char* indexName = index_get_name(dbName, radius, sequenceLength); - lshfid = open (indexName, O_RDONLY); - delete[] indexName; - close(lshfid); - - if(lshfid<0) - return false; - else +bool audiodb_index_exists(const char *dbName, double radius, Uns32T sequenceLength) { + char *indexName = audiodb_index_get_name(dbName, radius, sequenceLength); + if(!indexName) { + return false; + } + struct stat st; + if(stat(indexName, &st)) { + delete [] indexName; + return false; + } + /* FIXME: other stat checks here? */ + /* FIXME: is there any better way to check whether we can open a + * file for reading than by opening a file for reading? */ + int fd = open(indexName, O_RDONLY); + delete [] indexName; + if(fd < 0) { + return false; + } else { + close(fd); return true; + } } -// If we are a server and have a memory-resident index, check the indexName against the resident index (using get_indexName()) -// If they match, i.e. path+dbName_resident == path+dbName_requested, use -// the memory-resident index. -// Else allocate a new LSH instance and load the index from disk -LSH* audioDB::index_allocate(char* indexName, bool load_hashTables){ - LSH* gIndx=SERVER_LSH_INDEX_SINGLETON; - if(isServer && gIndx && (strncmp(gIndx->get_indexName(), indexName, MAXSTR)==0) ) - audioDB::lsh = gIndx; // Use the global SERVER resident index - else{ - if(audioDB::lsh) - delete audioDB::lsh; - audioDB::lsh = new LSH(indexName, load_hashTables); +/* FIXME: the indexName arg should be "const char *", but the LSH + * library doesn't like that. + */ +LSH *audiodb_index_allocate(adb_t *adb, char *indexName, bool load_tables) { + LSH *lsh; + if(adb->cached_lsh) { + if(!strncmp(adb->cached_lsh->get_indexName(), indexName, MAXSTR)) { + return adb->cached_lsh; + } else { + delete adb->cached_lsh; + } } - assert(audioDB::lsh); - return audioDB::lsh; + lsh = new LSH(indexName, load_tables); + if(load_tables) { + adb->cached_lsh = lsh; + } + return lsh; } -vector<vector<float> >* audioDB::index_initialize_shingles(Uns32T sz){ - if(vv) - delete vv; - vv = new vector<vector<float> >(sz); - for(Uns32T i=0 ; i < sz ; i++) - (*vv)[i]=vector<float>(dbH->dim*sequenceLength); // allocate shingle storage +vector<vector<float> > *audiodb_index_initialize_shingles(Uns32T sz, Uns32T dim, Uns32T seqLen) { + std::vector<std::vector<float> > *vv = new vector<vector<float> >(sz); + for(Uns32T i=0 ; i < sz ; i++) { + (*vv)[i]=vector<float>(dim * seqLen); + } return vv; } +void audiodb_index_delete_shingles(vector<vector<float> > *vv) { + delete vv; +} + /******************** LSH indexing audioDB database access forall s \in {S} ***********************/ // Prepare the AudioDB database for read access and allocate auxillary memory @@ -105,11 +115,11 @@ for(Uns32T i = 0; i < dbH->numFiles; i++){ if(trackTable[i] >= sequenceLength) { - sequence_sum(snpp, trackTable[i], sequenceLength); - sequence_sqrt(snpp, trackTable[i], sequenceLength); + audiodb_sequence_sum(snpp, trackTable[i], sequenceLength); + audiodb_sequence_sqrt(snpp, trackTable[i], sequenceLength); - sequence_sum(sppp, trackTable[i], sequenceLength); - sequence_average(sppp, trackTable[i], sequenceLength); + audiodb_sequence_sum(sppp, trackTable[i], sequenceLength); + audiodb_sequence_average(sppp, trackTable[i], sequenceLength); } snpp += trackTable[i]; sppp += trackTable[i]; @@ -123,6 +133,80 @@ } +/********************* LSH shingle construction ***************************/ + +// Construct shingles out of a feature matrix +// inputs: +// idx is vector index in feature matrix +// fvp is base feature matrix pointer double* [numVecs x dbH->dim] +// +// pre-conditions: +// dbH->dim +// sequenceLength +// idx < numVectors - sequenceLength + 1 +// +// post-conditions: +// (*vv)[idx] contains a shingle with dbH->dim*sequenceLength float values + +static void audiodb_index_make_shingle(vector<vector<float> >* vv, Uns32T idx, double* fvp, Uns32T dim, Uns32T seqLen){ + assert(idx<(*vv).size()); + vector<float>::iterator ve = (*vv)[idx].end(); + vector<float>::iterator vi = (*vv)[idx].begin(); + // First feature vector in shingle + if(idx == 0) { + while(vi!=ve) { + *vi++ = (float)(*fvp++); + } + } else { + // Not first feature vector in shingle + vector<float>::iterator ui=(*vv)[idx-1].begin() + dim; + // Previous seqLen-1 dim-vectors + while(vi!=ve-dim) { + *vi++ = *ui++; + } + // Move data pointer to next feature vector + fvp += ( seqLen + idx - 1 ) * dim ; + // New d-vector + while(vi!=ve) { + *vi++ = (float)(*fvp++); + } + } +} + +// norm shingles +// in-place norming, no deletions +// If using power, return number of shingles above power threshold +int audiodb_index_norm_shingles(vector<vector<float> >* vv, double* snp, double* spp, Uns32T dim, Uns32T seqLen, double radius, bool normed_vectors, bool use_pthreshold, float pthreshold) { + int z = 0; // number of above-threshold shingles + float l2norm; + double power; + float oneOverRadius = 1./(float)sqrt(radius); // Passed radius is really radius^2 + float oneOverSqrtl2NormDivRad = oneOverRadius; + Uns32T shingleSize = seqLen * dim; + + if(!spp) { + return -1; + } + for(Uns32T a=0; a<(*vv).size(); a++){ + l2norm = (float)(*snp++); + if(normed_vectors) + oneOverSqrtl2NormDivRad = (1./l2norm)*oneOverRadius; + + for(Uns32T b=0; b < shingleSize ; b++) + (*vv)[a][b]*=oneOverSqrtl2NormDivRad; + + power = *spp++; + if(use_pthreshold){ + if (power >= pthreshold) + z++; + } + else + z++; + } + return z; +} + + /************************ LSH indexing ***********************************/ void audioDB::index_index_db(const char* dbName){ char* newIndexName; @@ -141,7 +225,10 @@ if(dbH->flags & O2_FLAG_TIMES) usingTimes = true; - newIndexName = index_get_name(dbName, radius, sequenceLength); + newIndexName = audiodb_index_get_name(dbName, radius, sequenceLength); + if(!newIndexName) { + error("failed to get index name", dbName); + } // Set unit norming flag override audioDB::normalizedDistance = !audioDB::no_unit_norming; @@ -183,6 +270,7 @@ // Clean up delete lsh; lsh = 0; + } else { close(lshfid); } @@ -195,7 +283,7 @@ // Get the lsh header info and find how many tracks are inserted already lsh = new LSH(mergeIndexName, false); // lshInCore=false to avoid loading hashTables here assert(lsh); - Uns32T maxs = index_to_trackID(lsh->get_maxp(), lsh_n_point_bits)+1; + Uns32T maxs = audiodb_index_to_track_id(lsh->get_maxp(), audiodb_lsh_n_point_bits(adb))+1; delete lsh; lsh = 0; @@ -239,6 +327,28 @@ } +void audioDB::insertPowerData(unsigned numVectors, int powerfd, double *powerdata) { + if(usingPower){ + int one; + unsigned int count; + + count = read(powerfd, &one, sizeof(unsigned int)); + if (count != sizeof(unsigned int)) { + error("powerfd read failed", "int", "read"); + } + if (one != 1) { + error("dimensionality of power file not 1", powerFileName); + } + + // FIXME: should check that the powerfile is the right size for + // this. -- CSR, 2007-10-30 + count = read(powerfd, powerdata, numVectors * sizeof(double)); + if (count != numVectors * sizeof(double)) { + error("powerfd read failed", "double", "read"); + } + } +} + // initialize auxillary track data from filesystem // pre-conditions: // dbH->flags & O2_FLAG_LARGE_ADB @@ -281,17 +391,17 @@ close(powerfd); } - sequence_sum(*sPowerp, trackTable[trackID], sequenceLength); - sequence_average(*sPowerp, trackTable[trackID], sequenceLength); + audiodb_sequence_sum(*sPowerp, trackTable[trackID], sequenceLength); + audiodb_sequence_average(*sPowerp, trackTable[trackID], sequenceLength); powerTable = 0; // Allocate and calculate the l2norm sequence *sNormpp = new double[trackTable[trackID]]; assert(*sNormpp); *snPtrp = *sNormpp; - unitNorm(fvp, dbH->dim, trackTable[trackID], *sNormpp); - sequence_sum(*sNormpp, trackTable[trackID], sequenceLength); - sequence_sqrt(*sNormpp, trackTable[trackID], sequenceLength); + audiodb_l2norm_buffer(fvp, dbH->dim, trackTable[trackID], *sNormpp); + audiodb_sequence_sum(*sNormpp, trackTable[trackID], sequenceLength); + audiodb_sequence_sqrt(*sNormpp, trackTable[trackID], sequenceLength); } } @@ -314,10 +424,11 @@ prefix_name((char ** const) &prefixedString, adb_feature_root); if(prefixedString!=tmpStr) delete[] tmpStr; - initInputFile(prefixedString, false); // nommap, file pointer at correct position + initInputFile(prefixedString); trackfd = infid; } - read_data(trackfd, trackID, &fvp, &nfv); // over-writes fvp and nfv + if(audiodb_read_data(adb, trackfd, trackID, &fvp, &nfv)) + error("failed to read data"); *fvpp = fvp; // Protect memory allocation and free() for track data if( dbH->flags & O2_FLAG_LARGE_ADB ) @@ -355,17 +466,23 @@ Uns32T numVecsAboveThreshold = 0, collisionCount = 0; if(numVecs){ - vv = index_initialize_shingles(numVecs); + std::vector<std::vector<float> > *vv = audiodb_index_initialize_shingles(numVecs, dbH->dim, sequenceLength); for( Uns32T pointID = 0 ; pointID < numVecs; pointID++ ) - index_make_shingle(vv, pointID, *fvpp, dbH->dim, sequenceLength); - - numVecsAboveThreshold = index_norm_shingles(vv, *snpp, *sppp); + audiodb_index_make_shingle(vv, pointID, *fvpp, dbH->dim, sequenceLength); + int vcount = audiodb_index_norm_shingles(vv, *snpp, *sppp, dbH->dim, sequenceLength, radius, normalizedDistance, use_absolute_threshold, absolute_threshold); + if(vcount == -1) { + audiodb_index_delete_shingles(vv); + error("failed to norm shingles"); + } + numVecsAboveThreshold = vcount; collisionCount = index_insert_shingles(vv, trackID, *sppp); + audiodb_index_delete_shingles(vv); } + float meanCollisionCount = numVecsAboveThreshold?(float)collisionCount/numVecsAboveThreshold:0; - /* index_norm_shingles() only goes as far as the end of the + /* audiodb_index_norm_shingles() only goes as far as the end of the sequence, which is right, but the space allocated is for the whole track. */ @@ -389,270 +506,177 @@ cout << "[" << trackID << "]" << fileTable+trackID*O2_FILETABLE_ENTRY_SIZE; for( Uns32T pointID=0 ; pointID < (*vv).size(); pointID+=sequenceHop){ if(!use_absolute_threshold || (use_absolute_threshold && (*spp >= absolute_threshold))) - collisionCount += lsh->insert_point((*vv)[pointID], index_from_trackInfo(trackID, pointID, lsh_n_point_bits)); + collisionCount += lsh->insert_point((*vv)[pointID], audiodb_index_from_trackinfo(trackID, pointID, audiodb_lsh_n_point_bits(adb))); spp+=sequenceHop; } return collisionCount; } -/********************* LSH shingle construction ***************************/ - -// Construct shingles out of a feature matrix -// inputs: -// idx is vector index in feature matrix -// fvp is base feature matrix pointer double* [numVecs x dbH->dim] -// -// pre-conditions: -// dbH->dim -// sequenceLength -// idx < numVectors - sequenceLength + 1 -// -// post-conditions: -// (*vv)[idx] contains a shingle with dbH->dim*sequenceLength float values - -void audioDB::index_make_shingle(vector<vector<float> >* vv, Uns32T idx, double* fvp, Uns32T dim, Uns32T seqLen){ - assert(idx<(*vv).size()); - vector<float>::iterator ve = (*vv)[idx].end(); - vi=(*vv)[idx].begin(); // shingle iterator - // First feature vector in shingle - if(idx==0){ - while(vi!=ve) - *vi++ = (float)(*fvp++); - } - // Not first feature vector in shingle - else{ - vector<float>::iterator ui=(*vv)[idx-1].begin() + dim; // previous shingle iterator - // Previous seqLen-1 dim-vectors - while(vi!=ve-dim) - *vi++=*ui++; - // Move data pointer to next feature vector - fvp += ( seqLen + idx - 1 ) * dim ; - // New d-vector - while(vi!=ve) - *vi++ = (float)(*fvp++); - } -} - -// norm shingles -// in-place norming, no deletions -// If using power, return number of shingles above power threshold -int audioDB::index_norm_shingles(vector<vector<float> >* vv, double* snp, double* spp){ - int z = 0; // number of above-threshold shingles - float l2norm; - double power; - float oneOverRadius = 1./(float)sqrt(radius); // Passed radius is really radius^2 - float oneOverSqrtl2NormDivRad = oneOverRadius; - if(!spp) - error("LSH indexing and query requires a power feature using -w or -W"); - Uns32T shingleSize = sequenceLength*dbH->dim; - for(Uns32T a=0; a<(*vv).size(); a++){ - l2norm = (float)(*snp++); - if(audioDB::normalizedDistance) - oneOverSqrtl2NormDivRad = (1./l2norm)*oneOverRadius; - - for(Uns32T b=0; b < shingleSize ; b++) - (*vv)[a][b]*=oneOverSqrtl2NormDivRad; - - power = *spp++; - if(use_absolute_threshold){ - if ( power >= absolute_threshold ) - z++; - } - else - z++; - } - return z; -} - - /*********************** LSH retrieval ****************************/ // return true if indexed query performed else return false -int audioDB::index_init_query(const char* dbName){ +int audiodb_index_init_query(adb_t *adb, const adb_query_spec_t *spec, adb_qstate_internal_t *qstate, bool corep) { - if(!(index_exists(dbName, radius, sequenceLength))) + uint32_t sequence_length = spec->qid.sequence_length; + double radius = spec->refine.radius; + if(!(audiodb_index_exists(adb->path, radius, sequence_length))) return false; - char* indexName = index_get_name(dbName, radius, sequenceLength); - - // Test to see if file exists - if((lshfid = open (indexName, O_RDONLY)) < 0){ - delete[] indexName; - return false; + char *indexName = audiodb_index_get_name(adb->path, radius, sequence_length); + if(!indexName) { + return false; } - lsh = index_allocate(indexName, false); // Get the header only here - sequenceLength = lsh->get_lshHeader()->dataDim / dbH->dim; // shingleDim / vectorDim - - if(lsh!=SERVER_LSH_INDEX_SINGLETON){ - if( fabs(radius - lsh->get_radius())>fabs(O2_DISTANCE_TOLERANCE)) - printf("*** Warning: adb_radius (%f) != lsh_radius (%f) ***\n", radius, lsh->get_radius()); - VERB_LOG(1,"INDEX: dim %d\n", (int)dbH->dim); - VERB_LOG(1,"INDEX: R %f\n", lsh->get_radius()); - VERB_LOG(1,"INDEX: seqlen %d\n", sequenceLength); - VERB_LOG(1,"INDEX: w %f\n", lsh->get_lshHeader()->get_binWidth()); - VERB_LOG(1,"INDEX: k %d\n", lsh->get_lshHeader()->get_numFuns()); - VERB_LOG(1,"INDEX: L (m*(m-1))/2 %d\n", lsh->get_lshHeader()->get_numTables()); - VERB_LOG(1,"INDEX: N %d\n", lsh->get_lshHeader()->get_numRows()); - VERB_LOG(1,"INDEX: s %d\n", index_to_trackID(lsh->get_maxp(), lsh_n_point_bits)); - VERB_LOG(1,"INDEX: Opened LSH index file %s\n", indexName); + qstate->lsh = audiodb_index_allocate(adb, indexName, corep); + + /* FIXME: it would be nice if the LSH library didn't make me do + * this. */ + if((!corep) && (qstate->lsh->get_lshHeader()->flags & O2_SERIAL_FILEFORMAT2)) { + delete qstate->lsh; + qstate->lsh = audiodb_index_allocate(adb, indexName, true); } - // Check to see if we are loading hash tables into core, and do so if true - if((lsh->get_lshHeader()->flags&O2_SERIAL_FILEFORMAT2) || lsh_in_core){ - if(SERVER_LSH_INDEX_SINGLETON) - fprintf(stderr,"INDEX: using persistent hash tables: %s\n", lsh->get_indexName()); - else - VERB_LOG(1,"INDEX: loading hash tables into core %s\n", (lsh->get_lshHeader()->flags&O2_SERIAL_FILEFORMAT2)?"FORMAT2":"FORMAT1"); - lsh = index_allocate(indexName, true); - } - delete[] indexName; return true; } -// *Static* exact NN point reporter callback method for lshlib -void audioDB::index_add_point(void* instancePtr, Uns32T pointID, Uns32T qpos, float dist){ - assert(instancePtr); // We need an instance for this callback - audioDB* myself = (audioDB*) instancePtr; // Use explicit cast to recover "this" instance - Uns32T trackID = index_to_trackID(pointID, myself->lsh_n_point_bits); - Uns32T spos = index_to_trackPos(pointID, myself->lsh_n_point_bits); - // Skip identity in query_from_key - if( (!myself->query_from_key || (myself->query_from_key && ( trackID != myself->query_from_key_index ))) - && (!myself->trackFile || myself->is_in_allowed_keys(trackID)) ) - if(myself->lsh_exact) - myself->index_insert_exact_evaluation_queue(trackID, qpos, spos); - else - myself->reporter->add_point(trackID, qpos, spos, dist); -} - -int audioDB::is_in_allowed_keys(Uns32T trackID){ - std::set<Uns32T>::iterator it; - if(!allowed_keys) - return 0; - it = allowed_keys->find(trackID); - if(it == allowed_keys->end()) - return 0; - else - return 1; +void audiodb_index_add_point_approximate(void *user_data, Uns32T pointID, Uns32T qpos, float dist) { + adb_qcallback_t *data = (adb_qcallback_t *) user_data; + adb_t *adb = data->adb; + adb_qstate_internal_t *qstate = data->qstate; + uint32_t nbits = audiodb_lsh_n_point_bits(adb); + uint32_t trackID = audiodb_index_to_track_id(pointID, nbits); + uint32_t spos = audiodb_index_to_track_pos(pointID, nbits); + std::set<std::string>::iterator keys_end = qstate->allowed_keys->end(); + if(qstate->allowed_keys->find((*adb->keys)[trackID]) != keys_end) { + adb_result_t r; + r.key = (*adb->keys)[trackID].c_str(); + r.dist = dist; + r.qpos = qpos; + r.ipos = spos; + qstate->accumulator->add_point(&r); + } } -void audioDB::initialize_exact_evalutation_queue(){ - if(exact_evaluation_queue) - delete exact_evaluation_queue; - exact_evaluation_queue = new priority_queue<PointPair, std::vector<PointPair>, std::less<PointPair> >; +// Maintain a queue of points to pass to audiodb_query_queue_loop() +// for exact evaluation +void audiodb_index_add_point_exact(void *user_data, Uns32T pointID, Uns32T qpos, float dist) { + adb_qcallback_t *data = (adb_qcallback_t *) user_data; + adb_t *adb = data->adb; + adb_qstate_internal_t *qstate = data->qstate; + uint32_t nbits = audiodb_lsh_n_point_bits(adb); + uint32_t trackID = audiodb_index_to_track_id(pointID, nbits); + uint32_t spos = audiodb_index_to_track_pos(pointID, nbits); + std::set<std::string>::iterator keys_end = qstate->allowed_keys->end(); + if(qstate->allowed_keys->find((*adb->keys)[trackID]) != keys_end) { + PointPair p(trackID, qpos, spos); + qstate->exact_evaluation_queue->push(p); + } } -void audioDB::initialize_allowed_keys(std::ifstream* trackFile){ - Uns32T trackIndex; - char nextKey[MAXSTR]; - - allowed_keys = new std::set<Uns32T>; - // Read keys from file, look up the index for each and insert in allowed_keys set - do { - trackFile->getline(nextKey,MAXSTR); - if(!trackFile->eof()) { - trackIndex = getKeyPos(nextKey); - allowed_keys->insert(trackIndex); - } - } while(!trackFile->eof()); -} - -void audioDB::index_insert_exact_evaluation_queue(Uns32T trackID, Uns32T qpos, Uns32T spos){ - PointPair p(trackID, qpos, spos); - exact_evaluation_queue->push(p); -} - +// return -1 on error // return 0: if index does not exist // return nqv: if index exists -int audioDB::index_query_loop(const char* dbName, Uns32T queryIndex) { +int audiodb_index_query_loop(adb_t *adb, const adb_query_spec_t *spec, adb_qstate_internal_t *qstate) { - unsigned int numVectors = 0; double *query = 0, *query_data = 0; - double *qNorm = 0, *qnPtr = 0, *qPower = 0, *qpPtr = 0; - double meanQdur = 0; - void (*add_point_func)(void*,Uns32T,Uns32T,float); + adb_qpointers_internal_t qpointers = {0}; - // Set the point-reporter callback based on the value of lsh_exact - if(lsh_exact) - initialize_exact_evalutation_queue(); - add_point_func = &index_add_point; + adb_qcallback_t callback_data; + callback_data.adb = adb; + callback_data.qstate = qstate; - if(!index_init_query(dbName)) // sets-up LSH index structures for querying + void (*add_point_func)(void *, uint32_t, uint32_t, float); + + uint32_t sequence_length = spec->qid.sequence_length; + bool normalized = (spec->params.distance == ADB_DISTANCE_EUCLIDEAN_NORMED); + double radius = spec->refine.radius; + bool use_absolute_threshold = spec->refine.flags & ADB_REFINE_ABSOLUTE_THRESHOLD; + double absolute_threshold = spec->refine.absolute_threshold; + + if(spec->qid.flags & ADB_QID_FLAG_ALLOW_FALSE_POSITIVES) { + add_point_func = &audiodb_index_add_point_approximate; + } else { + qstate->exact_evaluation_queue = new std::priority_queue<PointPair>; + add_point_func = &audiodb_index_add_point_exact; + } + + /* FIXME: this hardwired lsh_in_core is here to allow for a + * transition period while the need for the argument is worked + * through. Hopefully it will disappear again eventually. */ + bool lsh_in_core = true; + + if(!audiodb_index_init_query(adb, spec, qstate, lsh_in_core)) { return 0; + } - char* database = index_get_name(dbName, radius, sequenceLength); + char *database = audiodb_index_get_name(adb->path, radius, sequence_length); + if(!database) { + return -1; + } - if(query_from_key) - set_up_query_from_key(&query_data, &query, &qNorm, &qnPtr, &qPower, &qpPtr, &meanQdur, &numVectors, queryIndex); - else - set_up_query(&query_data, &query, &qNorm, &qnPtr, &qPower, &qpPtr, &meanQdur, &numVectors); // get query vectors + if(audiodb_query_spec_qpointers(adb, spec, &query_data, &query, &qpointers)) { + delete [] database; + return -1; + } - close(infid); // Free up this file handle - VERB_LOG(1, "retrieving tracks..."); - - assert(pointNN>0 && pointNN<=O2_MAXNN); - assert(trackNN>0 && trackNN<=O2_MAXNN); - - gettimeofday(&tv1, NULL); - // query vector index - Uns32T Nq = (numVectors>O2_MAXTRACKLEN?O2_MAXTRACKLEN:numVectors) - sequenceLength + 1; - vv = index_initialize_shingles(Nq); // allocate memory to copy query vectors to shingles - VERB_LOG(1, "Nq=%d", Nq); - - // restrictList initialization - if(trackFile) - initialize_allowed_keys(trackFile); // trackFile is list of valid keys to admit in search + uint32_t Nq = (qpointers.nvectors > O2_MAXTRACKLEN ? O2_MAXTRACKLEN : qpointers.nvectors) - sequence_length + 1; + std::vector<std::vector<float> > *vv = audiodb_index_initialize_shingles(Nq, adb->header->dim, sequence_length); // Construct shingles from query features - for( Uns32T pointID = 0 ; pointID < Nq ; pointID++ ) - index_make_shingle(vv, pointID, query, dbH->dim, sequenceLength); + for(uint32_t pointID = 0; pointID < Nq; pointID++) { + audiodb_index_make_shingle(vv, pointID, query, adb->header->dim, sequence_length); + } // Normalize query vectors - Uns32T numVecsAboveThreshold = index_norm_shingles( vv, qnPtr, qpPtr ); - VERB_LOG(1, "Nq'=%d\n", numVecsAboveThreshold); + int vcount = audiodb_index_norm_shingles(vv, qpointers.l2norm, qpointers.power, adb->header->dim, sequence_length, radius, normalized, use_absolute_threshold, absolute_threshold); + if(vcount == -1) { + audiodb_index_delete_shingles(vv); + delete [] database; + return -1; + } + uint32_t numVecsAboveThreshold = vcount; // Nq contains number of inspected points in query file, // numVecsAboveThreshold is number of points with power >= absolute_threshold - double* qpp = qpPtr; // Keep original qpPtr for possible exact evaluation - if(usingQueryPoint && numVecsAboveThreshold){ - if((lsh->get_lshHeader()->flags&O2_SERIAL_FILEFORMAT2) || lsh_in_core) - lsh->retrieve_point((*vv)[0], queryPoint, add_point_func, (void*)this); - else - lsh->serial_retrieve_point(database, (*vv)[0], queryPoint, add_point_func, (void*)this); - } - else if(numVecsAboveThreshold) - for( Uns32T pointID = 0 ; pointID < Nq; pointID++ ) + double *qpp = qpointers.power; // Keep original qpPtr for possible exact evaluation + if(!(spec->qid.flags & ADB_QID_FLAG_EXHAUSTIVE) && numVecsAboveThreshold) { + if((qstate->lsh->get_lshHeader()->flags & O2_SERIAL_FILEFORMAT2) || lsh_in_core) { + qstate->lsh->retrieve_point((*vv)[0], spec->qid.sequence_start, add_point_func, &callback_data); + } else { + qstate->lsh->serial_retrieve_point(database, (*vv)[0], spec->qid.sequence_start, add_point_func, &callback_data); + } + } else if(numVecsAboveThreshold) { + for(uint32_t pointID = 0; pointID < Nq; pointID++) { if(!use_absolute_threshold || (use_absolute_threshold && (*qpp++ >= absolute_threshold))) { - if((lsh->get_lshHeader()->flags&O2_SERIAL_FILEFORMAT2) || lsh_in_core) { - lsh->retrieve_point((*vv)[pointID], pointID, add_point_func, (void*)this); + if((qstate->lsh->get_lshHeader()->flags & O2_SERIAL_FILEFORMAT2) || lsh_in_core) { + qstate->lsh->retrieve_point((*vv)[pointID], pointID, add_point_func, &callback_data); } else { - lsh->serial_retrieve_point(database, (*vv)[pointID], pointID, add_point_func, (void*)this); + qstate->lsh->serial_retrieve_point(database, (*vv)[pointID], pointID, add_point_func, &callback_data); } } + } + } + audiodb_index_delete_shingles(vv); - if(lsh_exact) - // Perform exact distance computation on point pairs in exact_evaluation_queue - query_loop_points(query, qnPtr, qpPtr, meanQdur, numVectors); + if(!(spec->qid.flags & ADB_QID_FLAG_ALLOW_FALSE_POSITIVES)) { + audiodb_query_queue_loop(adb, spec, qstate, query, &qpointers); + } - gettimeofday(&tv2,NULL); - VERB_LOG(1,"elapsed time: %ld msec\n", - (tv2.tv_sec*1000 + tv2.tv_usec/1000) - - (tv1.tv_sec*1000 + tv1.tv_usec/1000)) - - // Close the index file - close(lshfid); - // Clean up if(query_data) delete[] query_data; - if(qNorm) - delete[] qNorm; - if(qPower) - delete[] qPower; + if(qpointers.l2norm_data) + delete[] qpointers.l2norm_data; + if(qpointers.power_data) + delete[] qpointers.power_data; + if(qpointers.mean_duration) + delete[] qpointers.mean_duration; if(database) delete[] database; + if(qstate->lsh != adb->cached_lsh) + delete qstate->lsh; return Nq; }
--- a/insert.cpp Sat Jan 10 11:11:27 2009 +0000 +++ b/insert.cpp Sat Jan 10 16:47:57 2009 +0000 @@ -1,546 +1,443 @@ #include "audioDB.h" +extern "C" { +#include "audioDB_API.h" +} +#include "audioDB-internals.h" -bool audioDB::enough_per_file_space_free() { - unsigned int fmaxfiles, tmaxfiles; - unsigned int maxfiles; - - fmaxfiles = fileTableLength / O2_FILETABLE_ENTRY_SIZE; - tmaxfiles = trackTableLength / O2_TRACKTABLE_ENTRY_SIZE; - maxfiles = fmaxfiles > tmaxfiles ? tmaxfiles : fmaxfiles; - return(dbH->numFiles < maxfiles); -} - -bool audioDB::enough_data_space_free(off_t size) { - return(dbH->timesTableOffset > dbH->dataOffset + dbH->length + size); -} - -void audioDB::insert_data_vectors(off_t offset, void *buffer, size_t size) { - if(lseek(dbfid, dbH->dataOffset + offset, SEEK_SET) == (off_t) -1) { - error("error seeking to offset", "", "lseek"); - } - CHECKED_WRITE(dbfid, buffer, size); -} - -void audioDB::insert(const char* dbName, const char* inFile) { - forWrite = true; - initTables(dbName, inFile); - - if(dbH->flags & O2_FLAG_LARGE_ADB) - error("Single-feature inserts not allowed with LARGE audioDB instances"); - - if(!usingTimes && (dbH->flags & O2_FLAG_TIMES)) - error("Must use timestamps with timestamped database","use --times"); - - if(!usingPower && (dbH->flags & O2_FLAG_POWER)) - error("Must use power with power-enabled database", dbName); - - if(!enough_per_file_space_free()) { - error("Insert failed: no more room for metadata", inFile); - } - - if(!enough_data_space_free(statbuf.st_size - sizeof(int))) { - error("Insert failed: no more room in database", inFile); - } - - if(!key) - key=inFile; - // Linear scan of filenames check for pre-existing feature - unsigned alreadyInserted=0; - for(unsigned k=0; k<dbH->numFiles; k++) - if(strncmp(fileTable + k*O2_FILETABLE_ENTRY_SIZE, key, strlen(key)+1)==0){ - alreadyInserted=1; - break; - } - - if(alreadyInserted) { - VERB_LOG(0, "key already exists in database; ignoring: %s\n", inFile); - // FIXME: Do we need to munmap here (see below) ? MKC 18/08/08 - return; - } - - // Make a track index table of features to file indexes - unsigned numVectors = (statbuf.st_size-sizeof(int))/(sizeof(double)*dbH->dim); - if(!numVectors) { - VERB_LOG(0, "ignoring zero-length feature vector file: %s\n", key); - - // CLEAN UP - munmap(indata,statbuf.st_size); - munmap(db,dbH->dbSize); - close(infid); - return; - } - - INSERT_FILETABLE_STRING(fileTable, key); - - off_t insertoffset = dbH->length;// Store current state - - // Check times status and insert times from file - unsigned indexoffset = insertoffset/(dbH->dim*sizeof(double)); - double *timesdata = timesTable + 2*indexoffset; - - if(2*(indexoffset + numVectors) > timesTableLength) { - error("out of space for times", key); - } - - if (usingTimes) { - insertTimeStamps(numVectors, timesFile, timesdata); - } - - double *powerdata = powerTable + indexoffset; - insertPowerData(numVectors, powerfd, powerdata); - - // Increment file count - dbH->numFiles++; - - // Update Header information - dbH->length+=(statbuf.st_size-sizeof(int)); - - // Update track to file index map - memcpy(trackTable + dbH->numFiles - 1, &numVectors, sizeof(unsigned)); - - insert_data_vectors(insertoffset, indata + sizeof(int), statbuf.st_size - sizeof(int)); - - // Norm the vectors on input if the database is already L2 normed - if(dbH->flags & O2_FLAG_L2NORM) - unitNormAndInsertL2((double *)(indata + sizeof(int)), dbH->dim, numVectors, 1); // append - - // Report status - status(dbName); - VERB_LOG(0, "%s %s %u vectors %jd bytes.\n", COM_INSERT, dbName, numVectors, (intmax_t) (statbuf.st_size - sizeof(int))); - - // Copy the header back to the database - memcpy (db, dbH, sizeof(dbTableHeaderT)); - - // CLEAN UP - munmap(indata,statbuf.st_size); - close(infid); -} - -void audioDB::insertTimeStamps(unsigned numVectors, std::ifstream *timesFile, double *timesdata) { - assert(usingTimes); - - unsigned numtimes = 0; - - if(!(dbH->flags & O2_FLAG_TIMES) && !dbH->numFiles) { - dbH->flags=dbH->flags|O2_FLAG_TIMES; - } else if(!(dbH->flags & O2_FLAG_TIMES)) { - error("Timestamp file used with non-timestamped database", timesFileName); - } - - if(!timesFile->is_open()) { - error("problem opening times file on timestamped database", timesFileName); - } - - double timepoint, next; - *timesFile >> timepoint; - if (timesFile->eof()) { - error("no entries in times file", timesFileName); - } - numtimes++; - do { - *timesFile >> next; - if (timesFile->eof()) { - break; - } - numtimes++; - timesdata[0] = timepoint; - timepoint = (timesdata[1] = next); - timesdata += 2; - } while (numtimes < numVectors + 1); - - if (numtimes < numVectors + 1) { - error("too few timepoints in times file", timesFileName); - } - - *timesFile >> next; - if (!timesFile->eof()) { - error("too many timepoints in times file", timesFileName); +static bool audiodb_enough_data_space_free(adb_t *adb, off_t size) { + adb_header_t *header = adb->header; + if(header->flags & O2_FLAG_LARGE_ADB) { + return true; + } else { + /* FIXME: timesTableOffset isn't necessarily the next biggest + * offset after dataOffset. Maybe make the offsets into an array + * that we can iterate over... */ + return (header->timesTableOffset > + (header->dataOffset + header->length + size)); } } -void audioDB::insertPowerData(unsigned numVectors, int powerfd, double *powerdata) { - if(usingPower){ - if (!(dbH->flags & O2_FLAG_POWER)) { - error("Cannot insert power data on non-power DB", dbName); +static bool audiodb_enough_per_file_space_free(adb_t *adb) { + /* FIXME: the comment above about the ordering of the tables applies + here too. */ + adb_header_t *header = adb->header; + off_t file_table_length = header->trackTableOffset - header->fileTableOffset; + off_t track_table_length = header->dataOffset - header->trackTableOffset; + int fmaxfiles = file_table_length / O2_FILETABLE_ENTRY_SIZE; + int tmaxfiles = track_table_length / O2_TRACKTABLE_ENTRY_SIZE; + /* maxfiles is the _minimum_ of the two. Do not be confused... */ + int maxfiles = fmaxfiles > tmaxfiles ? tmaxfiles : fmaxfiles; + if(header->flags & O2_FLAG_LARGE_ADB) { + /* by default, these tables are created with the same size as the + * fileTable (which should be called key_table); relying on that + * always being the case, though, smacks of optimism, so instead + * we code defensively... */ + off_t data_table_length = header->timesTableOffset - header->dataOffset; + off_t times_table_length = header->powerTableOffset - header->timesTableOffset; + off_t power_table_length = header->dbSize - header->powerTableOffset; + int dmaxfiles = data_table_length / O2_FILETABLE_ENTRY_SIZE; + int timaxfiles = times_table_length / O2_FILETABLE_ENTRY_SIZE; + int pmaxfiles = power_table_length / O2_FILETABLE_ENTRY_SIZE; + /* ... even though it means a certain amount of tedium. */ + maxfiles = maxfiles > dmaxfiles ? dmaxfiles : maxfiles; + maxfiles = maxfiles > timaxfiles ? timaxfiles : maxfiles; + maxfiles = maxfiles > pmaxfiles ? pmaxfiles : maxfiles; + } + return (header->numFiles < (unsigned int) maxfiles); +} + +/* + * Hey, look, a comment. Normally I wouldn't bother, as the code + * should be self-documenting, but a lot of logic is concentrated in + * this one place, so let's give an overview beforehand. To insert a + * datum into the database, we: + * + * 1. check write permission; + * 2. check for enough space; + * 3. check that datum->dim and adb->header->dim agree (or that the + * header dimension is zero, in which case write datum->dim to + * adb->header->dim). + * 4. check for presence of datum->key in adb->keymap; + * 5. check for consistency between power and O2_FLAG_POWER, and + * times and O2_FLAG_TIMES; + * 6. write in data, power, times as appropriate; add to track + * and key tables too; + * 7. if O2_FLAG_L2NORM and !O2_FLAG_LARGE_ADB, compute norms and fill + * in table; + * 8. update adb->keys, adb->keymap, adb->track_lengths, + * adb->track_offsets and adb->header; + * 9. sync adb->header with disk. + * + * Step 9 essentially commits the transaction; until we update + * header->length, nothing will recognize the newly-written data. In + * principle, if it fails, we should roll back, which we can in fact + * do on the assumption that nothing in step 8 can ever fail; on the + * other hand, if it's failed, then it's unlikely that rolling back by + * syncing the original header back to disk is going to work + * desperately well. We should perhaps take an operating-system lock + * around step 9, so that we can't be interrupted part-way through + * (except of course for SIGKILL, but if we're hit with that we will + * always lose). + */ +static int audiodb_insert_datum_internal(adb_t *adb, adb_datum_internal_t *datum) { + + off_t size, offset, nfiles; + double *l2norm_buffer = NULL; + + /* 1. check write permission; */ + if(!(adb->flags & O_RDWR)) { + return 1; + } + /* 2. check for enough space; */ + size = sizeof(double) * datum->nvectors * datum->dim; + if(!audiodb_enough_data_space_free(adb, size)) { + return 1; + } + if(!audiodb_enough_per_file_space_free(adb)) { + return 1; + } + /* 3. check that datum->dim and adb->header->dim agree (or that the + * header dimension is zero, in which case write datum->dim to + * adb->header->dim). + */ + if(adb->header->dim == 0) { + adb->header->dim = datum->dim; + } else if (adb->header->dim != datum->dim) { + return 1; + } + /* 4. check for presence of datum->key in adb->keymap; */ + if(adb->keymap->count(datum->key)) { + /* not part of an explicit API/ABI, but we need a distinguished + value in this circumstance to preserve somewhat wonky behaviour + of audioDB::batchinsert. */ + return 2; + } + /* 5. check for consistency between power and O2_FLAG_POWER, and + * times and O2_FLAG_TIMES; + */ + if((datum->power && !(adb->header->flags & O2_FLAG_POWER)) || + ((adb->header->flags & O2_FLAG_POWER) && !datum->power)) { + return 1; + } + if(datum->times && !(adb->header->flags & O2_FLAG_TIMES)) { + if(adb->header->numFiles == 0) { + adb->header->flags |= O2_FLAG_TIMES; + } else { + return 1; } + } else if ((adb->header->flags & O2_FLAG_TIMES) && !datum->times) { + return 1; + } + /* 6. write in data, power, times as appropriate; add to track + * and key tables too; + */ + offset = adb->header->length; + nfiles = adb->header->numFiles; + + /* FIXME: checking for all these lseek()s */ + lseek(adb->fd, adb->header->fileTableOffset + nfiles * O2_FILETABLE_ENTRY_SIZE, SEEK_SET); + write_or_goto_error(adb->fd, datum->key, strlen(datum->key)+1); + lseek(adb->fd, adb->header->trackTableOffset + nfiles * O2_TRACKTABLE_ENTRY_SIZE, SEEK_SET); + write_or_goto_error(adb->fd, &datum->nvectors, O2_TRACKTABLE_ENTRY_SIZE); + if(adb->header->flags & O2_FLAG_LARGE_ADB) { + char cwd[PATH_MAX]; + char slash = '/'; + + if(!getcwd(cwd, PATH_MAX)) { + goto error; + } + lseek(adb->fd, adb->header->dataOffset + nfiles * O2_FILETABLE_ENTRY_SIZE, SEEK_SET); + if(*((char *) datum->data) != '/') { + write_or_goto_error(adb->fd, cwd, strlen(cwd)); + write_or_goto_error(adb->fd, &slash, 1); + } + write_or_goto_error(adb->fd, datum->data, strlen((const char *) datum->data)+1); + if(datum->power) { + lseek(adb->fd, adb->header->powerTableOffset + nfiles * O2_FILETABLE_ENTRY_SIZE, SEEK_SET); + if(*((char *) datum->power) != '/') { + write_or_goto_error(adb->fd, cwd, strlen(cwd)); + write_or_goto_error(adb->fd, &slash, 1); + } + write_or_goto_error(adb->fd, datum->power, strlen((const char *) datum->power)+1); + } + if(datum->times) { + lseek(adb->fd, adb->header->timesTableOffset + nfiles * O2_FILETABLE_ENTRY_SIZE, SEEK_SET); + if(*((char *) datum->times) != '/') { + write_or_goto_error(adb->fd, cwd, strlen(cwd)); + write_or_goto_error(adb->fd, &slash, 1); + } + write_or_goto_error(adb->fd, datum->times, strlen((const char *) datum->times)+1); + } + } else { + lseek(adb->fd, adb->header->dataOffset + offset, SEEK_SET); + write_or_goto_error(adb->fd, datum->data, sizeof(double) * datum->nvectors * datum->dim); + if(datum->power) { + lseek(adb->fd, adb->header->powerTableOffset + offset / datum->dim, SEEK_SET); + write_or_goto_error(adb->fd, datum->power, sizeof(double) * datum->nvectors); + } + if(datum->times) { + lseek(adb->fd, adb->header->timesTableOffset + offset / datum->dim * 2, SEEK_SET); + write_or_goto_error(adb->fd, datum->times, sizeof(double) * datum->nvectors * 2); + } + } + + /* 7. if O2_FLAG_L2NORM and !O2_FLAG_LARGE_ADB, compute norms and fill + * in table; + */ + if((adb->header->flags & O2_FLAG_L2NORM) && + !(adb->header->flags & O2_FLAG_LARGE_ADB)) { + l2norm_buffer = (double *) malloc(datum->nvectors * sizeof(double)); - int one; - unsigned int count; + audiodb_l2norm_buffer((double *) datum->data, datum->dim, datum->nvectors, l2norm_buffer); + lseek(adb->fd, adb->header->l2normTableOffset + offset / datum->dim, SEEK_SET); + write_or_goto_error(adb->fd, l2norm_buffer, sizeof(double) * datum->nvectors); + free(l2norm_buffer); + l2norm_buffer = NULL; + } + + /* 8. update adb->keys, adb->keymap, adb->track_lengths, + * adb->track_offsets and adb->header; + */ + adb->keys->push_back(datum->key); + (*adb->keymap)[datum->key] = adb->header->numFiles; + adb->track_lengths->push_back(datum->nvectors); + adb->track_offsets->push_back(offset); + adb->header->numFiles += 1; + adb->header->length += sizeof(double) * datum->nvectors * datum->dim; + + /* 9. sync adb->header with disk. */ + return audiodb_sync_header(adb); + + error: + if(l2norm_buffer) { + free(l2norm_buffer); + } + return 1; +} + +int audiodb_insert_datum(adb_t *adb, const adb_datum_t *datum) { + if(adb->header->flags & O2_FLAG_LARGE_ADB) { + return 1; + } else { + adb_datum_internal_t d; + d.nvectors = datum->nvectors; + d.dim = datum->dim; + d.key = datum->key; + d.data = datum->data; + d.times = datum->times; + d.power = datum->power; + return audiodb_insert_datum_internal(adb, &d); + } +} + +int audiodb_insert_reference(adb_t *adb, const adb_reference_t *reference) { + if(!(adb->header->flags & O2_FLAG_LARGE_ADB)) { + return 1; + } else { + adb_datum_internal_t d; + struct stat st; + int fd; + off_t size; - count = read(powerfd, &one, sizeof(unsigned int)); - if (count != sizeof(unsigned int)) { - error("powerfd read failed", "int", "read"); + if((fd = open(reference->features, O_RDONLY)) == -1) { + return 1; } - if (one != 1) { - error("dimensionality of power file not 1", powerFileName); + if(fstat(fd, &st)) { + goto error; } - - // FIXME: should check that the powerfile is the right size for - // this. -- CSR, 2007-10-30 - count = read(powerfd, powerdata, numVectors * sizeof(double)); - if (count != numVectors * sizeof(double)) { - error("powerfd read failed", "double", "read"); + read_or_goto_error(fd, &(d.dim), sizeof(uint32_t)); + close(fd); + fd = 0; + size = st.st_size - sizeof(uint32_t); + d.nvectors = size / (sizeof(double) * d.dim); + d.data = (void *) reference->features; + if(reference->power) { + if(stat(reference->power, &st)) { + return 1; + } + } + d.power = (void *) reference->power; + if(reference->times) { + if(stat(reference->times, &st)) { + return 1; + } + } + d.times = (void *) reference->times; + d.key = reference->key ? reference->key : reference->features; + return audiodb_insert_datum_internal(adb, &d); + error: + if(fd) { + close(fd); + } + return 1; + } +} + +int audiodb_free_datum(adb_datum_t *datum) { + if(datum->data) { + free(datum->data); + datum->data = NULL; + } + if(datum->power) { + free(datum->power); + datum->power = NULL; + } + if(datum->times) { + free(datum->times); + datum->times = NULL; + } + return 0; +} + +int audiodb_insert_create_datum(adb_insert_t *insert, adb_datum_t *datum) { + int fd = 0; + FILE *file = NULL; + struct stat st; + off_t size; + + datum->data = NULL; + datum->power = NULL; + datum->times = NULL; + if((fd = open(insert->features, O_RDONLY)) == -1) { + goto error; + } + if(fstat(fd, &st)) { + goto error; + } + read_or_goto_error(fd, &(datum->dim), sizeof(uint32_t)); + size = st.st_size - sizeof(uint32_t); + datum->nvectors = size / (sizeof(double) * datum->dim); + datum->data = (double *) malloc(size); + if(!datum->data) { + goto error; + } + read_or_goto_error(fd, datum->data, size); + close(fd); + fd = 0; + if(insert->power) { + int dim; + if((fd = open(insert->power, O_RDONLY)) == -1) { + goto error; + } + if(fstat(fd, &st)) { + goto error; + } + /* This cast is so non-trivial that it deserves a comment. + * + * The data types in this expression, left to right, are: off_t, + * size_t, off_t, uint32_t. The rules for conversions in + * arithmetic expressions with mixtures of integral types are + * essentially that the widest type wins, with unsigned types + * winning on a tie-break. + * + * Because we are enforcing (through the use of sufficient + * compiler flags, if necessary) that off_t be a (signed) 64-bit + * type, the only variability in this set of types is in fact the + * size_t. On 32-bit machines, size_t is uint32_t and so the + * coercions on both sides of the equality end up promoting + * everything to int64_t, which is fine. On 64-bit machines, + * however, the left hand side is promoted to a uint64_t, while + * the right hand side remains int64_t. + * + * The mixture of signed and unsigned types in comparisons is Evil + * Bad and Wrong, and gcc complains about it. (It's right to do + * so, actually). Of course in this case it will never matter + * because of the particular relationships between all of these + * numbers, so we just cast the left hand side to off_t, which + * will do the right thing for us on all platforms. + * + * I hate C. + */ + if(((off_t) (st.st_size - sizeof(uint32_t))) != (size / datum->dim)) { + goto error; + } + read_or_goto_error(fd, &dim, sizeof(uint32_t)); + if(dim != 1) { + goto error; + } + datum->power = (double *) malloc(size / datum->dim); + if(!datum->power) { + goto error; + } + read_or_goto_error(fd, datum->power, size / datum->dim); + close(fd); + } + if(insert->times) { + double t, *tp; + if(!(file = fopen(insert->times, "r"))) { + goto error; + } + datum->times = (double *) malloc(2 * size / datum->dim); + if(!datum->times) { + goto error; + } + if(fscanf(file, " %lf", &t) != 1) { + goto error; + } + tp = datum->times; + *tp++ = t; + for(unsigned int n = 0; n < datum->nvectors - 1; n++) { + if(fscanf(file, " %lf", &t) != 1) { + goto error; + } + *tp++ = t; + *tp++ = t; + } + if(fscanf(file, " %lf", &t) != 1) { + goto error; + } + *tp = t; + fclose(file); + } + datum->key = insert->key ? insert->key : insert->features; + return 0; + + error: + if(fd > 0) { + close(fd); + } + if(file) { + fclose(file); + } + audiodb_free_datum(datum); + return 1; +} + +int audiodb_insert(adb_t *adb, adb_insert_t *insert) { + if(adb->header->flags & O2_FLAG_LARGE_ADB) { + adb_reference_t *reference = insert; + int err; + err = audiodb_insert_reference(adb, reference); + + if(err == 2) { + return 0; + } else { + return err; + } + } else { + adb_datum_t datum; + int err; + + if(audiodb_insert_create_datum(insert, &datum)) { + return 1; + } + err = audiodb_insert_datum(adb, &datum); + audiodb_free_datum(&datum); + + if(err == 2) { + return 0; + } else { + return err; } } } -void audioDB::batchinsert(const char* dbName, const char* inFile) { - - forWrite = true; - initDBHeader(dbName); - - // Treat large ADB instances differently - if( dbH->flags & O2_FLAG_LARGE_ADB ){ - batchinsert_large_adb(dbName, inFile) ; - return; +int audiodb_batchinsert(adb_t *adb, adb_insert_t *insert, unsigned int size) { + int err; + for(unsigned int n = 0; n < size; n++) { + if((err = audiodb_insert(adb, &(insert[n])))) { + return err; + } } - - if(!key) - key=inFile; - std::ifstream *filesIn = 0; - std::ifstream *keysIn = 0; - std::ifstream* thisTimesFile = 0; - int thispowerfd = 0; - - if(!(filesIn = new std::ifstream(inFile))) - error("Could not open batch in file", inFile); - if(key && key!=inFile) - if(!(keysIn = new std::ifstream(key))) - error("Could not open batch key file",key); - - if(!usingTimes && (dbH->flags & O2_FLAG_TIMES)) - error("Must use timestamps with timestamped database","use --times"); - - if(!usingPower && (dbH->flags & O2_FLAG_POWER)) - error("Must use power with power-enabled database", dbName); - - unsigned totalVectors=0; - char *thisFile = new char[MAXSTR]; - char *thisKey = 0; - if (key && (key != inFile)) { - thisKey = new char[MAXSTR]; - } - char *thisTimesFileName = new char[MAXSTR]; - char *thisPowerFileName = new char[MAXSTR]; - - std::set<std::string> s; - - for (unsigned k = 0; k < dbH->numFiles; k++) { - s.insert(fileTable + k*O2_FILETABLE_ENTRY_SIZE); - } - - do { - filesIn->getline(thisFile,MAXSTR); - if(key && key!=inFile) { - keysIn->getline(thisKey,MAXSTR); - } else { - thisKey = thisFile; - } - if(usingTimes) { - timesFile->getline(thisTimesFileName,MAXSTR); - } - if(usingPower) { - powerFile->getline(thisPowerFileName, MAXSTR); - } - - if(filesIn->eof()) { - break; - } - initInputFile(thisFile); - - if(!enough_per_file_space_free()) { - error("batchinsert failed: no more room for metadata", thisFile); - } - - if(!enough_data_space_free(statbuf.st_size - sizeof(int))) { - error("batchinsert failed: no more room in database", thisFile); - } - - if(s.count(thisKey)) { - VERB_LOG(0, "key already exists in database: %s\n", thisKey); - } else { - s.insert(thisKey); - // Make a track index table of features to file indexes - unsigned numVectors = (statbuf.st_size-sizeof(int))/(sizeof(double)*dbH->dim); - if(!numVectors) { - VERB_LOG(0, "ignoring zero-length feature vector file: %s\n", thisKey); - } - else{ - if(usingTimes){ - if(timesFile->eof()) { - error("not enough timestamp files in timesList", timesFileName); - } - thisTimesFile = new std::ifstream(thisTimesFileName,std::ios::in); - if(!thisTimesFile->is_open()) { - error("Cannot open timestamp file", thisTimesFileName); - } - off_t insertoffset = dbH->length; - unsigned indexoffset = insertoffset / (dbH->dim*sizeof(double)); - double *timesdata = timesTable + 2*indexoffset; - if(2*(indexoffset + numVectors) > timesTableLength) { - error("out of space for times", key); - } - insertTimeStamps(numVectors, thisTimesFile, timesdata); - if(thisTimesFile) - delete thisTimesFile; - } - - if (usingPower) { - if(powerFile->eof()) { - error("not enough power files in powerList", powerFileName); - } - thispowerfd = open(thisPowerFileName, O_RDONLY); - if (thispowerfd < 0) { - error("failed to open power file", thisPowerFileName); - } - off_t insertoffset = dbH->length; - unsigned poweroffset = insertoffset / (dbH->dim * sizeof(double)); - double *powerdata = powerTable + poweroffset; - insertPowerData(numVectors, thispowerfd, powerdata); - if (0 < thispowerfd) { - close(thispowerfd); - } - } - - INSERT_FILETABLE_STRING(fileTable, thisKey); - - off_t insertoffset = dbH->length;// Store current state - - // Increment file count - dbH->numFiles++; - - // Update Header information - dbH->length+=(statbuf.st_size-sizeof(int)); - - // Update track to file index map - memcpy (trackTable+dbH->numFiles-1, &numVectors, sizeof(unsigned)); - - insert_data_vectors(insertoffset, indata + sizeof(int), statbuf.st_size - sizeof(int)); - - // Norm the vectors on input if the database is already L2 normed - if(dbH->flags & O2_FLAG_L2NORM) - unitNormAndInsertL2((double *)(indata + sizeof(int)), dbH->dim, numVectors, 1); // append - - totalVectors+=numVectors; - - // Copy the header back to the database - memcpy (db, dbH, sizeof(dbTableHeaderT)); - } - } - - // CLEAN UP - munmap(indata,statbuf.st_size); - indata = NULL; - close(infid); - infid = 0; - } while(!filesIn->eof()); - - VERB_LOG(0, "%s %s %u vectors %ju bytes.\n", COM_BATCHINSERT, dbName, totalVectors, (intmax_t) (totalVectors * dbH->dim * sizeof(double))); - - delete [] thisPowerFileName; - if(key && (key != inFile)) { - delete [] thisKey; - } - delete [] thisFile; - delete [] thisTimesFileName; - - delete filesIn; - delete keysIn; - - // Report status - status(dbName); + return 0; } - - -// BATCHINSERT_LARGE_ADB -// -// This method inserts file pointers into the ADB instance rather than the actual feature data -// -// This method is intended for databases that are large enough to only support indexed query -// So exhaustive searching across all feature vectors will not be performed -// -// We insert featureFileName, [powerFileName], [timesFileName] -// -// l2norms and power sequence sums are calculated on-the-fly at INDEX and --lsh_exact QUERY time -// -// LIMITS: -// -// We impose an upper limit of 1M keys, 1M featureFiles, 1M powerFiles and 1M timesFiles -// -void audioDB::batchinsert_large_adb(const char* dbName, const char* inFile) { - - if(!key) - key=inFile; - std::ifstream *filesIn = 0; - std::ifstream *keysIn = 0; - std::ifstream* thisTimesFile = 0; - int thispowerfd = 0; - - if(!(filesIn = new std::ifstream(inFile))) - error("Could not open batch in file", inFile); - if(key && key!=inFile) - if(!(keysIn = new std::ifstream(key))) - error("Could not open batch key file",key); - - if(!usingTimes && (dbH->flags & O2_FLAG_TIMES)) - error("Must use timestamps with timestamped database","use --times"); - - if(!usingPower && (dbH->flags & O2_FLAG_POWER)) - error("Must use power with power-enabled database", dbName); - - char *cwd = new char[PATH_MAX]; - - if ((getcwd(cwd, PATH_MAX)) == 0) { - error("error getting working directory", "", "getcwd"); - } - - unsigned totalVectors=0; - char *thisFile = new char[MAXSTR]; - char *thisKey = 0; - if (key && (key != inFile)) { - thisKey = new char[MAXSTR]; - } - char *thisTimesFileName = new char[MAXSTR]; - char *thisPowerFileName = new char[MAXSTR]; - - std::set<std::string> s; - - for (unsigned k = 0; k < dbH->numFiles; k++) { - s.insert(fileTable + k*O2_FILETABLE_ENTRY_SIZE); - } - - do { - filesIn->getline(thisFile,MAXSTR); - if(key && key!=inFile) { - keysIn->getline(thisKey,MAXSTR); - } else { - thisKey = thisFile; - } - if(usingTimes) { - timesFile->getline(thisTimesFileName,MAXSTR); - } - if(usingPower) { - powerFile->getline(thisPowerFileName, MAXSTR); - } - - if(filesIn->eof()) { - break; - } - - initInputFile(thisFile, false); - - if(!enough_per_file_space_free()) { - error("batchinsert failed: no more room for metadata", thisFile); - } - - if(s.count(thisKey)) { - VERB_LOG(0, "key already exists in database: %s\n", thisKey); - } else { - s.insert(thisKey); - // Make a track index table of features to file indexes - unsigned numVectors = (statbuf.st_size-sizeof(int))/(sizeof(double)*dbH->dim); - if(!numVectors) { - VERB_LOG(0, "ignoring zero-length feature vector file: %s\n", thisKey); - } - else{ - // Check that time-stamp file exists - if(usingTimes){ - if(timesFile->eof()) { - error("not enough timestamp files in timesList", timesFileName); - } - thisTimesFile = new std::ifstream(thisTimesFileName,std::ios::in); - if(!thisTimesFile->is_open()) { - error("Cannot open timestamp file", thisTimesFileName); - } - if(thisTimesFile) - delete thisTimesFile; - } - - // Check that power file exists - if (usingPower) { - if(powerFile->eof()) { - error("not enough power files in powerList", powerFileName); - } - thispowerfd = open(thisPowerFileName, O_RDONLY); - if (thispowerfd < 0) { - error("failed to open power file", thisPowerFileName); - } - if (0 < thispowerfd) { - close(thispowerfd); - } - } - - // persist links to the feature files for reading from filesystem later - - // Primary Keys - INSERT_FILETABLE_STRING(fileTable, thisKey); - - if(*thisFile != '/') { - /* FIXME: MAXSTR and O2_FILETABLE_ENTRY_SIZE should probably - be the same thing. Also, both are related to PATH_MAX, - which admittedly is not always defined or a - constant... */ - char tmp[MAXSTR]; - strncpy(tmp, thisFile, MAXSTR); - snprintf(thisFile, MAXSTR, "%s/%s", cwd, tmp); - } - // Feature Vector fileNames - INSERT_FILETABLE_STRING(featureFileNameTable, thisFile); - - // Time Stamp fileNames - if(usingTimes) { - if(*thisTimesFileName != '/') { - char tmp[MAXSTR]; - strncpy(tmp, thisTimesFileName, MAXSTR); - snprintf(thisTimesFileName, MAXSTR, "%s/%s", cwd, tmp); - } - INSERT_FILETABLE_STRING(timesFileNameTable, thisTimesFileName); - } - - // Power fileNames - if(usingPower) { - if(*thisPowerFileName != '/') { - char tmp[MAXSTR]; - strncpy(tmp, thisPowerFileName, MAXSTR); - snprintf(thisPowerFileName, MAXSTR, "%s/%s", cwd, tmp); - } - INSERT_FILETABLE_STRING(powerFileNameTable, thisPowerFileName); - } - - // Increment file count - dbH->numFiles++; - - // Update Header information - dbH->length+=(statbuf.st_size-sizeof(int)); - - // Update track to file index map - memcpy (trackTable+dbH->numFiles-1, &numVectors, sizeof(unsigned)); - - totalVectors+=numVectors; - - // Copy the header back to the database - memcpy (db, dbH, sizeof(dbTableHeaderT)); - } - } - // CLEAN UP - if(indata) - munmap(indata,statbuf.st_size); - if(infid>0) - close(infid); - } while(!filesIn->eof()); - - VERB_LOG(0, "%s %s %u vectors %ju bytes.\n", COM_BATCHINSERT, dbName, totalVectors, (intmax_t) (totalVectors * dbH->dim * sizeof(double))); - - delete [] thisPowerFileName; - if(key && (key != inFile)) { - delete [] thisKey; - } - delete [] thisFile; - delete [] thisTimesFileName; - - delete filesIn; - delete keysIn; - - // Report status - status(dbName); -}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/l2norm.cpp Sat Jan 10 16:47:57 2009 +0000 @@ -0,0 +1,60 @@ +#include "audioDB.h" +extern "C" { +#include "audioDB_API.h" +#include "audioDB-internals.h" +} + +static int audiodb_l2norm_existing(adb_t *adb) { + double *data_buffer, *l2norm_buffer; + adb_header_t *header = adb->header; + size_t data_buffer_size = ALIGN_PAGE_UP(header->length); + size_t nvectors = header->length / (sizeof(double) * header->dim); + /* FIXME: this map of the vector data will lose if we ever turn the + * l2norm flag on when we have already inserted a large number of + * vectors, as the mmap() will fail. "Don't do that, then" is one + * possible answer. */ + mmap_or_goto_error(double *, data_buffer, header->dataOffset, data_buffer_size); + l2norm_buffer = (double *) malloc(nvectors * sizeof(double)); + if(!l2norm_buffer) { + goto error; + } + audiodb_l2norm_buffer(data_buffer, header->dim, nvectors, l2norm_buffer); + if(lseek(adb->fd, adb->header->l2normTableOffset, SEEK_SET) == (off_t) -1) { + goto error; + } + write_or_goto_error(adb->fd, l2norm_buffer, nvectors * sizeof(double)); + + munmap(data_buffer, data_buffer_size); + free(l2norm_buffer); + + return 0; + + error: + maybe_munmap(data_buffer, data_buffer_size); + if(l2norm_buffer) { + free(l2norm_buffer); + } + return 1; +} + +int audiodb_l2norm(adb_t *adb) { + adb_header_t *header = adb->header; + if(!(adb->flags & O_RDWR)) { + return 1; + } + if(header->flags & O2_FLAG_L2NORM) { + /* non-error code for forthcoming backwards-compatibility + * reasons */ + return 0; + } + if((!(header->flags & O2_FLAG_LARGE_ADB)) && (header->length > 0)) { + if(audiodb_l2norm_existing(adb)) { + goto error; + } + } + adb->header->flags |= O2_FLAG_L2NORM; + return audiodb_sync_header(adb); + + error: + return 1; +}
--- a/libtests/0001/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0001/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,88 +1,34 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -#include <errno.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + struct stat st; + clean_remove_db(TESTDB); -int main(int argc, char **argv){ + adb = audiodb_open(TESTDB, O_RDWR); + if(adb) + return 1; - int returnval=0; - adb_ptr mydbp={0}; - adb_ptr mydbp2={0}; - struct stat statbuf; - int statval=0; + adb = audiodb_create(TESTDB, 0, 0, 0); + if (!adb) + return 1; - char * databasename="testdb"; + if(stat(TESTDB, &st)) + return 1; - //if [ -f testdb ]; then rm -f testdb; fi - /* remove old directory */ - clean_remove_db(databasename); + audiodb_close(adb); - /* create new db */ - //# creation - //${AUDIODB} -N -d testdb - mydbp=audiodb_open(databasename); + adb = audiodb_create(TESTDB, 0, 0, 0); + if(adb) + return 1; + adb = audiodb_open(TESTDB, O_RDONLY); + if (!adb) + return 1; - /* open should fail (return NULL), so create a new db */ - if (!mydbp){ - mydbp=audiodb_create(databasename,0,0,0); - } + audiodb_close(adb); - - - if (!mydbp){ - printf("fail\n"); - returnval=-1; - } - - - /* stat testdb - let's make sure that it is there */ - //stat testdb - statval=stat(databasename, &statbuf); - - if (statval){ - returnval=-1; - } - - audiodb_close(mydbp); - - /* try to create should fail, because db exists now */ - mydbp2=audiodb_create(databasename,0,0,0); - - if (mydbp2){ - returnval=-1; - } - - -/* should pass now - db exists */ -//expect_clean_error_exit ${AUDIODB} -N -d testdb - mydbp2=audiodb_open(databasename); - if (!mydbp2){ - returnval=-1; - } - -//this test would fail at compile time because of the API interface -//# should fail (no db given) -//expect_clean_error_exit ${AUDIODB} -N - - - audiodb_close(mydbp2); - -// printf("returnval:%d\n",returnval); - - return(returnval); + return 104; } -
--- a/libtests/0001/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0001/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -DB creation using lib +DB creation with library
--- a/libtests/0002/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0002/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,58 +1,22 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + adb_status_t status={0}; + clean_remove_db(TESTDB); -int main(int argc, char **argv){ + adb = audiodb_create(TESTDB, 0, 0, 0); - int returnval=0; - adb_ptr mydbp={0}; - adb_status_t mystatus={0}; + if(audiodb_status(adb, &status)){ + return 1; + } + if(status.numFiles != 0) + return 1; - char * databasename="testdb"; + audiodb_close(adb); -//. ../test-utils.sh -// -//if [ -f testdb ]; then rm -f testdb; fi -// - /* remove old directory */ - clean_remove_db(databasename); - - /* create new db */ -//${AUDIODB} -N -d testdb -// - mydbp=audiodb_create(databasename,0,0,0); - - -//# FIXME: at some point we will want to test that some relevant -//# information is being printed -//${AUDIODB} -S -d testdb -//${AUDIODB} -d testdb -S - - if(audiodb_status(mydbp,&mystatus)){ - returnval=-1; - } - -/* not relevent, caught by API */ -//# should fail (no db given) -//expect_clean_error_exit ${AUDIODB} -S - - - - audiodb_close(mydbp); - - return(returnval); + return 104; }
--- a/libtests/0002/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0002/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -DB status using lib +DB status with library
--- a/libtests/0003/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0003/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,91 +1,39 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; + if(audiodb_l2norm(adb)) + return 1; -int main(int argc, char **argv){ + adb_datum_t datum = {1, 1, "testfeature", (double[1]) {1}, NULL, NULL}; + if(audiodb_insert_datum(adb, &datum)) + return 1; - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]={0}; - double dvals[10]={0.0}; - adb_insert_t myinsert={0}; - unsigned int myerr=0; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - char * databasename="testdb"; - int size=0; + adb_query_id_t qid = {0}; + qid.datum = &datum; + qid.sequence_length = 1; + qid.sequence_start = 0; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_DB, ADB_DISTANCE_DOT_PRODUCT, 10, 0}; + adb_query_refine_t refine = {0}; + refine.hopsize = 1; + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; - /* remove old directory */ - //if [ -f testdb ]; then rm -f testdb; fi - // - clean_remove_db(databasename); + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 1, 0, 0); + audiodb_query_free_results(adb, &spec, results); + audiodb_close(adb); - /* create new db */ - //${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); - - /* turn on l2norm */ - //# point query now implemented as sequence search - //${AUDIODB} -d testdb -L - audiodb_l2norm(mydbp); - - /* make a test file */ - //# FIXME: endianness! - //intstring 1 > testfeature - //floatstring 1 >> testfeature - ivals[0]=1; - dvals[0]=1; - maketestfile("testfeature",ivals,dvals,1); - - /* insert */ - //${AUDIODB} -d testdb -I -f testfeature - myinsert.features="testfeature"; - myerr=audiodb_insert(mydbp,&myinsert); - - /* query */ - //${AUDIODB} -d testdb -Q point -f testfeature > test-query-output - myadbquery.querytype="point"; - myadbquery.feature="testfeature"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - - /* check the test values */ - //echo testfeature 1 0 0 > test-expected-query-output - //cmp test-query-output test-expected-query-output - size=myadbqueryresult.sizeRlist; - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,0)) {returnval = -1;}; - - -//# -//## failure cases -//expect_clean_error_exit ${AUDIODB} -d testdb -I -//expect_clean_error_exit ${AUDIODB} -d testdb -f testfeature -//expect_clean_error_exit ${AUDIODB} -I -f testfeature -//expect_clean_error_exit ${AUDIODB} -d testdb -Q notpoint -f testfeature -//expect_clean_error_exit ${AUDIODB} -Q point -f testfeature -/* all of these will fail at compile time because of API */ - - audiodb_close(mydbp); - - return(returnval); + return 104; } - - - - -
--- a/libtests/0004/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0004/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,140 +1,62 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; + if(audiodb_l2norm(adb)) + return 1; -int main(int argc, char **argv){ + adb_datum_t feature = {2, 2, "testfeature", (double[4]) {0, 1, 1, 0}}; + if(audiodb_insert_datum(adb, &feature)) + return 1; - int returnval=0; - adb_ptr mydbp={0}; - adb_status_t mystatus={0}; - int ivals[10]; - double dvals[10]; - FILE * myfile; - int ret=0; - adb_insert_t myinsert={0}; - unsigned int myerr=0; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - adb_query_t myadbquery2={0}; - adb_queryresult_t myadbqueryresult2={0}; - char * databasename="testdb"; - int i=0; - int size=0; + adb_datum_t query = {1, 2, "testquery", (double [2]) {0, 0.5}}; + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 1; + qid.sequence_start = 0; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_DB, ADB_DISTANCE_DOT_PRODUCT, 10, 0}; + adb_query_refine_t refine = {0}; + refine.hopsize = 1; - /* remove old directory */ - //if [ -f testdb ]; then rm -f testdb; fi - clean_remove_db(databasename); + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; - /* create new db */ - //${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0.5, 0, 0); + result_present_or_fail(results, "testfeature", 0, 0, 1); + audiodb_query_free_results(adb, &spec, results); - /* turn on l2norm */ - //${AUDIODB} -d testdb -L - audiodb_l2norm(mydbp); + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 0.5, 0, 0); + audiodb_query_free_results(adb, &spec, results); - /* make a test file */ - //intstring 2 > testfeature - //floatstring 0 1 >> testfeature - //floatstring 1 0 >> testfeature - ivals[0]=2; - dvals[0]=0; dvals[1]=1; dvals[2]=1; dvals[3]=0; - maketestfile("testfeature", ivals,dvals,4); + spec.qid.datum->data = (double[2]) {0.5, 0}; + spec.params.npoints = 10; + results = audiodb_query_spec(adb, &spec); + result_present_or_fail(results, "testfeature", 0.5, 0, 1); + result_present_or_fail(results, "testfeature", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 0.5, 0, 1); + audiodb_query_free_results(adb, &spec, results); - /* insert */ - //${AUDIODB} -d testdb -I -f testfeature - myinsert.features="testfeature"; - myerr=audiodb_insert(mydbp,&myinsert); + audiodb_close(adb); - /* testquery */ - //echo "query point (0.0,0.5)" - //intstring 2 > testquery - //floatstring 0 0.5 >> testquery - ivals[0]=2; - dvals[0]=0; dvals[1]=0.5; dvals[2]=0; dvals[3]=0; - maketestfile("testquery", ivals,dvals,4); - - /* query 1 */ - //${AUDIODB} -d testdb -Q point -f testquery > testoutput - //echo testfeature 0.5 0 0 > test-expected-output - //echo testfeature 0 0 1 >> test-expected-output - //cmp testoutput test-expected-output - myadbquery.querytype="point"; - myadbquery.feature="testquery"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 2) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",.5,0,0)) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,1,"testfeature",0,0,1)) {returnval = -1;}; - - /* query 2 - same but only first result */ -//${AUDIODB} -d testdb -Q point -f testquery -n 1 > testoutput -//echo testfeature 0.5 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.numpoints="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",.5,0,0)) {returnval = -1;}; - - - /* testquery2 */ - //echo "query point (0.5,0.0)" - //intstring 2 > testquery - //floatstring 0.5 0 >> testquery - ivals[0]=2; - dvals[0]=0.5; dvals[1]=0; dvals[2]=0; dvals[3]=0; - maketestfile("testquery", ivals,dvals,4); - - /* query 3 */ - //${AUDIODB} -d testdb -Q point -f testquery > testoutput - //echo testfeature 0.5 0 1 > test-expected-output - //echo testfeature 0 0 0 >> test-expected-output - //cmp testoutput test-expected-output - myadbquery2.querytype="point"; - myadbquery2.feature="testquery"; - myadbquery2.numpoints=NULL; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - /* check the test values */ - if (size != 2) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,0,"testfeature",.5,0,1)) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,1,"testfeature",0,0,0)) {returnval = -1;}; - - /* query 4 - same as 3 but only first result */ - //${AUDIODB} -d testdb -Q point -f testquery -n 1 > testoutput - //echo testfeature 0.5 0 1 > test-expected-output - //cmp testoutput test-expected-output - myadbquery2.numpoints="1"; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,0,"testfeature",.5,0,1)) {returnval = -1;}; - - audiodb_close(mydbp); - - - return(returnval); + return 104; }
--- a/libtests/0005/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0005/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,69 +1,22 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; -int main(int argc, char **argv){ + adb_datum_t datum = {2, 2, "testfeature", (double[4]) {0, 1, 1, 0}}; + if(audiodb_insert_datum(adb,&datum)) + return 1; - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - unsigned int myerr=0; - char * databasename="testdb"; - int myerror=0; + if(audiodb_l2norm(adb)) + return 1; + audiodb_close(adb); - /* remove old directory */ - //if [ -f testdb ]; then rm -f testdb; fi - clean_remove_db(databasename); - - /* create new db */ - //${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); - - - /* make a test file */ - //intstring 2 > testfeature - //floatstring 0 1 >> testfeature - //floatstring 1 0 >> testfeature - ivals[0]=2; - dvals[0]=0; dvals[1]=1; dvals[2]=1; dvals[3]=0; - maketestfile("testfeature",ivals,dvals,4); - - - /* insert */ - //${AUDIODB} -d testdb -I -f testfeature - myinsert.features="testfeature"; - myerr=audiodb_insert(mydbp,&myinsert); - - /* turn on l2norm */ - //echo running L2Norm - //${AUDIODB} -d testdb -L - myerror=audiodb_l2norm(mydbp); - if (myerror){ - returnval=-1; - } - - - /* close */ - audiodb_close(mydbp); - - - - return(returnval); + return 104; }
--- a/libtests/0005/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0005/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -L2 Norm with lib +l2 norming after insertion with library
--- a/libtests/0006/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0006/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,147 +1,60 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; -int main(int argc, char **argv){ + adb_datum_t feature = {2, 2, "testfeature", (double[4]) {0, 1, 1, 0}}; + if(audiodb_insert_datum(adb, &feature)) + return 1; + audiodb_l2norm(adb); - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - unsigned int myerr=0; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - adb_query_t myadbquery2={0}; - adb_queryresult_t myadbqueryresult2={0}; - int size=0; + adb_datum_t query = {1, 2, "testquery", (double[2]) {0, 0.5}}; + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 1; + qid.sequence_start = 0; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_EUCLIDEAN_NORMED, 10, 10}; + adb_query_refine_t refine = {0}; + refine.hopsize = 1; + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; - /* remove old directory */ - //if [ -f testdb ]; then rm -f testdb; fi - clean_remove_db(databasename); + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + result_present_or_fail(results, "testfeature", 2, 0, 1); + audiodb_query_free_results(adb, &spec, results); - /* create new db */ - //${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); - /* create testfeature file */ - //intstring 2 > testfeature - //floatstring 0 1 >> testfeature - //floatstring 1 0 >> testfeature - ivals[0]=2; - dvals[0]=0; dvals[1]=1; dvals[2]=1; dvals[3]=0; - maketestfile("testfeature",ivals,dvals,4); + spec.qid.datum->data = (double [2]) {0.5, 0}; + spec.params.npoints = 10; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 1); + result_present_or_fail(results, "testfeature", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 1); + audiodb_query_free_results(adb, &spec, results); - /* insert */ - //${AUDIODB} -d testdb -I -f testfeature - myinsert.features="testfeature"; - if(audiodb_insert(mydbp,&myinsert)){ - returnval=-1; - }; + audiodb_close(adb); - - /* turn on L2NORM */ - //# sequence queries require L2NORM - //${AUDIODB} -d testdb -L - audiodb_l2norm(mydbp); - - /* make a test query */ - //echo "query point (0.0,0.5)" - //intstring 2 > testquery - //floatstring 0 0.5 >> testquery - ivals[0]=2; - dvals[0]=0; dvals[1]=0.5; dvals[2]=0; dvals[3]=0; - maketestfile("testquery",ivals,dvals,2); - - /* test a sequence query */ - //${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput - //echo testfeature 1 0 0 > test-expected-output - //cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,0)) {returnval = -1;}; - - - /* same but with limites */ - //${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -n 1 > testoutput - //echo testfeature 0 0 0 > test-expected-output - //cmp testoutput test-expected-output - myadbquery.numpoints="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",0,0,0)) {returnval = -1;}; - - /* make another query */ - //echo "query point (0.5,0.0)" - //intstring 2 > testquery - //floatstring 0.5 0 >> testquery - ivals[0]=2; - dvals[0]=0.5; dvals[1]=0.0; dvals[2]=0; dvals[3]=0; - maketestfile("testquery",ivals,dvals,2); - - - /* test new query */ - //${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput - //echo testfeature 1 0 1 > test-expected-output - //cmp testoutput test-expected-output - - myadbquery2.querytype="sequence"; - myadbquery2.feature="testquery"; - myadbquery2.sequencelength="1"; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,0,"testfeature",1,0,1)) {returnval = -1;}; - - - /* test new query with limits */ - myadbquery2.numpoints="1"; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - //${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -n 1 > testoutput - //echo testfeature 0 0 1 > test-expected-output - //cmp testoutput test-expected-output - - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,0,"testfeature",0,0,1)) {returnval = -1;}; - - - - /* close */ - audiodb_close(mydbp); - - printf("returnval:%d\n",returnval); - return(returnval); + return 104; } - -
--- a/libtests/0006/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0006/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -sequence search / 1 track with lib +sequence search / 1 track with library
--- a/libtests/0007/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0007/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,124 +1,64 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; -int main(int argc, char **argv){ + adb_datum_t feature = {2, 2, "testfeature", (double[4]) {0, 1, 1, 0}}; + if(audiodb_insert_datum(adb, &feature)) + return 1; + audiodb_l2norm(adb); - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - unsigned int myerr=0; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - // adbquery myadbquery2={0}; - // adbqueryresult myadbqueryresult2={0}; - int size=0; + adb_datum_t query = {1, 2, "testquery", (double[2]) {0, 0.5}}; + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 16; + qid.sequence_start = 0; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_EUCLIDEAN_NORMED, 10, 10}; + adb_query_refine_t refine = {0}; + refine.hopsize = 1; - /* remove old directory */ - clean_remove_db(databasename); + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(results) return 1; - /* create new db */ - mydbp=audiodb_create(databasename,0,0,0); + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + if(results) return 1; -//# tests that the lack of -l when the query sequence is shorter doesn't -//# segfault. + spec.qid.datum->data = (double [2]) {0.5, 0}; + spec.params.npoints = 10; + results = audiodb_query_spec(adb, &spec); + if(results) return 1; + + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + if(results) return 1; - /* make test file */ - //intstring 2 > testfeature - //floatstring 0 1 >> testfeature - //floatstring 1 0 >> testfeature - ivals[0]=2; - dvals[0]=0; dvals[1]=1; dvals[2]=1; dvals[3]=0; - maketestfile("testfeature",ivals,dvals,4); + /* the above tests mirror those in the audioDB command-line test + * suite. We can test for additional bad input cases too: */ - /* insert */ - //${AUDIODB} -d testdb -I -f testfeature - myinsert.features="testfeature"; - myerr=audiodb_insert(mydbp,&myinsert); - if(myerr){ returnval=-1; }; + spec.qid.sequence_start = 1; + spec.qid.sequence_length = 1; + results = audiodb_query_spec(adb, &spec); + if(results) return 1; + /* and just sanity check that we haven't broken everything */ + spec.qid.sequence_start = 0; + spec.params.npoints = 2; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + audiodb_query_free_results(adb, &spec, results); - /* turn on l2norm */ - //# sequence queries require L2NORM - //${AUDIODB} -d testdb -L - if(audiodb_l2norm(mydbp)){ returnval=-1; }; + audiodb_close(adb); - - /* make query */ - //echo "query point (0.0,0.5)" - //intstring 2 > testquery - //floatstring 0 0.5 >> testquery - ivals[0]=2; - dvals[0]=0; dvals[1]=0.5; - maketestfile("testquery",ivals,dvals,2); - - -/* should fail */ - -//audioDB -Q sequence -d testdb -f testquery -//expect_clean_error_exit ${AUDIODB} -d testdb -Q sequence -f testquery - - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - //myadbquery.sequencelength="1"; - myerr=audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - if (!myerr){ returnval = -1;}; - - -///* should fail */ -//expect_clean_error_exit ${AUDIODB} -d testdb -Q sequence -f testquery -n 1 - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.numpoints="1"; - myerr=audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - if(!myerr){ returnval=-1; }; - -/* query 2 */ -//echo "query point (0.5,0.0)" -//intstring 2 > testquery -//floatstring 0.5 0 >> testquery - ivals[0]=2; - dvals[0]=0.5; dvals[1]=0.0; - maketestfile("testquery",ivals,dvals,2); - -/* should fail */ -//expect_clean_error_exit ${AUDIODB} -d testdb -Q sequence -f testquery - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.numpoints=NULL; - myerr=audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - if(!myerr){ returnval=-1; }; - -/* should fail */ -//expect_clean_error_exit ${AUDIODB} -d testdb -Q sequence -f testquery -n 1 - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.numpoints="1"; - myerr=audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - if(!myerr){ returnval=-1; }; - - - - - //printf("returnval:%d\n", returnval); - - return(returnval); + return 104; } -
--- a/libtests/0007/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0007/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -short query [no -l] error with lib +short query [no -l] error with library
--- a/libtests/0008/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0008/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,165 +1,66 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB,0,0,0))) + return 1; -int main(int argc, char **argv){ + adb_datum_t datum1 = {1, 2, "testfeature01", (double[4]) {0, 1}}; + adb_datum_t datum2 = {1, 2, "testfeature10", (double[4]) {1, 0}}; + if(audiodb_insert_datum(adb, &datum1)) + return 1; + if(audiodb_insert_datum(adb, &datum2)) + return 1; - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - unsigned int myerr=0; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - adb_query_t myadbquery2={0}; - adb_queryresult_t myadbqueryresult2={0}; - int size=0; + if(audiodb_l2norm(adb)) + return 1; + adb_datum_t query = {1, 2, "testquery", (double[2]) {0, 0.5}}; + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 1; + qid.sequence_start = 0; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_EUCLIDEAN_NORMED, 10, 10}; + adb_query_refine_t refine = {0}; + refine.hopsize = 1; + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; + + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + result_present_or_fail(results, "testfeature10", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); - /* remove old directory */ - clean_remove_db(databasename); + spec.params.ntracks = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); + spec.qid.datum->data = (double[2]) {0.5, 0}; + spec.params.ntracks = 10; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature10", 0, 0, 0); + result_present_or_fail(results, "testfeature01", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); + spec.params.ntracks = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature10", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); -//${AUDIODB} -d testdb -N - /* create new db */ - mydbp=audiodb_create(databasename,0,0,0); + audiodb_close(adb); - -//intstring 2 > testfeature01 -//floatstring 0 1 >> testfeature01 -//intstring 2 > testfeature10 -//floatstring 1 0 >> testfeature10 - - /* create testfeature01 file */ - ivals[0]=2; - dvals[0]=0; dvals[1]=1; dvals[2]=0; dvals[3]=0; - maketestfile("testfeature01",ivals,dvals,2); - - /* create testfeature10 file */ - ivals[0]=2; - dvals[0]=1; dvals[1]=0; dvals[2]=0; dvals[3]=0; - maketestfile("testfeature10",ivals,dvals,2); - -//${AUDIODB} -d testdb -I -f testfeature01 -//${AUDIODB} -d testdb -I -f testfeature10 - - /* insert */ - myinsert.features="testfeature01"; - myerr=audiodb_insert(mydbp,&myinsert); - - myinsert.features="testfeature10"; - myerr=audiodb_insert(mydbp,&myinsert); - -//# sequence queries require L2NORM -//${AUDIODB} -d testdb -L - audiodb_l2norm(mydbp); - -//echo "query point (0.0,0.5)" -//intstring 2 > testquery -//floatstring 0 0.5 >> testquery - - /* create testquery file */ - ivals[0]=2; - dvals[0]=0.0; dvals[1]=0.5; dvals[2]=0; dvals[3]=0; - maketestfile("testquery",ivals,dvals,2); - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput -//echo testfeature01 0 0 0 > test-expected-output -//echo testfeature10 2 0 0 >> test-expected-output -//cmp testoutput test-expected-output - - /* query */ - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - - - /* check the test values */ - if (size != 2) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature01",0,0,0)) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,1,"testfeature10",2,0,0)) {returnval = -1;}; - -////${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -r 1 > testoutput -////echo testfeature01 0 0 0 > test-expected-output -////cmp testoutput test-expected-output - - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.resultlength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature01",0,0,0)) {returnval = -1;}; - -//echo "query point (0.5,0.0)" -//intstring 2 > testquery -//floatstring 0.5 0 >> testquery - - /* create testquery file */ - ivals[0]=2; - dvals[0]=0.5; dvals[1]=0.0; dvals[2]=0; dvals[3]=0; - maketestfile("testquery",ivals,dvals,2); - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput -//echo testfeature10 0 0 0 > test-expected-output -//echo testfeature01 2 0 0 >> test-expected-output -//cmp testoutput test-expected-output - - myadbquery2.querytype="sequence"; - myadbquery2.feature="testquery"; - myadbquery2.sequencelength="1"; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - /* check the test values */ - if (size != 2) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,0,"testfeature10",0,0,0)) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,1,"testfeature01",2,0,0)) {returnval = -1;}; - - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -r 1 > testoutput -//echo testfeature10 0 0 0 > test-expected-output -//cmp testoutput test-expected-output - - - myadbquery2.querytype="sequence"; - myadbquery2.feature="testquery"; - myadbquery2.sequencelength="1"; - myadbquery2.resultlength="1"; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,0,"testfeature10",0,0,0)) {returnval = -1;}; - - - - return(returnval); + return 104; } -
--- a/libtests/0008/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0008/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -sequence search / 2 tracks with lib +sequence search / 2 tracks with library
--- a/libtests/0009/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0009/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,166 +1,65 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; + if(audiodb_l2norm(adb)) + return 1; -int main(int argc, char **argv){ + adb_datum_t datum1 = {1, 2, "testfeature01", (double[2]) {0, 1}}; + adb_datum_t datum2 = {1, 2, "testfeature10", (double[2]) {1, 0}}; + if(audiodb_insert_datum(adb, &datum1)) + return 1; + if(audiodb_insert_datum(adb, &datum2)) + return 1; - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - unsigned int myerr=0; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - adb_query_t myadbquery2={0}; - adb_queryresult_t myadbqueryresult2={0}; - int size=0; + adb_datum_t query = {1, 2, "testquery", (double[2]) {0, 0.5}}; + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 1; + qid.sequence_start = 0; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_DOT_PRODUCT, 10, 10}; + adb_query_refine_t refine = {0}; + refine.hopsize = 1; + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; - //if [ -f testdb ]; then rm -f testdb; fi + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature01", 0.5, 0, 0); + result_present_or_fail(results, "testfeature10", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); - /* remove old directory */ - clean_remove_db(databasename); + spec.params.ntracks = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature01", 0.5, 0, 0); + audiodb_query_free_results(adb, &spec, results); + spec.qid.datum->data = (double [2]) {0.5, 0}; + spec.params.ntracks = 10; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature10", 0.5, 0, 0); + result_present_or_fail(results, "testfeature01", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); - //${AUDIODB} -d testdb -N + spec.params.ntracks = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature10", 0.5, 0, 0); + audiodb_query_free_results(adb, &spec, results); - /* create new db */ - mydbp=audiodb_create(databasename,0,0,0); + audiodb_close(adb); - //${AUDIODB} -d testdb -L - if(audiodb_l2norm(mydbp)){ - returnval=-1; - }; - - //intstring 2 > testfeature01 - //floatstring 0 1 >> testfeature01 - //intstring 2 > testfeature10 - //floatstring 1 0 >> testfeature10 - - /* create testfeature01 file */ - ivals[0]=2; - dvals[0]=0; dvals[1]=1; dvals[2]=0; dvals[3]=0; - maketestfile("testfeature01",ivals,dvals,2); - - /* create testfeature10 file */ - ivals[0]=2; - dvals[0]=1; dvals[1]=0; dvals[2]=0; dvals[3]=0; - maketestfile("testfeature10",ivals,dvals,2); - - //${AUDIODB} -d testdb -I -f testfeature01 - //${AUDIODB} -d testdb -I -f testfeature10 - - /* insert */ - myinsert.features="testfeature01"; - myerr=audiodb_insert(mydbp,&myinsert); - myinsert.features="testfeature10"; - myerr=audiodb_insert(mydbp,&myinsert); - - //echo "query point (0.0,0.5)" - //intstring 2 > testquery - //floatstring 0 0.5 >> testquery - - - /* create testquery file */ - ivals[0]=2; - dvals[0]=0.0; dvals[1]=0.5; dvals[2]=0; dvals[3]=0; - maketestfile("testquery",ivals,dvals,2); - - //${AUDIODB} -d testdb -Q track -l 1 -f testquery > testoutput - //echo testfeature01 0.5 0 0 > test-expected-output - //echo testfeature10 0 0 0 >> test-expected-output - //cmp testoutput test-expected-output - - /* query */ - myadbquery.querytype="track"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 2) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature01",0.5,0,0)) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,1,"testfeature10",0,0,0)) {returnval = -1;}; - - //${AUDIODB} -d testdb -Q track -l 1 -f testquery -r 1 > testoutput - //echo testfeature01 0.5 0 0 > test-expected-output - //cmp testoutput test-expected-output - - /* query */ - myadbquery.querytype="track"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.resultlength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature01",0.5,0,0)) {returnval = -1;}; - - //echo "query point (0.5,0.0)" - //intstring 2 > testquery - //floatstring 0.5 0 >> testquery - - /* create testquery file */ - ivals[0]=2; - dvals[0]=0.5; dvals[1]=0.0; - maketestfile("testquery",ivals,dvals,2); - - //${AUDIODB} -d testdb -Q track -l 1 -f testquery > testoutput - //echo testfeature10 0.5 0 0 > test-expected-output - //echo testfeature01 0 0 0 >> test-expected-output - //cmp testoutput test-expected-output - - /* query */ - myadbquery2.querytype="track"; - myadbquery2.feature="testquery"; - myadbquery2.sequencelength="1"; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - - /* check the test values */ - if (size != 2) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,0,"testfeature10",0.5,0,0)) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,1,"testfeature01",0,0,0)) {returnval = -1;}; - - - //${AUDIODB} -d testdb -Q track -l 1 -f testquery -r 1 > testoutput - //echo testfeature10 0.5 0 0 > test-expected-output - //cmp testoutput test-expected-output - - /* query */ - myadbquery2.querytype="track"; - myadbquery2.feature="testquery"; - myadbquery2.sequencelength="1"; - myadbquery2.resultlength="1"; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,0,"testfeature10",0.5,0,0)) {returnval = -1;}; - - - return(returnval); + return 104; } -
--- a/libtests/0009/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0009/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -track search with lib +track search with library
--- a/libtests/0010/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0010/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,125 +1,71 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; + adb_datum_t datum1 = {1, 2, "testfeature01", (double[2]) {0, 1}}; + adb_datum_t datum2 = {1, 2, "testfeature10", (double[2]) {1, 0}}; + if(audiodb_insert_datum(adb, &datum1)) + return 1; + if(audiodb_insert_datum(adb, &datum2)) + return 1; + if(audiodb_l2norm(adb)) + return 1; -int main(int argc, char **argv){ + adb_datum_t query = {1, 2, "testquery", (double[2]) {0, 0.5}}; - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - unsigned int myerr=0; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - int size=0; + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 1; + qid.sequence_start = 0; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_EUCLIDEAN_NORMED, 10, 10}; + adb_query_refine_t refine = {0}; + refine.flags |= ADB_REFINE_RADIUS; + refine.radius = 5; + refine.hopsize = 1; -/* clean */ -//if [ -f testdb ]; then rm -f testdb; fi - clean_remove_db(databasename); + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; -/* new db */ -//${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + result_present_or_fail(results, "testfeature10", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); + + /* the test in the original test suite for + * audioDB-the-command-line-program alters the parms.ntracks, which + * is not very meaningful in this context (given that we don't do + * aggregation, but simply return valid points); here instead we + * check that radius filtering works. */ + spec.refine.radius = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); -/* test feature files */ -//intstring 2 > testfeature01 -//floatstring 0 1 >> testfeature01 - ivals[0]=2; - dvals[0]=0; dvals[1]=1; - maketestfile("testfeature01",ivals,dvals,2); -//intstring 2 > testfeature10 -//floatstring 1 0 >> testfeature10 - ivals[0]=2; - dvals[0]=1; dvals[1]=0; - maketestfile("testfeature10",ivals,dvals,2); + spec.qid.datum->data = (double[2]) {0.5, 0}; + spec.refine.radius = 5; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature10", 0, 0, 0); + result_present_or_fail(results, "testfeature01", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); -/* inserts */ -//${AUDIODB} -d testdb -I -f testfeature01 -//${AUDIODB} -d testdb -I -f testfeature10 - myinsert.features="testfeature01"; - myerr=audiodb_insert(mydbp,&myinsert); - myinsert.features="testfeature10"; - myerr=audiodb_insert(mydbp,&myinsert); + spec.refine.radius = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature10", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); -/* l2norm */ -//# sequence queries require L2NORM -//${AUDIODB} -d testdb -L - audiodb_l2norm(mydbp); + audiodb_close(adb); -/* query 1 */ -//echo "query point (0.0,0.5)" -//intstring 2 > testquery -//floatstring 0 0.5 >> testquery - ivals[0]=2; - dvals[0]=0; dvals[1]=0.5; dvals[2]=0; dvals[3]=0; - maketestfile("testquery",ivals,dvals,2); - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -R 5 > testoutput -//echo testfeature01 1 > test-expected-output -//echo testfeature10 1 >> test-expected-output -//cmp testoutput test-expected-output - - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.radius="5"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - dump_query(&myadbquery,&myadbqueryresult); - /* check the test values */ - if (size != 2) {returnval = -1;}; - //if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,0)) {returnval = -1;}; - //if (testoneresult(&myadbqueryresult,1,"testfeature",1,0,0)) {returnval = -1;}; - - -/* query 2 */ -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -r 1 -R 5 > testoutput -//echo testfeature01 1 > test-expected-output -//cmp testoutput test-expected-output -// -//echo "query point (0.5,0.0)" -//intstring 2 > testquery -//floatstring 0.5 0 >> testquery - - -/* query 3 */ -//# FIXME: because there's only one point in each track (and the query), -//# the ordering is essentially database order. We need these test -//# cases anyway because we need to test non-segfaulting, non-empty -//# results... -// -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -R 5 > testoutput -//echo testfeature01 1 > test-expected-output -//echo testfeature10 1 >> test-expected-output -//cmp testoutput test-expected-output - - - -/* query 4 */ -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -r 1 -R 5 > testoutput -//echo testfeature01 1 > test-expected-output -//cmp testoutput test-expected-output - - - printf("returnval:%d\n",returnval); - //returnval=-1; - - return(returnval); + return 104; } -
--- a/libtests/0010/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0010/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -sequence radius search / 1 point with lib +sequence radius search / 1 point with library
--- a/libtests/0011/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0011/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,141 +1,61 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; -int main(int argc, char **argv){ + adb_datum_t datum = {2, 2, "testfeature", (double[4]) {0, 0.5, 0.5, 0}}; + if(audiodb_insert_datum(adb, &datum)) + return 1; + if(audiodb_l2norm(adb)) + return 1; - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - unsigned int myerr=0; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - adb_query_t myadbquery2={0}; - adb_queryresult_t myadbqueryresult2={0}; - int size=0; + adb_datum_t query = {1, 2, "testquery", (double[2]) {0, 0.5}}; + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 1; + qid.sequence_start = 0; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_EUCLIDEAN_NORMED, 10, 10}; + adb_query_refine_t refine = {0}; + refine.hopsize = 1; + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; - //if [ -f testdb ]; then rm -f testdb; fi - /* remove old directory */ - clean_remove_db(databasename); + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + result_present_or_fail(results, "testfeature", 2, 0, 1); + audiodb_query_free_results(adb, &spec, results); + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); - //${AUDIODB} -d testdb -N - /* create new db */ - mydbp=audiodb_create(databasename,0,0,0); + spec.qid.datum->data = (double[2]) {0.5, 0}; + spec.params.npoints = 10; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 1); + result_present_or_fail(results, "testfeature", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); + + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 1); + audiodb_query_free_results(adb, &spec, results); + audiodb_close(adb); - /* create testfeature01 file */ - //intstring 2 > testfeature - //floatstring 0 0.5 >> testfeature - //floatstring 0.5 0 >> testfeature - ivals[0]=2; - dvals[0]=0; dvals[1]=0.5; dvals[2]=0.5; dvals[3]=0; - maketestfile("testfeature",ivals,dvals,4); - - /* insert */ - //${AUDIODB} -d testdb -I -f testfeature - myinsert.features="testfeature"; - myerr=audiodb_insert(mydbp,&myinsert); - - /* create testquery file */ - //echo "query point (0.0,0.5)" - //intstring 2 > testquery - //floatstring 0 0.5 >> testquery - ivals[0]=2; - dvals[0]=0.0; dvals[1]=0.5; dvals[2]=0; dvals[3]=0; - maketestfile("testquery",ivals,dvals,2); - - /* l2norm */ - //# sequence queries require L2NORM - //${AUDIODB} -d testdb -L - audiodb_l2norm(mydbp); - - /* query */ - //${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput - //echo testfeature 1 0 0 > test-expected-output - //cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,0)) {returnval = -1;}; - - /* query2 */ - //${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -n 1 > testoutput - //echo testfeature 0 0 0 > test-expected-output - //cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.numpoints="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",0,0,0)) {returnval = -1;}; - - /* query3 */ - //echo "query point (0.5,0.0)" - //intstring 2 > testquery - //floatstring 0.5 0 >> testquery - ivals[0]=2; - dvals[0]=0.5; dvals[1]=0; - maketestfile("testquery",ivals,dvals,2); - - //${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput - //echo testfeature 1 0 1 > test-expected-output - //cmp testoutput test-expected-output - myadbquery2.querytype="sequence"; - myadbquery2.feature="testquery"; - myadbquery2.sequencelength="1"; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,0,"testfeature",1,0,1)) {returnval = -1;}; - - - /* query4 */ - //${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -n 1 > testoutput - //echo testfeature 0 0 1 > test-expected-output - //cmp testoutput test-expected-output - myadbquery2.querytype="sequence"; - myadbquery2.feature="testquery"; - myadbquery2.sequencelength="1"; - myadbquery2.numpoints="1"; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,0,"testfeature",0,0,1)) {returnval = -1;}; - - - - return(returnval); + return 104; } -
--- a/libtests/0011/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0011/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -sequence search / 1 track / non-normed features using lib +sequence search / 1 track / non-normed features using library
--- a/libtests/0012/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0012/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,138 +1,61 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; + if(audiodb_l2norm(adb)) + return 1; -int main(int argc, char **argv){ + adb_datum_t datum = {2, 2, "testfeature", (double[4]) {0, 0.5, 0.5, 0}}; + if(audiodb_insert_datum(adb, &datum)) + return 1; - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - unsigned int myerr=0; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - adb_query_t myadbquery2={0}; - adb_queryresult_t myadbqueryresult2={0}; - int size=0; + adb_datum_t query = {1, 2, "testquery", (double[2]) {0, 0.5}}; + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 1; + qid.sequence_start = 0; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_EUCLIDEAN_NORMED, 10, 10}; + adb_query_refine_t refine = {0}; + refine.hopsize = 1; + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; - /* remove old directory */ - //if [ -f testdb ]; then rm -f testdb; fi - clean_remove_db(databasename); + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + result_present_or_fail(results, "testfeature", 2, 0, 1); + audiodb_query_free_results(adb, &spec, results); - /* create new db */ - //${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); - //intstring 2 > testfeature - //floatstring 0 0.5 >> testfeature - //floatstring 0.5 0 >> testfeature - ivals[0]=2; - dvals[0]=0; dvals[1]=0.5; dvals[2]=0.5; dvals[3]=0; - maketestfile("testfeature",ivals,dvals,4); + spec.qid.datum->data = (double[2]) {0.5, 0}; + spec.params.npoints = 10; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 1); + result_present_or_fail(results, "testfeature", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); + + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 1); + audiodb_query_free_results(adb, &spec, results); - //# sequence queries require L2NORM; check that we can still insert - //# after turning flag on - //${AUDIODB} -d testdb -L - if(audiodb_l2norm(mydbp)){ - returnval=-1; - }; + audiodb_close(adb); - //${AUDIODB} -d testdb -I -f testfeature - myinsert.features="testfeature"; - myerr=audiodb_insert(mydbp,&myinsert); - - //echo "query point (0.0,0.5)" - //intstring 2 > testquery - //floatstring 0 0.5 >> testquery - ivals[0]=2; - dvals[0]=0; dvals[1]=0.5; - maketestfile("testquery",ivals,dvals,2); - - /* query 1 */ -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput -//echo testfeature 1 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,0)) {returnval = -1;}; - - - /* query 2 */ - //${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -n 1 > testoutput - //echo testfeature 0 0 0 > test-expected-output - //cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.numpoints="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",0,0,0)) {returnval = -1;}; - - /* testquery */ -//echo "query point (0.5,0.0)" -//intstring 2 > testquery -//floatstring 0.5 0 >> testquery - /* create testquery file */ - ivals[0]=2; - dvals[0]=0.5; dvals[1]=0.0; dvals[2]=0; dvals[3]=0; - maketestfile("testquery",ivals,dvals,2); - - /* query 3 file */ -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput -//echo testfeature 1 0 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery2.querytype="sequence"; - myadbquery2.feature="testquery"; - myadbquery2.sequencelength="1"; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,0,"testfeature",1,0,1)) {returnval = -1;}; - - /* query 4 */ -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -n 1 > testoutput -//echo testfeature 0 0 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery2.querytype="sequence"; - myadbquery2.feature="testquery"; - myadbquery2.sequencelength="1"; - myadbquery2.numpoints="1"; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,0,"testfeature",0,0,1)) {returnval = -1;}; - - - return(returnval); + return 104; } -
--- a/libtests/0012/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0012/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -insert after L2Norm with lib +insert after L2Norm with library
--- a/libtests/0022/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0022/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,152 +1,66 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + adb_insert_t batch[2] = {{0}, {0}}; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; -int main(int argc, char **argv){ + maketestfile("testfeature01", 2, (double[2]) {0, 1}, 2); + maketestfile("testfeature10", 2, (double[2]) {1, 0}, 2); - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - adb_query_t myadbquery2={0}; - adb_queryresult_t myadbqueryresult2={0}; - adb_insert_t ins1[2]={{0},{0}}; - int size=0; + batch[0].features="testfeature01"; + batch[1].features="testfeature10"; + if(audiodb_batchinsert(adb, batch, 2)) + return 1; + if(audiodb_l2norm(adb)) + return 1; - /* remove old directory */ - //if [ -f testdb ]; then rm -f testdb; fi - clean_remove_db(databasename); + adb_datum_t query = {1, 2, "testquery", (double[2]) {0, 0.5}}; + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 1; + qid.sequence_start = 0; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_EUCLIDEAN_NORMED, 10, 10}; + adb_query_refine_t refine = {0}; + refine.hopsize = 1; - /* create new db */ - //${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; - if (!mydbp){ returnval=-1;}; + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + result_present_or_fail(results, "testfeature10", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); + + spec.params.ntracks = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); -//intstring 2 > testfeature01 -//floatstring 0 1 >> testfeature01 -//intstring 2 > testfeature10 -//floatstring 1 0 >> testfeature10 - ivals[0]=2; - dvals[0]=0; dvals[1]=1; - maketestfile("testfeature01",ivals,dvals,2); - ivals[0]=2; - dvals[0]=1; dvals[1]=0; - maketestfile("testfeature10",ivals,dvals,2); + spec.qid.datum->data = (double [2]) {0.5, 0}; + spec.params.ntracks = 10; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature10", 0, 0, 0); + result_present_or_fail(results, "testfeature01", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); + spec.params.ntracks = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature10", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); -//cat > testfeaturefiles <<EOF -//testfeature01 -//testfeature10 -//EOF + audiodb_close(adb); - ins1[0].features="testfeature01"; - ins1[1].features="testfeature10"; - -//audioDB --BATCHINSERT -d testdb --featureList tempfeatures -//${AUDIODB} -d testdb -B -F testfeaturefiles - - if(audiodb_batchinsert(mydbp,ins1,2)){ - returnval=-1; - }; - - -//# sequence queries require L2NORM -//${AUDIODB} -d testdb -L - if(audiodb_l2norm(mydbp)){ - returnval=-1; - }; - -//echo "query point (0.0,0.5)" -//intstring 2 > testquery -//floatstring 0 0.5 >> testquery - ivals[0]=2; - dvals[0]=0; dvals[1]=0.5; - maketestfile("testquery",ivals,dvals,2); - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput -//echo testfeature01 0 0 0 > test-expected-output -//echo testfeature10 2 0 0 >> test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - //dump_query(&myadbquery,&myadbqueryresult); - ///* check the test values */ - if (size != 2) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature01",0,0,0)) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,1,"testfeature10",2,0,0)) {returnval = -1;}; - - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -r 1 > testoutput -//echo testfeature01 0 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.resultlength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature01",0,0,0)) {returnval = -1;}; -//echo "query point (0.5,0.0)" -//intstring 2 > testquery -//floatstring 0.5 0 >> testquery - ivals[0]=2; - dvals[0]=0.5; dvals[1]=0; - maketestfile("testquery",ivals,dvals,2); - -////${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput -////echo testfeature10 0 0 0 > test-expected-output -////echo testfeature01 2 0 0 >> test-expected-output -////cmp testoutput test-expected-output - myadbquery2.querytype="sequence"; - myadbquery2.feature="testquery"; - myadbquery2.sequencelength="1"; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - if (size != 2) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,0,"testfeature10",0,0,0)) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,1,"testfeature01",2,0,0)) {returnval = -1;}; - - -////${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -r 1 > testoutput -////echo testfeature10 0 0 0 > test-expected-output -////cmp testoutput test-expected-output - myadbquery2.querytype="sequence"; - myadbquery2.feature="testquery"; - myadbquery2.sequencelength="1"; - myadbquery2.resultlength="1"; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,0,"testfeature10",0,0,0)) {returnval = -1;}; - - audiodb_close(mydbp); -// fprintf(stderr,"returnval:%d\n",returnval); - - return(returnval); + return 104; } -
--- a/libtests/0022/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0022/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -Batchinsert / sequence search with lib +Batchinsert / sequence search with library
--- a/libtests/0023/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0023/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,160 +1,66 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + adb_insert_t batch[2]={{0},{0}}; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; -int main(int argc, char **argv){ + maketestfile("testfeature01", 2, (double[2]) {0, 1}, 2); + maketestfile("testfeature10", 2, (double[2]) {1, 0}, 2); - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - adb_query_t myadbquery2={0}; - adb_queryresult_t myadbqueryresult2={0}; - adb_query_t myadbquery3={0}; - adb_queryresult_t myadbqueryresult3={0}; - int size=0; - adb_insert_t ins1[2]={{0},{0}}; + batch[0].features="testfeature01"; + batch[1].features="testfeature10"; + if(audiodb_batchinsert(adb, batch, 2)) + return 1; + if(audiodb_l2norm(adb)) + return 1; - /* remove old directory */ - //if [ -f testdb ]; then rm -f testdb; fi - clean_remove_db(databasename); + adb_datum_t query = {2, 2, "testquery", (double[4]) {0, 0.5, 0.5, 0}}; + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 1; + qid.sequence_start = 0; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_EUCLIDEAN_NORMED, 10, 10}; + adb_query_refine_t refine = {0}; + refine.hopsize = 1; - /* create new db */ - //${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; + + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + result_present_or_fail(results, "testfeature10", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); -//intstring 2 > testfeature01 -//floatstring 0 1 >> testfeature01 -//intstring 2 > testfeature10 -//floatstring 1 0 >> testfeature10 - ivals[0]=2; - dvals[0]=0; dvals[1]=1; - maketestfile("testfeature01",ivals,dvals,2); - ivals[0]=2; - dvals[0]=1; dvals[1]=0; - maketestfile("testfeature10",ivals,dvals,2); + spec.params.ntracks = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); + + spec.qid.sequence_start = 1; + spec.params.ntracks = 10; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature10", 0, 1, 0); + result_present_or_fail(results, "testfeature01", 2, 1, 0); + audiodb_query_free_results(adb, &spec, results); -//cat > testfeaturefiles <<EOF -//testfeature01 -//testfeature10 -//EOF - ins1[0].features="testfeature01"; - ins1[1].features="testfeature10"; + spec.params.ntracks = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature10", 0, 1, 0); + audiodb_query_free_results(adb, &spec, results); -//audioDB -B -d testdb -F tempfeatures -//${AUDIODB} -d testdb -B -F testfeaturefiles - if(audiodb_batchinsert(mydbp,ins1,2)){ - returnval=-1; - }; + audiodb_close(adb); -//# sequence queries require L2NORM -//${AUDIODB} -d testdb -L - if(audiodb_l2norm(mydbp)){ - returnval=-1; - }; - -////echo "query point (0.0,0.5)" -////intstring 2 > testquery -////floatstring 0 0.5 >> testquery -////floatstring 0.5 0 >> testquery - - ivals[0]=2; - dvals[0]=0; dvals[1]=0.5; - dvals[2]=0.5; dvals[3]=0; - maketestfile("testquery",ivals,dvals,4); - - - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -p 0 > testoutput -//echo testfeature01 0 0 0 > test-expected-output -//echo testfeature10 2 0 0 >> test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.qpoint="0"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - //printf("size:%d\n",size); - - //dump_query(&myadbquery,&myadbqueryresult); - ///* check the test values */ - if (size != 2) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature01",0,0,0)) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,1,"testfeature10",2,0,0)) {returnval = -1;}; - - - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -r 1 -p 0 > testoutput -//echo testfeature01 0 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery2.querytype="sequence"; - myadbquery2.feature="testquery"; - myadbquery2.sequencelength="1"; - myadbquery2.resultlength="1"; - myadbquery2.qpoint="0"; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - ///* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,0,"testfeature01",0,0,0)) {returnval = -1;}; - - -//echo "query point (0.5,0.0)" - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -p 1 > testoutput -//echo testfeature10 0 1 0 > test-expected-output -//echo testfeature01 2 1 0 >> test-expected-output -//cmp testoutput test-expected-output - myadbquery3.querytype="sequence"; - myadbquery3.feature="testquery"; - myadbquery3.sequencelength="1"; - myadbquery3.qpoint="1"; - audiodb_query(mydbp,&myadbquery3,&myadbqueryresult3); - size=myadbqueryresult3.sizeRlist; - -// dump_query(&myadbquery3,&myadbqueryresult3); - if (size != 2) {returnval = -1;}; - if (testoneresult(&myadbqueryresult3,0,"testfeature10",0,1,0)) {returnval = -1;}; - if (testoneresult(&myadbqueryresult3,1,"testfeature01",2,1,0)) {returnval = -1;}; - - -////${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -r 1 -p 1 > testoutput -////echo testfeature10 0 1 0 > test-expected-output -////cmp testoutput test-expected-output - myadbquery2.querytype="sequence"; - myadbquery2.feature="testquery"; - myadbquery2.sequencelength="1"; - myadbquery2.qpoint="1"; - myadbquery2.resultlength="1"; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,0,"testfeature10",0,1,0)) {returnval = -1;}; - - -// printf("returnval:%d\n",returnval); - - return(returnval); + return 104; } -
--- a/libtests/0023/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0023/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -as 0022 but with -p with lib +as 0022 but with -p with library
--- a/libtests/0024/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0024/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,118 +1,55 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + adb_insert_t batch[2]={{0}, {0}}; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; -int main(int argc, char **argv){ + maketestfile("testfeature01", 2, (double[2]) {0, 1}, 2); + maketestfile("testfeature10", 2, (double[2]) {1, 0}, 2); - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - unsigned int myerr=0; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - adb_query_t myadbquery2={0}; - adb_queryresult_t myadbqueryresult2={0}; - int size=0; - adb_insert_t ins1[2]={0}; + batch[0].features="testfeature01"; + batch[1].features="testfeature10"; + if(audiodb_batchinsert(adb, batch, 2)) + return 1; + if(audiodb_l2norm(adb)) + return 1; - /* remove old directory */ - //if [ -f testdb ]; then rm -f testdb; fi - clean_remove_db(databasename); + adb_datum_t query = {2, 2, "testquery", (double[4]) {0, 0.5, 0.5, 0}}; + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 1; + qid.flags = ADB_QID_FLAG_EXHAUSTIVE; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_EUCLIDEAN_NORMED, 10, 10}; + adb_query_refine_t refine = {0}; + refine.hopsize = 1; - /* create new db */ - //${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; + + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 4) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + result_present_or_fail(results, "testfeature01", 2, 1, 0); + result_present_or_fail(results, "testfeature10", 2, 0, 0); + result_present_or_fail(results, "testfeature10", 0, 1, 0); + audiodb_query_free_results(adb, &spec, results); -//intstring 2 > testfeature01 -//floatstring 0 1 >> testfeature01 -//intstring 2 > testfeature10 -//floatstring 1 0 >> testfeature10 - ivals[0]=2; - dvals[0]=0; dvals[1]=1; - maketestfile("testfeature01",ivals,dvals,2); - ivals[0]=2; - dvals[0]=1; dvals[1]=0; - maketestfile("testfeature10",ivals,dvals,2); + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + result_present_or_fail(results, "testfeature10", 0, 1, 0); + audiodb_query_free_results(adb, &spec, results); -//cat > testfeaturefiles <<EOF -//testfeature01 -//testfeature10 -//EOF - ins1[0].features="testfeature01"; - ins1[1].features="testfeature10"; + audiodb_close(adb); - -//${AUDIODB} -d testdb -B -F testfeaturefiles - returnval=audiodb_batchinsert(mydbp,ins1,2); - -//# sequence queries require L2NORM -//${AUDIODB} -d testdb -L - audiodb_l2norm(mydbp); - -//echo "exhaustive search" -//intstring 2 > testquery -//floatstring 0 0.5 >> testquery -//floatstring 0.5 0 >> testquery - ivals[0]=2; - dvals[0]=0; dvals[1]=0.5; - dvals[2]=0.5; dvals[3]=0; - maketestfile("testquery",ivals,dvals,4); - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -e > testoutput -//echo testfeature01 1 0 0 > test-expected-output -//echo testfeature10 1 1 0 >> test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.exhaustive=1; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - //printf("size:%d\n",size); - //dump_query(&myadbquery,&myadbqueryresult); - ///* check the test values */ - if (size != 2) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature01",1,0,0)) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,1,"testfeature10",1,1,0)) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -n 1 -e > testoutput -//echo testfeature01 0 0 0 > test-expected-output -//echo testfeature10 0 1 0 >> test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.exhaustive=1; - myadbquery.numpoints="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - //printf("size:%d\n",size); - //dump_query(&myadbquery,&myadbqueryresult); - ///* check the test values */ - if (size != 2) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature01",0,0,0)) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,1,"testfeature10",0,1,0)) {returnval = -1;}; - - - printf("returnval:%d\n",returnval); - return(returnval); + return 104; } -
--- a/libtests/0024/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0024/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ --l 1 exhaustive search like 0022 with lib +-l 1 exhaustive search like 0022 with library
--- a/libtests/0025/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0025/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,158 +1,62 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; -void maketimesfile(char * filename); + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; -int main(int argc, char **argv){ + adb_datum_t datum = {2, 2, "testfeature", (double[4]) {0, 0.5, 0.5, 0}, + NULL, (double[4]) {0, 1, 1, 2}}; + if(audiodb_insert_datum(adb, &datum)) + return 1; + if(audiodb_l2norm(adb)) + return 1; - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - int size=0; + adb_datum_t query = {1, 2, "testquery", (double[2]) {0, 0.5}}; + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 1; + qid.sequence_start = 0; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_EUCLIDEAN_NORMED, 10, 10}; + adb_query_refine_t refine = {0}; + refine.hopsize = 1; - /* remove old directory */ - //if [ -f testdb ]; then rm -f testdb; fi - clean_remove_db(databasename); + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; - /* create new db */ - //${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + result_present_or_fail(results, "testfeature", 2, 0, 1); + audiodb_query_free_results(adb, &spec, results); -//intstring 2 > testfeature -//floatstring 0 0.5 >> testfeature -//floatstring 0.5 0 >> testfeature - ivals[0]=2; - dvals[0]=0; dvals[1]=0.5; dvals[2]=0.5; dvals[3]=0; - maketestfile("testfeature",ivals,dvals,4); + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); + spec.qid.datum->data = (double[2]) {0.5, 0}; + spec.params.npoints = 10; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 1); + result_present_or_fail(results, "testfeature", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); + + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 1); + audiodb_query_free_results(adb, &spec, results); -//cat > testtimes <<EOF -//0 -//1 -//2 -//EOF - maketimesfile("testtimes"); + audiodb_close(adb); -//${AUDIODB} -d testdb -I -f testfeature -t testtimes - myinsert.features="testfeature"; - if(audiodb_insert(mydbp,&myinsert)){ - returnval=-1; - }; - -//# sequence queries require L2NORM -//${AUDIODB} -d testdb -L - if(audiodb_l2norm(mydbp)){ - returnval=-1; - }; - -//echo "query point (0.0,0.5)" -//intstring 2 > testquery -//floatstring 0 0.5 >> testquery - ivals[0]=2; - dvals[0]=0; dvals[1]=0.5; - maketestfile("testquery",ivals,dvals,2); - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput -//echo testfeature 1 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - //dump_query(&myadbquery,&myadbqueryresult); - ///* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,0)) {returnval = -1;}; - - -////${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -n 1 > testoutput -////echo testfeature 0 0 0 > test-expected-output -////cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.numpoints="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - //dump_query(&myadbquery,&myadbqueryresult); - ///* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",0,0,0)) {returnval = -1;}; - -////echo "query point (0.5,0.0)" -////intstring 2 > testquery -////floatstring 0.5 0 >> testquery - ivals[0]=2; - dvals[0]=0.5; dvals[1]=0.0; - maketestfile("testquery",ivals,dvals,2); - -////${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput -////echo testfeature 1 0 1 > test-expected-output -////cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.numpoints=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - //dump_query(&myadbquery,&myadbqueryresult); - ///* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,1)) {returnval = -1;}; - - -////${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -n 1 > testoutput -////echo testfeature 0 0 1 > test-expected-output -////cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.numpoints="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - //dump_query(&myadbquery,&myadbqueryresult); - ///* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",0,0,1)) {returnval = -1;}; - - - - //printf("returnval:%d\n",returnval); - return(returnval); + return 104; } - -void maketimesfile(char * filename){ - - FILE * myfile; - - myfile=fopen(filename,"w"); - fprintf(myfile,"0\n"); - fprintf(myfile,"1\n"); - fprintf(myfile,"2\n"); - fflush(myfile); - fclose(myfile); - -}
--- a/libtests/0025/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0025/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -insertion with times with lib +insertion with times with library
--- a/libtests/0026/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0026/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,56 +1,19 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; + if(audiodb_power(adb)) + return 1; + if(audiodb_power(adb)) + return 1; -int main(int argc, char **argv){ + audiodb_close(adb); - int returnval=0; - adb_ptr mydbp={0}; - // int ivals[10]; - // double dvals[10]; - // adbinsert myinsert={0}; - // unsigned int myerr=0; - char * databasename="testdb"; - // adbquery myadbquery={0}; - // adbqueryresult myadbqueryresult={0}; - // adbquery myadbquery2={0}; - // adbqueryresult myadbqueryresult2={0}; - int myerror=0; - - - /* remove old directory */ - //if [ -f testdb ]; then rm -f testdb; fi - clean_remove_db(databasename); - - /* create new db */ - //${AUDIODB} -N -d testdb - mydbp=audiodb_create(databasename,0,0,0); - - /* power flag on */ - //${AUDIODB} -P -d testdb - //${AUDIODB} -d testdb -P - myerror=audiodb_power(mydbp); - if (myerror){ - returnval=-1; - } - - //# should fail (no db given) - //expect_clean_error_exit ${AUDIODB} -P - /* not relevent, API wouldn't compile */ - - return(returnval); + return 104; } -
--- a/libtests/0026/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0026/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -command-line -P handling with lib +power-flag handling with library
--- a/libtests/0027/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0027/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,362 +1,144 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; -int main(int argc, char **argv){ + adb_datum_t feature = {2, 2, "testfeature", (double[4]){0, 1, 1, 0}, + (double[2]){-0.5, -1}}; + if(!audiodb_insert_datum(adb, &feature)) + return 1; + if(audiodb_power(adb)) + return 1; + feature.power = NULL; + if(!audiodb_insert_datum(adb, &feature)) + return 1; + feature.power = (double[2]){-0.5, -1}; + if(audiodb_insert_datum(adb, &feature)) + return 1; - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - unsigned int myerr=0; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - adb_query_t myadbquery2={0}; - adb_queryresult_t myadbqueryresult2={0}; - int size=0; + if(audiodb_l2norm(adb)) + return 1; + adb_datum_t query = {1, 2, "testquery", (double[2]) {0, 0.5}}; + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 1; + qid.sequence_start = 0; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_EUCLIDEAN_NORMED, 10, 10}; + adb_query_refine_t refine = {0}; + refine.hopsize = 1; - /* remove old directory */ - //if [ -f testdb ]; then rm -f testdb; fi - clean_remove_db(databasename); + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; - /* create new db */ - //${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + result_present_or_fail(results, "testfeature", 2, 0, 1); + audiodb_query_free_results(adb, &spec, results); - //intstring 2 > testfeature - //floatstring 0 1 >> testfeature - //floatstring 1 0 >> testfeature - ivals[0]=2; - dvals[0]=0; dvals[1]=1; dvals[2]=1; dvals[3]=0; - maketestfile("testfeature",ivals,dvals,4); + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); - //intstring 1 > testpower - //floatstring -0.5 >> testpower - //floatstring -1 >> testpower - ivals[0]=1; - dvals[0]=-0.5; dvals[1]=-1; - maketestfile("testpower",ivals,dvals,2); + spec.qid.datum->data = (double [2]) {0.5, 0}; + spec.params.npoints = 10; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 1); + result_present_or_fail(results, "testfeature", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); - /* insert breaks because power flag off and power insert */ - //expect_clean_error_exit ${AUDIODB} -d testdb -I -f testfeature -w testpower - myinsert.features="testfeature"; - myinsert.power="testpower"; - if (!audiodb_insert(mydbp,&myinsert)){ returnval=-1; } + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 1); + audiodb_query_free_results(adb, &spec, results); - - /* make sure power can be turned on */ -//${AUDIODB} -d testdb -P - if(audiodb_power(mydbp)){ returnval=-1; }; + /* queries with associated power data */ + spec.qid.datum->data = (double [2]) {0, 0.5}; + spec.qid.datum->power = (double [1]) {-0.5}; + spec.params.npoints = 10; + spec.refine.flags = ADB_REFINE_ABSOLUTE_THRESHOLD; + spec.refine.absolute_threshold = -1.4; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + result_present_or_fail(results, "testfeature", 2, 0, 1); + audiodb_query_free_results(adb, &spec, results); + spec.refine.absolute_threshold = -0.6; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); - /* inserts now require power also */ -////expect_clean_error_exit ${AUDIODB} -d testdb -I -f testfeature - myinsert.features="testfeature"; - myinsert.power=NULL; - //myerr=audiodb_insert(mydbp,&myinsert); - if (!audiodb_insert(mydbp,&myinsert)){ returnval=-1; } + spec.refine.absolute_threshold = -0.2; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 0) return 1; + audiodb_query_free_results(adb, &spec, results); - /* now make a real insert */ -//${AUDIODB} -d testdb -I -f testfeature -w testpower - myinsert.features="testfeature"; - myinsert.power="testpower"; - myerr=audiodb_insert(mydbp,&myinsert); - if (audiodb_insert(mydbp,&myinsert)){ returnval=-1; } + spec.refine.absolute_threshold = 0; + spec.refine.flags = ADB_REFINE_RELATIVE_THRESHOLD; + spec.refine.relative_threshold = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + result_present_or_fail(results, "testfeature", 2, 0, 1); + audiodb_query_free_results(adb, &spec, results); + spec.refine.relative_threshold = 0.2; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); - //# sequence queries require L2NORM - //${AUDIODB} -d testdb -L - if(audiodb_l2norm(mydbp)){ returnval=-1; }; + spec.qid.datum->data = (double [2]) {0.5, 0}; + spec.refine.flags = ADB_REFINE_ABSOLUTE_THRESHOLD; + spec.refine.absolute_threshold = -1.4; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 1); + result_present_or_fail(results, "testfeature", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); + spec.refine.absolute_threshold = -0.6; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); - //# queries without power files should run as before - //echo "query point (0.0,0.5)" - //intstring 2 > testquery - //floatstring 0 0.5 >> testquery - ivals[0]=2; - dvals[0]=-0; dvals[1]=0.5; - maketestfile("testquery",ivals,dvals,2); + spec.refine.absolute_threshold = -0.2; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 0) return 1; + audiodb_query_free_results(adb, &spec, results); + spec.refine.absolute_threshold = 0; + spec.refine.flags = ADB_REFINE_RELATIVE_THRESHOLD; + spec.refine.relative_threshold = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 1); + result_present_or_fail(results, "testfeature", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); - /* query 1 */ - //${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput - //echo testfeature 1 0 0 > test-expected-output - //cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; + spec.refine.relative_threshold = 0.2; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,0)) {returnval = -1;}; + audiodb_close(adb); -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -n 1 > testoutput -//echo testfeature 0 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.numpoints="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",0,0,0)) {returnval = -1;}; - -//echo "query point (0.5,0.0)" -//intstring 2 > testquery -//floatstring 0.5 0 >> testquery - ivals[0]=2; - dvals[0]=0.5; dvals[1]=0.0; - maketestfile("testquery",ivals,dvals,2); - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput -//echo testfeature 1 0 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.numpoints=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,1)) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -n 1 > testoutput -//echo testfeature 0 0 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.numpoints="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",0,0,1)) {returnval = -1;}; - - - -//# queries with power files might do something different -//echo "query point (0.0,0.5), p=-0.5" -//intstring 2 > testquery -//floatstring 0 0.5 >> testquery -//intstring 1 > testquerypower -//floatstring -0.5 >> testquerypower - ivals[0]=2; - dvals[0]=-0; dvals[1]=0.5; - maketestfile("testquery",ivals,dvals,2); - ivals[0]=1; - dvals[0]=-0.5; - maketestfile("testquerypower",ivals,dvals,1); - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testquerypower --absolute-threshold=-1.4 > testoutput -//echo testfeature 1 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.power="testquerypower"; - myadbquery.numpoints=NULL; - myadbquery.absolute_threshold=-1.4; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,0)) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testquerypower --absolute-threshold=-0.6 > testoutput -//echo testfeature 0 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.power="testquerypower"; - myadbquery.numpoints=NULL; - myadbquery.absolute_threshold=-0.6; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",0,0,0)) {returnval = -1;}; - - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testquerypower --absolute-threshold=-0.2 > testoutput -//cat /dev/null > test-expected-output -//cmp testoutput test-expected-output - - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.power="testquerypower"; - myadbquery.numpoints=NULL; - myadbquery.absolute_threshold=-0.2; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - //dump_query(&myadbquery,&myadbqueryresult); - - /* check the test values */ - if (size != 0) {returnval = -1;}; - - - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testquerypower --relative-threshold=1 > testoutput -//echo testfeature 1 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.power="testquerypower"; - myadbquery.numpoints=NULL; - myadbquery.absolute_threshold=0; - myadbquery.relative_threshold=1; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,0)) {returnval = -1;}; - - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testquerypower --relative-threshold=0.2 > testoutput -//echo testfeature 0 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.power="testquerypower"; - myadbquery.numpoints=NULL; - myadbquery.absolute_threshold=0; - myadbquery.relative_threshold=0.2; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",0,0,0)) {returnval = -1;}; - - -//echo "query point (0.5,0.0), p=-0.5" -//intstring 2 > testquery -//floatstring 0.5 0 >> testquery - ivals[0]=2; - dvals[0]=0.5; dvals[1]=0.0; - maketestfile("testquery",ivals,dvals,2); - - -//audioDB -Q sequence -d testdb -f testquery -w testquerypower -l 1 --absolute-threshold -1.400000 -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testquerypower --absolute-threshold=-1.4 > testoutput -//echo testfeature 1 0 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery2.querytype="sequence"; - myadbquery2.feature="testquery"; - myadbquery2.sequencelength="1"; - myadbquery2.power="testquerypower"; - myadbquery2.numpoints=NULL; - myadbquery2.absolute_threshold=-1.4; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,0,"testfeature",1,0,1)) {returnval = -1;}; - - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testquerypower --absolute-threshold=-0.6 > testoutput -//echo testfeature 2 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery2.querytype="sequence"; - myadbquery2.feature="testquery"; - myadbquery2.sequencelength="1"; - myadbquery2.power="testquerypower"; - myadbquery2.numpoints=NULL; - myadbquery2.absolute_threshold=-0.6; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,0,"testfeature",2,0,0)) {returnval = -1;}; - - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testquerypower --absolute-threshold=-0.2 > testoutput -//cat /dev/null > test-expected-output -//cmp testoutput test-expected-output - myadbquery2.querytype="sequence"; - myadbquery2.feature="testquery"; - myadbquery2.sequencelength="1"; - myadbquery2.power="testquerypower"; - myadbquery2.numpoints=NULL; - myadbquery2.absolute_threshold=-0.2; - myadbquery2.relative_threshold=0.0; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - /* check the test values */ - if (size != 0) {returnval = -1;}; - - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testquerypower --relative-threshold=1 > testoutput -//echo testfeature 1 0 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery2.querytype="sequence"; - myadbquery2.feature="testquery"; - myadbquery2.sequencelength="1"; - myadbquery2.power="testquerypower"; - myadbquery2.numpoints=NULL; - myadbquery2.absolute_threshold=0.0; - myadbquery2.relative_threshold=1; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,0,"testfeature",1,0,1)) {returnval = -1;}; - - - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testquerypower --relative-threshold=0.2 > testoutput -//echo testfeature 2 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery2.querytype="sequence"; - myadbquery2.feature="testquery"; - myadbquery2.sequencelength="1"; - myadbquery2.power="testquerypower"; - myadbquery2.numpoints=NULL; - myadbquery2.absolute_threshold=-0.0; - myadbquery2.relative_threshold=0.1; - audiodb_query(mydbp,&myadbquery2,&myadbqueryresult2); - size=myadbqueryresult2.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult2,0,"testfeature",2,0,0)) {returnval = -1;}; - -// printf("returnval:%d\n",returnval); - return(returnval); + return 104; } -
--- a/libtests/0027/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0027/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -0006 with power with lib +0006 with power with library
--- a/libtests/0028/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0028/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,364 +1,151 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + adb_insert_t batch[1]={{0}}; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB,0,0,0))) + return 1; -int main(int argc, char **argv){ + maketestfile("testfeature", 2, (double[4]){0, 1, 1, 0}, 4); + maketestfile("testpower", 1, (double[2]) {-0.5, -1}, 2); - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - int size=0; - adb_insert_t ins1[2]={{0},{0}}; + batch[0].features="testfeature"; + batch[0].power="testpower"; + if(!audiodb_batchinsert(adb, batch, 1)) + return 1; + if(audiodb_power(adb)) + return 1; - /* remove old directory */ - //if [ -f testdb ]; then rm -f testdb; fi - clean_remove_db(databasename); + batch[0].power=NULL; + if(!audiodb_batchinsert(adb, batch, 1)) + return 1; - /* create new db */ - //${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); + batch[0].power="testpower"; + if(audiodb_batchinsert(adb, batch, 1)) + return 1; -//intstring 2 > testfeature -//floatstring 0 1 >> testfeature -//floatstring 1 0 >> testfeature - ivals[0]=2; - dvals[0]=0; dvals[1]=1; dvals[2]=1; dvals[3]=0; - maketestfile("testfeature",ivals,dvals,4); + if(audiodb_l2norm(adb)) + return 1; -//intstring 1 > testpower -//floatstring -0.5 >> testpower -//floatstring -1 >> testpower - ivals[0]=1; - dvals[0]=-0.5; dvals[1]=-1; - maketestfile("testpower",ivals,dvals,2); + adb_datum_t query = {1, 2, "testquery", (double[2]) {0, 0.5}}; + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 1; + qid.sequence_start = 0; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_EUCLIDEAN_NORMED, 10, 10}; + adb_query_refine_t refine = {0}; + refine.hopsize = 1; -//echo testfeature > testFeatureList.txt -//echo testpower > testPowerList.txt - ins1[0].features="testfeature"; - ins1[0].power="testpower"; + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; -//expect_clean_error_exit ${AUDIODB} -d testdb -B -F testFeatureList.txt -W testPowerList.txt - if(!audiodb_batchinsert(mydbp,ins1,1)){ - returnval=-1; - }; + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + result_present_or_fail(results, "testfeature", 2, 0, 1); + audiodb_query_free_results(adb, &spec, results); -//${AUDIODB} -d testdb -P - if(audiodb_power(mydbp)){ - returnval=-1; - }; + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); -//expect_clean_error_exit ${AUDIODB} -d testdb -B -F testFeatureList.txt - ins1[0].features="testfeature"; - ins1[0].power=NULL; - if(!audiodb_batchinsert(mydbp,ins1,1)){ - returnval=-1; - }; + spec.qid.datum->data = (double [2]) {0.5, 0}; + spec.params.npoints = 10; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 1); + result_present_or_fail(results, "testfeature", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); -//${AUDIODB} -d testdb -B -F testFeatureList.txt -W testPowerList.txt - ins1[0].features="testfeature"; - ins1[0].power="testpower"; - if(audiodb_batchinsert(mydbp,ins1,1)){ - returnval=-1; - }; + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 1); + audiodb_query_free_results(adb, &spec, results); + + /* queries with associated power data */ + spec.qid.datum->data = (double [2]) {0, 0.5}; + spec.qid.datum->power = (double [1]) {-0.5}; + spec.params.npoints = 10; + spec.refine.flags = ADB_REFINE_ABSOLUTE_THRESHOLD; + spec.refine.absolute_threshold = -1.4; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + result_present_or_fail(results, "testfeature", 2, 0, 1); + audiodb_query_free_results(adb, &spec, results); -//# sequence queries require L2NORM -//${AUDIODB} -d testdb -L - if(audiodb_l2norm(mydbp)){ - returnval=-1; - }; + spec.refine.absolute_threshold = -0.6; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); -//# queries without power files should run as before -//echo "query point (0.0,0.5)" -//intstring 2 > testquery -//floatstring 0 0.5 >> testquery - ivals[0]=2; - dvals[0]=-0.0; dvals[1]=0.5; - maketestfile("testquery",ivals,dvals,2); + spec.refine.absolute_threshold = -0.2; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 0) return 1; + audiodb_query_free_results(adb, &spec, results); -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput -//echo testfeature 1 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; + spec.refine.absolute_threshold = 0; + spec.refine.flags = ADB_REFINE_RELATIVE_THRESHOLD; + spec.refine.relative_threshold = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + result_present_or_fail(results, "testfeature", 2, 0, 1); + audiodb_query_free_results(adb, &spec, results); - //dump_query(&myadbquery,&myadbqueryresult); - ///* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,0)) {returnval = -1;}; + spec.refine.relative_threshold = 0.2; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -n 1 > testoutput -//echo testfeature 0 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.numpoints="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; + spec.qid.datum->data = (double [2]) {0.5, 0}; + spec.refine.flags = ADB_REFINE_ABSOLUTE_THRESHOLD; + spec.refine.absolute_threshold = -1.4; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 1); + result_present_or_fail(results, "testfeature", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); - //dump_query(&myadbquery,&myadbqueryresult); - ///* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",0,0,0)) {returnval = -1;}; + spec.refine.absolute_threshold = -0.6; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); -//echo "query point (0.5,0.0)" -//intstring 2 > testquery -//floatstring 0.5 0 >> testquery - ivals[0]=2; - dvals[0]=0.5; dvals[1]=0.0; - maketestfile("testquery",ivals,dvals,2); + spec.refine.absolute_threshold = -0.2; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 0) return 1; + audiodb_query_free_results(adb, &spec, results); + spec.refine.absolute_threshold = 0; + spec.refine.flags = ADB_REFINE_RELATIVE_THRESHOLD; + spec.refine.relative_threshold = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 1); + result_present_or_fail(results, "testfeature", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput -//echo testfeature 1 0 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.numpoints=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; + spec.refine.relative_threshold = 0.2; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); - //dump_query(&myadbquery,&myadbqueryresult); - ///* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,1)) {returnval = -1;}; + audiodb_close(adb); -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -n 1 > testoutput -//echo testfeature 0 0 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.numpoints="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - //dump_query(&myadbquery,&myadbqueryresult); - ///* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",0,0,1)) {returnval = -1;}; - -//# queries with power files might do something different -//echo "query point (0.0,0.5), p=-0.5" -//intstring 2 > testquery -//floatstring 0 0.5 >> testquery - ivals[0]=2; - dvals[0]=0.0; dvals[1]=0.5; - maketestfile("testquery",ivals,dvals,2); - -//intstring 1 > testquerypower -//floatstring -0.5 >> testquerypower - ivals[0]=1; - dvals[0]=-0.5; - maketestfile("testquerypower",ivals,dvals,1); - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testquerypower --absolute-threshold=-1.4 > testoutput -//echo testfeature 1 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="1"; - myadbquery.absolute_threshold=-1.4; - myadbquery.numpoints=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - ///* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,0)) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testquerypower --absolute-threshold=-0.6 > testoutput -//echo testfeature 0 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="1"; - myadbquery.absolute_threshold=-0.6; - myadbquery.numpoints=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - //dump_query(&myadbquery,&myadbqueryresult); - ///* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",0,0,0)) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testquerypower --absolute-threshold=-0.2 > testoutput -//cat /dev/null > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="1"; - myadbquery.absolute_threshold=-0.2; - myadbquery.numpoints=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - //dump_query(&myadbquery,&myadbqueryresult); - ///* check the test values */ - if (size != 0) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testquerypower --relative-threshold=1 > testoutput -//audioDB -Q sequence -d testdb -f testquery -w testquerypower -l 1 --relative-threshold 1.000000 -//echo testfeature 1 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="1"; - myadbquery.absolute_threshold=0.0; - myadbquery.relative_threshold=1.0; - myadbquery.numpoints=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - //dump_query(&myadbquery,&myadbqueryresult); - ///* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,0)) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testquerypower --relative-threshold=0.2 > testoutput -//echo testfeature 0 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="1"; - myadbquery.absolute_threshold=0.0; - myadbquery.relative_threshold=0.2; - myadbquery.numpoints=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - //dump_query(&myadbquery,&myadbqueryresult); - ///* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",0,0,0)) {returnval = -1;}; - -////echo "query point (0.5,0.0), p=-0.5" -////intstring 2 > testquery -////floatstring 0.5 0 >> testquery - ivals[0]=2; - dvals[0]=0.5; dvals[1]=0.0; - maketestfile("testquery",ivals,dvals,2); - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testquerypower --absolute-threshold=-1.4 > testoutput -//echo testfeature 1 0 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="1"; - myadbquery.absolute_threshold=-1.4; - myadbquery.relative_threshold=0.0; - myadbquery.numpoints=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - -// dump_query(&myadbquery,&myadbqueryresult); - ///* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,1)) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testquerypower --absolute-threshold=-0.6 > testoutput -//echo testfeature 2 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="1"; - myadbquery.absolute_threshold=-0.6; - myadbquery.relative_threshold=0.0; - myadbquery.numpoints=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - //dump_query(&myadbquery,&myadbqueryresult); - ///* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",2,0,0)) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testquerypower --absolute-threshold=-0.2 > testoutput -//cat /dev/null > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="1"; - myadbquery.absolute_threshold=-0.2; - myadbquery.relative_threshold=0.0; - myadbquery.numpoints=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - //dump_query(&myadbquery,&myadbqueryresult); - ///* check the test values */ - if (size != 0) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testquerypower --relative-threshold=1 > testoutput -//echo testfeature 1 0 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="1"; - myadbquery.relative_threshold=1.0; - myadbquery.absolute_threshold=0.0; - myadbquery.numpoints=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - //dump_query(&myadbquery,&myadbqueryresult); - ///* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,1)) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testquerypower --relative-threshold=0.2 > testoutput -//echo testfeature 2 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="1"; - myadbquery.relative_threshold=0.2; - myadbquery.absolute_threshold=0.0; - myadbquery.numpoints=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - ///* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",2,0,0)) {returnval = -1;}; - - - audiodb_close(mydbp); - //fprintf(stderr,"returnval:%d\n",returnval); - - return(returnval); + return 104; } -
--- a/libtests/0028/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0028/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -batchinsert version of 0027 with lib +batchinsert version of 0027 with library
--- a/libtests/0029/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0029/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,270 +1,133 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; -int main(int argc, char **argv){ + adb_datum_t feature = {4, 2, "testfeature", + (double [8]) {0, 1, 1, 0, 1, 0, 0, 1}, + (double [4]) {-0.5, -1, -1, -0.5}}; + if(!audiodb_insert_datum(adb, &feature)) + return 1; + audiodb_power(adb); + feature.power = NULL; + if(!audiodb_insert_datum(adb, &feature)) + return 1; + feature.power = (double [4]) {-0.5, -1, -1, -0.5}; + if(audiodb_insert_datum(adb, &feature)) + return 1; - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - int size=0; + if(audiodb_l2norm(adb)) + return 1; + adb_datum_t query = {3, 2, "testquery", + (double [6]) {0, 0.5, 0, 0.5, 0.5, 0}, + (double [3]) {-0.5, -1, -1}}; + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 1; + qid.sequence_start = 0; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_EUCLIDEAN_NORMED, 10, 10}; + adb_query_refine_t refine = {0}; + refine.hopsize = 1; - /* remove old directory */ - //if [ -f testdb ]; then rm -f testdb; fi - clean_remove_db(databasename); + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; - /* create new db */ - //${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 4) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + result_present_or_fail(results, "testfeature", 2, 0, 1); + result_present_or_fail(results, "testfeature", 2, 0, 2); + result_present_or_fail(results, "testfeature", 0, 0, 3); + audiodb_query_free_results(adb, &spec, results); -//intstring 2 > testfeature -//floatstring 0 1 >> testfeature -//floatstring 1 0 >> testfeature -//floatstring 1 0 >> testfeature -//floatstring 0 1 >> testfeature - ivals[0]=2; - dvals[0]=0; dvals[1]=1; dvals[2]=1; dvals[3]=0; - dvals[4]=1; dvals[5]=0; dvals[6]=0; dvals[7]=1; - maketestfile("testfeature",ivals,dvals,8); + spec.qid.sequence_start = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 4) return 1; + result_present_or_fail(results, "testfeature", 0, 1, 0); + result_present_or_fail(results, "testfeature", 2, 1, 1); + result_present_or_fail(results, "testfeature", 2, 1, 2); + result_present_or_fail(results, "testfeature", 0, 1, 3); + audiodb_query_free_results(adb, &spec, results); -//intstring 1 > testpower -//floatstring -0.5 >> testpower -//floatstring -1 >> testpower -//floatstring -1 >> testpower -//floatstring -0.5 >> testpower - ivals[0]=1; - dvals[0]=-0.5; dvals[1]=-1; dvals[2]=-1; dvals[3]=-0.5; - maketestfile("testpower",ivals,dvals,4); + spec.qid.sequence_start = 2; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 4) return 1; + result_present_or_fail(results, "testfeature", 2, 2, 0); + result_present_or_fail(results, "testfeature", 0, 2, 1); + result_present_or_fail(results, "testfeature", 0, 2, 2); + result_present_or_fail(results, "testfeature", 2, 2, 3); + audiodb_query_free_results(adb, &spec, results); -//expect_clean_error_exit ${AUDIODB} -d testdb -I -f testfeature -w testpower - myinsert.features="testfeature"; - myinsert.power="testpower"; - if (!audiodb_insert(mydbp,&myinsert)){ returnval=-1; } + spec.qid.sequence_length = 2; + spec.qid.sequence_start = 0; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 3) return 1; + result_present_or_fail(results, "testfeature", 1, 0, 0); + result_present_or_fail(results, "testfeature", 2, 0, 1); + result_present_or_fail(results, "testfeature", 1, 0, 2); + audiodb_query_free_results(adb, &spec, results); -//${AUDIODB} -d testdb -P - if(audiodb_power(mydbp)){ returnval=-1; }; + spec.qid.sequence_start = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 3) return 1; + result_present_or_fail(results, "testfeature", 0, 1, 0); + result_present_or_fail(results, "testfeature", 1, 1, 1); + result_present_or_fail(results, "testfeature", 2, 1, 2); + audiodb_query_free_results(adb, &spec, results); -//expect_clean_error_exit ${AUDIODB} -d testdb -I -f testfeature - myinsert.features="testfeature"; - myinsert.power=NULL; - if (!audiodb_insert(mydbp,&myinsert)){ returnval=-1; } + spec.refine.flags = ADB_REFINE_ABSOLUTE_THRESHOLD; + spec.refine.absolute_threshold = -1.4; + spec.qid.sequence_length = 2; + spec.qid.sequence_start = 0; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 3) return 1; + result_present_or_fail(results, "testfeature", 1, 0, 0); + result_present_or_fail(results, "testfeature", 2, 0, 1); + result_present_or_fail(results, "testfeature", 1, 0, 2); + audiodb_query_free_results(adb, &spec, results); -//${AUDIODB} -d testdb -I -f testfeature -w testpower - myinsert.features="testfeature"; - myinsert.power="testpower"; - if (audiodb_insert(mydbp,&myinsert)){ returnval=-1; } + spec.qid.sequence_start = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 3) return 1; + result_present_or_fail(results, "testfeature", 0, 1, 0); + result_present_or_fail(results, "testfeature", 1, 1, 1); + result_present_or_fail(results, "testfeature", 2, 1, 2); + audiodb_query_free_results(adb, &spec, results); + spec.refine.absolute_threshold = -0.8; + spec.qid.sequence_start = 0; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 1, 0, 0); + result_present_or_fail(results, "testfeature", 1, 0, 2); + audiodb_query_free_results(adb, &spec, results); -//# sequence queries require L2NORM -//${AUDIODB} -d testdb -L - if(audiodb_l2norm(mydbp)){ returnval=-1; }; + spec.qid.sequence_start = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 0) return 1; + audiodb_query_free_results(adb, &spec, results); -//echo "query points (0.0,0.5),(0.0,0.5),(0.5,0.0)" -//intstring 2 > testquery -//floatstring 0 0.5 >> testquery -//floatstring 0 0.5 >> testquery -//floatstring 0.5 0 >> testquery - ivals[0]=2; - dvals[0]=0; dvals[1]=0.5; dvals[2]=0; dvals[3]=0.5; dvals[4]=0.5; dvals[5]=0; - maketestfile("testquery",ivals,dvals,6); + spec.refine.absolute_threshold = 0; + spec.refine.flags = ADB_REFINE_RELATIVE_THRESHOLD; + spec.refine.relative_threshold = 0.1; + spec.qid.sequence_start = 0; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 1, 0, 0); + result_present_or_fail(results, "testfeature", 1, 0, 2); + audiodb_query_free_results(adb, &spec, results); -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput -//audioDB -Q sequence -d testdb -f testquery -l 1 -//echo testfeature 1 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; + audiodb_close(adb); - dump_query(&myadbquery,&myadbqueryresult); - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,0)) {returnval = -1;}; - - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -p 0 > testoutput -//echo testfeature 1 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.qpoint="0"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,0)) {returnval = -1;}; - - printf("returnval:%d\n",returnval); - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -p 1 > testoutput -//echo testfeature 1 1 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.qpoint="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,1,0)) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -p 0 > testoutput -//echo testfeature 1.33333 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="2"; - myadbquery.qpoint="0"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1.33333,0,0)) {returnval = -1;}; - - - -//${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -p 1 > testoutput -//echo testfeature 1 1 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="2"; - myadbquery.qpoint="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,1,0)) {returnval = -1;}; - - -//echo "query points (0.0,0.5)p=-0.5,(0.0,0.5)p=-1,(0.5,0.0)p=-1" -//intstring 1 > testquerypower -//floatstring -0.5 -1 -1 >> testquerypower - ivals[0]=1; - dvals[0]=-0.5; dvals[1]=-1; dvals[2]=-1; - maketestfile("testquerypower",ivals,dvals,3); - - -//${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -w testquerypower --absolute-threshold=-1.4 -p 0 > testoutput -//echo testfeature 1.33333 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="2"; - myadbquery.qpoint=NULL; - myadbquery.absolute_threshold=-1.4; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1.33333,0,0)) {returnval = -1;}; - - - -//${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -w testquerypower --absolute-threshold=-1.4 -p 1 > testoutput -//echo testfeature 1 1 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="2"; - myadbquery.qpoint="1"; - myadbquery.absolute_threshold=-1.4; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,1,0)) {returnval = -1;}; - - -//${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -w testquerypower --absolute-threshold=-0.8 -p 0 > testoutput -//echo testfeature 1 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="2"; - myadbquery.qpoint="0"; - myadbquery.absolute_threshold=-0.8; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,0)) {returnval = -1;}; - - -//${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -w testquerypower --absolute-threshold=-0.8 -p 1 > testoutput -//cat /dev/null > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="2"; - myadbquery.qpoint="1"; - myadbquery.absolute_threshold=-0.8; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 0) {returnval = -1;}; - - - -//${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -w testquerypower --relative-threshold=0.1 -p 0 > testoutput -//echo testfeature 1 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="2"; - myadbquery.qpoint="0"; - myadbquery.absolute_threshold=0.0; - myadbquery.relative_threshold=0.1; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature",1,0,0)) {returnval = -1;}; - - - -//// returnval=-1; - printf("returnval:%d\n",returnval); - - return(returnval); + return 104; } -
--- a/libtests/0029/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0029/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ --l 2 searches with power with lib +-l 2 searches with power with library
--- a/libtests/0030/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0030/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,334 +1,137 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; -int main(int argc, char **argv){ + adb_datum_t feature = {4, 2, "testfeature", + (double [8]) {0, 1, 1, 0, 1, 0, 0, 1}, + (double [4]) {-0.5, -1, -1, -0.5}}; + audiodb_power(adb); + if(audiodb_insert_datum(adb, &feature)) + return 1; + if(audiodb_l2norm(adb)) + return 1; - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - int size=0; + adb_datum_t query = {3, 2, "testquery", + (double [6]) {0, 0.5, 0, 0.5, 0.5, 0}, + (double [3]) {-0.5, -1, -1}}; + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 1; + qid.sequence_start = 0; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_EUCLIDEAN_NORMED, 10, 10}; + adb_query_refine_t refine = {0}; + refine.flags = ADB_REFINE_RADIUS; + refine.radius = 0.1; + refine.hopsize = 1; - /* remove old directory */ - //if [ -f testdb ]; then rm -f testdb; fi - clean_remove_db(databasename); + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; - /* create new db */ - //${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 0, 0); + result_present_or_fail(results, "testfeature", 0, 0, 3); + audiodb_query_free_results(adb, &spec, results); -//intstring 2 > testfeature -//floatstring 0 1 >> testfeature -//floatstring 1 0 >> testfeature -//floatstring 1 0 >> testfeature -//floatstring 0 1 >> testfeature - ivals[0]=2; - dvals[0]=0; dvals[1]=1; dvals[2]=1; dvals[3]=0; - dvals[4]=1; dvals[5]=0; dvals[6]=0; dvals[7]=1; - maketestfile("testfeature",ivals,dvals,8); + spec.qid.sequence_start = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 1, 0); + result_present_or_fail(results, "testfeature", 0, 1, 3); + audiodb_query_free_results(adb, &spec, results); -//intstring 1 > testpower -//floatstring -0.5 >> testpower -//floatstring -1 >> testpower -//floatstring -1 >> testpower -//floatstring -0.5 >> testpower - ivals[0]=1; - dvals[0]=-0.5; dvals[1]=-1; dvals[2]=-1; dvals[3]=-0.5; - maketestfile("testpower",ivals,dvals,4); + spec.qid.sequence_start = 2; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 0, 2, 1); + result_present_or_fail(results, "testfeature", 0, 2, 2); + audiodb_query_free_results(adb, &spec, results); -//expect_clean_error_exit ${AUDIODB} -d testdb -I -f testfeature -w testpower - myinsert.features="testfeature"; - myinsert.power="testpower"; - if (!audiodb_insert(mydbp,&myinsert)){ returnval=-1; } + spec.qid.sequence_start = 0; + spec.qid.sequence_length = 2; + spec.refine.radius = 1.1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 1, 0, 0); + result_present_or_fail(results, "testfeature", 1, 0, 2); + audiodb_query_free_results(adb, &spec, results); -//${AUDIODB} -d testdb -P - if(audiodb_power(mydbp)){ returnval=-1; }; + spec.refine.radius = 0.9; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 0) return 1; + audiodb_query_free_results(adb, &spec, results); -//expect_clean_error_exit ${AUDIODB} -d testdb -I -f testfeature - myinsert.features="testfeature"; - myinsert.power=NULL; - if (!audiodb_insert(mydbp,&myinsert)){ returnval=-1; } + spec.qid.sequence_start = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 0, 1, 0); + audiodb_query_free_results(adb, &spec, results); -//${AUDIODB} -d testdb -I -f testfeature -w testpower - myinsert.features="testfeature"; - myinsert.power="testpower"; - if (audiodb_insert(mydbp,&myinsert)){ returnval=-1; } - printf("returnval:%d\n",returnval); + /* tests with power from here on down */ -//# sequence queries require L2NORM -//${AUDIODB} -d testdb -L - if(audiodb_l2norm(mydbp)){ returnval=-1; }; + spec.qid.sequence_length = 2; + spec.qid.sequence_start = 0; + spec.refine.flags = ADB_REFINE_RADIUS|ADB_REFINE_ABSOLUTE_THRESHOLD; + spec.refine.radius = 1.1; + spec.refine.absolute_threshold = -1.4; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 1, 0, 0); + result_present_or_fail(results, "testfeature", 1, 0, 2); + audiodb_query_free_results(adb, &spec, results); -//echo "query points (0.0,0.5),(0.0,0.5),(0.5,0.0)" -//intstring 2 > testquery -//floatstring 0 0.5 >> testquery -//floatstring 0 0.5 >> testquery -//floatstring 0.5 0 >> testquery - ivals[0]=2; - dvals[0]=0; dvals[1]=0.5; dvals[2]=0; dvals[3]=0.5; dvals[4]=0.5; dvals[5]=0; - maketestfile("testquery",ivals,dvals,6); + spec.refine.absolute_threshold = -0.8; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 1, 0, 0); + result_present_or_fail(results, "testfeature", 1, 0, 2); + audiodb_query_free_results(adb, &spec, results); -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -R 0.1 > testoutput -//audioDB -Q sequence -d testdb -f testquery -R 0.1 -l 1 -//echo testfeature 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - //myadbquery.power="testquerypower"; - myadbquery.sequencelength="1"; - //myadbquery.qpoint="0"; - //myadbquery.absolute_threshold=0.0; - //myadbquery.relative_threshold=0.1; - myadbquery.radius="0.1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; + spec.refine.absolute_threshold = -0.7; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 0) return 1; + audiodb_query_free_results(adb, &spec, results); - dump_query(&myadbquery,&myadbqueryresult); + spec.qid.sequence_start = 1; + spec.refine.absolute_threshold = -1.4; + spec.refine.radius = 0.9; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature", 0, 1, 0); + audiodb_query_free_results(adb, &spec, results); - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneradiusresult(&myadbqueryresult,0,"testfeature",1)) {returnval = -1;}; + spec.refine.absolute_threshold = -0.9; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 0) return 1; + audiodb_query_free_results(adb, &spec, results); -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -p 0 -R 0.1 > testoutput -//echo testfeature 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - //myadbquery.power="testquerypower"; - myadbquery.sequencelength="1"; - myadbquery.qpoint="0"; - //myadbquery.absolute_threshold=0.0; - //myadbquery.relative_threshold=0.1; - myadbquery.radius="0.1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; + spec.qid.sequence_length = 2; + spec.qid.sequence_start = 0; + spec.refine.absolute_threshold = 0; + spec.refine.flags = ADB_REFINE_RADIUS|ADB_REFINE_RELATIVE_THRESHOLD; + spec.refine.radius = 1.1; + spec.refine.relative_threshold = 0.1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature", 1, 0, 0); + result_present_or_fail(results, "testfeature", 1, 0, 2); + audiodb_query_free_results(adb, &spec, results); - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneradiusresult(&myadbqueryresult,0,"testfeature",1)) {returnval = -1;}; + spec.refine.radius = 0.9; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 0) return 1; + audiodb_query_free_results(adb, &spec, results); -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -p 1 -R 0.1 > testoutput -//echo testfeature 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - //myadbquery.power="testquerypower"; - myadbquery.sequencelength="1"; - myadbquery.qpoint="1"; - //myadbquery.absolute_threshold=0.0; - //myadbquery.relative_threshold=0.1; - myadbquery.radius="0.1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; + audiodb_close(adb); - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneradiusresult(&myadbqueryresult,0,"testfeature",1)) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -p 0 -R 1.1 > testoutput -//echo testfeature 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - //myadbquery.power="testquerypower"; - myadbquery.sequencelength="2"; - myadbquery.qpoint="0"; - //myadbquery.absolute_threshold=0.0; - //myadbquery.relative_threshold=0.1; - myadbquery.radius="1.1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneradiusresult(&myadbqueryresult,0,"testfeature",1)) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -p 0 -R 0.9 > testoutput -//cat /dev/null > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - //myadbquery.power="testquerypower"; - myadbquery.sequencelength="2"; - myadbquery.qpoint="0"; - //myadbquery.absolute_threshold=0.0; - //myadbquery.relative_threshold=0.1; - myadbquery.radius="0.9"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 0) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -p 1 -R 0.9 > testoutput -//echo testfeature 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - //myadbquery.power="testquerypower"; - myadbquery.sequencelength="2"; - myadbquery.qpoint="1"; - myadbquery.radius="0.9"; - //myadbquery.absolute_threshold=0.0; - //myadbquery.relative_threshold=0.1; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneradiusresult(&myadbqueryresult,0,"testfeature",1)) {returnval = -1;}; - -//echo "query points (0.0,0.5)p=-0.5,(0.0,0.5)p=-1,(0.5,0.0)p=-1" -//intstring 1 > testquerypower -//floatstring -0.5 -1 -1 >> testquerypower - ivals[0]=1; - dvals[0]=-0.5; dvals[1]=-1; dvals[2]=-1; - maketestfile("testquerypower",ivals,dvals,3); - -//${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -w testquerypower --absolute-threshold=-1.4 -p 0 -R 1.1 > testoutput -//echo testfeature 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="2"; - myadbquery.qpoint="0"; - myadbquery.absolute_threshold=-1.4; - myadbquery.relative_threshold=0.0; - myadbquery.radius="1.1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneradiusresult(&myadbqueryresult,0,"testfeature",1)) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -w testquerypower --absolute-threshold=-0.8 -p 0 -R 1.1 > testoutput -//echo testfeature 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="2"; - myadbquery.qpoint="0"; - myadbquery.absolute_threshold=-0.8; - //myadbquery.relative_threshold=0.1; - myadbquery.radius="1.1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneradiusresult(&myadbqueryresult,0,"testfeature",1)) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -w testquerypower --absolute-threshold=-0.7 -p 0 -R 1.1 > testoutput -//cat /dev/null > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="2"; - myadbquery.qpoint="0"; - myadbquery.absolute_threshold=-0.7; - myadbquery.relative_threshold=0.0; - myadbquery.radius="1.1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 0) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -w testquerypower --absolute-threshold=-1.4 -p 1 -R 0.9 > testoutput -//echo testfeature 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="2"; - myadbquery.qpoint="1"; - myadbquery.absolute_threshold=-1.4; - myadbquery.relative_threshold=0.0; - myadbquery.radius="0.9"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneradiusresult(&myadbqueryresult,0,"testfeature",1)) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -w testquerypower --absolute-threshold=-0.9 -p 1 -R 0.9 > testoutput -//cat /dev/null > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="2"; - myadbquery.qpoint="1"; - myadbquery.absolute_threshold=-0.9; - myadbquery.relative_threshold=0.0; - myadbquery.radius="0.9"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 0) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -w testquerypower --relative-threshold=0.1 -p 0 -R 1.1 > testoutput -//echo testfeature 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="2"; - myadbquery.qpoint="0"; - myadbquery.absolute_threshold=0.0; - myadbquery.relative_threshold=0.1; - myadbquery.radius="1.1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneradiusresult(&myadbqueryresult,0,"testfeature",1)) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -w testquerypower --relative-threshold=0.1 -p 0 -R 0.9 > testoutput -//cat /dev/null > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.power="testquerypower"; - myadbquery.sequencelength="2"; - myadbquery.qpoint="0"; - myadbquery.absolute_threshold=0.0; - myadbquery.relative_threshold=0.1; - myadbquery.radius="0.9"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 0) {returnval = -1;}; - - //returnval=-1; - printf("returnval:%d\n",returnval); - - return(returnval); + return 104; } -
--- a/libtests/0030/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0030/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -single-track sequence radius search with power with lib +single-track sequence radius search with power with library
--- a/libtests/0031/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0031/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,266 +1,85 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + const char *keys[2]; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB,0,0,0))) + return 1; -int main(int argc, char **argv){ + adb_datum_t datum1 = {1, 2, "testfeature01", (double[4]) {0, 1}}; + adb_datum_t datum2 = {1, 2, "testfeature10", (double[4]) {1, 0}}; + if(audiodb_insert_datum(adb, &datum1)) + return 1; + if(audiodb_insert_datum(adb, &datum2)) + return 1; - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - int size=0; + if(audiodb_l2norm(adb)) + return 1; + adb_datum_t query = {1, 2, "testquery", (double[2]) {0, 0.5}}; - /* remove old directory */ - //if [ -f testdb ]; then rm -f testdb; fi - clean_remove_db(databasename); + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 1; + qid.sequence_start = 0; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_EUCLIDEAN_NORMED, 10, 10}; + adb_query_refine_t refine = {0}; + refine.hopsize = 1; - /* create new db */ - //${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; -///intstring 2 > testfeature01 -///floatstring 0 1 >> testfeature01 -///intstring 2 > testfeature10 -///floatstring 1 0 >> testfeature10 - ivals[0]=2; - dvals[0]=0; dvals[1]=1; - maketestfile("testfeature01",ivals,dvals,2); - ivals[0]=2; - dvals[0]=1; dvals[1]=0; - maketestfile("testfeature10",ivals,dvals,2); + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + result_present_or_fail(results, "testfeature10", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); -///${AUDIODB} -d testdb -I -f testfeature01 -///${AUDIODB} -d testdb -I -f testfeature10 + spec.refine.flags = ADB_REFINE_INCLUDE_KEYLIST; + spec.refine.include.nkeys = 0; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 0) return 1; + audiodb_query_free_results(adb, &spec, results); - myinsert.features="testfeature01"; - if (audiodb_insert(mydbp,&myinsert)){ returnval=-1; } + spec.refine.include.nkeys = 1; + spec.refine.include.keys = keys; + spec.refine.include.keys[0] = "testfeature01"; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); - myinsert.features="testfeature10"; - if (audiodb_insert(mydbp,&myinsert)){ returnval=-1; } + spec.refine.include.keys[0] = "testfeature10"; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature10", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); -///# sequence queries require L2NORM -///${AUDIODB} -d testdb -L - if(audiodb_l2norm(mydbp)){ returnval=-1; }; + spec.params.ntracks = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature10", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); -///echo "query point (0.0,0.5)" -///intstring 2 > testquery -///floatstring 0 0.5 >> testquery - ivals[0]=2; - dvals[0]=0; dvals[1]=0.5; - maketestfile("testquery",ivals,dvals,2); + spec.params.ntracks = 10; + spec.refine.flags = ADB_REFINE_EXCLUDE_KEYLIST; + spec.refine.include.nkeys = 0; + spec.refine.include.keys = NULL; + spec.refine.exclude.nkeys = 1; + spec.refine.exclude.keys = keys; + spec.refine.exclude.keys[0] = "testfeature01"; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature10", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); -///${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput -///echo testfeature01 0 0 0 > test-expected-output -///echo testfeature10 2 0 0 >> test-expected-output -///cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; + audiodb_close(adb); - /* check the test values */ - if (size != 2) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature01",0,0,0)) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,1,"testfeature10",2,0,0)) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -K /dev/null > testoutput -//cat /dev/null > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.keylist="/dev/null"; - myadbquery.sequencelength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - //dump_query(&myadbquery,&myadbqueryresult); - - /* check the test values */ - if (size != 0) {returnval = -1;}; - -///echo testfeature01 > testkl.txt - makekeylistfile("testkl.txt","testfeature01"); - -///${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -K testkl.txt > testoutput -///echo testfeature01 0 0 0 > test-expected-output -///cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.keylist="testkl.txt"; - myadbquery.sequencelength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - - size=myadbqueryresult.sizeRlist; - //dump_query(&myadbquery,&myadbqueryresult); - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature01",0,0,0)) {returnval = -1;}; - -///echo testfeature10 > testkl.txt - makekeylistfile("testkl.txt","testfeature10"); - -///${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -K testkl.txt > testoutput -///echo testfeature10 2 0 0 > test-expected-output -///cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.keylist="testkl.txt"; - myadbquery.sequencelength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - - size=myadbqueryresult.sizeRlist; - //dump_query(&myadbquery,&myadbqueryresult); - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature10",2,0,0)) {returnval = -1;}; - -///echo testfeature10 > testkl.txt - makekeylistfile("testkl.txt","testfeature10"); - -///${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -K testkl.txt -r 1 > testoutput -///echo testfeature10 2 0 0 > test-expected-output -///cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.keylist="testkl.txt"; - myadbquery.sequencelength="1"; - myadbquery.resultlength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - - size=myadbqueryresult.sizeRlist; - //dump_query(&myadbquery,&myadbqueryresult); - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature10",2,0,0)) {returnval = -1;}; - -///echo "query point (0.5,0.0)" -///intstring 2 > testquery -///floatstring 0.5 0 >> testquery - ivals[0]=2; - dvals[0]=0.5; dvals[1]=0.0; - maketestfile("testquery",ivals,dvals,2); - -///${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput -///echo testfeature10 0 0 0 > test-expected-output -///echo testfeature01 2 0 0 >> test-expected-output -///cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.keylist=NULL; - myadbquery.sequencelength="1"; - myadbquery.resultlength=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - - size=myadbqueryresult.sizeRlist; - //dump_query(&myadbquery,&myadbqueryresult); - - /* check the test values */ - if (size != 2) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature10",0,0,0)) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,1,"testfeature01",2,0,0)) {returnval = -1;}; - -///${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -K /dev/null > testoutput -///cat /dev/null > test-expected-output -///cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.keylist="/dev/null"; - myadbquery.sequencelength="1"; - myadbquery.resultlength=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - - size=myadbqueryresult.sizeRlist; - //dump_query(&myadbquery,&myadbqueryresult); - - /* check the test values */ - if (size != 0) {returnval = -1;}; - -///echo testfeature10 > testkl.txt - makekeylistfile("testkl.txt","testfeature10"); - -///${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -K testkl.txt > testoutput -///echo testfeature10 0 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.keylist="testkl.txt"; - myadbquery.sequencelength="1"; - myadbquery.resultlength=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - - size=myadbqueryresult.sizeRlist; - //dump_query(&myadbquery,&myadbqueryresult); - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature10",0,0,0)) {returnval = -1;}; - - -///echo testfeature01 > testkl.txt - makekeylistfile("testkl.txt","testfeature01"); - -///${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -K testkl.txt > testoutput -///echo testfeature01 2 0 0 > test-expected-output -///cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.keylist="testkl.txt"; - myadbquery.sequencelength="1"; - myadbquery.resultlength=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - - size=myadbqueryresult.sizeRlist; - //dump_query(&myadbquery,&myadbqueryresult); - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature01",2,0,0)) {returnval = -1;}; - -///echo testfeature01 > testkl.txt - makekeylistfile("testkl.txt","testfeature01"); - -///${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -K testkl.txt -r 1 > testoutput -///echo testfeature01 2 0 0 > test-expected-output -///cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.keylist="testkl.txt"; - myadbquery.sequencelength="1"; - myadbquery.resultlength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - - size=myadbqueryresult.sizeRlist; - //dump_query(&myadbquery,&myadbqueryresult); - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature01",2,0,0)) {returnval = -1;}; - - - - //printf("returnval:%d\n",returnval); - - return(returnval); + return 104; } -
--- a/libtests/0031/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0031/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -0008 with -K restriction with lib +0008 and include and exclude keys with library
--- a/libtests/0032/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0032/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,269 +1,92 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + const char *keys[2]; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; + if(audiodb_l2norm(adb)) + return 1; -int main(int argc, char **argv){ + adb_datum_t datum1 = {1, 2, "testfeature01", (double[2]) {0, 1}}; + adb_datum_t datum2 = {1, 2, "testfeature10", (double[2]) {1, 0}}; + if(audiodb_insert_datum(adb, &datum1)) + return 1; + if(audiodb_insert_datum(adb, &datum2)) + return 1; - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - int size=0; + adb_datum_t query = {1, 2, "testquery", (double[2]) {0, 0.5}}; + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 1; + qid.sequence_start = 0; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_DOT_PRODUCT, 10, 10}; + adb_query_refine_t refine = {0}; + refine.hopsize = 1; - /* remove old directory */ -//if [ -f testdb ]; then rm -f testdb; fi - clean_remove_db(databasename); + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; - /* create new db */ -//${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature01", 0.5, 0, 0); + result_present_or_fail(results, "testfeature10", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); - /* turn on l2 power */ -//${AUDIODB} -d testdb -L - if (audiodb_l2norm(mydbp)) {returnval=-1;}; + spec.refine.flags = ADB_REFINE_INCLUDE_KEYLIST; + spec.refine.include.nkeys = 0; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 0) return 1; + audiodb_query_free_results(adb, &spec, results); - /* make feature files */ -//intstring 2 > testfeature01 -//floatstring 0 1 >> testfeature01 -//intstring 2 > testfeature10 -//floatstring 1 0 >> testfeature10 - ivals[0]=2; - dvals[0]=0; dvals[1]=1; - maketestfile("testfeature01",ivals,dvals,2); - ivals[0]=2; - dvals[0]=1; dvals[1]=0; - maketestfile("testfeature10",ivals,dvals,2); + spec.refine.include.nkeys = 1; + spec.refine.include.keys = keys; + spec.refine.include.keys[0] = "testfeature01"; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature01", 0.5, 0, 0); + audiodb_query_free_results(adb, &spec, results); + spec.refine.include.keys[0] = "testfeature10"; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature10", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); - /* insertions */ - //${AUDIODB} -d testdb -I -f testfeature01 - //${AUDIODB} -d testdb -I -f testfeature10 - myinsert.features="testfeature01"; - if(audiodb_insert(mydbp,&myinsert)) {returnval = -1; }; - myinsert.features="testfeature10"; - if(audiodb_insert(mydbp,&myinsert)) {returnval = -1; }; - + spec.refine.include.nkeys = 2; + spec.refine.include.keys[0] = "testfeature01"; + spec.refine.include.keys[1] = "testfeature10"; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature01", 0.5, 0, 0); + result_present_or_fail(results, "testfeature10", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); - /* query */ - //echo "query point (0.0,0.5)" - //intstring 2 > testquery - //floatstring 0 0.5 >> testquery - ivals[0]=2; - dvals[0]=0; dvals[1]=0.5; - maketestfile("testquery",ivals,dvals,2); + spec.refine.include.nkeys = 0; + spec.refine.include.keys = NULL; + spec.refine.flags = ADB_REFINE_EXCLUDE_KEYLIST; + spec.refine.exclude.nkeys = 1; + spec.refine.exclude.keys = keys; + spec.refine.exclude.keys[0] = "testfeature10"; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature01", 0.5, 0, 0); + audiodb_query_free_results(adb, &spec, results); - /* test a sequence query */ - //${AUDIODB} -d testdb -Q track -l 1 -f testquery > testoutput - //echo testfeature01 0.5 0 0 > test-expected-output - //echo testfeature10 0 0 0 >> test-expected-output - //cmp testoutput test-expected-output - myadbquery.querytype="track"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; + spec.refine.exclude.keys[0] = "testfeature01"; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature10", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); - /* check the test values */ - if (size != 2) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature01",0.5,0,0)) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,1,"testfeature10",0,0,0)) {returnval = -1;}; + audiodb_close(adb); -//${AUDIODB} -d testdb -Q track -l 1 -f testquery -K /dev/null > testoutput -//cat /dev/null > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="track"; - myadbquery.feature="testquery"; - myadbquery.keylist="/dev/null"; - myadbquery.sequencelength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 0) {returnval = -1;}; - - - -//echo testfeature01 > testkl.txt - makekeylistfile("testkl.txt","testfeature01"); -//${AUDIODB} -d testdb -Q track -l 1 -f testquery -K testkl.txt > testoutput -//echo testfeature01 0.5 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="track"; - myadbquery.feature="testquery"; - myadbquery.keylist="testkl.txt"; - myadbquery.sequencelength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - - size=myadbqueryresult.sizeRlist; - //dump_query(&myadbquery,&myadbqueryresult); - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature01",0.5,0,0)) {returnval = -1;}; - - - - -//echo testfeature10 > testkl.txt - makekeylistfile("testkl.txt","testfeature10"); -//${AUDIODB} -d testdb -Q track -l 1 -f testquery -K testkl.txt > testoutput -//echo testfeature10 0 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="track"; - myadbquery.feature="testquery"; - myadbquery.keylist="testkl.txt"; - myadbquery.sequencelength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - - size=myadbqueryresult.sizeRlist; - //dump_query(&myadbquery,&myadbqueryresult); - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature10",0,0,0)) {returnval = -1;}; - - - -//echo testfeature10 > testkl.txt - makekeylistfile("testkl.txt","testfeature10"); -//${AUDIODB} -d testdb -Q track -l 1 -f testquery -K testkl.txt -r 1 > testoutput -//echo testfeature10 0 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="track"; - myadbquery.feature="testquery"; - myadbquery.keylist="testkl.txt"; - myadbquery.sequencelength="1"; - myadbquery.resultlength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - - size=myadbqueryresult.sizeRlist; - //dump_query(&myadbquery,&myadbqueryresult); - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature10",0,0,0)) {returnval = -1;}; - -//echo "query point (0.5,0.0)" -//intstring 2 > testquery -//floatstring 0.5 0 >> testquery - ivals[0]=2; - dvals[0]=0.5; dvals[1]=0.0; - maketestfile("testquery",ivals,dvals,2); - -//${AUDIODB} -d testdb -Q track -l 1 -f testquery > testoutput -//echo testfeature10 0.5 0 0 > test-expected-output -//echo testfeature01 0 0 0 >> test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="track"; - myadbquery.feature="testquery"; - myadbquery.keylist=NULL; - myadbquery.sequencelength="1"; - myadbquery.resultlength=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - - size=myadbqueryresult.sizeRlist; - //dump_query(&myadbquery,&myadbqueryresult); - - /* check the test values */ - if (size != 2) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature10",0.5,0,0)) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,1,"testfeature01",0,0,0)) {returnval = -1;}; - -//${AUDIODB} -d testdb -Q track -l 1 -f testquery -K /dev/null > testoutput -//cat /dev/null > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="track"; - myadbquery.feature="testquery"; - myadbquery.keylist="/dev/null"; - myadbquery.sequencelength="1"; - myadbquery.resultlength=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - - size=myadbqueryresult.sizeRlist; - //dump_query(&myadbquery,&myadbqueryresult); - - /* check the test values */ - if (size != 0) {returnval = -1;}; - -//echo testfeature10 > testkl.txt - makekeylistfile("testkl.txt","testfeature10"); -//${AUDIODB} -d testdb -Q track -l 1 -f testquery -K testkl.txt > testoutput -//echo testfeature10 0.5 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="track"; - myadbquery.feature="testquery"; - myadbquery.keylist="testkl.txt"; - myadbquery.sequencelength="1"; - myadbquery.resultlength=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - - size=myadbqueryresult.sizeRlist; - //dump_query(&myadbquery,&myadbqueryresult); - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature10",0.5,0,0)) {returnval = -1;}; - -//echo testfeature01 > testkl.txt - makekeylistfile("testkl.txt","testfeature01"); -//${AUDIODB} -d testdb -Q track -l 1 -f testquery -K testkl.txt > testoutput -//echo testfeature01 0 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="track"; - myadbquery.feature="testquery"; - myadbquery.keylist="testkl.txt"; - myadbquery.sequencelength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - - size=myadbqueryresult.sizeRlist; - //dump_query(&myadbquery,&myadbqueryresult); - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature01",0,0,0)) {returnval = -1;}; - -//echo testfeature01 > testkl.txt - makekeylistfile("testkl.txt","testfeature01"); -//${AUDIODB} -d testdb -Q track -l 1 -f testquery -K testkl.txt -r 1 > testoutput -//echo testfeature01 0 0 0 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="track"; - myadbquery.feature="testquery"; - myadbquery.keylist="testkl.txt"; - myadbquery.sequencelength="1"; - myadbquery.resultlength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - - size=myadbqueryresult.sizeRlist; - //dump_query(&myadbquery,&myadbqueryresult); - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneresult(&myadbqueryresult,0,"testfeature01",0,0,0)) {returnval = -1;}; - - - - -// printf("returnval:%d\n", returnval); - - return(returnval); + return 104; } -
--- a/libtests/0032/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0032/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -0009 with -K restriction with lib +0009 and include / exclude keys with library
--- a/libtests/0033/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0033/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,206 +1,92 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + const char *keys[2]; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; + adb_datum_t datum1 = {1, 2, "testfeature01", (double[2]) {0, 1}}; + adb_datum_t datum2 = {1, 2, "testfeature10", (double[2]) {1, 0}}; + if(audiodb_insert_datum(adb, &datum1)) + return 1; + if(audiodb_insert_datum(adb, &datum2)) + return 1; + if(audiodb_l2norm(adb)) + return 1; -int main(int argc, char **argv){ + adb_datum_t query = {1, 2, "testquery", (double[2]) {0, 0.5}}; - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - int size=0; + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 1; + qid.sequence_start = 0; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_EUCLIDEAN_NORMED, 10, 10}; + adb_query_refine_t refine = {0}; + refine.flags |= ADB_REFINE_RADIUS; + refine.radius = 5; + refine.hopsize = 1; - /* remove old directory */ -//if [ -f testdb ]; then rm -f testdb; fi - clean_remove_db(databasename); + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; - /* create new db */ -//${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + result_present_or_fail(results, "testfeature10", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); -//intstring 2 > testfeature01 -//floatstring 0 1 >> testfeature01 -//intstring 2 > testfeature10 -//floatstring 1 0 >> testfeature10 - ivals[0]=2; - dvals[0]=0; dvals[1]=1; - maketestfile("testfeature01",ivals,dvals,2); - ivals[0]=2; - dvals[0]=1; dvals[1]=0; - maketestfile("testfeature10",ivals,dvals,2); + /* the test in the original test suite for + * audioDB-the-command-line-program alters the parms.ntracks, which + * is not very meaningful in this context (given that we don't do + * aggregation, but simply return valid points); here instead we + * check that radius filtering works. */ + spec.refine.radius = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); -//${AUDIODB} -d testdb -I -f testfeature01 -//${AUDIODB} -d testdb -I -f testfeature10 - myinsert.features="testfeature01"; - if(audiodb_insert(mydbp,&myinsert)) {returnval = -1; }; - myinsert.features="testfeature10"; - if(audiodb_insert(mydbp,&myinsert)) {returnval = -1; }; + spec.refine.radius = 5; + spec.refine.flags |= ADB_REFINE_INCLUDE_KEYLIST; + spec.refine.include.nkeys = 0; + spec.refine.include.keys = keys; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 0) return 1; + audiodb_query_free_results(adb, &spec, results); -//# sequence queries require L2NORM -//${AUDIODB} -d testdb -L - if (audiodb_l2norm(mydbp)) {returnval=-1;}; + spec.refine.include.nkeys = 1; + spec.refine.include.keys[0] = "testfeature01"; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); -//echo "query point (0.0,0.5)" -//intstring 2 > testquery -//floatstring 0 0.5 >> testquery - ivals[0]=2; - dvals[0]=0; dvals[1]=0.5; - maketestfile("testquery",ivals,dvals,2); + spec.refine.radius = 1; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -R 5 > testoutput -//audioDB -Q sequence -d testdb -f testquery -R 5 -l 1 -//echo testfeature01 1 > test-expected-output -//echo testfeature10 1 >> test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="1"; - myadbquery.radius="5"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; + spec.refine.radius = 5; + spec.refine.include.keys[0] = "testfeature10"; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature10", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); - /* check the test values */ - if (size != 2) {returnval = -1;}; - if (testoneradiusresult(&myadbqueryresult,0,"testfeature01",1)) {returnval = -1;}; - if (testoneradiusresult(&myadbqueryresult,1,"testfeature10",1)) {returnval = -1;}; + spec.refine.radius = 1; + spec.refine.include.keys[0] = "testfeature10"; + results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 0) return 1; + audiodb_query_free_results(adb, &spec, results); -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -K /dev/null -R 5 > testoutput -//cat /dev/null > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.keylist="/dev/null"; - myadbquery.sequencelength="1"; - myadbquery.radius="5"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; + audiodb_close(adb); - /* check the test values */ - if (size != 0) {returnval = -1;}; - - - -//echo testfeature01 > testkl.txt - makekeylistfile("testkl.txt","testfeature01"); -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -K testkl.txt -R 5 > testoutput -//echo testfeature01 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.keylist="testkl.txt"; - myadbquery.sequencelength="1"; - myadbquery.radius="5"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneradiusresult(&myadbqueryresult,0,"testfeature01",1)) {returnval = -1;}; - -//echo testfeature10 > testkl.txt - makekeylistfile("testkl.txt","testfeature10"); -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -K testkl.txt -R 5 > testoutput -//echo testfeature10 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.keylist="testkl.txt"; - myadbquery.sequencelength="1"; - myadbquery.radius="5"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneradiusresult(&myadbqueryresult,0,"testfeature10",1)) {returnval = -1;}; - -//echo testfeature10 > testkl.txt -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -K testkl.txt -r 1 -R 5 > testoutput -//echo testfeature10 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.keylist="testkl.txt"; - myadbquery.sequencelength="1"; - myadbquery.radius="5"; - myadbquery.resultlength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneradiusresult(&myadbqueryresult,0,"testfeature10",1)) {returnval = -1;}; - - -//# NB: one might be tempted to insert a test here for having both keys -//# in the keylist, but in non-database order, and then checking that -//# the result list is also in that non-database order. I think that -//# would be misguided, as the efficient way of dealing with such a -//# keylist is to advance as-sequentially-as-possible through the -//# database; it just so happens that our current implementation is not -//# so smart. - -//echo "query point (0.5,0.0)" -//intstring 2 > testquery -//floatstring 0.5 0 >> testquery - ivals[0]=2; - dvals[0]=0.5; dvals[1]=0.0; - maketestfile("testquery",ivals,dvals,2); - -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -R 5 > testoutput -//echo testfeature01 1 > test-expected-output -//echo testfeature10 1 >> test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.keylist=NULL; - myadbquery.sequencelength="1"; - myadbquery.radius="5"; - myadbquery.resultlength=NULL; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 2) {returnval = -1;}; - if (testoneradiusresult(&myadbqueryresult,0,"testfeature01",1)) {returnval = -1;}; - if (testoneradiusresult(&myadbqueryresult,1,"testfeature10",1)) {returnval = -1;}; - -//echo testfeature10 > testkl.txt -//${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -K testkl.txt -r 1 -R 5 > testoutput -//echo testfeature10 1 > test-expected-output -//cmp testoutput test-expected-output - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.keylist="testkl.txt"; - myadbquery.sequencelength="1"; - myadbquery.radius="5"; - myadbquery.resultlength="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - /* check the test values */ - if (size != 1) {returnval = -1;}; - if (testoneradiusresult(&myadbqueryresult,0,"testfeature10",1)) {returnval = -1;}; - - - //fprintf(stderr,"returnval:%d\n",returnval); - return(returnval); + return 104; } -
--- a/libtests/0033/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0033/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -0010 with -K restriction with lib +0010 and include / exclude keylists with library
--- a/libtests/0034/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0034/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,183 +1,66 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + adb_insert_t insert = {0}; + adb_status_t status = {0}; + adb_insert_t batch[4] = {{0},{0},{0},{0}}; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; -int main(int argc, char **argv){ + maketestfile("testfeature", 2, (double[2]) {1, 1}, 2); + maketestfile("testfeature01", 2, (double[2]) {0, 1}, 2); + maketestfile("testfeature10", 2, (double[2]) {1, 0}, 2); - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - char * databasename="testdb"; - adb_status_t mystatus={0}; - adb_insert_t ins1[3]={{0},{0},{0}}; + insert.features = "testfeature"; + if(audiodb_insert(adb, &insert)) + return 1; + if(audiodb_status(adb, &status) || status.numFiles != 1) + return 1; + + /* reinserts using audiodb_insert() should silently not fail and + * silently not insert, to support legacy command-line behaviour. */ + if(audiodb_insert(adb, &insert)) + return 1; + if(audiodb_status(adb, &status) || status.numFiles != 1) + return 1; + /* reinserts using audiodb_insert_datum() should fail. */ + adb_datum_t datum = {1, 2, "testfeature", (double[2]) {1, 1}}; + if(!audiodb_insert_datum(adb, &datum)) + return 1; + insert.features = "testfeature01"; + if(audiodb_insert(adb, &insert)) + return 1; + if(audiodb_status(adb, &status) || status.numFiles != 2) + return 1; - /* remove old directory */ -//if [ -f testdb ]; then rm -f testdb; fi - clean_remove_db(databasename); + insert.features = "testfeature10"; + if(audiodb_insert(adb, &insert)) + return 1; + if(audiodb_status(adb, &status) || status.numFiles != 3) + return 1; + + audiodb_close(adb); - /* create new db */ -//${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; + batch[0].features = "testfeature"; + batch[1].features = "testfeature01"; + batch[2].features = "testfeature10"; + batch[3].features = "testfeature10"; + if(audiodb_batchinsert(adb, batch, 4)) + return 1; + if(audiodb_status(adb, &status) || status.numFiles != 3) + return 1; -//intstring 2 > testfeature -//floatstring 1 1 >> testfeature -//intstring 2 > testfeature01 -//floatstring 0 1 >> testfeature01 -//intstring 2 > testfeature10 -//floatstring 1 0 >> testfeature10 - ivals[0]=2; - dvals[0]=1; dvals[1]=1; - maketestfile("testfeature",ivals,dvals,2); - ivals[0]=2; - dvals[0]=0; dvals[1]=1; - maketestfile("testfeature01",ivals,dvals,2); - ivals[0]=2; - dvals[0]=1; dvals[1]=0; - maketestfile("testfeature10",ivals,dvals,2); + audiodb_close(adb); -//${AUDIODB} -d testdb -I -f testfeature - myinsert.features="testfeature"; - if(audiodb_insert(mydbp,&myinsert)) {returnval = -1; }; - -//${AUDIODB} -d testdb -S | grep "num files:1" - if(audiodb_status(mydbp,&mystatus)) {returnval = -1; }; - if(mystatus.numFiles != 1) { returnval = -1; } - -//${AUDIODB} -d testdb -I -f testfeature - myinsert.features="testfeature"; - if(audiodb_insert(mydbp,&myinsert)) {returnval = -1; }; - -//${AUDIODB} -d testdb -S | grep "num files:1" - if(audiodb_status(mydbp,&mystatus)) {returnval = -1; }; - if(mystatus.numFiles != 1) { returnval = -1; } - -//${AUDIODB} -d testdb -I -f testfeature01 - myinsert.features="testfeature01"; - if(audiodb_insert(mydbp,&myinsert)) {returnval = -1; }; - -//${AUDIODB} -d testdb -S | grep "num files:2" - if(audiodb_status(mydbp,&mystatus)) {returnval = -1; }; - if(mystatus.numFiles != 2) { returnval = -1; } - -//${AUDIODB} -d testdb -I -f testfeature10 - myinsert.features="testfeature10"; - if(audiodb_insert(mydbp,&myinsert)) {returnval = -1; }; - -//${AUDIODB} -d testdb -S | grep "num files:3" - if(audiodb_status(mydbp,&mystatus)) {returnval = -1; }; - if(mystatus.numFiles != 3) { returnval = -1; } - -//rm -f testdb - clean_remove_db(databasename); - - /* create new db */ -//${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); - - -//${AUDIODB} -d testdb -I -f testfeature01 - myinsert.features="testfeature01"; - if(audiodb_insert(mydbp,&myinsert)) {returnval = -1; }; - -//${AUDIODB} -d testdb -S | grep "num files:1" - if(audiodb_status(mydbp,&mystatus)) {returnval = -1; }; - if(mystatus.numFiles != 1) { returnval = -1; } - -//${AUDIODB} -d testdb -I -f testfeature01 - myinsert.features="testfeature01"; - if(audiodb_insert(mydbp,&myinsert)) {returnval = -1; }; - -//${AUDIODB} -d testdb -S | grep "num files:1" - if(audiodb_status(mydbp,&mystatus)) {returnval = -1; }; - if(mystatus.numFiles != 1) { returnval = -1; } - -//${AUDIODB} -d testdb -I -f testfeature10 - myinsert.features="testfeature10"; - if(audiodb_insert(mydbp,&myinsert)) {returnval = -1; }; - -//${AUDIODB} -d testdb -S | grep "num files:2" - if(audiodb_status(mydbp,&mystatus)) {returnval = -1; }; - if(mystatus.numFiles != 2) { returnval = -1; } - -//${AUDIODB} -d testdb -I -f testfeature - myinsert.features="testfeature"; - if(audiodb_insert(mydbp,&myinsert)) {returnval = -1; }; - -//${AUDIODB} -d testdb -S | grep "num files:3" - if(audiodb_status(mydbp,&mystatus)) {returnval = -1; }; - if(mystatus.numFiles != 3) { returnval = -1; } - - - - -//rm -f testdb - clean_remove_db(databasename); - - /* create new db */ -//${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); - -//echo testfeature > testfeaturelist.txt -//echo testfeature01 >> testfeaturelist.txt -//echo testfeature10 >> testfeaturelist.txt -//${AUDIODB} -B -F testfeaturelist.txt -d testdb - ins1[0].features="testfeature"; - ins1[1].features="testfeature01"; - ins1[2].features="testfeature10"; - if(audiodb_batchinsert(mydbp,ins1,3)){ - returnval=-1; - }; - -//${AUDIODB} -d testdb -S | grep "num files:3" - if(audiodb_status(mydbp,&mystatus)) {returnval = -1; }; - if(mystatus.numFiles != 3) { returnval = -1; } - - - - -//rm -f testdb - clean_remove_db(databasename); - - /* create new db */ -//${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); - - -//echo testfeature01 > testfeaturelist.txt -//echo testfeature10 >> testfeaturelist.txt -//echo testfeature >> testfeaturelist.txt -//${AUDIODB} -B -F testfeaturelist.txt -d testdb - ins1[0].features="testfeature"; - ins1[1].features="testfeature01"; - ins1[2].features="testfeature10"; - if(audiodb_batchinsert(mydbp,ins1,3)){ - returnval=-1; - }; - -//${AUDIODB} -d testdb -S | grep "num files:3" - if(audiodb_status(mydbp,&mystatus)) {returnval = -1; }; - if(mystatus.numFiles != 3) { returnval = -1; } - - - - fprintf(stderr,"returnval:%d\n",returnval); - return(returnval); + return 104; }
--- a/libtests/0034/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0034/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -key duplicate test with lib +key duplicate test with library
--- a/libtests/0035/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0035/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,95 +1,44 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; -int main(int argc, char **argv){ + adb_datum_t datum1 = {1, 2, "testfeature1", (double[2]) {0, 1}}; + adb_datum_t datum3 = {3, 2, "testfeature3", (double[6]) {1, 0, 0, 1, 1, 0}}; - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - int size=0; + if(audiodb_insert_datum(adb, &datum1)) + return 1; + if(audiodb_insert_datum(adb, &datum3)) + return 1; + if(audiodb_l2norm(adb)) + return 1; + adb_datum_t query = {2, 2, "testquery", (double[4]) {0, 1, 1, 0}}; + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 2; + qid.sequence_start = 0; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_EUCLIDEAN, 1, 10}; + adb_query_refine_t refine = {0}; + refine.hopsize = 1; - /* remove old directory */ - //if [ -f testdb ]; then rm -f testdb; fi - clean_remove_db(databasename); + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; - /* create new db */ - //${AUDIODB} -d testdb -N - mydbp=audiodb_create(databasename,0,0,0); + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + if(!results || results->nresults != 1) return 1; + result_present_or_fail(results, "testfeature3", 0, 0, 1); + audiodb_query_free_results(adb, &spec, results); -//intstring 2 > testfeature1 -//floatstring 0 1 >> testfeature1 - ivals[0]=2; - dvals[0]=0; dvals[1]=1; - maketestfile("testfeature1",ivals,dvals,2); + audiodb_close(adb); -//intstring 2 > testfeature3 -//floatstring 1 0 >> testfeature3 -//floatstring 0 1 >> testfeature3 -//floatstring 1 0 >> testfeature3 - ivals[0]=2; - dvals[0]=1; dvals[1]=0; - dvals[2]=0; dvals[3]=1; - dvals[4]=1; dvals[5]=0; - maketestfile("testfeature3",ivals,dvals,6); - -//${AUDIODB} -d testdb -I -f testfeature1 - myinsert.features="testfeature1"; - if (audiodb_insert(mydbp,&myinsert)){ returnval=-1; } -//${AUDIODB} -d testdb -I -f testfeature3 - myinsert.features="testfeature3"; - if (audiodb_insert(mydbp,&myinsert)){ returnval=-1; } - -//# sequence queries require L2NORM -//${AUDIODB} -d testdb -L - if(audiodb_l2norm(mydbp)){ returnval=-1; }; - -//echo "query point (0 1, 1 0)" -//intstring 2 > testquery -//floatstring 0 1 >> testquery -//floatstring 1 0 >> testquery - ivals[0]=2; - dvals[0]=0; dvals[1]=1; - dvals[2]=1; dvals[3]=0; - maketestfile("testquery",ivals,dvals,4); - -//audioDB -Q sequence -d testdb -f testquery -n 1 -l 2 -//${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -n 1 > testoutput - myadbquery.querytype="sequence"; - myadbquery.feature="testquery"; - myadbquery.sequencelength="2"; - myadbquery.numpoints="1"; - audiodb_query(mydbp,&myadbquery,&myadbqueryresult); - size=myadbqueryresult.sizeRlist; - - //dump_query(&myadbquery,&myadbqueryresult); - - /* check the test values */ -////wc -l testoutput | grep "1 testoutput" -////grep "^testfeature3 .* 0 1$" testoutput - if (size != 1) {returnval = -1;}; - if (strcmp(myadbqueryresult.Rlist[0],"testfeature3")){ returnval = -1; }; - - //printf("returnval:%d\n",returnval); - return(returnval); + return 104; } -
--- a/libtests/0035/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0035/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ --l 2 search with short tracks with lib +search with short tracks using library
--- a/libtests/0036/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0036/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,117 +1,115 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" +int main(int argc, char **argv) { + adb_t *adb; + clean_remove_db(TESTDB); + if(!(adb = audiodb_create(TESTDB, 0, 0, 0))) + return 1; -int main(int argc, char **argv){ + adb_datum_t datum1 = {2, 2, "testfeature01", (double[4]) {0, 1, 1, 0}}; + adb_datum_t datum2 = {2, 2, "testfeature10", (double[4]) {1, 0, 0, 1}}; - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - unsigned int myerr=0; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - adb_query_t myadbquery2={0}; - adb_queryresult_t myadbqueryresult2={0}; - int size=0; + if(audiodb_insert_datum(adb, &datum1)) + return 1; + if(audiodb_insert_datum(adb, &datum2)) + return 1; + if(audiodb_l2norm(adb)) + return 1; + adb_datum_t query = {1, 2, NULL, (double [2]){0, 0.5}, NULL, NULL}; + adb_query_id_t qid = {0}; + qid.datum = &query; + qid.sequence_length = 1; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_EUCLIDEAN_NORMED, 10, 10}; + adb_query_refine_t refine = {0}; + refine.hopsize = 1; -//#! /bin/bash -// -//. ../test-utils.sh -// -//if [ -f testdb ]; then rm -f testdb; fi -// -//${AUDIODB} -d testdb -N -// -//intstring 2 > testfeature01 -//floatstring 0 1 >> testfeature01 -//floatstring 1 0 >> testfeature01 -//intstring 2 > testfeature10 -//floatstring 1 0 >> testfeature10 -//floatstring 0 1 >> testfeature10 -// -//cat > testfeaturefiles <<EOF -//testfeature01 -//testfeature10 -//EOF -// -//${AUDIODB} -d testdb -B -F testfeaturefiles -// -//# sequence queries require L2NORM -//${AUDIODB} -d testdb -L -// -//echo "query point (0.0,0.5)" -//intstring 2 > testquery -//floatstring 0 0.5 >> testquery -// -//${AUDIODB} -d testdb -Q nsequence -l 1 -f testquery > testoutput -//echo testfeature01 1 > test-expected-output -//echo 0 0 0 >> test-expected-output -//echo 2 0 1 >> test-expected-output -//echo testfeature10 1 >> test-expected-output -//echo 0 0 1 >> test-expected-output -//echo 2 0 0 >> test-expected-output -//cmp testoutput test-expected-output -// -//${AUDIODB} -d testdb -Q nsequence -l 1 -f testquery -n 2 > testoutput -//cmp testoutput test-expected-output -// -//${AUDIODB} -d testdb -Q nsequence -l 1 -f testquery -n 5 > testoutput -//cmp testoutput test-expected-output -// -//${AUDIODB} -d testdb -Q nsequence -l 1 -f testquery -n 1 > testoutput -//echo testfeature01 0 > test-expected-output -//echo 0 0 0 >> test-expected-output -//echo testfeature10 0 >> test-expected-output -//echo 0 0 1 >> test-expected-output -//cmp testoutput test-expected-output -// -//echo "query point (0.5,0.0)" -//intstring 2 > testquery -//floatstring 0.5 0 >> testquery -// -//${AUDIODB} -d testdb -Q nsequence -l 1 -f testquery > testoutput -//echo testfeature01 1 > test-expected-output -//echo 0 0 1 >> test-expected-output -//echo 2 0 0 >> test-expected-output -//echo testfeature10 1 >> test-expected-output -//echo 0 0 0 >> test-expected-output -//echo 2 0 1 >> test-expected-output -//cmp testoutput test-expected-output -// -//${AUDIODB} -d testdb -Q nsequence -l 1 -f testquery -n 2 > testoutput -//cmp testoutput test-expected-output -// -//${AUDIODB} -d testdb -Q nsequence -l 1 -f testquery -n 5 > testoutput -//cmp testoutput test-expected-output -// -//${AUDIODB} -d testdb -Q nsequence -l 1 -f testquery -n 1 > testoutput -//echo testfeature01 0 > test-expected-output -//echo 0 0 1 >> test-expected-output -//echo testfeature10 0 >> test-expected-output -//echo 0 0 0 >> test-expected-output -//cmp testoutput test-expected-output -// -//exit 104 + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; - returnval=-1; - - return(returnval); + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + + if(!results || results->nresults != 4) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + result_present_or_fail(results, "testfeature01", 2, 0, 1); + result_present_or_fail(results, "testfeature10", 0, 0, 1); + result_present_or_fail(results, "testfeature10", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); + + spec.params.npoints = 2; + results = audiodb_query_spec(adb, &spec); + + if(!results || results->nresults != 4) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + result_present_or_fail(results, "testfeature01", 2, 0, 1); + result_present_or_fail(results, "testfeature10", 0, 0, 1); + result_present_or_fail(results, "testfeature10", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); + + spec.params.npoints = 5; + results = audiodb_query_spec(adb, &spec); + + if(!results || results->nresults != 4) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + result_present_or_fail(results, "testfeature01", 2, 0, 1); + result_present_or_fail(results, "testfeature10", 0, 0, 1); + result_present_or_fail(results, "testfeature10", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); + + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + result_present_or_fail(results, "testfeature10", 0, 0, 1); + + audiodb_query_free_results(adb, &spec, results); + + spec.qid.datum->data = (double [2]) {0.5, 0}; + spec.params.npoints = 10; + results = audiodb_query_spec(adb, &spec); + + if(!results || results->nresults != 4) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 1); + result_present_or_fail(results, "testfeature01", 2, 0, 0); + result_present_or_fail(results, "testfeature10", 0, 0, 0); + result_present_or_fail(results, "testfeature10", 2, 0, 1); + audiodb_query_free_results(adb, &spec, results); + + spec.params.npoints = 2; + results = audiodb_query_spec(adb, &spec); + + if(!results || results->nresults != 4) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 1); + result_present_or_fail(results, "testfeature01", 2, 0, 0); + result_present_or_fail(results, "testfeature10", 0, 0, 0); + result_present_or_fail(results, "testfeature10", 2, 0, 1); + audiodb_query_free_results(adb, &spec, results); + + spec.params.npoints = 5; + results = audiodb_query_spec(adb, &spec); + + if(!results || results->nresults != 4) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 1); + result_present_or_fail(results, "testfeature01", 2, 0, 0); + result_present_or_fail(results, "testfeature10", 0, 0, 0); + result_present_or_fail(results, "testfeature10", 2, 0, 1); + audiodb_query_free_results(adb, &spec, results); + + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 1); + result_present_or_fail(results, "testfeature10", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); + + audiodb_close(adb); + + return 104; } -
--- a/libtests/0036/short-description Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/0036/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -1,1 +1,1 @@ -nsequence search with lib +nsequence search using library
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/libtests/0037/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -0,0 +1,132 @@ +#include "audioDB_API.h" +#include "test_utils_lib.h" + +int main(int argc, char *argv[]) { + adb_t *adb; + adb_insert_t *batch = 0; + adb_status_t status; + + clean_remove_db(TESTDB); + adb = audiodb_create(TESTDB, 0, 0, 0); + if(!adb) { + return 1; + } + + maketestfile("testfeature01", 2, (double[4]) {0,1,1,0}, 4); + maketestfile("testfeature10", 2, (double[4]) {1,0,0,1}, 4); + + batch = (adb_insert_t *) calloc(6, sizeof(adb_insert_t)); + if(!batch) { + return 1; + } + batch[0].features = "testfeature01"; + batch[1].features = "testfeature01"; + batch[2].features = "testfeature10"; + batch[3].features = "testfeature10"; + batch[4].features = "testfeature01"; + batch[5].features = "testfeature10"; + + audiodb_batchinsert(adb, batch, 6); + free(batch); + + if(audiodb_status(adb, &status) || status.numFiles != 2) + return 1; + + if(audiodb_l2norm(adb)) + return 1; + + adb_datum_t datum = {1, 2, NULL, (double [2]){0, 0.5}, NULL, NULL}; + adb_query_id_t qid = {0}; + qid.datum = &datum; + qid.sequence_length = 1; + adb_query_parameters_t parms = + {ADB_ACCUMULATION_PER_TRACK, ADB_DISTANCE_EUCLIDEAN_NORMED, 10, 10}; + adb_query_refine_t refine = {0}; + refine.hopsize = 1; + + adb_query_spec_t spec; + spec.qid = qid; + spec.params = parms; + spec.refine = refine; + + adb_query_results_t *results = audiodb_query_spec(adb, &spec); + + if(!results || results->nresults != 4) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + result_present_or_fail(results, "testfeature01", 2, 0, 1); + result_present_or_fail(results, "testfeature10", 0, 0, 1); + result_present_or_fail(results, "testfeature10", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); + + spec.params.npoints = 2; + results = audiodb_query_spec(adb, &spec); + + if(!results || results->nresults != 4) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + result_present_or_fail(results, "testfeature01", 2, 0, 1); + result_present_or_fail(results, "testfeature10", 0, 0, 1); + result_present_or_fail(results, "testfeature10", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); + + spec.params.npoints = 5; + results = audiodb_query_spec(adb, &spec); + + if(!results || results->nresults != 4) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + result_present_or_fail(results, "testfeature01", 2, 0, 1); + result_present_or_fail(results, "testfeature10", 0, 0, 1); + result_present_or_fail(results, "testfeature10", 2, 0, 0); + audiodb_query_free_results(adb, &spec, results); + + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 0); + result_present_or_fail(results, "testfeature10", 0, 0, 1); + + audiodb_query_free_results(adb, &spec, results); + + spec.qid.datum->data = (double [2]) {0.5, 0}; + spec.params.npoints = 10; + results = audiodb_query_spec(adb, &spec); + + if(!results || results->nresults != 4) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 1); + result_present_or_fail(results, "testfeature01", 2, 0, 0); + result_present_or_fail(results, "testfeature10", 0, 0, 0); + result_present_or_fail(results, "testfeature10", 2, 0, 1); + audiodb_query_free_results(adb, &spec, results); + + spec.params.npoints = 2; + results = audiodb_query_spec(adb, &spec); + + if(!results || results->nresults != 4) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 1); + result_present_or_fail(results, "testfeature01", 2, 0, 0); + result_present_or_fail(results, "testfeature10", 0, 0, 0); + result_present_or_fail(results, "testfeature10", 2, 0, 1); + audiodb_query_free_results(adb, &spec, results); + + spec.params.npoints = 5; + results = audiodb_query_spec(adb, &spec); + + if(!results || results->nresults != 4) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 1); + result_present_or_fail(results, "testfeature01", 2, 0, 0); + result_present_or_fail(results, "testfeature10", 0, 0, 0); + result_present_or_fail(results, "testfeature10", 2, 0, 1); + audiodb_query_free_results(adb, &spec, results); + + spec.params.npoints = 1; + results = audiodb_query_spec(adb, &spec); + + if(!results || results->nresults != 2) return 1; + result_present_or_fail(results, "testfeature01", 0, 0, 1); + result_present_or_fail(results, "testfeature10", 0, 0, 0); + audiodb_query_free_results(adb, &spec, results); + + audiodb_close(adb); + + return 104; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/libtests/0037/short-description Sat Jan 10 16:47:57 2009 +0000 @@ -0,0 +1,1 @@ +batchinsert repeated key handling with library \ No newline at end of file
--- a/libtests/9000/prog1.c Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/9000/prog1.c Sat Jan 10 16:47:57 2009 +0000 @@ -1,44 +1,7 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sysexits.h> -#include <fcntl.h> -#include <dirent.h> -#include <unistd.h> -#include <sys/stat.h> -/* - * * #define NDEBUG - * * */ -#include <assert.h> +#include "audioDB_API.h" +#include "test_utils_lib.h" -#include "../../audioDB_API.h" -#include "../test_utils_lib.h" - - -int main(int argc, char **argv){ - - int returnval=0; - adb_ptr mydbp={0}; - int ivals[10]; - double dvals[10]; - adb_insert_t myinsert={0}; - unsigned int myerr=0; - char * databasename="testdb"; - adb_query_t myadbquery={0}; - adb_queryresult_t myadbqueryresult={0}; - adb_query_t myadbquery2={0}; - adb_queryresult_t myadbqueryresult2={0}; - int size=0; - - - - - - - - - returnval=-1; - - return(returnval); +int main(int argc, char **argv) { + return 14; }
--- a/libtests/libtest.mk Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/libtest.mk Sat Jan 10 16:47:57 2009 +0000 @@ -28,7 +28,7 @@ -ln -s $< $@ test1: prog1.c ../test_utils_lib.h ../../audioDB_API.h - gcc -Wall $(ARCH_FLAGS) -laudioDB -L. -Wl,-rpath,. -o $@ $< + gcc -g -std=c99 -Wall -Werror $(ARCH_FLAGS) -I.. -I../.. -laudioDB -L. -Wl,-rpath,. -o $@ $< clean: - -rm $(LIBRARY_FULL) $(LIBRARY_VERS) $(LIBRARY) \ No newline at end of file + -rm $(LIBRARY_FULL) $(LIBRARY_VERS) $(LIBRARY)
--- a/libtests/notes Sat Jan 10 11:11:27 2009 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,3 +0,0 @@ -test 23 in original: is there a testquery point missing? - -ask Christophe about 24: testtimes file has 1 entry too many
--- a/libtests/run-tests.sh Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/run-tests.sh Sat Jan 10 16:47:57 2009 +0000 @@ -1,15 +1,6 @@ #! /bin/bash -AUDIODB=../../${EXECUTABLE:-audioDB} -export AUDIODB - -if [ -x ${AUDIODB#../} ]; then - : -else - echo Cannot execute audioDB: ${AUDIODB#../} - exit 1 -fi - +# FIXME: work out how to do proper getopt in bash if [ "$1" = "--full" ]; then pattern="[0-9][0-9][0-9][0-9]*" else @@ -23,8 +14,15 @@ if [ -f ${file}/short-description ]; then awk '{ printf(" (%s)",$0) }' < ${file}/short-description fi + if [ "$1" = "--valgrind" ]; then + echo -n \ under valgrind + fi echo -n : - (cd ${file} && make -f ../libtest.mk >/dev/null 2>&1 && ./test1 > test.out 2> test.err && exit 104) + if [ "$1" = "--valgrind" ]; then + (cd ${file} && make -f ../libtest.mk >/dev/null 2>&1 && valgrind --leak-check=full --show-reachable=yes --error-exitcode=1 --tool=memcheck ./test1 > test.out 2> test.err) + else + (cd ${file} && make -f ../libtest.mk >/dev/null 2>&1 && ./test1 > test.out 2> test.err) + fi EXIT_STATUS=$? if [ ${EXIT_STATUS} -eq 14 ]; then echo " n/a."
--- a/libtests/test_utils_lib.h Sat Jan 10 11:11:27 2009 +0000 +++ b/libtests/test_utils_lib.h Sat Jan 10 16:47:57 2009 +0000 @@ -1,214 +1,42 @@ -void delete_dir(char * dirname); -void clean_remove_db(char * dirname); -void test_status(adb_ptr d, adb_status_ptr b); -unsigned int test_insert( adb_ptr d, char * features, char * power, char * key); -void dump_query(adb_query_ptr adbq, adb_queryresult_ptr myadbqueryresult); -int testoneresult(adb_queryresult_ptr myadbqueryresult, int i, char * Rlist, double Dist,double Qpos,double Spos); -double doubleabs(double foo); -void maketestfile(char * filename, int * ivals, double * dvals, int dvalsize); -int testoneradiusresult(adb_queryresult_ptr myadbqueryresult, int i, char * Rlist, int count); -void makekeylistfile(char * filename, char * item); +#include <sys/types.h> +#include <sys/stat.h> +#include <math.h> +#include <unistd.h> +#include <fcntl.h> +#include <string.h> +#include <stdio.h> +#include <stdlib.h> +#define TESTDB "testdb" - - -/* clean remove */ -void clean_remove_db(char * dbname){ - - FILE* db=0; - - db=fopen(dbname,"r"); - - if (!db){ - return; - } - - - fclose(db); - remove(dbname); - - return; - +void clean_remove_db(char * dbname) { + unlink(dbname); } +void maketestfile(const char *path, int dim, double *doubles, int ndoubles) { + FILE *file; -/* delete directory */ -void delete_dir(char * dirname){ - - struct dirent *d; - DIR *dir; - char buf[256]; - - printf("Deleting directory '%s' and all files\n", dirname); - dir = opendir(dirname); - - if (dir){ - while((d = readdir(dir))) { - //printf("Deleting %s in %s\n",d->d_name, dirname); - sprintf(buf, "%s/%s", dirname, d->d_name); - remove(buf); - } - } - closedir(dir); - - rmdir(dirname); - - - return; - + file = fopen(path, "w"); + fwrite(&dim, sizeof(int), 1, file); + fwrite(doubles, sizeof(double), ndoubles, file); + fflush(file); + fclose(file); } - -unsigned int test_insert( - adb_ptr d, - char * features, - char * power, - char * key -){ - - adb_insert_t myinsert={0}; - unsigned int myerr=0; - - printf("Insert:\n"); - myinsert.features=features; - myinsert.power=power; - myinsert.key=key; - myerr=audiodb_insert(d,&myinsert); - printf("\n"); - - return myerr; - +int close_enough(double a, double b, double epsilon) { + return (fabs(a-b) < epsilon); } -void test_status(adb_ptr d, adb_status_ptr b){ - - /* get the status of the database */ - audiodb_status(d,b); - - /* could probably make this look a bit more clever, but it works for now */ - printf("numFiles:\t%d\n",b->numFiles); - printf("dim:\t%d\n",b->dim); - printf("length:\t%d\n",b->length); - printf("dudCount:\t%d\n",b->dudCount); - printf("nullCount:\t%d\n",b->nullCount); - printf("flags:\t%d\n",b->flags); - - return; +int result_position(adb_query_results_t *r, const char *key, float dist, uint32_t qpos, uint32_t ipos) { + for(uint32_t k = 0; k < r->nresults; k++) { + adb_result_t result = r->results[k]; + if(close_enough(dist, result.dist, 1e-4) && (qpos == result.qpos) && + (ipos == result.ipos) && !(strcmp(key, result.key))) { + return k; + } + } + return -1; } - -void dump_query(adb_query_ptr adbq, adb_queryresult_ptr myadbqueryresult){ - - int size=0; - int i=0; - - size=myadbqueryresult->sizeRlist; - - printf("Dumping query:\n"); - for(i=0; i<size; i++){ - printf("\t'%s' query: Result %02d:%s is dist:%f qpos:%d spos:%d\n", - adbq->querytype, - i, - myadbqueryresult->Rlist[i], - myadbqueryresult->Dist[i], - myadbqueryresult->Qpos[i], - myadbqueryresult->Spos[i] - ); - } - printf("\n"); - -} - - - -int testoneresult(adb_queryresult_ptr myadbqueryresult, int i, char * Rlist, double Dist,double Qpos,double Spos){ - - int ret=0; - double tolerance=.0001; - - - - if (strcmp(Rlist,myadbqueryresult->Rlist[i])){ - ret=-1; - } - - - if (doubleabs((double)Dist - (double)myadbqueryresult->Dist[i]) > tolerance){ - ret=-1; - } - - if (doubleabs((double)Qpos - (double)myadbqueryresult->Qpos[i]) > tolerance){ - ret=-1; - } - - if (doubleabs((double)Spos - (double)myadbqueryresult->Spos[i]) > tolerance){ - ret=-1; - } - - return ret; -} - - -int testoneradiusresult(adb_queryresult_ptr myadbqueryresult, int i, char * Rlist, int count){ - - int ret=0; - - if (strcmp(Rlist,myadbqueryresult->Rlist[i])){ - ret=-1; - } - - /* KLUDGE: at the moment, the structure returned from "sequence" - queries with a radius has two unused fields, Dist and Qpos, and - the Spos field is punned to indicate the count of hits from - that track. This is really ugly and needs to die. */ - if (count != myadbqueryresult->Spos[i]) { - ret=-1; - } - - return ret; -} - - -double doubleabs(double foo){ - - double retval=foo; - - if (foo < 0.0) { - retval=foo * -1.0; - } - - return retval; -} - - - -void maketestfile(char * filename, int * ivals, double * dvals, int dvalsize) { - - FILE * myfile; - - myfile=fopen(filename,"w"); - fwrite(ivals,sizeof(int),1,myfile); - fwrite(dvals,sizeof(double),dvalsize,myfile); - fflush(myfile); - fclose(myfile); - - /* should probably test for success, but then it is a test suite already... */ -} - - - -void makekeylistfile(char * filename, char * item){ - - FILE * myfile; - - myfile=fopen(filename,"w"); - fprintf(myfile,"%s\n",item); - fflush(myfile); - fclose(myfile); - -} - - - - - +#define result_present_or_fail(r, k, d, q, i) \ + if(result_position(r, k, d, q, i) < 0) return 1;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/liszt.cpp Sat Jan 10 16:47:57 2009 +0000 @@ -0,0 +1,34 @@ +#include "audioDB.h" + +void audioDB::liszt(const char* dbName, unsigned offset, unsigned numLines, adb__lisztResponse* adbLisztResponse){ + if(!dbH) { + initTables(dbName, 0); + } + + assert(trackTable && fileTable); + + if(offset>dbH->numFiles){ + char tmpStr[MAXSTR]; + sprintf(tmpStr, "numFiles=%u, lisztOffset=%u", dbH->numFiles, offset); + error("listKeys offset out of range", tmpStr); + } + + if(!adbLisztResponse){ + for(Uns32T k=0; k<numLines && offset+k<dbH->numFiles; k++){ + fprintf(stdout, "[%d] %s (%d)\n", offset+k, fileTable+(offset+k)*O2_FILETABLE_ENTRY_SIZE, trackTable[offset+k]); + } + } + else{ + adbLisztResponse->result.Rkey = new char*[numLines]; + adbLisztResponse->result.Rlen = new unsigned int[numLines]; + Uns32T k = 0; + for( ; k<numLines && offset+k<dbH->numFiles; k++){ + adbLisztResponse->result.Rkey[k] = new char[MAXSTR]; + snprintf(adbLisztResponse->result.Rkey[k], O2_MAXFILESTR, "%s", fileTable+(offset+k)*O2_FILETABLE_ENTRY_SIZE); + adbLisztResponse->result.Rlen[k] = trackTable[offset+k]; + } + adbLisztResponse->result.__sizeRkey = k; + adbLisztResponse->result.__sizeRlen = k; + } + +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lock.cpp Sat Jan 10 16:47:57 2009 +0000 @@ -0,0 +1,37 @@ +#include "audioDB.h" + +int acquire_lock(int fd, bool exclusive) { + struct flock lock; + int status; + + lock.l_type = exclusive ? F_WRLCK : F_RDLCK; + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 0; /* "the whole file" */ + + retry: + do { + status = fcntl(fd, F_SETLKW, &lock); + } while (status != 0 && errno == EINTR); + + if (status) { + if (errno == EAGAIN) { + sleep(1); + goto retry; + } else { + return status; + } + } + return 0; +} + +int divest_lock(int fd) { + struct flock lock; + + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 0; + + return fcntl(fd, F_SETLKW, &lock); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/nearestaccumulator.h Sat Jan 10 16:47:57 2009 +0000 @@ -0,0 +1,57 @@ +template <class T> class NearestAccumulator : public Accumulator { +public: + NearestAccumulator(); + ~NearestAccumulator(); + void add_point(adb_result_t *r); + adb_query_results_t *get_points(); +private: + std::set< adb_result_t, adb_result_triple_lt > *set; + std::set< adb_result_t, adb_result_qpos_lt > *points; +}; + +template <class T> NearestAccumulator<T>::NearestAccumulator() + : set(0), points(0) { + set = new std::set< adb_result_t, adb_result_triple_lt >; + points = new std::set< adb_result_t, adb_result_qpos_lt >; +} + +template <class T> NearestAccumulator<T>::~NearestAccumulator() { + if(set) { + delete set; + } + if(points) { + delete points; + } +} + +template <class T> void NearestAccumulator<T>::add_point(adb_result_t *r) { + if(!isnan(r->dist)) { + if(set->find(*r) == set->end()) { + set->insert(*r); + + std::set< adb_result_t, adb_result_qpos_lt >::iterator it; + it = points->find(*r); + if(it == points->end()) { + points->insert(*r); + } else if(T()(*(const adb_result_t *)r,(*it))) { + points->erase(it); + points->insert(*r); + } + } + } +} + +template <class T> adb_query_results_t *NearestAccumulator<T>::get_points() { + unsigned int nresults = points->size(); + adb_query_results_t *r = (adb_query_results_t *) malloc(sizeof(adb_query_results_t)); + adb_result_t *rs = (adb_result_t *) calloc(nresults, sizeof(adb_result_t)); + r->nresults = nresults; + r->results = rs; + std::set< adb_result_t, adb_result_qpos_lt >::iterator it; + unsigned int k = 0; + for(it = points->begin(); it != points->end(); it++) { + rs[k++] = *it; + } + return r; +} +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/open.cpp Sat Jan 10 16:47:57 2009 +0000 @@ -0,0 +1,157 @@ +#include "audioDB.h" +extern "C" { +#include "audioDB_API.h" +#include "audioDB-internals.h" +} + +static bool audiodb_check_header(adb_header_t *header) { + /* FIXME: use syslog() or write to stderr or something to give the + poor user some diagnostics. */ + if(header->magic == O2_OLD_MAGIC) { + return false; + } + if(header->magic != O2_MAGIC) { + return false; + } + if(header->version != O2_FORMAT_VERSION) { + return false; + } + if(header->headerSize != O2_HEADERSIZE) { + return false; + } + return true; +} + +static int audiodb_collect_keys(adb_t *adb) { + char *key_table = 0; + size_t key_table_length = 0; + + if(adb->header->length > 0) { + unsigned nfiles = adb->header->numFiles; + key_table_length = ALIGN_PAGE_UP(nfiles * O2_FILETABLE_ENTRY_SIZE); + mmap_or_goto_error(char *, key_table, adb->header->fileTableOffset, key_table_length); + for (unsigned int k = 0; k < nfiles; k++) { + adb->keys->push_back(key_table + k*O2_FILETABLE_ENTRY_SIZE); + (*adb->keymap)[(key_table + k*O2_FILETABLE_ENTRY_SIZE)] = k; + } + munmap(key_table, key_table_length); + } + + return 0; + + error: + maybe_munmap(key_table, key_table_length); + return 1; +} + +static int audiodb_collect_track_lengths(adb_t *adb) { + uint32_t *track_table = 0; + size_t track_table_length = 0; + if(adb->header->length > 0) { + unsigned nfiles = adb->header->numFiles; + track_table_length = ALIGN_PAGE_UP(nfiles * O2_TRACKTABLE_ENTRY_SIZE); + mmap_or_goto_error(uint32_t *, track_table, adb->header->trackTableOffset, track_table_length); + off_t offset = 0; + for (unsigned int k = 0; k < nfiles; k++) { + uint32_t track_length = track_table[k]; + adb->track_lengths->push_back(track_length); + adb->track_offsets->push_back(offset); + offset += track_length * adb->header->dim * sizeof(double); + } + munmap(track_table, track_table_length); + } + + return 0; + + error: + maybe_munmap(track_table, track_table_length); + return 1; +} + +adb_t *audiodb_open(const char *path, int flags) { + adb_t *adb = 0; + int fd = -1; + + flags &= (O_RDONLY|O_RDWR); + fd = open(path, flags); + if(fd == -1) { + goto error; + } + if(acquire_lock(fd, flags == O_RDWR)) { + goto error; + } + + adb = (adb_t *) calloc(1, sizeof(adb_t)); + if(!adb) { + goto error; + } + adb->fd = fd; + adb->flags = flags; + adb->path = (char *) malloc(1+strlen(path)); + if(!(adb->path)) { + goto error; + } + strcpy(adb->path, path); + + adb->header = (adb_header_t *) malloc(sizeof(adb_header_t)); + if(!(adb->header)) { + goto error; + } + if(read(fd, (char *) adb->header, O2_HEADERSIZE) != O2_HEADERSIZE) { + goto error; + } + if(!audiodb_check_header(adb->header)) { + goto error; + } + + adb->keys = new std::vector<std::string>; + if(!adb->keys) { + goto error; + } + adb->keymap = new std::map<std::string,uint32_t>; + if(!adb->keymap) { + goto error; + } + if(audiodb_collect_keys(adb)) { + goto error; + } + adb->track_lengths = new std::vector<uint32_t>; + if(!adb->track_lengths) { + goto error; + } + adb->track_lengths->reserve(adb->header->numFiles); + adb->track_offsets = new std::vector<off_t>; + if(!adb->track_offsets) { + goto error; + } + adb->track_offsets->reserve(adb->header->numFiles); + if(audiodb_collect_track_lengths(adb)) { + goto error; + } + adb->cached_lsh = 0; + return adb; + + error: + if(adb) { + if(adb->header) { + free(adb->header); + } + if(adb->path) { + free(adb->path); + } + if(adb->keys) { + delete adb->keys; + } + if(adb->keymap) { + delete adb->keymap; + } + if(adb->track_lengths) { + delete adb->track_lengths; + } + free(adb); + } + if(fd != -1) { + close(fd); + } + return NULL; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pertrackaccumulator.h Sat Jan 10 16:47:57 2009 +0000 @@ -0,0 +1,102 @@ +template <class T> class PerTrackAccumulator : public Accumulator { +public: + PerTrackAccumulator(unsigned int pointNN, unsigned int trackNN); + ~PerTrackAccumulator(); + void add_point(adb_result_t *r); + adb_query_results_t *get_points(); +private: + unsigned int pointNN; + unsigned int trackNN; + std::map<adb_result_t, std::priority_queue< adb_result_t, std::vector<adb_result_t>, T > *, adb_result_key_lt> *queues; + std::set< adb_result_t, adb_result_triple_lt > *set; +}; + +template <class T> PerTrackAccumulator<T>::PerTrackAccumulator(unsigned int pointNN, unsigned int trackNN) + : pointNN(pointNN), trackNN(trackNN), queues(0), set(0) { + queues = new std::map<adb_result_t, std::priority_queue< adb_result_t, std::vector<adb_result_t>, T > *, adb_result_key_lt>; + set = new std::set< adb_result_t, adb_result_triple_lt >; +} + +template <class T> PerTrackAccumulator<T>::~PerTrackAccumulator() { + if(queues) { + typename std::map< adb_result_t, std::priority_queue< adb_result_t, std::vector< adb_result_t >, T > *, adb_result_key_lt>::iterator it; + for(it = queues->begin(); it != queues->end(); it++) { + delete (*it).second; + } + delete queues; + } + if(set) { + delete set; + } +} + +template <class T> void PerTrackAccumulator<T>::add_point(adb_result_t *r) { + if(!isnan(r->dist)) { + if(set->find(*r) == set->end()) { + set->insert(*r); + + typename std::map< adb_result_t, std::priority_queue< adb_result_t, std::vector< adb_result_t >, T > *, adb_result_key_lt>::iterator it; + std::priority_queue< adb_result_t, std::vector< adb_result_t >, T > *queue; + it = queues->find(*r); + if(it == queues->end()) { + queue = new std::priority_queue< adb_result_t, std::vector< adb_result_t >, T >; + (*queues)[*r] = queue; + } else { + queue = (*it).second; + } + + queue->push(*r); + if(queue->size() > pointNN) { + queue->pop(); + } + } + } +} + +template <class T> adb_query_results_t *PerTrackAccumulator<T>::get_points() { + typename std::map< adb_result_t, std::vector< adb_result_t >, adb_result_key_lt> points; + typename std::priority_queue< adb_result_t, std::vector< adb_result_t >, T> queue; + typename std::map< adb_result_t, std::priority_queue< adb_result_t, std::vector< adb_result_t >, T > *, adb_result_key_lt>::iterator it; + + unsigned int size = 0; + for(it = queues->begin(); it != queues->end(); it++) { + unsigned int n = ((*it).second)->size(); + std::vector<adb_result_t> v; + adb_result_t r; + double dist = 0; + for(unsigned int k = 0; k < n; k++) { + r = ((*it).second)->top(); + dist += r.dist; + v.push_back(r); + ((*it).second)->pop(); + } + points[r] = v; + dist /= n; + size += n; + r.dist = dist; + /* I will burn in hell */ + r.ipos = n; + queue.push(r); + if(queue.size() > trackNN) { + size -= queue.top().ipos; + queue.pop(); + } + } + + adb_query_results_t *r = (adb_query_results_t *) malloc(sizeof(adb_query_results_t)); + adb_result_t *rs = (adb_result_t *) calloc(size, sizeof(adb_result_t)); + r->nresults = size; + r->results = rs; + + unsigned int k = 0; + while(queue.size() > 0) { + std::vector<adb_result_t> v = points[queue.top()]; + queue.pop(); + while(v.size() > 0) { + rs[k++] = v.back(); + v.pop_back(); + } + } + return r; +} +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pointpair.cpp Sat Jan 10 16:47:57 2009 +0000 @@ -0,0 +1,27 @@ +#include "audioDB.h" +extern "C" { +#include "audioDB_API.h" +#include "audioDB-internals.h" +} + +PointPair::PointPair(Uns32T a, Uns32T b, Uns32T c) : + trackID(a), qpos(b), spos(c) { +}; + +bool operator<(const PointPair& a, const PointPair& b) { + return ((a.trackID < b.trackID) || + ((a.trackID == b.trackID) && + ((a.spos < b.spos) || ((a.spos == b.spos) && (a.qpos < b.qpos))))); +} + +bool operator>(const PointPair& a, const PointPair& b) { + return ((a.trackID > b.trackID) || + ((a.trackID == b.trackID) && + ((a.spos > b.spos) || ((a.spos == b.spos) && (a.qpos > b.qpos))))); +} + +bool operator==(const PointPair& a, const PointPair& b) { + return ((a.trackID == b.trackID) && + (a.qpos == b.qpos) && + (a.spos == b.spos)); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/power.cpp Sat Jan 10 16:47:57 2009 +0000 @@ -0,0 +1,17 @@ +#include "audioDB.h" +extern "C" { +#include "audioDB_API.h" +#include "audioDB-internals.h" +} + +int audiodb_power(adb_t *adb) { + if(!(adb->flags & O_RDWR)) { + return 1; + } + if(adb->header->length > 0) { + return 1; + } + + adb->header->flags |= O2_FLAG_POWER; + return audiodb_sync_header(adb); +}
--- a/query.cpp Sat Jan 10 11:11:27 2009 +0000 +++ b/query.cpp Sat Jan 10 16:47:57 2009 +0000 @@ -1,194 +1,132 @@ #include "audioDB.h" -#include "reporter.h" +#include "audioDB-internals.h" +#include "accumulators.h" -bool audioDB::powers_acceptable(double p1, double p2) { - if (use_absolute_threshold) { - if ((p1 < absolute_threshold) || (p2 < absolute_threshold)) { +bool audiodb_powers_acceptable(const adb_query_refine_t *r, double p1, double p2) { + if (r->flags & ADB_REFINE_ABSOLUTE_THRESHOLD) { + if ((p1 < r->absolute_threshold) || (p2 < r->absolute_threshold)) { return false; } } - if (use_relative_threshold) { - if (fabs(p1-p2) > fabs(relative_threshold)) { + if (r->flags & ADB_REFINE_RELATIVE_THRESHOLD) { + if (fabs(p1-p2) > fabs(r->relative_threshold)) { return false; } } return true; } -void audioDB::query(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse) { - // init database tables and dbH first - if(query_from_key) - initTables(dbName); - else - initTables(dbName, inFile); +adb_query_results_t *audiodb_query_spec(adb_t *adb, const adb_query_spec_t *qspec) { + adb_qstate_internal_t qstate = {0}; + qstate.allowed_keys = new std::set<std::string>; + adb_query_results_t *results; + if(qspec->refine.flags & ADB_REFINE_INCLUDE_KEYLIST) { + for(unsigned int k = 0; k < qspec->refine.include.nkeys; k++) { + qstate.allowed_keys->insert(qspec->refine.include.keys[k]); + } + } else { + for(unsigned int k = 0; k < adb->header->numFiles; k++) { + qstate.allowed_keys->insert((*adb->keys)[k]); + } + } + if(qspec->refine.flags & ADB_REFINE_EXCLUDE_KEYLIST) { + for(unsigned int k = 0; k < qspec->refine.exclude.nkeys; k++) { + qstate.allowed_keys->erase(qspec->refine.exclude.keys[k]); + } + } - // keyKeyPos requires dbH to be initialized - if(query_from_key && (!key || (query_from_key_index = getKeyPos((char*)key))==O2_ERR_KEYNOTFOUND)) - error("Query key not found :",key); - - switch (queryType) { - case O2_POINT_QUERY: - sequenceLength = 1; - normalizedDistance = false; - reporter = new pointQueryReporter< std::greater < NNresult > >(pointNN); - break; - case O2_TRACK_QUERY: - sequenceLength = 1; - normalizedDistance = false; - reporter = new trackAveragingReporter< std::greater< NNresult > >(pointNN, trackNN, dbH->numFiles); - break; - case O2_SEQUENCE_QUERY: - if(no_unit_norming) - normalizedDistance = false; - if(radius == 0) { - reporter = new trackAveragingReporter< std::less< NNresult > >(pointNN, trackNN, dbH->numFiles); - } else { - if(index_exists(dbName, radius, sequenceLength)){ - char* indexName = index_get_name(dbName, radius, sequenceLength); - lsh = index_allocate(indexName, false); - reporter = new trackSequenceQueryRadReporter(trackNN, index_to_trackID(lsh->get_maxp(), lsh_n_point_bits)+1); - delete[] indexName; - } - else - reporter = new trackSequenceQueryRadReporter(trackNN, dbH->numFiles); + switch(qspec->params.distance) { + case ADB_DISTANCE_DOT_PRODUCT: + switch(qspec->params.accumulation) { + case ADB_ACCUMULATION_DB: + qstate.accumulator = new DBAccumulator<adb_result_dist_gt>(qspec->params.npoints); + break; + case ADB_ACCUMULATION_PER_TRACK: + qstate.accumulator = new PerTrackAccumulator<adb_result_dist_gt>(qspec->params.npoints, qspec->params.ntracks); + break; + case ADB_ACCUMULATION_ONE_TO_ONE: + qstate.accumulator = new NearestAccumulator<adb_result_dist_gt>(); + break; + default: + goto error; } break; - case O2_N_SEQUENCE_QUERY: - if(no_unit_norming) - normalizedDistance = false; - if(radius == 0) { - reporter = new trackSequenceQueryNNReporter< std::less < NNresult > >(pointNN, trackNN, dbH->numFiles); - } else { - if(index_exists(dbName, radius, sequenceLength)){ - char* indexName = index_get_name(dbName, radius, sequenceLength); - lsh = index_allocate(indexName, false); - reporter = new trackSequenceQueryRadNNReporter(pointNN,trackNN, index_to_trackID(lsh->get_maxp(), lsh_n_point_bits)+1); - delete[] indexName; - } - else - reporter = new trackSequenceQueryRadNNReporter(pointNN,trackNN, dbH->numFiles); - } - break; - case O2_ONE_TO_ONE_N_SEQUENCE_QUERY : - if(no_unit_norming) - normalizedDistance = false; - if(radius == 0) { - error("query-type not yet supported"); - } - else { - if(index_exists(dbName, radius, sequenceLength)){ - char* indexName = index_get_name(dbName, radius, sequenceLength); - lsh = index_allocate(indexName, false); - reporter = new trackSequenceQueryRadNNReporterOneToOne(pointNN,trackNN, index_to_trackID(lsh->get_maxp(), lsh_n_point_bits)+1); - delete[] indexName; - } - else - reporter = new trackSequenceQueryRadNNReporterOneToOne(pointNN,trackNN, dbH->numFiles); + case ADB_DISTANCE_EUCLIDEAN_NORMED: + case ADB_DISTANCE_EUCLIDEAN: + switch(qspec->params.accumulation) { + case ADB_ACCUMULATION_DB: + qstate.accumulator = new DBAccumulator<adb_result_dist_lt>(qspec->params.npoints); + break; + case ADB_ACCUMULATION_PER_TRACK: + qstate.accumulator = new PerTrackAccumulator<adb_result_dist_lt>(qspec->params.npoints, qspec->params.ntracks); + break; + case ADB_ACCUMULATION_ONE_TO_ONE: + qstate.accumulator = new NearestAccumulator<adb_result_dist_lt>(); + break; + default: + goto error; } break; default: - error("unrecognized queryType in query()"); - } - - // Test for index (again) here - if(radius && index_exists(dbName, radius, sequenceLength)){ - VERB_LOG(1, "Calling indexed query on database %s, radius=%f, sequenceLength=%d\n", dbName, radius, sequenceLength); - index_query_loop(dbName, query_from_key_index); + goto error; } - else{ - VERB_LOG(1, "Calling brute-force query on database %s\n", dbName); - query_loop(dbName, query_from_key_index); + + if((qspec->refine.flags & ADB_REFINE_RADIUS) && audiodb_index_exists(adb->path, qspec->refine.radius, qspec->qid.sequence_length)) { + if(audiodb_index_query_loop(adb, qspec, &qstate) < 0) { + goto error; + } + } else { + if(audiodb_query_loop(adb, qspec, &qstate)) { + goto error; + } } - reporter->report(fileTable, adbQueryResponse); + results = qstate.accumulator->get_points(); + + delete qstate.accumulator; + delete qstate.allowed_keys; + + return results; + + error: + if(qstate.accumulator) + delete qstate.accumulator; + if(qstate.allowed_keys) + delete qstate.allowed_keys; + return NULL; } -// return ordinal position of key in keyTable -// this should really be a STL hash map search -unsigned audioDB::getKeyPos(char* key){ - if(!dbH) - error("dbH not initialized","getKeyPos"); - for(unsigned k=0; k<dbH->numFiles; k++) - if(strncmp(fileTable + k*O2_FILETABLE_ENTRY_SIZE, key, strlen(key))==0) - return k; - error("Key not found",key); - return O2_ERR_KEYNOTFOUND; +int audiodb_query_free_results(adb_t *adb, const adb_query_spec_t *spec, adb_query_results_t *rs) { + free(rs->results); + free(rs); + return 0; } -// This is a common pattern in sequence queries: what we are doing is -// taking a window of length seqlen over a buffer of length length, -// and placing the sum of the elements in that window in the first -// element of the window: thus replacing all but the last seqlen -// elements in the buffer with the corresponding windowed sum. -void audioDB::sequence_sum(double *buffer, int length, int seqlen) { - double tmp1, tmp2, *ps; - int j, w; - - tmp1 = *buffer; - j = 1; - w = seqlen - 1; - while(w--) { - *buffer += buffer[j++]; - } - ps = buffer + 1; - w = length - seqlen; // +1 - 1 - while(w--) { - tmp2 = *ps; - if(isfinite(tmp1)) { - *ps = *(ps - 1) - tmp1 + *(ps + seqlen - 1); - } else { - for(int i = 1; i < seqlen; i++) { - *ps += *(ps + i); - } - } - tmp1 = tmp2; - ps++; - } -} - -// In contrast to sequence_sum() above, sequence_sqrt() and -// sequence_average() below are simple mappers across the sequence. -void audioDB::sequence_sqrt(double *buffer, int length, int seqlen) { - int w = length - seqlen + 1; - while(w--) { - *buffer = sqrt(*buffer); - buffer++; - } -} - -void audioDB::sequence_average(double *buffer, int length, int seqlen) { - int w = length - seqlen + 1; - while(w--) { - *buffer /= seqlen; - buffer++; - } -} - -void audioDB::initialize_arrays(int track, unsigned int numVectors, double *query, double *data_buffer, double **D, double **DD) { +static void audiodb_initialize_arrays(adb_t *adb, const adb_query_spec_t *spec, int track, unsigned int numVectors, double *query, double *data_buffer, double **D, double **DD) { unsigned int j, k, l, w; double *dp, *qp, *sp; - const unsigned HOP_SIZE = sequenceHop; - const unsigned wL = sequenceLength; + const unsigned HOP_SIZE = spec->refine.hopsize; + const unsigned wL = spec->qid.sequence_length; for(j = 0; j < numVectors; j++) { // Sum products matrix - D[j] = new double[trackTable[track]]; + D[j] = new double[(*adb->track_lengths)[track]]; assert(D[j]); // Matched filter matrix - DD[j]=new double[trackTable[track]]; + DD[j]=new double[(*adb->track_lengths)[track]]; assert(DD[j]); } // Dot product for(j = 0; j < numVectors; j++) - for(k = 0; k < trackTable[track]; k++){ - qp = query + j * dbH->dim; - sp = data_buffer + k * dbH->dim; + for(k = 0; k < (*adb->track_lengths)[track]; k++){ + qp = query + j * adb->header->dim; + sp = data_buffer + k * adb->header->dim; DD[j][k] = 0.0; // Initialize matched filter array dp = &D[j][k]; // point to correlation cell j,k *dp = 0.0; // initialize correlation cell - l = dbH->dim; // size of vectors + l = adb->header->dim; // size of vectors while(l--) *dp += *qp++ * *sp++; } @@ -201,7 +139,7 @@ for(j = 0; j < numVectors - w; j++) { sp = DD[j]; spd = D[j+w] + w; - k = trackTable[track] - w; + k = (*adb->track_lengths)[track] - w; while(k--) *sp++ += *spd++; } @@ -211,7 +149,7 @@ for(j = 0; j < numVectors - w; j += HOP_SIZE) { sp = DD[j]; spd = D[j+w]+w; - for(k = 0; k < trackTable[track] - w; k += HOP_SIZE) { + for(k = 0; k < (*adb->track_lengths)[track] - w; k += HOP_SIZE) { *sp += *spd; sp += HOP_SIZE; spd += HOP_SIZE; @@ -221,7 +159,7 @@ } } -void audioDB::delete_arrays(int track, unsigned int numVectors, double **D, double **DD) { +static void audiodb_delete_arrays(int track, unsigned int numVectors, double **D, double **DD) { if(D != NULL) { for(unsigned int j = 0; j < numVectors; j++) { delete[] D[j]; @@ -234,555 +172,467 @@ } } -void audioDB::read_data(int trkfid, int track, double **data_buffer_p, size_t *data_buffer_size_p) { - if (trackTable[track] * sizeof(double) * dbH->dim > *data_buffer_size_p) { +int audiodb_read_data(adb_t *adb, int trkfid, int track, double **data_buffer_p, size_t *data_buffer_size_p) { + uint32_t track_length = (*adb->track_lengths)[track]; + size_t track_size = track_length * sizeof(double) * adb->header->dim; + if (track_size > *data_buffer_size_p) { if(*data_buffer_p) { free(*data_buffer_p); } { - *data_buffer_size_p = trackTable[track] * sizeof(double) * dbH->dim; - void *tmp = malloc(*data_buffer_size_p); + *data_buffer_size_p = track_size; + void *tmp = malloc(track_size); if (tmp == NULL) { - error("error allocating data buffer"); + goto error; } *data_buffer_p = (double *) tmp; } } - CHECKED_READ(trkfid, *data_buffer_p, trackTable[track] * sizeof(double) * dbH->dim); + read_or_goto_error(trkfid, *data_buffer_p, track_size); + return 0; + + error: + return 1; } -// These names deserve some unpicking. The names starting with a "q" -// are pointers to the query, norm and power vectors; the names -// starting with "v" are things that will end up pointing to the -// actual query point's information. -- CSR, 2007-12-05 -void audioDB::set_up_query(double **qp, double **vqp, double **qnp, double **vqnp, double **qpp, double **vqpp, double *mqdp, unsigned *nvp) { - *nvp = (statbuf.st_size - sizeof(int)) / (dbH->dim * sizeof(double)); +int audiodb_track_id_datum(adb_t *adb, uint32_t track_id, adb_datum_t *d) { + off_t track_offset = (*adb->track_offsets)[track_id]; + if(adb->header->flags & O2_FLAG_LARGE_ADB) { + /* create a reference/insert, then use adb_insert_create_datum() */ + adb_reference_t reference = {0}; + char features[MAXSTR], power[MAXSTR], times[MAXSTR]; + lseek(adb->fd, adb->header->dataOffset + track_id * O2_FILETABLE_ENTRY_SIZE, SEEK_SET); + read_or_goto_error(adb->fd, features, MAXSTR); + reference.features = features; + if(adb->header->flags & O2_FLAG_POWER) { + lseek(adb->fd, adb->header->powerTableOffset + track_id * O2_FILETABLE_ENTRY_SIZE, SEEK_SET); + read_or_goto_error(adb->fd, power, MAXSTR); + reference.power = power; + } + if(adb->header->flags & O2_FLAG_TIMES) { + lseek(adb->fd, adb->header->timesTableOffset + track_id * O2_FILETABLE_ENTRY_SIZE, SEEK_SET); + read_or_goto_error(adb->fd, times, MAXSTR); + reference.times = times; + } + return audiodb_insert_create_datum(&reference, d); + } else { + /* initialize from sources of data that we already have */ + d->nvectors = (*adb->track_lengths)[track_id]; + d->dim = adb->header->dim; + d->key = (*adb->keys)[track_id].c_str(); + /* read out stuff from the database tables */ + d->data = (double *) malloc(d->nvectors * d->dim * sizeof(double)); + lseek(adb->fd, adb->header->dataOffset + track_offset, SEEK_SET); + read_or_goto_error(adb->fd, d->data, d->nvectors * d->dim * sizeof(double)); + if(adb->header->flags & O2_FLAG_POWER) { + d->power = (double *) malloc(d->nvectors * sizeof(double)); + lseek(adb->fd, adb->header->powerTableOffset + track_offset / d->dim, SEEK_SET); + read_or_goto_error(adb->fd, d->power, d->nvectors * sizeof(double)); + } + if(adb->header->flags & O2_FLAG_TIMES) { + d->times = (double *) malloc(2 * d->nvectors * sizeof(double)); + lseek(adb->fd, adb->header->timesTableOffset + track_offset / d->dim, SEEK_SET); + read_or_goto_error(adb->fd, d->times, 2 * d->nvectors * sizeof(double)); + } + return 0; + } + error: + audiodb_free_datum(d); + return 1; +} - if(!(dbH->flags & O2_FLAG_L2NORM)) { - error("Database must be L2 normed for sequence query","use -L2NORM"); +int audiodb_datum_qpointers(adb_datum_t *d, uint32_t sequence_length, double **vector_data, double **vector, adb_qpointers_internal_t *qpointers) { + uint32_t nvectors = d->nvectors; + + qpointers->nvectors = nvectors; + + size_t vector_size = nvectors * sizeof(double) * d->dim; + *vector_data = new double[vector_size]; + memcpy(*vector_data, d->data, vector_size); + + qpointers->l2norm_data = new double[vector_size / d->dim]; + audiodb_l2norm_buffer(*vector_data, d->dim, nvectors, qpointers->l2norm_data); + audiodb_sequence_sum(qpointers->l2norm_data, nvectors, sequence_length); + audiodb_sequence_sqrt(qpointers->l2norm_data, nvectors, sequence_length); + + if(d->power) { + qpointers->power_data = new double[vector_size / d->dim]; + memcpy(qpointers->power_data, d->power, vector_size / d->dim); + audiodb_sequence_sum(qpointers->power_data, nvectors, sequence_length); + audiodb_sequence_average(qpointers->power_data, nvectors, sequence_length); } - if(*nvp < sequenceLength) { - error("Query shorter than requested sequence length", "maybe use -l"); - } - - VERB_LOG(1, "performing norms... "); - - *qp = new double[*nvp * dbH->dim]; - memcpy(*qp, indata+sizeof(int), *nvp * dbH->dim * sizeof(double)); - *qnp = new double[*nvp]; - unitNorm(*qp, dbH->dim, *nvp, *qnp); - - sequence_sum(*qnp, *nvp, sequenceLength); - sequence_sqrt(*qnp, *nvp, sequenceLength); - - if (usingPower) { - *qpp = new double[*nvp]; - if (lseek(powerfd, sizeof(int), SEEK_SET) == (off_t) -1) { - error("error seeking to data", powerFileName, "lseek"); + if(d->times) { + qpointers->mean_duration = new double[1]; + *qpointers->mean_duration = 0; + for(unsigned int k = 0; k < nvectors; k++) { + *qpointers->mean_duration += d->times[2*k+1] - d->times[2*k]; } - int count = read(powerfd, *qpp, *nvp * sizeof(double)); - if (count == -1) { - error("error reading data", powerFileName, "read"); - } - if ((unsigned) count != *nvp * sizeof(double)) { - error("short read", powerFileName); - } - - sequence_sum(*qpp, *nvp, sequenceLength); - sequence_average(*qpp, *nvp, sequenceLength); + *qpointers->mean_duration /= nvectors; } - if (usingTimes) { - unsigned int k; - *mqdp = 0.0; - double *querydurs = new double[*nvp]; - double *timesdata = new double[*nvp*2]; - insertTimeStamps(*nvp, timesFile, timesdata); - for(k = 0; k < *nvp; k++) { - querydurs[k] = timesdata[2*k+1] - timesdata[2*k]; - *mqdp += querydurs[k]; + *vector = *vector_data; + qpointers->l2norm = qpointers->l2norm_data; + qpointers->power = qpointers->power_data; + return 0; +} + +int audiodb_query_spec_qpointers(adb_t *adb, const adb_query_spec_t *spec, double **vector_data, double **vector, adb_qpointers_internal_t *qpointers) { + adb_datum_t *datum; + adb_datum_t d = {0}; + uint32_t sequence_length; + uint32_t sequence_start; + + datum = spec->qid.datum; + sequence_length = spec->qid.sequence_length; + sequence_start = spec->qid.sequence_start; + + if(datum->data) { + if(datum->dim != adb->header->dim) { + return 1; } - *mqdp /= k; - - VERB_LOG(1, "mean query file duration: %f\n", *mqdp); - - delete [] querydurs; - delete [] timesdata; + /* initialize d, and mark that nothing needs freeing later. */ + d = *datum; + datum = &d; + } else if (datum->key) { + uint32_t track_id; + if((track_id = audiodb_key_index(adb, datum->key)) == (uint32_t) -1) { + return 1; + } + audiodb_track_id_datum(adb, track_id, &d); + } else { + return 1; } - // Defaults, for exhaustive search (!usingQueryPoint) - *vqp = *qp; - *vqnp = *qnp; - *vqpp = *qpp; + /* FIXME: check the overflow logic here */ + if(sequence_start + sequence_length > d.nvectors) { + if(datum != &d) { + audiodb_free_datum(&d); + } + return 1; + } - if(usingQueryPoint) { - if( !(queryPoint < *nvp && queryPoint < *nvp - sequenceLength + 1) ) { - error("queryPoint >= numVectors-sequenceLength+1 in query"); - } else { - VERB_LOG(1, "query point: %u\n", queryPoint); - *vqp = *qp + queryPoint * dbH->dim; - *vqnp = *qnp + queryPoint; - if (usingPower) { - *vqpp = *qpp + queryPoint; - } - *nvp = sequenceLength; + audiodb_datum_qpointers(&d, sequence_length, vector_data, vector, qpointers); + + /* Finally, if applicable, set up the moving qpointers. */ + if(spec->qid.flags & ADB_QID_FLAG_EXHAUSTIVE) { + /* the qpointers are already at the start, and so correct. */ + } else { + /* adjust the qpointers to point to the correct place in the sequence */ + *vector = *vector_data + spec->qid.sequence_start * d.dim; + qpointers->l2norm = qpointers->l2norm_data + spec->qid.sequence_start; + if(d.power) { + qpointers->power = qpointers->power_data + spec->qid.sequence_start; } + qpointers->nvectors = sequence_length; } + + /* Clean up: free any bits of datum that we have ourselves + * allocated. */ + if(datum != &d) { + audiodb_free_datum(&d); + } + + return 0; } -// Does the same as set_up_query(...) but from database features instead of from a file -// Constructs the same outputs as set_up_query -void audioDB::set_up_query_from_key(double **qp, double **vqp, double **qnp, double **vqnp, double **qpp, double **vqpp, double *mqdp, unsigned *nvp, Uns32T queryIndex) { - if(!trackTable) - error("trackTable not initialized","set_up_query_from_key"); +static int audiodb_set_up_dbpointers(adb_t *adb, const adb_query_spec_t *spec, adb_qpointers_internal_t *dbpointers) { + uint32_t nvectors = adb->header->length / (adb->header->dim * sizeof(double)); + uint32_t sequence_length = spec->qid.sequence_length; - if(!(dbH->flags & O2_FLAG_L2NORM)) { - error("Database must be L2 normed for sequence query","use -L2NORM"); + bool using_power = spec->refine.flags & (ADB_REFINE_ABSOLUTE_THRESHOLD|ADB_REFINE_RELATIVE_THRESHOLD); + bool using_times = spec->refine.flags & ADB_REFINE_DURATION_RATIO; + double *times_table = NULL; + + + dbpointers->nvectors = nvectors; + dbpointers->l2norm_data = new double[nvectors]; + + double *snpp = dbpointers->l2norm_data, *sppp = 0; + lseek(adb->fd, adb->header->l2normTableOffset, SEEK_SET); + read_or_goto_error(adb->fd, dbpointers->l2norm_data, nvectors * sizeof(double)); + + if (using_power) { + if (!(adb->header->flags & O2_FLAG_POWER)) { + goto error; + } + dbpointers->power_data = new double[nvectors]; + sppp = dbpointers->power_data; + lseek(adb->fd, adb->header->powerTableOffset, SEEK_SET); + read_or_goto_error(adb->fd, dbpointers->power_data, nvectors * sizeof(double)); } - - if(dbH->flags & O2_FLAG_POWER) - usingPower = true; - - if(dbH->flags & O2_FLAG_TIMES) - usingTimes = true; - *nvp = trackTable[queryIndex]; - if(*nvp < sequenceLength) { - error("Query shorter than requested sequence length", "maybe use -l"); - } - - VERB_LOG(1, "performing norms... "); - - // For LARGE_ADB load query features from file - if( dbH->flags & O2_FLAG_LARGE_ADB ){ - if(infid>0) - close(infid); - char* prefixedString = new char[O2_MAXFILESTR]; - char* tmpStr = prefixedString; - strncpy(prefixedString, featureFileNameTable+queryIndex*O2_FILETABLE_ENTRY_SIZE, O2_MAXFILESTR); - prefix_name(&prefixedString, adb_feature_root); - if(tmpStr!=prefixedString) - delete[] tmpStr; - initInputFile(prefixedString, false); // nommap, file pointer at correct position - size_t allocatedSize = 0; - read_data(infid, queryIndex, qp, &allocatedSize); // over-writes qp and allocatedSize - // Consistency check on allocated memory and query feature size - if(*nvp*sizeof(double)*dbH->dim != allocatedSize) - error("Query memory allocation failed consitency check","set_up_query_from_key"); - // Allocated and calculate auxillary sequences: l2norm and power - init_track_aux_data(queryIndex, *qp, qnp, vqnp, qpp, vqpp); - } - else{ // Load from self-contained ADB database - // Read query feature vectors from database - *qp = NULL; - lseek(dbfid, dbH->dataOffset + trackOffsetTable[queryIndex] * sizeof(double), SEEK_SET); - size_t allocatedSize = 0; - read_data(dbfid, queryIndex, qp, &allocatedSize); - // Consistency check on allocated memory and query feature size - if(*nvp*sizeof(double)*dbH->dim != allocatedSize) - error("Query memory allocation failed consitency check","set_up_query_from_key"); - - Uns32T trackIndexOffset = trackOffsetTable[queryIndex]/dbH->dim; // Convert num data elements to num vectors - // Copy L2 norm partial-sum coefficients - assert(*qnp = new double[*nvp]); - memcpy(*qnp, l2normTable+trackIndexOffset, *nvp*sizeof(double)); - sequence_sum(*qnp, *nvp, sequenceLength); - sequence_sqrt(*qnp, *nvp, sequenceLength); - - if( usingPower ){ - // Copy Power partial-sum coefficients - assert(*qpp = new double[*nvp]); - memcpy(*qpp, powerTable+trackIndexOffset, *nvp*sizeof(double)); - sequence_sum(*qpp, *nvp, sequenceLength); - sequence_average(*qpp, *nvp, sequenceLength); + for(unsigned int i = 0; i < adb->header->numFiles; i++){ + size_t track_length = (*adb->track_lengths)[i]; + if(track_length >= sequence_length) { + audiodb_sequence_sum(snpp, track_length, sequence_length); + audiodb_sequence_sqrt(snpp, track_length, sequence_length); + if (using_power) { + audiodb_sequence_sum(sppp, track_length, sequence_length); + audiodb_sequence_average(sppp, track_length, sequence_length); + } } - - if (usingTimes) { - unsigned int k; - *mqdp = 0.0; - double *querydurs = new double[*nvp]; - double *timesdata = new double[*nvp*2]; - assert(querydurs && timesdata); - memcpy(timesdata, timesTable+trackIndexOffset, *nvp*sizeof(double)); - for(k = 0; k < *nvp; k++) { - querydurs[k] = timesdata[2*k+1] - timesdata[2*k]; - *mqdp += querydurs[k]; - } - *mqdp /= k; - - VERB_LOG(1, "mean query file duration: %f\n", *mqdp); - - delete [] querydurs; - delete [] timesdata; + snpp += track_length; + if (using_power) { + sppp += track_length; } } - // Defaults, for exhaustive search (!usingQueryPoint) - *vqp = *qp; - *vqnp = *qnp; - *vqpp = *qpp; + if (using_times) { + if(!(adb->header->flags & O2_FLAG_TIMES)) { + goto error; + } - if(usingQueryPoint) { - if( !(queryPoint < *nvp && queryPoint < *nvp - sequenceLength + 1) ) { - error("queryPoint >= numVectors-sequenceLength+1 in query"); - } else { - VERB_LOG(1, "query point: %u\n", queryPoint); - *vqp = *qp + queryPoint * dbH->dim; - *vqnp = *qnp + queryPoint; - if (usingPower) { - *vqpp = *qpp + queryPoint; + dbpointers->mean_duration = new double[adb->header->numFiles]; + + times_table = (double *) malloc(2 * nvectors * sizeof(double)); + if(!times_table) { + goto error; + } + lseek(adb->fd, adb->header->timesTableOffset, SEEK_SET); + read_or_goto_error(adb->fd, times_table, 2 * nvectors * sizeof(double)); + for(unsigned int k = 0; k < adb->header->numFiles; k++) { + size_t track_length = (*adb->track_lengths)[k]; + unsigned int j; + dbpointers->mean_duration[k] = 0.0; + for(j = 0; j < track_length; j++) { + dbpointers->mean_duration[k] += times_table[2*j+1] - times_table[2*j]; } - *nvp = sequenceLength; + dbpointers->mean_duration[k] /= j; } + + free(times_table); + times_table = NULL; } + + dbpointers->l2norm = dbpointers->l2norm_data; + dbpointers->power = dbpointers->power_data; + return 0; + + error: + if(dbpointers->l2norm_data) { + delete [] dbpointers->l2norm_data; + } + if(dbpointers->power_data) { + delete [] dbpointers->power_data; + } + if(dbpointers->mean_duration) { + delete [] dbpointers->mean_duration; + } + if(times_table) { + free(times_table); + } + return 1; + } +int audiodb_query_queue_loop(adb_t *adb, const adb_query_spec_t *spec, adb_qstate_internal_t *qstate, double *query, adb_qpointers_internal_t *qpointers) { + adb_qpointers_internal_t dbpointers = {0}; -// FIXME: this is not the right name; we're not actually setting up -// the database, but copying various bits of it out of mmap()ed tables -// in order to reduce seeks. -void audioDB::set_up_db(double **snp, double **vsnp, double **spp, double **vspp, double **mddp, unsigned int *dvp) { - *dvp = dbH->length / (dbH->dim * sizeof(double)); - *snp = new double[*dvp]; + uint32_t sequence_length = spec->qid.sequence_length; + bool power_refine = spec->refine.flags & (ADB_REFINE_ABSOLUTE_THRESHOLD|ADB_REFINE_RELATIVE_THRESHOLD); - double *snpp = *snp, *sppp = 0; - memcpy(*snp, l2normTable, *dvp * sizeof(double)); - - if (usingPower) { - if (!(dbH->flags & O2_FLAG_POWER)) { - error("database not power-enabled", dbName); - } - *spp = new double[*dvp]; - sppp = *spp; - memcpy(*spp, powerTable, *dvp * sizeof(double)); + if(qstate->exact_evaluation_queue->size() == 0) { + return 0; } - for(unsigned int i = 0; i < dbH->numFiles; i++){ - if(trackTable[i] >= sequenceLength) { - sequence_sum(snpp, trackTable[i], sequenceLength); - sequence_sqrt(snpp, trackTable[i], sequenceLength); - - if (usingPower) { - sequence_sum(sppp, trackTable[i], sequenceLength); - sequence_average(sppp, trackTable[i], sequenceLength); + /* We are guaranteed that the order of points is sorted by: + * {trackID, spos, qpos} so we can be relatively efficient in + * initialization of track data. We assume that points usually + * don't overlap, so we will use exhaustive dot product evaluation + * (instead of memoization of partial sums, as in query_loop()). + */ + double dist; + double *dbdata = 0, *dbdata_pointer; + Uns32T currentTrack = 0x80000000; // KLUDGE: Initialize with a value outside of track index range + Uns32T npairs = qstate->exact_evaluation_queue->size(); + while(npairs--) { + PointPair pp = qstate->exact_evaluation_queue->top(); + if(currentTrack != pp.trackID) { + SAFE_DELETE_ARRAY(dbdata); + SAFE_DELETE_ARRAY(dbpointers.l2norm_data); + SAFE_DELETE_ARRAY(dbpointers.power_data); + SAFE_DELETE_ARRAY(dbpointers.mean_duration); + currentTrack = pp.trackID; + adb_datum_t d = {0}; + if(audiodb_track_id_datum(adb, pp.trackID, &d)) { + delete qstate->exact_evaluation_queue; + return 1; + } + if(audiodb_datum_qpointers(&d, sequence_length, &dbdata, &dbdata_pointer, &dbpointers)) { + delete qstate->exact_evaluation_queue; + audiodb_free_datum(&d); + return 1; + } + audiodb_free_datum(&d); + } + Uns32T qPos = (spec->qid.flags & ADB_QID_FLAG_EXHAUSTIVE) ? pp.qpos : 0; + Uns32T sPos = pp.spos; // index into l2norm table + // Test power thresholds before computing distance + if( ( (!power_refine) || audiodb_powers_acceptable(&spec->refine, qpointers->power[qPos], dbpointers.power[sPos])) && + ( qPos<qpointers->nvectors-sequence_length+1 && sPos<(*adb->track_lengths)[pp.trackID]-sequence_length+1 ) ){ + // Compute distance + dist = audiodb_dot_product(query + qPos*adb->header->dim, dbdata + sPos*adb->header->dim, adb->header->dim*sequence_length); + double qn = qpointers->l2norm[qPos]; + double sn = dbpointers.l2norm[sPos]; + switch(spec->params.distance) { + case ADB_DISTANCE_EUCLIDEAN_NORMED: + dist = 2 - (2/(qn*sn))*dist; + break; + case ADB_DISTANCE_EUCLIDEAN: + dist = qn*qn + sn*sn - 2*dist; + break; + } + if((!(spec->refine.flags & ADB_REFINE_RADIUS)) || + dist <= (spec->refine.radius+O2_DISTANCE_TOLERANCE)) { + adb_result_t r; + r.key = (*adb->keys)[pp.trackID].c_str(); + r.dist = dist; + r.qpos = pp.qpos; + r.ipos = pp.spos; + qstate->accumulator->add_point(&r); } } - snpp += trackTable[i]; - if (usingPower) { - sppp += trackTable[i]; - } - } - - if (usingTimes) { - if(!(dbH->flags & O2_FLAG_TIMES)) { - error("query timestamps provided for non-timed database", dbName); - } - - *mddp = new double[dbH->numFiles]; - - for(unsigned int k = 0; k < dbH->numFiles; k++) { - unsigned int j; - (*mddp)[k] = 0.0; - for(j = 0; j < trackTable[k]; j++) { - (*mddp)[k] += timesTable[2*j+1] - timesTable[2*j]; - } - (*mddp)[k] /= j; - } - } - - *vsnp = *snp; - *vspp = *spp; -} - -// query_points() -// -// using PointPairs held in the exact_evaluation_queue compute squared distance for each PointPair -// and insert result into the current reporter. -// -// Preconditions: -// A query inFile has been opened with setup_query(...) and query pointers initialized -// The database contains some points -// An exact_evaluation_queue has been allocated and populated -// A reporter has been allocated -// -// Postconditions: -// reporter contains the points and distances that meet the reporter constraints - -void audioDB::query_loop_points(double* query, double* qnPtr, double* qpPtr, double meanQdur, Uns32T numVectors){ - unsigned int dbVectors; - double *sNorm = 0, *snPtr, *sPower = 0, *spPtr = 0; - double *meanDBdur = 0; - - // check pre-conditions - assert(exact_evaluation_queue&&reporter); - if(!exact_evaluation_queue->size()) // Exit if no points to evaluate - return; - - // Compute database info - // FIXME: we more than likely don't need very much of the database - // so make a new method to build these values per-track or, even better, per-point - if( !( dbH->flags & O2_FLAG_LARGE_ADB) ) - set_up_db(&sNorm, &snPtr, &sPower, &spPtr, &meanDBdur, &dbVectors); - - VERB_LOG(1, "matching points..."); - - assert(pointNN>0 && pointNN<=O2_MAXNN); - assert(trackNN>0 && trackNN<=O2_MAXNN); - - // We are guaranteed that the order of points is sorted by: - // trackID, spos, qpos - // so we can be relatively efficient in initialization of track data. - // Here we assume that points don't overlap, so we will use exhaustive dot - // product evaluation instead of memoization of partial sums which is used - // for exhaustive brute-force evaluation from smaller databases: e.g. query_loop() - double dist; - size_t data_buffer_size = 0; - double *data_buffer = 0; - Uns32T trackOffset = 0; - Uns32T trackIndexOffset = 0; - Uns32T currentTrack = 0x80000000; // Initialize with a value outside of track index range - Uns32T npairs = exact_evaluation_queue->size(); - while(npairs--){ - PointPair pp = exact_evaluation_queue->top(); - // Large ADB track data must be loaded here for sPower - if(dbH->flags & O2_FLAG_LARGE_ADB){ - trackOffset=0; - trackIndexOffset=0; - if(currentTrack!=pp.trackID){ - char* prefixedString = new char[O2_MAXFILESTR]; - char* tmpStr = prefixedString; - // On currentTrack change, allocate and load track data - currentTrack=pp.trackID; - SAFE_DELETE_ARRAY(sNorm); - SAFE_DELETE_ARRAY(sPower); - if(infid>0) - close(infid); - // Open and check dimensions of feature file - strncpy(prefixedString, featureFileNameTable+pp.trackID*O2_FILETABLE_ENTRY_SIZE, O2_MAXFILESTR); - prefix_name((char ** const) &prefixedString, adb_feature_root); - if (prefixedString!=tmpStr) - delete[] tmpStr; - initInputFile(prefixedString, false); // nommap, file pointer at correct position - // Load the feature vector data for current track into data_buffer - read_data(infid, pp.trackID, &data_buffer, &data_buffer_size); - // Load power and calculate power and l2norm sequence sums - init_track_aux_data(pp.trackID, data_buffer, &sNorm, &snPtr, &sPower, &spPtr); - } - } - else{ - // These offsets are w.r.t. the entire database of feature vectors and auxillary variables - trackOffset=trackOffsetTable[pp.trackID]; // num data elements offset - trackIndexOffset=trackOffset/dbH->dim; // num vectors offset - } - Uns32T qPos = usingQueryPoint?0:pp.qpos;// index for query point - Uns32T sPos = trackIndexOffset+pp.spos; // index into l2norm table - // Test power thresholds before computing distance - if( ( !usingPower || powers_acceptable(qpPtr[qPos], sPower[sPos])) && - ( qPos<numVectors-sequenceLength+1 && pp.spos<trackTable[pp.trackID]-sequenceLength+1 ) ){ - // Non-large ADB track data is loaded inside power test for efficiency - if( !(dbH->flags & O2_FLAG_LARGE_ADB) && (currentTrack!=pp.trackID) ){ - // On currentTrack change, allocate and load track data - currentTrack=pp.trackID; - lseek(dbfid, dbH->dataOffset + trackOffset * sizeof(double), SEEK_SET); - read_data(dbfid, currentTrack, &data_buffer, &data_buffer_size); - } - // Compute distance - dist = dot_product_points(query+qPos*dbH->dim, data_buffer+pp.spos*dbH->dim, dbH->dim*sequenceLength); - double qn = qnPtr[qPos]; - double sn = sNorm[sPos]; - if(normalizedDistance) - dist = 2 - (2/(qn*sn))*dist; - else - if(no_unit_norming) - dist = qn*qn + sn*sn - 2*dist; - // else - // dist = dist; - if((!radius) || dist <= (radius+O2_DISTANCE_TOLERANCE)) - reporter->add_point(pp.trackID, pp.qpos, pp.spos, dist); - } - exact_evaluation_queue->pop(); + qstate->exact_evaluation_queue->pop(); } // Cleanup - free(data_buffer); - SAFE_DELETE_ARRAY(sNorm); - SAFE_DELETE_ARRAY(sPower); - SAFE_DELETE_ARRAY(meanDBdur); + SAFE_DELETE_ARRAY(dbdata); + SAFE_DELETE_ARRAY(dbpointers.l2norm_data); + SAFE_DELETE_ARRAY(dbpointers.power_data); + SAFE_DELETE_ARRAY(dbpointers.mean_duration); + delete qstate->exact_evaluation_queue; + return 0; } -// A completely unprotected dot-product method -// Caller is responsible for ensuring that memory is within bounds -inline double audioDB::dot_product_points(double* q, double* p, Uns32T L){ - double dist = 0.0; - while(L--) - dist += *q++ * *p++; - return dist; -} +int audiodb_query_loop(adb_t *adb, const adb_query_spec_t *spec, adb_qstate_internal_t *qstate) { + + double *query, *query_data; + adb_qpointers_internal_t qpointers = {0}, dbpointers = {0}; -void audioDB::query_loop(const char* dbName, Uns32T queryIndex) { - - unsigned int numVectors; - double *query, *query_data; - double *qNorm, *qnPtr, *qPower = 0, *qpPtr = 0; - double meanQdur; + bool power_refine = spec->refine.flags & (ADB_REFINE_ABSOLUTE_THRESHOLD|ADB_REFINE_RELATIVE_THRESHOLD); - if( dbH->flags & O2_FLAG_LARGE_ADB ) - error("error: LARGE_ADB requires indexed query"); + if(adb->header->flags & O2_FLAG_LARGE_ADB) { + /* FIXME: actually it would be nice to support this mode of + * operation, but for now... */ + return 1; + } - if(query_from_key) - set_up_query_from_key(&query_data, &query, &qNorm, &qnPtr, &qPower, &qpPtr, &meanQdur, &numVectors, queryIndex); - else - set_up_query(&query_data, &query, &qNorm, &qnPtr, &qPower, &qpPtr, &meanQdur, &numVectors); + if(audiodb_query_spec_qpointers(adb, spec, &query_data, &query, &qpointers)) { + return 1; + } - unsigned int dbVectors; - double *sNorm, *snPtr, *sPower = 0, *spPtr = 0; - double *meanDBdur = 0; + if(audiodb_set_up_dbpointers(adb, spec, &dbpointers)) { + return 1; + } - set_up_db(&sNorm, &snPtr, &sPower, &spPtr, &meanDBdur, &dbVectors); - - VERB_LOG(1, "matching tracks..."); - - assert(pointNN>0 && pointNN<=O2_MAXNN); - assert(trackNN>0 && trackNN<=O2_MAXNN); - - unsigned j,k,track,trackOffset=0, HOP_SIZE=sequenceHop, wL=sequenceLength; + unsigned j,k,track,trackOffset=0, HOP_SIZE = spec->refine.hopsize; + unsigned wL = spec->qid.sequence_length; double **D = 0; // Differences query and target double **DD = 0; // Matched filter distance - D = new double*[numVectors]; // pre-allocate - DD = new double*[numVectors]; + D = new double*[qpointers.nvectors]; // pre-allocate + DD = new double*[qpointers.nvectors]; - gettimeofday(&tv1, NULL); - unsigned processedTracks = 0; off_t trackIndexOffset; - char nextKey[MAXSTR]; // Track loop size_t data_buffer_size = 0; double *data_buffer = 0; - lseek(dbfid, dbH->dataOffset, SEEK_SET); + lseek(adb->fd, adb->header->dataOffset, SEEK_SET); - for(processedTracks=0, track=0 ; processedTracks < dbH->numFiles ; track++, processedTracks++) { - - trackOffset = trackOffsetTable[track]; // numDoubles offset - - // get trackID from file if using a control file - if(trackFile) { - trackFile->getline(nextKey,MAXSTR); - if(!trackFile->eof()) { - track = getKeyPos(nextKey); - trackOffset = trackOffsetTable[track]; - lseek(dbfid, dbH->dataOffset + trackOffset * sizeof(double), SEEK_SET); - } else { - break; + std::set<std::string>::iterator keys_end = qstate->allowed_keys->end(); + for(track = 0; track < adb->header->numFiles; track++) { + unsigned t = track; + + while (qstate->allowed_keys->find((*adb->keys)[track]) == keys_end) { + track++; + if(track == adb->header->numFiles) { + goto loop_finish; } } + trackOffset = (*adb->track_offsets)[track]; + if(track != t) { + lseek(adb->fd, adb->header->dataOffset + trackOffset, SEEK_SET); + } + trackIndexOffset = trackOffset / (adb->header->dim * sizeof(double)); // dbpointers.nvectors offset - // skip identity on query_from_key - if( query_from_key && (track == queryIndex) ) { - if(queryIndex!=dbH->numFiles-1){ - track++; - trackOffset = trackOffsetTable[track]; - lseek(dbfid, dbH->dataOffset + trackOffset * sizeof(double), SEEK_SET); - } - else{ - break; - } + if(audiodb_read_data(adb, adb->fd, track, &data_buffer, &data_buffer_size)) { + return 1; } + if(wL <= (*adb->track_lengths)[track]) { // test for short sequences + + audiodb_initialize_arrays(adb, spec, track, qpointers.nvectors, query, data_buffer, D, DD); - trackIndexOffset=trackOffset/dbH->dim; // numVectors offset - - read_data(dbfid, track, &data_buffer, &data_buffer_size); - if(sequenceLength <= trackTable[track]) { // test for short sequences - - VERB_LOG(7,"%u.%jd.%u | ", track, (intmax_t) trackIndexOffset, trackTable[track]); - - initialize_arrays(track, numVectors, query, data_buffer, D, DD); - - if(usingTimes) { - VERB_LOG(3,"meanQdur=%f meanDBdur=%f\n", meanQdur, meanDBdur[track]); - } - - if((!usingTimes) || fabs(meanDBdur[track]-meanQdur) < meanQdur*timesTol) { - if(usingTimes) { - VERB_LOG(3,"within duration tolerance.\n"); - } + if((!(spec->refine.flags & ADB_REFINE_DURATION_RATIO)) || + fabs(dbpointers.mean_duration[track]-qpointers.mean_duration[0]) < qpointers.mean_duration[0]*spec->refine.duration_ratio) { // Search for minimum distance by shingles (concatenated vectors) - for(j = 0; j <= numVectors - wL; j += HOP_SIZE) { - for(k = 0; k <= trackTable[track] - wL; k += HOP_SIZE) { - double thisDist; - if(normalizedDistance) - thisDist = 2-(2/(qnPtr[j]*sNorm[trackIndexOffset+k]))*DD[j][k]; - else - if(no_unit_norming) - thisDist = qnPtr[j]*qnPtr[j]+sNorm[trackIndexOffset+k]*sNorm[trackIndexOffset+k] - 2*DD[j][k]; - else - thisDist = DD[j][k]; - + for(j = 0; j <= qpointers.nvectors - wL; j += HOP_SIZE) { + for(k = 0; k <= (*adb->track_lengths)[track] - wL; k += HOP_SIZE) { + double thisDist = 0; + double qn = qpointers.l2norm[j]; + double sn = dbpointers.l2norm[trackIndexOffset + k]; + switch(spec->params.distance) { + case ADB_DISTANCE_EUCLIDEAN_NORMED: + thisDist = 2-(2/(qn*sn))*DD[j][k]; + break; + case ADB_DISTANCE_EUCLIDEAN: + thisDist = qn*qn + sn*sn - 2*DD[j][k]; + break; + case ADB_DISTANCE_DOT_PRODUCT: + thisDist = DD[j][k]; + break; + } // Power test - if ((!usingPower) || powers_acceptable(qpPtr[j], sPower[trackIndexOffset + k])) { + if ((!power_refine) || audiodb_powers_acceptable(&spec->refine, qpointers.power[j], dbpointers.power[trackIndexOffset + k])) { // radius test - if((!radius) || thisDist <= (radius+O2_DISTANCE_TOLERANCE)) { - reporter->add_point(track, usingQueryPoint ? queryPoint : j, k, thisDist); + if((!(spec->refine.flags & ADB_REFINE_RADIUS)) || + thisDist <= (spec->refine.radius+O2_DISTANCE_TOLERANCE)) { + adb_result_t r; + r.key = (*adb->keys)[track].c_str(); + r.dist = thisDist; + if(spec->qid.flags & ADB_QID_FLAG_EXHAUSTIVE) { + r.qpos = j; + } else { + r.qpos = spec->qid.sequence_start; + } + r.ipos = k; + qstate->accumulator->add_point(&r); } } } } } // Duration match - delete_arrays(track, numVectors, D, DD); + audiodb_delete_arrays(track, qpointers.nvectors, D, DD); } } + loop_finish: + free(data_buffer); - gettimeofday(&tv2,NULL); - VERB_LOG(1,"elapsed time: %ld msec\n", - (tv2.tv_sec*1000 + tv2.tv_usec/1000) - - (tv1.tv_sec*1000 + tv1.tv_usec/1000)) - // Clean up if(query_data) delete[] query_data; - if(qNorm) - delete[] qNorm; - if(sNorm) - delete[] sNorm; - if(qPower) - delete[] qPower; - if(sPower) - delete[] sPower; + if(qpointers.l2norm_data) + delete[] qpointers.l2norm_data; + if(qpointers.power_data) + delete[] qpointers.power_data; + if(qpointers.mean_duration) + delete[] qpointers.mean_duration; + if(dbpointers.power_data) + delete[] dbpointers.power_data; + if(dbpointers.l2norm_data) + delete[] dbpointers.l2norm_data; if(D) delete[] D; if(DD) delete[] DD; - if(meanDBdur) - delete[] meanDBdur; + if(dbpointers.mean_duration) + delete[] dbpointers.mean_duration; + + return 0; } - -// Unit norm block of features -void audioDB::unitNorm(double* X, unsigned dim, unsigned n, double* qNorm){ - unsigned d; - double L2, *p; - - VERB_LOG(2, "norming %u vectors...", n); - while(n--) { - p = X; - L2 = 0.0; - d = dim; - while(d--) { - L2 += *p * *p; - p++; - } - if(qNorm) { - *qNorm++=L2; - } - X += dim; - } - VERB_LOG(2, "done.\n"); -} - -
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/query.txt Sat Jan 10 16:47:57 2009 +0000 @@ -0,0 +1,93 @@ +Currently supported query types: + +O2_POINT_QUERY + * dot_product + + Find and report, from the database, up to "pointNN" + near-neighbours of length-1 query sequences. + +O2_TRACK_QUERY + * dot_product + + Find, in each track, up to "pointNN" near-neighbours of length-1 + query sequences, reporting the top "trackNN" tracks, ordered by + the average distance of the pairwise matches. + +O2_SEQUENCE_QUERY + - radius, + radius + * euclidean_normed, euclidean + +O2_N_SEQUENCE_QUERY + - radius, + radius + * euclidean_normed, euclidean + + Find, in each track, up to "pointNN" near-neighbours of query + sequences. Report the results from the "trackNN" top tracks, + where the tracks are ordered by the average distance of the + retrieved pairwise matches. The difference between SEQUENCE and + N_SEQUENCE is that the SEQUENCE case reports only the average, + while the N_SEQUENCE reports the individual points too. + + (Ordering by average is arbitrary, and it's not hard to construct + cases where it is suboptimal. The two cases where it is not + arbitrary are when pointNN is 1, and when trackNN is equal to the + number of files in the database.) + +O2_ONE_TO_ONE_N_SEQUENCE_QUERY + + radius + * euclidean_normed + + For all applicable query sequences, find and report the closest + target instance point. Each query sequence is responsible for + exactly one result. + + (This feels like it should be more orthogonal than a separate + query type; the restriction on using a target instance point only + once in a match seems like it should compose with the sequencing + query above.) + +Plan: + +We have + + reporter->add_point(), + reporter->report(). + +Insert into the whole shebang a new class Accumulator, with methods + + void accumulator->add_point() + adb_query_results *accumulator->get_points() + +The accumulator has to be responsible for keeping track of how many +points (total, or per track) there are so far; ->get_points() has to +make the final decision about which points to preserve. So sadly we +can't be completely on the side of the angels and have only one single +accumulator class, as POINT_QUERY is different from all the others. +(Though maybe we can with a suitably careful use of the "if" +construct). + +We don't have to alter the Reporter class at all. The query loop goes +roughly + + choose point pair + if(everything OK with point pair) + accumulator->add_point() + loop + + results = accumulator->get_points() + + for matches in results + reporter->add_point(match) + loop + + reporter->report() + +This separation is engineered (ha) such that everything after the last +use of the accumulator doesn't need to be in libaudiodb; the return +value from audiodb_query() can be "results" in the above, and then the +command-line binary and SOAP server can do whatever weird mangling to +the results they want to. + +We still need to be careful in the accumulator to defend against some +of the weird things that our query implementation might choose to do: +insert the same hit multiple times or some such.
--- a/reporter.h Sat Jan 10 11:11:27 2009 +0000 +++ b/reporter.h Sat Jan 10 16:47:57 2009 +0000 @@ -44,10 +44,9 @@ // FIXME: this interface is a bit wacky: a relic of previous, more // confused times. Really it might make sense to have separate // reporter classes for WS and for stdout, rather than passing this - // adbQueryResponse thing everywhere; the fileTable argument is - // there solely for convertion trackIDs into names. -- CSR, - // 2007-12-10. - virtual void report(char *fileTable, void* adbQueryResponse) = 0; + // adbQueryResponse thing everywhere; the adb argument is there + // solely for converting trackIDs into names. -- CSR, 2007-12-10. + virtual void report(adb_t *adb, void* adbQueryResponse) = 0; }; template <class T> class pointQueryReporter : public Reporter { @@ -55,7 +54,7 @@ pointQueryReporter(unsigned int pointNN); ~pointQueryReporter(); void add_point(unsigned int trackID, unsigned int qpos, unsigned int spos, double dist); - void report(char *fileTable, void* adbQueryResponse); + void report(adb_t *adb, void* adbQueryResponse); private: unsigned int pointNN; std::priority_queue< NNresult, std::vector< NNresult >, T> *queue; @@ -84,7 +83,7 @@ } } -template <class T> void pointQueryReporter<T>::report(char *fileTable, void *adbQueryResponse) { +template <class T> void pointQueryReporter<T>::report(adb_t *adb, void *adbQueryResponse) { NNresult r; std::vector<NNresult> v; unsigned int size = queue->size(); @@ -98,32 +97,33 @@ if(adbQueryResponse==0) { for(rit = v.rbegin(); rit < v.rend(); rit++) { r = *rit; - if(fileTable) - std::cout << fileTable + r.trackID*O2_FILETABLE_ENTRY_SIZE << " "; + if(adb) + std::cout << audiodb_index_key(adb, r.trackID) << " "; else std::cout << r.trackID << " "; std::cout << r.dist << " " << r.qpos << " " << r.spos << std::endl; } } else { - ((adb__queryResponse*)adbQueryResponse)->result.__sizeRlist=size; - ((adb__queryResponse*)adbQueryResponse)->result.__sizeDist=size; - ((adb__queryResponse*)adbQueryResponse)->result.__sizeQpos=size; - ((adb__queryResponse*)adbQueryResponse)->result.__sizeSpos=size; - ((adb__queryResponse*)adbQueryResponse)->result.Rlist= new char*[size]; - ((adb__queryResponse*)adbQueryResponse)->result.Dist = new double[size]; - ((adb__queryResponse*)adbQueryResponse)->result.Qpos = new unsigned int[size]; - ((adb__queryResponse*)adbQueryResponse)->result.Spos = new unsigned int[size]; + adb__queryResponse *response = (adb__queryResponse *) adbQueryResponse; + response->result.__sizeRlist=size; + response->result.__sizeDist=size; + response->result.__sizeQpos=size; + response->result.__sizeSpos=size; + response->result.Rlist= new char*[size]; + response->result.Dist = new double[size]; + response->result.Qpos = new unsigned int[size]; + response->result.Spos = new unsigned int[size]; unsigned int k = 0; for(rit = v.rbegin(); rit < v.rend(); rit++, k++) { r = *rit; - ((adb__queryResponse*)adbQueryResponse)->result.Rlist[k] = new char[O2_MAXFILESTR]; - ((adb__queryResponse*)adbQueryResponse)->result.Dist[k] = r.dist; - ((adb__queryResponse*)adbQueryResponse)->result.Qpos[k] = r.qpos; - ((adb__queryResponse*)adbQueryResponse)->result.Spos[k] = r.spos; - if(fileTable) - snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "%s", fileTable+r.trackID*O2_FILETABLE_ENTRY_SIZE); + response->result.Rlist[k] = new char[O2_MAXFILESTR]; + response->result.Dist[k] = r.dist; + response->result.Qpos[k] = r.qpos; + response->result.Spos[k] = r.spos; + if(adb) + snprintf(response->result.Rlist[k], O2_MAXFILESTR, "%s", audiodb_index_key(adb, r.trackID)); else - snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "%d", r.trackID); + snprintf(response->result.Rlist[k], O2_MAXFILESTR, "%d", r.trackID); } } } @@ -133,7 +133,7 @@ trackAveragingReporter(unsigned int pointNN, unsigned int trackNN, unsigned int numFiles); ~trackAveragingReporter(); void add_point(unsigned int trackID, unsigned int qpos, unsigned int spos, double dist); - void report(char *fileTable, void *adbQueryResponse); + void report(adb_t *adb, void *adbQueryResponse); protected: unsigned int pointNN; unsigned int trackNN; @@ -164,7 +164,7 @@ } } -template <class T> void trackAveragingReporter<T>::report(char *fileTable, void *adbQueryResponse) { +template <class T> void trackAveragingReporter<T>::report(adb_t *adb, void *adbQueryResponse) { std::priority_queue < NNresult, std::vector< NNresult>, T> result; for (int i = numFiles-1; i >= 0; i--) { unsigned int size = queues[i].size(); @@ -205,32 +205,33 @@ if(adbQueryResponse==0) { for(rit = v.rbegin(); rit < v.rend(); rit++) { r = *rit; - if(fileTable) - std::cout << fileTable + r.trackID*O2_FILETABLE_ENTRY_SIZE << " "; + if(adb) + std::cout << audiodb_index_key(adb, r.trackID) << " "; else std::cout << r.trackID << " "; std::cout << r.dist << " " << r.qpos << " " << r.spos << std::endl; } } else { - ((adb__queryResponse*)adbQueryResponse)->result.__sizeRlist=size; - ((adb__queryResponse*)adbQueryResponse)->result.__sizeDist=size; - ((adb__queryResponse*)adbQueryResponse)->result.__sizeQpos=size; - ((adb__queryResponse*)adbQueryResponse)->result.__sizeSpos=size; - ((adb__queryResponse*)adbQueryResponse)->result.Rlist= new char*[size]; - ((adb__queryResponse*)adbQueryResponse)->result.Dist = new double[size]; - ((adb__queryResponse*)adbQueryResponse)->result.Qpos = new unsigned int[size]; - ((adb__queryResponse*)adbQueryResponse)->result.Spos = new unsigned int[size]; + adb__queryResponse *response = (adb__queryResponse *) adbQueryResponse; + response->result.__sizeRlist=size; + response->result.__sizeDist=size; + response->result.__sizeQpos=size; + response->result.__sizeSpos=size; + response->result.Rlist= new char*[size]; + response->result.Dist = new double[size]; + response->result.Qpos = new unsigned int[size]; + response->result.Spos = new unsigned int[size]; unsigned int k = 0; for(rit = v.rbegin(); rit < v.rend(); rit++, k++) { r = *rit; - ((adb__queryResponse*)adbQueryResponse)->result.Rlist[k] = new char[O2_MAXFILESTR]; - ((adb__queryResponse*)adbQueryResponse)->result.Dist[k] = r.dist; - ((adb__queryResponse*)adbQueryResponse)->result.Qpos[k] = r.qpos; - ((adb__queryResponse*)adbQueryResponse)->result.Spos[k] = r.spos; - if(fileTable) - snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "%s", fileTable+r.trackID*O2_FILETABLE_ENTRY_SIZE); + response->result.Rlist[k] = new char[O2_MAXFILESTR]; + response->result.Dist[k] = r.dist; + response->result.Qpos[k] = r.qpos; + response->result.Spos[k] = r.spos; + if(adb) + snprintf(response->result.Rlist[k], O2_MAXFILESTR, "%s", audiodb_index_key(adb, r.trackID)); else - snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "%d", r.trackID); + snprintf(response->result.Rlist[k], O2_MAXFILESTR, "%d", r.trackID); } } } @@ -244,13 +245,13 @@ using trackAveragingReporter<T>::pointNN; public: trackSequenceQueryNNReporter(unsigned int pointNN, unsigned int trackNN, unsigned int numFiles); - void report(char *fileTable, void *adbQueryResponse); + void report(adb_t *adb, void *adbQueryResponse); }; template <class T> trackSequenceQueryNNReporter<T>::trackSequenceQueryNNReporter(unsigned int pointNN, unsigned int trackNN, unsigned int numFiles) :trackAveragingReporter<T>(pointNN, trackNN, numFiles){} -template <class T> void trackSequenceQueryNNReporter<T>::report(char *fileTable, void *adbQueryResponse) { +template <class T> void trackSequenceQueryNNReporter<T>::report(adb_t *adb, void *adbQueryResponse) { std::priority_queue < NNresult, std::vector< NNresult>, T> result; std::priority_queue< NNresult, std::vector< NNresult>, std::less<NNresult> > *point_queues = new std::priority_queue< NNresult, std::vector< NNresult>, std::less<NNresult> >[numFiles]; @@ -297,11 +298,11 @@ if(adbQueryResponse==0) { for(rit = v.rbegin(); rit < v.rend(); rit++) { r = *rit; - if(fileTable) - std::cout << fileTable + r.trackID*O2_FILETABLE_ENTRY_SIZE << " "; + if(adb) + std::cout << audiodb_index_key(adb, r.trackID) << " "; else std::cout << r.trackID << " "; - std::cout << r.dist << std::endl; + std::cout << r.dist << std::endl; unsigned int qsize = point_queues[r.trackID].size(); // Reverse the order of the points stored in point_queues for(unsigned int k=0; k < qsize; k++){ @@ -316,14 +317,15 @@ } } } else { - ((adb__queryResponse*)adbQueryResponse)->result.__sizeRlist=size*pointNN; - ((adb__queryResponse*)adbQueryResponse)->result.__sizeDist=size*pointNN; - ((adb__queryResponse*)adbQueryResponse)->result.__sizeQpos=size*pointNN; - ((adb__queryResponse*)adbQueryResponse)->result.__sizeSpos=size*pointNN; - ((adb__queryResponse*)adbQueryResponse)->result.Rlist= new char*[size*pointNN]; - ((adb__queryResponse*)adbQueryResponse)->result.Dist = new double[size*pointNN]; - ((adb__queryResponse*)adbQueryResponse)->result.Qpos = new unsigned int[size*pointNN]; - ((adb__queryResponse*)adbQueryResponse)->result.Spos = new unsigned int[size*pointNN]; + adb__queryResponse *response = (adb__queryResponse *) adbQueryResponse; + response->result.__sizeRlist=size*pointNN; + response->result.__sizeDist=size*pointNN; + response->result.__sizeQpos=size*pointNN; + response->result.__sizeSpos=size*pointNN; + response->result.Rlist= new char*[size*pointNN]; + response->result.Dist = new double[size*pointNN]; + response->result.Qpos = new unsigned int[size*pointNN]; + response->result.Spos = new unsigned int[size*pointNN]; unsigned int k = 0; // Loop over returned tracks for(rit = v.rbegin(); rit < v.rend(); rit++) { @@ -345,20 +347,20 @@ rk.spos = 0xFFFFFFFF; } - ((adb__queryResponse*)adbQueryResponse)->result.Rlist[k] = new char[O2_MAXFILESTR]; - ((adb__queryResponse*)adbQueryResponse)->result.Dist[k] = rk.dist; - ((adb__queryResponse*)adbQueryResponse)->result.Qpos[k] = rk.qpos; - ((adb__queryResponse*)adbQueryResponse)->result.Spos[k] = rk.spos; + response->result.Rlist[k] = new char[O2_MAXFILESTR]; + response->result.Dist[k] = rk.dist; + response->result.Qpos[k] = rk.qpos; + response->result.Spos[k] = rk.spos; if(qsize){ - if(fileTable) - snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "%s", fileTable+r.trackID*O2_FILETABLE_ENTRY_SIZE); + if(adb) + snprintf(response->result.Rlist[k], O2_MAXFILESTR, "%s", audiodb_index_key(adb, r.trackID)); else - snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "%d", r.trackID); + snprintf(response->result.Rlist[k], O2_MAXFILESTR, "%d", r.trackID); point_queue.pop(); qsize--; } else - snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "NULL"); + snprintf(response->result.Rlist[k], O2_MAXFILESTR, "NULL"); k++; } } @@ -413,7 +415,7 @@ trackSequenceQueryRadReporter(unsigned int trackNN, unsigned int numFiles); ~trackSequenceQueryRadReporter(); void add_point(unsigned int trackID, unsigned int qpos, unsigned int spos, double dist); - void report(char *fileTable, void *adbQueryResponse); + void report(adb_t *adb, void *adbQueryResponse); protected: unsigned int trackNN; unsigned int numFiles; @@ -460,7 +462,7 @@ } } -void trackSequenceQueryRadReporter::report(char *fileTable, void *adbQueryResponse) { +void trackSequenceQueryRadReporter::report(adb_t *adb, void *adbQueryResponse) { std::priority_queue < Radresult, std::vector<Radresult>, std::greater<Radresult> > result; // KLUDGE: doing this backwards in an attempt to get the same // tiebreak behaviour as before. @@ -489,33 +491,33 @@ if(adbQueryResponse==0) { for(rit = v.rbegin(); rit < v.rend(); rit++) { r = *rit; - if(fileTable) - std::cout << fileTable + r.trackID*O2_FILETABLE_ENTRY_SIZE << " "; + if(adb) + std::cout << audiodb_index_key(adb, r.trackID) << " "; else std::cout << r.trackID << " "; std::cout << r.count << std::endl; } - } - else { - ((adb__queryResponse*)adbQueryResponse)->result.__sizeRlist=size; - ((adb__queryResponse*)adbQueryResponse)->result.__sizeDist=size; - ((adb__queryResponse*)adbQueryResponse)->result.__sizeQpos=size; - ((adb__queryResponse*)adbQueryResponse)->result.__sizeSpos=size; - ((adb__queryResponse*)adbQueryResponse)->result.Rlist= new char*[size]; - ((adb__queryResponse*)adbQueryResponse)->result.Dist = new double[size]; - ((adb__queryResponse*)adbQueryResponse)->result.Qpos = new unsigned int[size]; - ((adb__queryResponse*)adbQueryResponse)->result.Spos = new unsigned int[size]; + } else { + adb__queryResponse *response = (adb__queryResponse *) adbQueryResponse; + response->result.__sizeRlist=size; + response->result.__sizeDist=size; + response->result.__sizeQpos=size; + response->result.__sizeSpos=size; + response->result.Rlist= new char*[size]; + response->result.Dist = new double[size]; + response->result.Qpos = new unsigned int[size]; + response->result.Spos = new unsigned int[size]; unsigned int k = 0; for(rit = v.rbegin(); rit < v.rend(); rit++, k++) { r = *rit; - ((adb__queryResponse*)adbQueryResponse)->result.Rlist[k] = new char[O2_MAXFILESTR]; - ((adb__queryResponse*)adbQueryResponse)->result.Dist[k] = 0; - ((adb__queryResponse*)adbQueryResponse)->result.Qpos[k] = 0; - ((adb__queryResponse*)adbQueryResponse)->result.Spos[k] = r.count; - if(fileTable) - snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "%s", fileTable+r.trackID*O2_FILETABLE_ENTRY_SIZE); + response->result.Rlist[k] = new char[O2_MAXFILESTR]; + response->result.Dist[k] = 0; + response->result.Qpos[k] = 0; + response->result.Spos[k] = r.count; + if(adb) + snprintf(response->result.Rlist[k], O2_MAXFILESTR, "%s", audiodb_index_key(adb, r.trackID)); else - snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "%d", r.trackID); + snprintf(response->result.Rlist[k], O2_MAXFILESTR, "%d", r.trackID); } } } @@ -529,7 +531,7 @@ trackSequenceQueryRadNNReporter(unsigned int pointNN, unsigned int trackNN, unsigned int numFiles); ~trackSequenceQueryRadNNReporter(); void add_point(unsigned int trackID, unsigned int qpos, unsigned int spos, double dist); - void report(char *fileTable, void *adbQueryResponse); + void report(adb_t *adb, void *adbQueryResponse); protected: unsigned int pointNN; unsigned int trackNN; @@ -592,7 +594,7 @@ } } -void trackSequenceQueryRadNNReporter::report(char *fileTable, void *adbQueryResponse) { +void trackSequenceQueryRadNNReporter::report(adb_t *adb, void *adbQueryResponse) { std::priority_queue < Radresult, std::vector<Radresult>, std::greater<Radresult> > result; // KLUDGE: doing this backwards in an attempt to get the same // tiebreak behaviour as before. @@ -634,7 +636,7 @@ } } // Report - rep->report(fileTable, adbQueryResponse); + rep->report(adb, adbQueryResponse); // Exit delete[] point_queues; return; @@ -648,8 +650,8 @@ if(adbQueryResponse==0) { for(rit = v.rbegin(); rit < v.rend(); rit++) { r = *rit; - if(fileTable) - std::cout << fileTable + r.trackID*O2_FILETABLE_ENTRY_SIZE << " "; + if(adb) + std::cout << audiodb_index_key(adb, r.trackID) << " "; else std::cout << r.trackID << " "; std::cout << r.count << std::endl; @@ -666,16 +668,16 @@ point_queue.pop(); } } - } - else { - ((adb__queryResponse*)adbQueryResponse)->result.__sizeRlist=size*pointNN; - ((adb__queryResponse*)adbQueryResponse)->result.__sizeDist=size*pointNN; - ((adb__queryResponse*)adbQueryResponse)->result.__sizeQpos=size*pointNN; - ((adb__queryResponse*)adbQueryResponse)->result.__sizeSpos=size*pointNN; - ((adb__queryResponse*)adbQueryResponse)->result.Rlist= new char*[size*pointNN]; - ((adb__queryResponse*)adbQueryResponse)->result.Dist = new double[size*pointNN]; - ((adb__queryResponse*)adbQueryResponse)->result.Qpos = new unsigned int[size*pointNN]; - ((adb__queryResponse*)adbQueryResponse)->result.Spos = new unsigned int[size*pointNN]; + } else { + adb__queryResponse *response = (adb__queryResponse *) adbQueryResponse; + response->result.__sizeRlist=size*pointNN; + response->result.__sizeDist=size*pointNN; + response->result.__sizeQpos=size*pointNN; + response->result.__sizeSpos=size*pointNN; + response->result.Rlist= new char*[size*pointNN]; + response->result.Dist = new double[size*pointNN]; + response->result.Qpos = new unsigned int[size*pointNN]; + response->result.Spos = new unsigned int[size*pointNN]; unsigned int k = 0; // Loop over returned tracks for(rit = v.rbegin(); rit < v.rend(); rit++) { @@ -697,20 +699,20 @@ rk.spos = 0xFFFFFFFF; } - ((adb__queryResponse*)adbQueryResponse)->result.Rlist[k] = new char[O2_MAXFILESTR]; - ((adb__queryResponse*)adbQueryResponse)->result.Dist[k] = rk.dist; - ((adb__queryResponse*)adbQueryResponse)->result.Qpos[k] = rk.qpos; - ((adb__queryResponse*)adbQueryResponse)->result.Spos[k] = rk.spos; + response->result.Rlist[k] = new char[O2_MAXFILESTR]; + response->result.Dist[k] = rk.dist; + response->result.Qpos[k] = rk.qpos; + response->result.Spos[k] = rk.spos; if(qsize){ - if(fileTable) - snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "%s", fileTable+r.trackID*O2_FILETABLE_ENTRY_SIZE); + if(adb) + snprintf(response->result.Rlist[k], O2_MAXFILESTR, "%s", audiodb_index_key(adb, r.trackID)); else - snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "%d", r.trackID); + snprintf(response->result.Rlist[k], O2_MAXFILESTR, "%d", r.trackID); point_queue.pop(); qsize--; } else - snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "NULL"); + snprintf(response->result.Rlist[k], O2_MAXFILESTR, "NULL"); k++; } } @@ -728,7 +730,7 @@ trackSequenceQueryRadNNReporterOneToOne(unsigned int pointNN, unsigned int trackNN, unsigned int numFiles); ~trackSequenceQueryRadNNReporterOneToOne(); void add_point(unsigned int trackID, unsigned int qpos, unsigned int spos, double dist); - void report(char *fileTable, void *adbQueryResponse); + void report(adb_t *adb, void *adbQueryResponse); protected: unsigned int pointNN; unsigned int trackNN; @@ -776,7 +778,7 @@ } -void trackSequenceQueryRadNNReporterOneToOne::report(char *fileTable, void *adbQueryResponse) { +void trackSequenceQueryRadNNReporterOneToOne::report(adb_t *adb, void *adbQueryResponse) { if(adbQueryResponse==0) { std::vector< NNresult >::iterator vit; NNresult rk; @@ -785,8 +787,8 @@ std::cout << rk.dist << " " << rk.qpos << " " << rk.spos << " "; - if(fileTable) - std::cout << fileTable + rk.trackID*O2_FILETABLE_ENTRY_SIZE << " "; + if(adb) + std::cout << audiodb_index_key(adb, rk.trackID) << " "; else std::cout << rk.trackID << " "; std::cout << std::endl;
--- a/sample.cpp Sat Jan 10 11:11:27 2009 +0000 +++ b/sample.cpp Sat Jan 10 16:47:57 2009 +0000 @@ -60,7 +60,6 @@ error("error: sample not yet supported for LARGE_ADB"); } - // build track offset table (FIXME: cut'n'pasted from query.cpp) off_t *trackOffsetTable = new off_t[dbH->numFiles]; unsigned cumTrack=0; for(unsigned int k = 0; k < dbH->numFiles; k++){
--- a/soap.cpp Sat Jan 10 11:11:27 2009 +0000 +++ b/soap.cpp Sat Jan 10 16:47:57 2009 +0000 @@ -1,4 +1,5 @@ #include "audioDB.h" +#include "audioDB-internals.h" #include "adb.nsmap" /* Command-line client definitions */ @@ -454,21 +455,12 @@ else { fprintf(stderr, "Socket connection successful: master socket = %d\n", m); - // Make a global Web Services LSH Index (SINGLETON) - if(WS_load_index && dbName && !index_exists(dbName, radius, sequenceLength)){ - error("Can't find requested index file:", index_get_name(dbName,radius,sequenceLength)); - } - if(WS_load_index && dbName && index_exists(dbName, radius, sequenceLength)){ - char* indexName = index_get_name(dbName, radius, sequenceLength); - fprintf(stderr, "Loading LSH hashtables: %s...\n", indexName); - lsh = new LSH(indexName, true); - assert(lsh); - SERVER_LSH_INDEX_SINGLETON = lsh; - fprintf(stderr, "LSH INDEX READY\n"); - fflush(stderr); - delete[] indexName; - } - + /* FIXME: we used to have a global cache of a single LSH index + * here. CSR removed it because it interacted badly with + * APIification of querying, replacing it with a per-open-adb + * cache; we should try to take advantage of that instead. + */ + // Server-side path prefix to databases and features if(adb_root) SERVER_ADB_ROOT = (char*)adb_root; // Server-side database root
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/status.cpp Sat Jan 10 16:47:57 2009 +0000 @@ -0,0 +1,48 @@ +#include "audioDB.h" +extern "C" { +#include "audioDB_API.h" +#include "audioDB-internals.h" +} + +int audiodb_status(adb_t *adb, adb_status_t *status) { + /* FIXME: it would be nice to be able to test for "is this database + pointer valid", but at the moment we punt that to memory + discipline. */ + + unsigned dudCount = 0; + unsigned nullCount = 0; + + for(unsigned k = 0; k < adb->header->numFiles; k++) { + /* FIXME: this bare "16" here reveals a problem (or maybe two). + * 16 here means the default value of the sequenceLength parameter + * initializer (both in C++ and corresponding to the "-l" or + * "--sequencelength" command-line argument). + * + * The problem is that the API as currently designed provides no + * way to pass that information in to this routine; there's no + * input parameter; nor is there in the SOAP version of this + * query. However, there /is/ a way to pass that information on + * the command-line -- though that codepath is completely + * untested. I can see that it might be useful to provide this + * information, but at present it's probably completely unused, so + * the compromise for now is to hardwire the 16. + */ + if((*adb->track_lengths)[k] < 16) { + dudCount++; + if(!(*adb->track_lengths)[k]) { + nullCount++; + } + } + } + + status->numFiles = adb->header->numFiles; + status->dim = adb->header->dim; + status->length = adb->header->length; + status->dudCount = dudCount; + status->nullCount = nullCount; + status->flags = adb->header->flags; + status->data_region_size = adb->header->timesTableOffset - adb->header->dataOffset; + + return 0; +} +
--- a/tests/0010/run-test.sh Sat Jan 10 11:11:27 2009 +0000 +++ b/tests/0010/run-test.sh Sat Jan 10 16:47:57 2009 +0000 @@ -21,29 +21,34 @@ intstring 2 > testquery floatstring 0 0.5 >> testquery +# because we have a tie, we treat both possible answers as correct. +# This is the only way to preserve my sanity right now. -- CSR, +# 2008-12-15. + ${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -R 5 > testoutput -echo testfeature01 1 > test-expected-output -echo testfeature10 1 >> test-expected-output -cmp testoutput test-expected-output +echo testfeature01 1 > test-expected-output1 +echo testfeature10 1 >> test-expected-output1 +echo testfeature10 1 > test-expected-output2 +echo testfeature01 1 >> test-expected-output2 +cmp testoutput test-expected-output1 || cmp testoutput test-expected-output2 ${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -r 1 -R 5 > testoutput -echo testfeature01 1 > test-expected-output -cmp testoutput test-expected-output +echo testfeature01 1 > test-expected-output1 +echo testfeature10 1 > test-expected-output2 +cmp testoutput test-expected-output1 || cmp testoutput test-expected-output2 echo "query point (0.5,0.0)" intstring 2 > testquery floatstring 0.5 0 >> testquery -# FIXME: because there's only one point in each track (and the query), -# the ordering is essentially database order. We need these test -# cases anyway because we need to test non-segfaulting, non-empty -# results... - ${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -R 5 > testoutput -echo testfeature01 1 > test-expected-output -echo testfeature10 1 >> test-expected-output -cmp testoutput test-expected-output +echo testfeature01 1 > test-expected-output1 +echo testfeature10 1 >> test-expected-output1 +echo testfeature10 1 > test-expected-output2 +echo testfeature01 1 >> test-expected-output2 +cmp testoutput test-expected-output1 || cmp testoutput test-expected-output2 ${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -r 1 -R 5 > testoutput -echo testfeature01 1 > test-expected-output -cmp testoutput test-expected-output +echo testfeature01 1 > test-expected-output1 +echo testfeature10 1 > test-expected-output2 +cmp testoutput test-expected-output1 || cmp testoutput test-expected-output2 exit 104
--- a/tests/0020/run-test.sh Sat Jan 10 11:11:27 2009 +0000 +++ b/tests/0020/run-test.sh Sat Jan 10 16:47:57 2009 +0000 @@ -24,12 +24,15 @@ floatstring 0 0.5 >> testquery ${AUDIODB} -c localhost:10020 -d testdb -Q sequence -l 1 -f testquery -R 5 > testoutput -echo testfeature01 1 > test-expected-output -echo testfeature10 1 >> test-expected-output -cmp testoutput test-expected-output +echo testfeature01 1 > test-expected-output1 +echo testfeature10 1 >> test-expected-output1 +echo testfeature10 1 > test-expected-output2 +echo testfeature01 1 >> test-expected-output2 +cmp testoutput test-expected-output1 || cmp testoutput test-expected-output2 ${AUDIODB} -c localhost:10020 -d testdb -Q sequence -l 1 -f testquery -r 1 -R 5 > testoutput -echo testfeature01 1 > test-expected-output -cmp testoutput test-expected-output +echo testfeature01 1 > test-expected-output1 +echo testfeature10 1 > test-expected-output2 +cmp testoutput test-expected-output1 || cmp testoutput test-expected-output2 check_server $! @@ -37,18 +40,16 @@ intstring 2 > testquery floatstring 0.5 0 >> testquery -# FIXME: because there's only one point in each track (and the query), -# the ordering is essentially database order. We need these test -# cases anyway because we need to test non-segfaulting, non-empty -# results... - ${AUDIODB} -c localhost:10020 -d testdb -Q sequence -l 1 -f testquery -R 5 > testoutput -echo testfeature01 1 > test-expected-output -echo testfeature10 1 >> test-expected-output -cmp testoutput test-expected-output +echo testfeature01 1 > test-expected-output1 +echo testfeature10 1 >> test-expected-output1 +echo testfeature10 1 > test-expected-output2 +echo testfeature01 1 >> test-expected-output2 +cmp testoutput test-expected-output1 || cmp testoutput test-expected-output2 ${AUDIODB} -c localhost:10020 -d testdb -Q sequence -l 1 -f testquery -r 1 -R 5 > testoutput -echo testfeature01 1 > test-expected-output -cmp testoutput test-expected-output +echo testfeature01 1 > test-expected-output1 +echo testfeature10 1 > test-expected-output2 +cmp testoutput test-expected-output1 || cmp testoutput test-expected-output2 stop_server $!
--- a/tests/0029/run-test.sh Sat Jan 10 11:11:27 2009 +0000 +++ b/tests/0029/run-test.sh Sat Jan 10 16:47:57 2009 +0000 @@ -62,14 +62,16 @@ cmp testoutput test-expected-output ${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -w testquerypower --absolute-threshold=-0.8 -p 0 > testoutput -echo testfeature 1 0 0 > test-expected-output -cmp testoutput test-expected-output +echo testfeature 1 0 0 > test-expected-output1 +echo testfeature 1 0 2 > test-expected-output2 +cmp testoutput test-expected-output1 || cmp testoutput test-expected-output2 ${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -w testquerypower --absolute-threshold=-0.8 -p 1 > testoutput cat /dev/null > test-expected-output cmp testoutput test-expected-output ${AUDIODB} -d testdb -Q sequence -l 2 -f testquery -w testquerypower --relative-threshold=0.1 -p 0 > testoutput -echo testfeature 1 0 0 > test-expected-output -cmp testoutput test-expected-output +echo testfeature 1 0 0 > test-expected-output1 +echo testfeature 1 0 2 > test-expected-output2 +cmp testoutput test-expected-output1 || cmp testoutput test-expected-output2 exit 104
--- a/tests/0041/run-test.sh Sat Jan 10 11:11:27 2009 +0000 +++ b/tests/0041/run-test.sh Sat Jan 10 16:47:57 2009 +0000 @@ -86,10 +86,11 @@ ${AUDIODB} -d testdb -X -l 1 -R 0.9 --lsh_k 1 --no_unit_norming ${AUDIODB} -c localhost:${WSPORT} -d testdb -Q nsequence -l 1 -f testquery -w testpower -p 0 -R 0.9 -n 2 --no_unit_norming --lsh_exact > testoutput -echo testfeature 3 > test-expected-output -echo testfeature 0 >> test-expected-output -cmp testoutput test-expected-output - +echo testfeature 3 > test-expected-output1 +echo testfeature 0 >> test-expected-output1 +echo testfeature 0 > test-expected-output2 +echo testfeature 3 >> test-expected-output2 +cmp testoutput test-expected-output1 || cmp testoutput test-expected-output2 stop_server $!