changeset 468:4dbd7917bf9e api-inversion

YAY!

audioDB::index_query_loop is now disentangled.  Hardwired lsh_in_core to
true, and invented a new QID flag for !lsh_exact.  All tests continue to
pass.

The plans now:
- extract the audiodb_query_spec() function from inside audioDB::query,
  and move audioDB::query (and the timestamps function) to audioDB.cpp;
- write libtests/0036 and libtests/0037 in terms of audiodb_query_spec();
- rewrite all the other libtests in terms of audiodb_query_spec();
- delete audiodb_query() [ and maybe rename audiodb_query_spec(), I
  dunno];
- implement example bindings (probably to Lisp, because that's what I
  know best);
- see if anyone other than me can work out how the API works.  If not,
  provide documentation;
- revise API in the light of user feedback.

--- a/audioDB-internals.h	Wed Dec 31 12:25:22 2008 +0000
+++ b/audioDB-internals.h	Wed Dec 31 15:44:12 2008 +0000
@@ -41,8 +41,14 @@
  * using adb->fd, but changing to use pread() might win us
  * threadsafety eventually.
  */
-// FIXME moved to audioDB.h for now
+typedef struct adb_qstate_internal {
+  Accumulator *accumulator;
+  std::set<std::string> *allowed_keys;
+  std::priority_queue<PointPair> *exact_evaluation_queue;
+  LSH *lsh;
+} adb_qstate_internal_t;
 
+/* the transparent version of the opaque (forward-declared) adb_t. */
 struct adb {
   char *path;
   int fd;
@@ -247,3 +253,4 @@
 int audiodb_query_loop(adb_t *, adb_query_spec_t *, adb_qstate_internal_t *);
 char *audiodb_index_get_name(const char *, double, uint32_t);
 bool audiodb_index_exists(const char *, double, uint32_t);
+int audiodb_index_query_loop(adb_t *, adb_query_spec_t *, adb_qstate_internal_t *);

--- a/audioDB.h	Wed Dec 31 12:25:22 2008 +0000
+++ b/audioDB.h	Wed Dec 31 15:44:12 2008 +0000
@@ -42,14 +42,6 @@
 };
 bool operator<(const PointPair& a, const PointPair& b);
 
-// should be in -internals.h
-typedef struct adb_qstate_internal {
-  Accumulator *accumulator;
-  std::set<std::string> *allowed_keys;
-  std::priority_queue<PointPair> *exact_evaluation_queue;
-  LSH *lsh;
-} adb_qstate_internal_t;
-
 #define MAXSTR 512
 
 // Databse PRIMARY commands
@@ -364,7 +356,6 @@
   void index_insert_tracks(Uns32T start_track, Uns32T end_track, double** fvpp, double** sNormpp,double** snPtrp, double** sPowerp, double** spPtrp);
   int index_insert_track(Uns32T trackID, double** fvpp, double** snpp, double** sppp);
   Uns32T index_insert_shingles(vector<vector<float> >*, Uns32T trackID, double* spp);
-  int index_query_loop(adb_t *adb, adb_query_spec_t *spec, adb_qstate_internal_t *qstate);
   void insertPowerData(unsigned n, int powerfd, double *powerdata);
   void init_track_aux_data(Uns32T trackID, double* fvp, double** sNormpp,double** snPtrp, double** sPowerp, double** spPtrp);
   

--- a/audioDB_API.h	Wed Dec 31 12:25:22 2008 +0000
+++ b/audioDB_API.h	Wed Dec 31 15:44:12 2008 +0000
@@ -146,7 +146,8 @@
   adb_result_t *results;
 } adb_query_results_t;
 
-#define ADB_QUERY_ID_FLAG_EXHAUSTIVE 1
+#define ADB_QID_FLAG_EXHAUSTIVE 1
+#define ADB_QID_FLAG_ALLOW_FALSE_POSITIVES 2
 
 typedef struct adbqueryid {
   adb_datum_t *datum;

--- a/index.cpp	Wed Dec 31 12:25:22 2008 +0000
+++ b/index.cpp	Wed Dec 31 15:44:12 2008 +0000
@@ -578,7 +578,7 @@
 // return -1 on error
 // return 0: if index does not exist
 // return nqv: if index exists
-int audioDB::index_query_loop(adb_t *adb, adb_query_spec_t *spec, adb_qstate_internal_t *qstate) {
+int audiodb_index_query_loop(adb_t *adb, adb_query_spec_t *spec, adb_qstate_internal_t *qstate) {
   
   double *query = 0, *query_data = 0;
   adb_qpointers_internal_t qpointers = {0};
@@ -595,14 +595,18 @@
   bool use_absolute_threshold = spec->refine.flags & ADB_REFINE_ABSOLUTE_THRESHOLD;
   double absolute_threshold = spec->refine.absolute_threshold;
 
-  // Set the point-reporter callback based on the value of lsh_exact
-  if(lsh_exact) {
+  if(spec->qid.flags & ADB_QID_FLAG_ALLOW_FALSE_POSITIVES) {
+    add_point_func = &audiodb_index_add_point_approximate;  
+  } else {
     qstate->exact_evaluation_queue = new std::priority_queue<PointPair>;
     add_point_func = &audiodb_index_add_point_exact;
-  } else {
-    add_point_func = &audiodb_index_add_point_approximate;  
   }
 
+  /* FIXME: this hardwired lsh_in_core is here to allow for a
+   * transition period while the need for the argument is worked
+   * through.  Hopefully it will disappear again eventually. */
+  bool lsh_in_core = true;
+
   if(!audiodb_index_init_query(adb, spec, qstate, lsh_in_core)) {
     return 0;
   }
@@ -621,8 +625,9 @@
   std::vector<std::vector<float> > *vv = audiodb_index_initialize_shingles(Nq, adb->header->dim, sequence_length);
 
   // Construct shingles from query features  
-  for(uint32_t pointID = 0; pointID < Nq; pointID++) 
+  for(uint32_t pointID = 0; pointID < Nq; pointID++) {
     audiodb_index_make_shingle(vv, pointID, query, adb->header->dim, sequence_length);
+  }
   
   // Normalize query vectors
   int vcount = audiodb_index_norm_shingles(vv, qpointers.l2norm, qpointers.power, adb->header->dim, sequence_length, radius, normalized, use_absolute_threshold, absolute_threshold);
@@ -636,7 +641,7 @@
   // Nq contains number of inspected points in query file, 
   // numVecsAboveThreshold is number of points with power >= absolute_threshold
   double *qpp = qpointers.power; // Keep original qpPtr for possible exact evaluation
-  if(!(spec->qid.flags & ADB_QUERY_ID_FLAG_EXHAUSTIVE) && numVecsAboveThreshold) {
+  if(!(spec->qid.flags & ADB_QID_FLAG_EXHAUSTIVE) && numVecsAboveThreshold) {
     if((qstate->lsh->get_lshHeader()->flags & O2_SERIAL_FILEFORMAT2) || lsh_in_core) {
       qstate->lsh->retrieve_point((*vv)[0], spec->qid.sequence_start, add_point_func, &callback_data);
     } else {
@@ -655,9 +660,7 @@
   }
   audiodb_index_delete_shingles(vv);
 
-  if(lsh_exact) {
-    // Perform exact distance computation on point pairs in
-    // exact_evaluation_queue
+  if(!(spec->qid.flags & ADB_QID_FLAG_ALLOW_FALSE_POSITIVES)) {
     audiodb_query_queue_loop(adb, spec, qstate, query, &qpointers);
   }
   

--- a/query.cpp	Wed Dec 31 12:25:22 2008 +0000
+++ b/query.cpp	Wed Dec 31 15:44:12 2008 +0000
@@ -116,7 +116,9 @@
 
   qspec.qid.datum = &datum;
   qspec.qid.sequence_length = sequenceLength;
-  qspec.qid.flags = usingQueryPoint ? 0 : ADB_QUERY_ID_FLAG_EXHAUSTIVE;
+  qspec.qid.flags = 0;
+  qspec.qid.flags |= usingQueryPoint ? 0 : ADB_QID_FLAG_EXHAUSTIVE;
+  qspec.qid.flags |= lsh_exact ? 0 : ADB_QID_FLAG_ALLOW_FALSE_POSITIVES;
   qspec.qid.sequence_start = queryPoint;
 
   switch(queryType) {
@@ -228,7 +230,7 @@
   // Test for index (again) here
   if((qspec.refine.flags & ADB_REFINE_RADIUS) && audiodb_index_exists(adb->path, qspec.refine.radius, qspec.qid.sequence_length)){ 
     VERB_LOG(1, "Calling indexed query on database %s, radius=%f, sequence_length=%d\n", adb->path, qspec.refine.radius, qspec.qid.sequence_length);
-    if(index_query_loop(adb, &qspec, &qstate) < 0) {
+    if(audiodb_index_query_loop(adb, &qspec, &qstate) < 0) {
       error("index_query_loop failed");
     }
   } else {
@@ -508,7 +510,7 @@
   audiodb_datum_qpointers(&d, sequence_length, vector_data, vector, qpointers);
 
   /* Finally, if applicable, set up the moving qpointers. */
-  if(spec->qid.flags & ADB_QUERY_ID_FLAG_EXHAUSTIVE) {
+  if(spec->qid.flags & ADB_QID_FLAG_EXHAUSTIVE) {
     /* the qpointers are already at the start, and so correct. */
   } else {
     /* adjust the qpointers to point to the correct place in the sequence */
@@ -659,7 +661,7 @@
       }
       audiodb_free_datum(&d);
     }
-    Uns32T qPos = (spec->qid.flags & ADB_QUERY_ID_FLAG_EXHAUSTIVE) ? pp.qpos : 0;
+    Uns32T qPos = (spec->qid.flags & ADB_QID_FLAG_EXHAUSTIVE) ? pp.qpos : 0;
     Uns32T sPos = pp.spos; // index into l2norm table
     // Test power thresholds before computing distance
     if( ( (!power_refine) || audiodb_powers_acceptable(&spec->refine, qpointers->power[qPos], dbpointers.power[sPos])) &&
@@ -784,7 +786,7 @@
                 adb_result_t r;
                 r.key = (*adb->keys)[track].c_str();
                 r.dist = thisDist;
-                if(spec->qid.flags & ADB_QUERY_ID_FLAG_EXHAUSTIVE) {
+                if(spec->qid.flags & ADB_QID_FLAG_EXHAUSTIVE) {
                   r.qpos = j;
                 } else {
                   r.qpos = spec->qid.sequence_start;