comparison index.cpp @ 458:913a95f06998 api-inversion

Start using the query state structure.

Actually using it means moving it around in the source code a little
bit, thanks to entanglements.  It'll all be alright on the night.  Now
the accumulator, allowed_keys and exact_evaluation_queue are all part of
the query state, and can therefore be passed around with minimal effort
(and deleted in the appropriate place).

Now a whole bunch of static methods (the callbacks, basically) in
index.cpp can be rewritten as plain old C functions.  The callbacks need
both an adb_t and a query state structure to function (the adb_t to get
at things like lsh_n_point_bits and the track->key table; the qstate to
get at the accumulator and allowed_keys list).

Rearrange audioDB::query a little bit, and mark the beginning and the
end of the putative audiodb_query_spec() API function implementation.

 // 19th August 2008 - added O2_FLAG_LARGE_ADB support
 
 #include "audioDB.h"
 #include "audioDB-internals.h"
 
-/************************* LSH point index to audioDB conversion  *****************/
-Uns32T audioDB::index_to_trackID(Uns32T lshID, Uns32T nPntBits){
-  assert(nPntBits);
-  return lshID>>nPntBits;
-}
-
-Uns32T audioDB::index_to_trackPos(Uns32T lshID, Uns32T nPntBits){
-  assert(nPntBits);
-  return lshID&((1<<nPntBits)-1);
-}
-
-Uns32T audioDB::index_from_trackInfo(Uns32T trackID, Uns32T spos, Uns32T nPntBits){
-  assert(nPntBits);
-  return (trackID << nPntBits) | spos;
-}
+typedef struct adb_qcallback {
+  adb_t *adb;
+  adb_qstate_internal_t *qstate;
+} adb_qcallback_t;
 
 /************************* LSH indexing and query initialization  *****************/
 
 char* audioDB::index_get_name(const char*dbName, double radius, Uns32T sequenceLength){
   char* indexName = new char[MAXSTR];  

     char* mergeIndexName = newIndexName;
 
     // Get the lsh header info and find how many tracks are inserted already
     lsh = new LSH(mergeIndexName, false); // lshInCore=false to avoid loading hashTables here
     assert(lsh);
-    Uns32T maxs = index_to_trackID(lsh->get_maxp(), lsh_n_point_bits)+1;
+    Uns32T maxs = audiodb_index_to_track_id(lsh->get_maxp(), audiodb_lsh_n_point_bits(adb))+1;
     delete lsh;
     lsh = 0;
 
     // Insert up to lsh_param_b tracks
     if(  !sNorm && !(dbH->flags & O2_FLAG_LARGE_ADB) ){

 Uns32T audioDB::index_insert_shingles(vector<vector<float> >* vv, Uns32T trackID, double* spp){
   Uns32T collisionCount = 0;
   cout << "[" << trackID << "]" << fileTable+trackID*O2_FILETABLE_ENTRY_SIZE;
   for( Uns32T pointID=0 ; pointID < (*vv).size(); pointID+=sequenceHop){
     if(!use_absolute_threshold || (use_absolute_threshold && (*spp >= absolute_threshold)))
-      collisionCount += lsh->insert_point((*vv)[pointID], index_from_trackInfo(trackID, pointID, lsh_n_point_bits));
+      collisionCount += lsh->insert_point((*vv)[pointID], audiodb_index_from_trackinfo(trackID, pointID, audiodb_lsh_n_point_bits(adb)));
     spp+=sequenceHop;
     }
   return collisionCount;
 }
 

     VERB_LOG(1,"INDEX: seqlen %d\n", sequenceLength);
     VERB_LOG(1,"INDEX: w %f\n", lsh->get_lshHeader()->get_binWidth());
     VERB_LOG(1,"INDEX: k %d\n", lsh->get_lshHeader()->get_numFuns());
     VERB_LOG(1,"INDEX: L (m*(m-1))/2 %d\n", lsh->get_lshHeader()->get_numTables());
     VERB_LOG(1,"INDEX: N %d\n", lsh->get_lshHeader()->get_numRows());
-    VERB_LOG(1,"INDEX: s %d\n", index_to_trackID(lsh->get_maxp(), lsh_n_point_bits));
+    VERB_LOG(1,"INDEX: s %d\n", audiodb_index_to_track_id(lsh->get_maxp(), audiodb_lsh_n_point_bits(adb)));
     VERB_LOG(1,"INDEX: Opened LSH index file %s\n", indexName);
   }
 
   // Check to see if we are loading hash tables into core, and do so if true
   if((lsh->get_lshHeader()->flags&O2_SERIAL_FILEFORMAT2) || lsh_in_core){

   
   delete[] indexName;
   return true;
 }
 
-// *Static* approximate NN point reporter callback method for lshlib
-void audioDB::index_add_point_approximate(void* instancePtr, Uns32T pointID, Uns32T qpos, float dist){
-  assert(instancePtr); // We need an instance for this callback
-  audioDB* myself = (audioDB*) instancePtr; // Use explicit cast to recover "this" instance
-  Uns32T trackID = index_to_trackID(pointID, myself->lsh_n_point_bits);
-  Uns32T spos = index_to_trackPos(pointID, myself->lsh_n_point_bits);
-  // Skip identity in query_from_key
-  if( !myself->query_from_key || (myself->query_from_key && ( trackID != myself->query_from_key_index )) ) {
+void audiodb_index_add_point_approximate(void *user_data, Uns32T pointID, Uns32T qpos, float dist) {
+  adb_qcallback_t *data = (adb_qcallback_t *) user_data;
+  adb_t *adb = data->adb;
+  adb_qstate_internal_t *qstate = data->qstate;
+  uint32_t nbits = audiodb_lsh_n_point_bits(adb);
+  uint32_t trackID = audiodb_index_to_track_id(pointID, nbits);
+  uint32_t spos = audiodb_index_to_track_pos(pointID, nbits);
+  std::set<std::string>::iterator keys_end = qstate->allowed_keys->end();
+  if(qstate->allowed_keys->find((*adb->keys)[trackID]) != keys_end) {
     adb_result_t r;
-    r.key = myself->fileTable + trackID * O2_FILETABLE_ENTRY_SIZE;
+    r.key = (*adb->keys)[trackID].c_str();
     r.dist = dist;
     r.qpos = qpos;
     r.ipos = spos;
-    myself->accumulator->add_point(&r);
-  }
-}
-
-// *Static* exact NN point reporter callback method for lshlib
-// Maintain a queue of points to pass to query_points() for exact evaluation
-void audioDB::index_add_point_exact(void* instancePtr, Uns32T pointID, Uns32T qpos, float dist){
-  assert(instancePtr); // We need an instance for this callback
-  audioDB* myself = (audioDB*) instancePtr; // Use explicit cast to recover "this" instance  
-  Uns32T trackID = index_to_trackID(pointID, myself->lsh_n_point_bits);
-  Uns32T spos = index_to_trackPos(pointID, myself->lsh_n_point_bits);
-  // Skip identity in query_from_key
-  if( !myself->query_from_key || (myself->query_from_key && ( trackID != myself->query_from_key_index )) )
-    myself->index_insert_exact_evaluation_queue(trackID, qpos, spos);
-}
-
-void audioDB::initialize_exact_evalutation_queue(){
-  if(exact_evaluation_queue)
-    delete exact_evaluation_queue;
-  exact_evaluation_queue = new priority_queue<PointPair, std::vector<PointPair>, std::less<PointPair> >;
-}
-
-void audioDB::index_insert_exact_evaluation_queue(Uns32T trackID, Uns32T qpos, Uns32T spos){
-  PointPair p(trackID, qpos, spos);
-  exact_evaluation_queue->push(p);
+    qstate->accumulator->add_point(&r);
+  }
+}
+
+// Maintain a queue of points to pass to query_loop_points() for exact
+// evaluation
+void audiodb_index_add_point_exact(void *user_data, Uns32T pointID, Uns32T qpos, float dist) {
+  adb_qcallback_t *data = (adb_qcallback_t *) user_data;
+  adb_t *adb = data->adb;
+  adb_qstate_internal_t *qstate = data->qstate;
+  uint32_t nbits = audiodb_lsh_n_point_bits(adb);
+  uint32_t trackID = audiodb_index_to_track_id(pointID, nbits);
+  uint32_t spos = audiodb_index_to_track_pos(pointID, nbits);
+  std::set<std::string>::iterator keys_end = qstate->allowed_keys->end();
+  if(qstate->allowed_keys->find((*adb->keys)[trackID]) != keys_end) {
+    PointPair p(trackID, qpos, spos);
+    qstate->exact_evaluation_queue->push(p);
+  }
 }
 
 // return 0: if index does not exist
 // return nqv: if index exists
-int audioDB::index_query_loop(adb_t *adb, adb_query_spec_t *spec) {
+int audioDB::index_query_loop(adb_t *adb, adb_query_spec_t *spec, adb_qstate_internal_t *qstate) {
   
   double *query = 0, *query_data = 0;
   adb_qpointers_internal_t qpointers = {0};
 
+  adb_qcallback_t callback_data;
+  callback_data.adb = adb;
+  callback_data.qstate = qstate;
+
   void (*add_point_func)(void*,Uns32T,Uns32T,float);
 
   sequenceLength = spec->qid.sequence_length;
   normalizedDistance = (spec->params.distance == ADB_DISTANCE_EUCLIDEAN_NORMED);
 
   // Set the point-reporter callback based on the value of lsh_exact
-  if(lsh_exact){
-    initialize_exact_evalutation_queue();
-    add_point_func = &index_add_point_exact;
-  }
-  else
-    add_point_func = &index_add_point_approximate;  
+  if(lsh_exact) {
+    qstate->exact_evaluation_queue = new std::priority_queue<PointPair>;
+    add_point_func = &audiodb_index_add_point_exact;
+  } else {
+    add_point_func = &audiodb_index_add_point_approximate;  
+  }
 
   if(!index_init_query(adb->path)) // sets-up LSH index structures for querying
     return 0;
 
   char* database = index_get_name(adb->path, radius, sequenceLength);

   // Nq contains number of inspected points in query file, 
   // numVecsAboveThreshold is number of points with power >= absolute_threshold
   double* qpp = qpointers.power; // Keep original qpPtr for possible exact evaluation
   if(usingQueryPoint && numVecsAboveThreshold){
     if((lsh->get_lshHeader()->flags&O2_SERIAL_FILEFORMAT2) || lsh_in_core)
-      lsh->retrieve_point((*vv)[0], queryPoint, add_point_func, (void*)this);
+      lsh->retrieve_point((*vv)[0], queryPoint, add_point_func, &callback_data);
     else
-      lsh->serial_retrieve_point(database, (*vv)[0], queryPoint, add_point_func, (void*)this);
+      lsh->serial_retrieve_point(database, (*vv)[0], queryPoint, add_point_func, &callback_data);
   }
   else if(numVecsAboveThreshold)
     for( Uns32T pointID = 0 ; pointID < Nq; pointID++ )
       if(!use_absolute_threshold || (use_absolute_threshold && (*qpp++ >= absolute_threshold))) {
 	if((lsh->get_lshHeader()->flags&O2_SERIAL_FILEFORMAT2) || lsh_in_core) {
-	  lsh->retrieve_point((*vv)[pointID], pointID, add_point_func, (void*)this);
+	  lsh->retrieve_point((*vv)[pointID], pointID, add_point_func, &callback_data);
         } else {
-	  lsh->serial_retrieve_point(database, (*vv)[pointID], pointID, add_point_func, (void*)this);   
+	  lsh->serial_retrieve_point(database, (*vv)[pointID], pointID, add_point_func, &callback_data);   
         }
       }
 
   if(lsh_exact)
     // Perform exact distance computation on point pairs in exact_evaluation_queue
-    query_loop_points(adb, spec, query, &qpointers);
+    query_loop_points(adb, spec, qstate, query, &qpointers);
   
   // Close the index file
   close(lshfid);
     
  // Clean up