changeset 458:913a95f06998 api-inversion

Start using the query state structure.

Actually using it means moving it around in the source code a little
bit, thanks to entanglements.  It'll all be alright on the night.  Now
the accumulator, allowed_keys and exact_evaluation_queue are all part of
the query state, and can therefore be passed around with minimal effort
(and deleted in the appropriate place).

Now a whole bunch of static methods (the callbacks, basically) in
index.cpp can be rewritten as plain old C functions.  The callbacks need
both an adb_t and a query state structure to function (the adb_t to get
at things like lsh_n_point_bits and the track->key table; the qstate to
get at the accumulator and allowed_keys list).

Rearrange audioDB::query a little bit, and mark the beginning and the
end of the putative audiodb_query_spec() API function implementation.

--- a/audioDB-internals.h	Wed Dec 24 10:57:30 2008 +0000
+++ b/audioDB-internals.h	Sun Dec 28 18:44:08 2008 +0000
@@ -24,12 +24,7 @@
  * using adb->fd, but changing to use pread() might win us
  * threadsafety eventually.
  */
-typedef struct adb_qstate_internal {
-  Accumulator *accumulator;
-  adb_qpointers_internal_t *qpointers;
-  adb_qpointers_internal_t *dbpointers;
-  std::set<std::string> *allowed_keys;
-} adb_qstate_internal_t;
+// FIXME moved to audioDB.h for now
 
 struct adb {
   char *path;
@@ -207,6 +202,24 @@
   }
 }
 
+/************************* LSH point index to audioDB conversion  *****************/
+static inline uint32_t audiodb_index_to_track_id(uint32_t lshid, uint32_t n_point_bits) {
+  return (lshid >> n_point_bits);
+}
+
+static inline uint32_t audiodb_index_to_track_pos(uint32_t lshid, uint32_t n_point_bits) {
+  return (lshid & ((1 << n_point_bits) - 1));
+}
+
+static inline uint32_t audiodb_index_from_trackinfo(uint32_t track_id, uint32_t track_pos, uint32_t n_point_bits) {
+  return ((track_id << n_point_bits) | track_pos);
+}
+
+static inline uint32_t audiodb_lsh_n_point_bits(adb_t *adb) {
+  uint32_t nbits = adb->header->flags >> 28;
+  return (nbits ? nbits : O2_DEFAULT_LSH_N_POINT_BITS);
+}
+
 int audiodb_read_data(adb_t *, int, int, double **, size_t *);
 int audiodb_insert_create_datum(adb_insert_t *, adb_datum_t *);
 int audiodb_free_datum(adb_datum_t *);

--- a/audioDB.cpp	Wed Dec 24 10:57:30 2008 +0000
+++ b/audioDB.cpp	Sun Dec 28 18:44:08 2008 +0000
@@ -217,10 +217,6 @@
     munmap(powerFileNameTable, fileTableLength);
   if(reporter)
     delete reporter;
-  if(accumulator)
-    delete accumulator;
-  if(exact_evaluation_queue)
-    delete exact_evaluation_queue;
   if(rng)
     gsl_rng_free(rng);
   if(vv)

--- a/audioDB.h	Wed Dec 24 10:57:30 2008 +0000
+++ b/audioDB.h	Sun Dec 28 18:44:08 2008 +0000
@@ -32,6 +32,16 @@
 #include "soapH.h"
 #include "cmdline.h"
 
+// should probably be rewritten
+class PointPair{
+ public:
+  Uns32T trackID;
+  Uns32T qpos;
+  Uns32T spos;
+  PointPair(Uns32T a, Uns32T b, Uns32T c);
+};
+bool operator<(const PointPair& a, const PointPair& b);
+
 // should be in -internals.h
 typedef struct adb_qpointers_internal {
   uint32_t nvectors;
@@ -42,6 +52,12 @@
   double *mean_duration;
 } adb_qpointers_internal_t;
 
+typedef struct adb_qstate_internal {
+  Accumulator *accumulator;
+  std::set<std::string> *allowed_keys;
+  std::priority_queue<PointPair> *exact_evaluation_queue;
+} adb_qstate_internal_t;
+
 #define MAXSTR 512
 
 // Databse PRIMARY commands
@@ -219,16 +235,6 @@
   off_t dbSize;
 } dbTableHeaderT, *dbTableHeaderPtr;
 
-class PointPair{
- public:
-  Uns32T trackID;
-  Uns32T qpos;
-  Uns32T spos;
-  PointPair(Uns32T a, Uns32T b, Uns32T c);
-};
-
-bool operator<(const PointPair& a, const PointPair& b);
-
 class audioDB{  
  private:
   gengetopt_args_info args_info;
@@ -309,8 +315,6 @@
   double relative_threshold;
   
   ReporterBase* reporter;  // track/point reporter
-  Accumulator *accumulator;
-  priority_queue<PointPair, std::vector<PointPair>, std::less<PointPair> >* exact_evaluation_queue;
 
   // LISZT parameters
   unsigned lisztOffset;
@@ -324,8 +328,8 @@
   void error(const char* a, const char* b = "", const char *sysFunc = 0);
 
   void insertTimeStamps(unsigned n, std::ifstream* timesFile, double* timesdata);
-  int query_loop(adb_t *adb, adb_query_spec_t *spec);
-  void query_loop_points(adb_t *adb, adb_query_spec_t *spec, double* query, adb_qpointers_internal_t *qpointers);
+  int query_loop(adb_t *adb, adb_query_spec_t *spec, adb_qstate_internal_t *qstate);
+  void query_loop_points(adb_t *adb, adb_query_spec_t *spec, adb_qstate_internal_t *qstate, double *query, adb_qpointers_internal_t *qpointers);
   void initRNG();
   void initDBHeader(const char *dbName);
   void initInputFile(const char *inFile);
@@ -371,7 +375,6 @@
   Uns32T lsh_param_N; // Number of rows per hash table
   Uns32T lsh_param_b; // Batch size, in number of tracks, per indexing iteration
   Uns32T lsh_param_ncols; // Maximum number of collision in a hash-table row
-  Uns32T lsh_n_point_bits; // How many bits to use to encode point ID within a track
 
 
   // LSH vector<> containers for one in-core copy of a set of feature vectors
@@ -384,18 +387,11 @@
   int index_insert_track(Uns32T trackID, double** fvpp, double** snpp, double** sppp);
   Uns32T index_insert_shingles(vector<vector<float> >*, Uns32T trackID, double* spp);
   int index_norm_shingles(vector<vector<float> >*, double* snp, double* spp);
-  int index_query_loop(adb_t *adb, adb_query_spec_t *spec);
+  int index_query_loop(adb_t *adb, adb_query_spec_t *spec, adb_qstate_internal_t *qstate);
   vector<vector<float> >* index_initialize_shingles(Uns32T sz);
   int index_init_query(const char* dbName);
   int index_exists(const char* dbName, double radius, Uns32T sequenceLength);
   char* index_get_name(const char*dbName, double radius, Uns32T sequenceLength);
-  static void index_add_point_approximate(void* instance, Uns32T pointID, Uns32T qpos, float dist); // static point reporter callback method
-  static void index_add_point_exact(void* instance, Uns32T pointID, Uns32T qpos, float dist); // static point reporter callback method
-  static Uns32T index_to_trackID(Uns32T lshID, Uns32T nPntBits);  // Convert lsh point index to audioDB trackID
-  static Uns32T index_to_trackPos(Uns32T lshID, Uns32T nPntBits); // Convert lsh point index to audioDB trackPos (spos)
-  static Uns32T index_from_trackInfo(Uns32T trackID, Uns32T pntID, Uns32T nPntBits); // Convert audioDB trackID and trackPos to an lsh point index
-  void initialize_exact_evalutation_queue();
-  void index_insert_exact_evaluation_queue(Uns32T trackID, Uns32T qpos, Uns32T spos);
   LSH* index_allocate(char* indexName, bool load_hashTables);
   void insertPowerData(unsigned n, int powerfd, double *powerdata);
   void init_track_aux_data(Uns32T trackID, double* fvp, double** sNormpp,double** snPtrp, double** sPowerp, double** spPtrp);
@@ -474,8 +470,6 @@
     use_relative_threshold(false),		\
     relative_threshold(0.0),			\
     reporter(0),                                \
-    accumulator(0),                             \
-    exact_evaluation_queue(0),                  \
     lisztOffset(0),                             \
     lisztLength(0),                             \
     apierrortemp(0),                            \
@@ -490,6 +484,5 @@
     lsh_param_N(0),				\
     lsh_param_b(0),				\
     lsh_param_ncols(0),                         \
-    lsh_n_point_bits(0),                        \
     vv(0)
 #endif

--- a/common.cpp	Wed Dec 24 10:57:30 2008 +0000
+++ b/common.cpp	Sun Dec 28 18:44:08 2008 +0000
@@ -145,11 +145,6 @@
       CHECKED_MMAP(double *, l2normTable, dbH->l2normTableOffset, l2normTableLength);
     }
   }
-
-  // Assign correct number of point bits per track in LSH indexing / retrieval
-  lsh_n_point_bits = dbH->flags >> 28;
-  if( !lsh_n_point_bits )
-    lsh_n_point_bits = O2_DEFAULT_LSH_N_POINT_BITS;
 }
 
 void audioDB::initInputFile (const char *inFile) {

--- a/create.cpp	Wed Dec 24 10:57:30 2008 +0000
+++ b/create.cpp	Sun Dec 28 18:44:08 2008 +0000
@@ -70,14 +70,14 @@
   databytes = ((off_t) datasize) * 1024 * 1024;
   auxbytes = databytes / datadim;
 
-  /* FIXME: what's going on here?  There are two distinct
-     preprocessor constants (O2_LSH_N_POINT_BITS, LSH_N_POINT_BITS);
-     a third is presumably some default
-     (O2_DEFAULT_LSH_N_POINT_BITS), and then there's this magic 28
-     bits.  Should this really be part of the flags structure at
-     all?  Putting it elsewhere will of course break backwards
-     compatibility, unless 14 is the only value that's been used
-     anywhere... */
+  /* FIXME: what's going on here?  There are two distinct preprocessor
+     constants (O2_LSH_N_POINT_BITS, LSH_N_POINT_BITS); a third is
+     presumably some default (O2_DEFAULT_LSH_N_POINT_BITS), and then
+     there's this magic 28 bits [which needs to be the same as the 28
+     in audiodb_lsh_n_point_bits()].  Should this really be part of
+     the flags structure at all?  Putting it elsewhere will of course
+     break backwards compatibility, unless 14 is the only value that's
+     been used anywhere... */
 
   // For backward-compatibility, Record the point-encoding parameter for LSH indexing in the adb header
   // If this value is 0 then it will be set to 14

--- a/index.cpp	Wed Dec 24 10:57:30 2008 +0000
+++ b/index.cpp	Sun Dec 28 18:44:08 2008 +0000
@@ -14,21 +14,10 @@
 #include "audioDB.h"
 #include "audioDB-internals.h"
 
-/************************* LSH point index to audioDB conversion  *****************/
-Uns32T audioDB::index_to_trackID(Uns32T lshID, Uns32T nPntBits){
-  assert(nPntBits);
-  return lshID>>nPntBits;
-}
-
-Uns32T audioDB::index_to_trackPos(Uns32T lshID, Uns32T nPntBits){
-  assert(nPntBits);
-  return lshID&((1<<nPntBits)-1);
-}
-
-Uns32T audioDB::index_from_trackInfo(Uns32T trackID, Uns32T spos, Uns32T nPntBits){
-  assert(nPntBits);
-  return (trackID << nPntBits) | spos;
-}
+typedef struct adb_qcallback {
+  adb_t *adb;
+  adb_qstate_internal_t *qstate;
+} adb_qcallback_t;
 
 /************************* LSH indexing and query initialization  *****************/
 
@@ -266,7 +255,7 @@
     // Get the lsh header info and find how many tracks are inserted already
     lsh = new LSH(mergeIndexName, false); // lshInCore=false to avoid loading hashTables here
     assert(lsh);
-    Uns32T maxs = index_to_trackID(lsh->get_maxp(), lsh_n_point_bits)+1;
+    Uns32T maxs = audiodb_index_to_track_id(lsh->get_maxp(), audiodb_lsh_n_point_bits(adb))+1;
     delete lsh;
     lsh = 0;
 
@@ -483,7 +472,7 @@
   cout << "[" << trackID << "]" << fileTable+trackID*O2_FILETABLE_ENTRY_SIZE;
   for( Uns32T pointID=0 ; pointID < (*vv).size(); pointID+=sequenceHop){
     if(!use_absolute_threshold || (use_absolute_threshold && (*spp >= absolute_threshold)))
-      collisionCount += lsh->insert_point((*vv)[pointID], index_from_trackInfo(trackID, pointID, lsh_n_point_bits));
+      collisionCount += lsh->insert_point((*vv)[pointID], audiodb_index_from_trackinfo(trackID, pointID, audiodb_lsh_n_point_bits(adb)));
     spp+=sequenceHop;
     }
   return collisionCount;
@@ -519,7 +508,7 @@
     VERB_LOG(1,"INDEX: k %d\n", lsh->get_lshHeader()->get_numFuns());
     VERB_LOG(1,"INDEX: L (m*(m-1))/2 %d\n", lsh->get_lshHeader()->get_numTables());
     VERB_LOG(1,"INDEX: N %d\n", lsh->get_lshHeader()->get_numRows());
-    VERB_LOG(1,"INDEX: s %d\n", index_to_trackID(lsh->get_maxp(), lsh_n_point_bits));
+    VERB_LOG(1,"INDEX: s %d\n", audiodb_index_to_track_id(lsh->get_maxp(), audiodb_lsh_n_point_bits(adb)));
     VERB_LOG(1,"INDEX: Opened LSH index file %s\n", indexName);
   }
 
@@ -536,65 +525,63 @@
   return true;
 }
 
-// *Static* approximate NN point reporter callback method for lshlib
-void audioDB::index_add_point_approximate(void* instancePtr, Uns32T pointID, Uns32T qpos, float dist){
-  assert(instancePtr); // We need an instance for this callback
-  audioDB* myself = (audioDB*) instancePtr; // Use explicit cast to recover "this" instance
-  Uns32T trackID = index_to_trackID(pointID, myself->lsh_n_point_bits);
-  Uns32T spos = index_to_trackPos(pointID, myself->lsh_n_point_bits);
-  // Skip identity in query_from_key
-  if( !myself->query_from_key || (myself->query_from_key && ( trackID != myself->query_from_key_index )) ) {
+void audiodb_index_add_point_approximate(void *user_data, Uns32T pointID, Uns32T qpos, float dist) {
+  adb_qcallback_t *data = (adb_qcallback_t *) user_data;
+  adb_t *adb = data->adb;
+  adb_qstate_internal_t *qstate = data->qstate;
+  uint32_t nbits = audiodb_lsh_n_point_bits(adb);
+  uint32_t trackID = audiodb_index_to_track_id(pointID, nbits);
+  uint32_t spos = audiodb_index_to_track_pos(pointID, nbits);
+  std::set<std::string>::iterator keys_end = qstate->allowed_keys->end();
+  if(qstate->allowed_keys->find((*adb->keys)[trackID]) != keys_end) {
     adb_result_t r;
-    r.key = myself->fileTable + trackID * O2_FILETABLE_ENTRY_SIZE;
+    r.key = (*adb->keys)[trackID].c_str();
     r.dist = dist;
     r.qpos = qpos;
     r.ipos = spos;
-    myself->accumulator->add_point(&r);
+    qstate->accumulator->add_point(&r);
   }
 }
 
-// *Static* exact NN point reporter callback method for lshlib
-// Maintain a queue of points to pass to query_points() for exact evaluation
-void audioDB::index_add_point_exact(void* instancePtr, Uns32T pointID, Uns32T qpos, float dist){
-  assert(instancePtr); // We need an instance for this callback
-  audioDB* myself = (audioDB*) instancePtr; // Use explicit cast to recover "this" instance  
-  Uns32T trackID = index_to_trackID(pointID, myself->lsh_n_point_bits);
-  Uns32T spos = index_to_trackPos(pointID, myself->lsh_n_point_bits);
-  // Skip identity in query_from_key
-  if( !myself->query_from_key || (myself->query_from_key && ( trackID != myself->query_from_key_index )) )
-    myself->index_insert_exact_evaluation_queue(trackID, qpos, spos);
-}
-
-void audioDB::initialize_exact_evalutation_queue(){
-  if(exact_evaluation_queue)
-    delete exact_evaluation_queue;
-  exact_evaluation_queue = new priority_queue<PointPair, std::vector<PointPair>, std::less<PointPair> >;
-}
-
-void audioDB::index_insert_exact_evaluation_queue(Uns32T trackID, Uns32T qpos, Uns32T spos){
-  PointPair p(trackID, qpos, spos);
-  exact_evaluation_queue->push(p);
+// Maintain a queue of points to pass to query_loop_points() for exact
+// evaluation
+void audiodb_index_add_point_exact(void *user_data, Uns32T pointID, Uns32T qpos, float dist) {
+  adb_qcallback_t *data = (adb_qcallback_t *) user_data;
+  adb_t *adb = data->adb;
+  adb_qstate_internal_t *qstate = data->qstate;
+  uint32_t nbits = audiodb_lsh_n_point_bits(adb);
+  uint32_t trackID = audiodb_index_to_track_id(pointID, nbits);
+  uint32_t spos = audiodb_index_to_track_pos(pointID, nbits);
+  std::set<std::string>::iterator keys_end = qstate->allowed_keys->end();
+  if(qstate->allowed_keys->find((*adb->keys)[trackID]) != keys_end) {
+    PointPair p(trackID, qpos, spos);
+    qstate->exact_evaluation_queue->push(p);
+  }
 }
 
 // return 0: if index does not exist
 // return nqv: if index exists
-int audioDB::index_query_loop(adb_t *adb, adb_query_spec_t *spec) {
+int audioDB::index_query_loop(adb_t *adb, adb_query_spec_t *spec, adb_qstate_internal_t *qstate) {
   
   double *query = 0, *query_data = 0;
   adb_qpointers_internal_t qpointers = {0};
 
+  adb_qcallback_t callback_data;
+  callback_data.adb = adb;
+  callback_data.qstate = qstate;
+
   void (*add_point_func)(void*,Uns32T,Uns32T,float);
 
   sequenceLength = spec->qid.sequence_length;
   normalizedDistance = (spec->params.distance == ADB_DISTANCE_EUCLIDEAN_NORMED);
 
   // Set the point-reporter callback based on the value of lsh_exact
-  if(lsh_exact){
-    initialize_exact_evalutation_queue();
-    add_point_func = &index_add_point_exact;
+  if(lsh_exact) {
+    qstate->exact_evaluation_queue = new std::priority_queue<PointPair>;
+    add_point_func = &audiodb_index_add_point_exact;
+  } else {
+    add_point_func = &audiodb_index_add_point_approximate;  
   }
-  else
-    add_point_func = &index_add_point_approximate;  
 
   if(!index_init_query(adb->path)) // sets-up LSH index structures for querying
     return 0;
@@ -621,23 +608,23 @@
   double* qpp = qpointers.power; // Keep original qpPtr for possible exact evaluation
   if(usingQueryPoint && numVecsAboveThreshold){
     if((lsh->get_lshHeader()->flags&O2_SERIAL_FILEFORMAT2) || lsh_in_core)
-      lsh->retrieve_point((*vv)[0], queryPoint, add_point_func, (void*)this);
+      lsh->retrieve_point((*vv)[0], queryPoint, add_point_func, &callback_data);
     else
-      lsh->serial_retrieve_point(database, (*vv)[0], queryPoint, add_point_func, (void*)this);
+      lsh->serial_retrieve_point(database, (*vv)[0], queryPoint, add_point_func, &callback_data);
   }
   else if(numVecsAboveThreshold)
     for( Uns32T pointID = 0 ; pointID < Nq; pointID++ )
       if(!use_absolute_threshold || (use_absolute_threshold && (*qpp++ >= absolute_threshold))) {
 	if((lsh->get_lshHeader()->flags&O2_SERIAL_FILEFORMAT2) || lsh_in_core) {
-	  lsh->retrieve_point((*vv)[pointID], pointID, add_point_func, (void*)this);
+	  lsh->retrieve_point((*vv)[pointID], pointID, add_point_func, &callback_data);
         } else {
-	  lsh->serial_retrieve_point(database, (*vv)[pointID], pointID, add_point_func, (void*)this);   
+	  lsh->serial_retrieve_point(database, (*vv)[pointID], pointID, add_point_func, &callback_data);   
         }
       }
 
   if(lsh_exact)
     // Perform exact distance computation on point pairs in exact_evaluation_queue
-    query_loop_points(adb, spec, query, &qpointers);
+    query_loop_points(adb, spec, qstate, query, &qpointers);
   
   // Close the index file
   close(lshfid);

--- a/query.cpp	Wed Dec 24 10:57:30 2008 +0000
+++ b/query.cpp	Sun Dec 28 18:44:08 2008 +0000
@@ -149,7 +149,7 @@
       } else if (index_exists(adb->path, qspec.refine.radius, qspec.qid.sequence_length)) {
 	char* indexName = index_get_name(adb->path, qspec.refine.radius, qspec.qid.sequence_length);
 	lsh = index_allocate(indexName, false);
-	reporter = new trackSequenceQueryRadReporter(trackNN, index_to_trackID(lsh->get_maxp(), lsh_n_point_bits)+1);
+	reporter = new trackSequenceQueryRadReporter(trackNN, audiodb_index_to_track_id(lsh->get_maxp(), audiodb_lsh_n_point_bits(adb))+1);
 	delete[] indexName;
       } else {
 	reporter = new trackSequenceQueryRadReporter(trackNN, dbH->numFiles);
@@ -161,7 +161,7 @@
       } else if (index_exists(adb->path, qspec.refine.radius, qspec.qid.sequence_length)){
 	char* indexName = index_get_name(adb->path, qspec.refine.radius, qspec.qid.sequence_length);
 	lsh = index_allocate(indexName, false);
-	reporter = new trackSequenceQueryRadNNReporter(pointNN,trackNN, index_to_trackID(lsh->get_maxp(), lsh_n_point_bits)+1);
+	reporter = new trackSequenceQueryRadNNReporter(pointNN,trackNN, audiodb_index_to_track_id(lsh->get_maxp(), audiodb_lsh_n_point_bits(adb))+1);
 	delete[] indexName;
       } else {
 	reporter = new trackSequenceQueryRadNNReporter(pointNN,trackNN, dbH->numFiles);
@@ -179,21 +179,37 @@
     error("unrecognized queryType");
   }
 
-  // keyKeyPos requires dbH to be initialized
-  if(query_from_key && (!key || (query_from_key_index = audiodb_key_index(adb, key)) == (uint32_t) -1)) 
-    error("Query key not found", key);  
+  /* Somewhere around here is where the implementation of
+   * audiodb_query_spec() starts. */
+
+  adb_qstate_internal_t qstate;
+  qstate.allowed_keys = new std::set<std::string>;
+  if(qspec.refine.flags & ADB_REFINE_INCLUDE_KEYLIST) {
+    for(unsigned int k = 0; k < qspec.refine.include.nkeys; k++) {
+      qstate.allowed_keys->insert(qspec.refine.include.keys[k]);
+    }
+  } else {
+    for(unsigned int k = 0; k < adb->header->numFiles; k++) {
+      qstate.allowed_keys->insert((*adb->keys)[k]);
+    }
+  }
+  if(qspec.refine.flags & ADB_REFINE_EXCLUDE_KEYLIST) {
+    for(unsigned int k = 0; k < qspec.refine.exclude.nkeys; k++) {
+      qstate.allowed_keys->erase(qspec.refine.exclude.keys[k]);
+    }
+  }
 
   switch(qspec.params.distance) {
   case ADB_DISTANCE_DOT_PRODUCT:
     switch(qspec.params.accumulation) {
     case ADB_ACCUMULATION_DB:
-      accumulator = new DBAccumulator<adb_result_dist_gt>(qspec.params.npoints);
+      qstate.accumulator = new DBAccumulator<adb_result_dist_gt>(qspec.params.npoints);
       break;
     case ADB_ACCUMULATION_PER_TRACK:
-      accumulator = new PerTrackAccumulator<adb_result_dist_gt>(qspec.params.npoints, qspec.params.ntracks);
+      qstate.accumulator = new PerTrackAccumulator<adb_result_dist_gt>(qspec.params.npoints, qspec.params.ntracks);
       break;
     case ADB_ACCUMULATION_ONE_TO_ONE:
-      accumulator = new NearestAccumulator<adb_result_dist_gt>();
+      qstate.accumulator = new NearestAccumulator<adb_result_dist_gt>();
       break;
     default:
       error("unknown accumulation");
@@ -203,13 +219,13 @@
   case ADB_DISTANCE_EUCLIDEAN:
     switch(qspec.params.accumulation) {
     case ADB_ACCUMULATION_DB:
-      accumulator = new DBAccumulator<adb_result_dist_lt>(qspec.params.npoints);
+      qstate.accumulator = new DBAccumulator<adb_result_dist_lt>(qspec.params.npoints);
       break;
     case ADB_ACCUMULATION_PER_TRACK:
-      accumulator = new PerTrackAccumulator<adb_result_dist_lt>(qspec.params.npoints, qspec.params.ntracks);
+      qstate.accumulator = new PerTrackAccumulator<adb_result_dist_lt>(qspec.params.npoints, qspec.params.ntracks);
       break;
     case ADB_ACCUMULATION_ONE_TO_ONE:
-      accumulator = new NearestAccumulator<adb_result_dist_lt>();
+      qstate.accumulator = new NearestAccumulator<adb_result_dist_lt>();
       break;
     default:
       error("unknown accumulation");
@@ -222,16 +238,22 @@
   // Test for index (again) here
   if((qspec.refine.flags & ADB_REFINE_RADIUS) && index_exists(adb->path, qspec.refine.radius, qspec.qid.sequence_length)){ 
     VERB_LOG(1, "Calling indexed query on database %s, radius=%f, sequence_length=%d\n", adb->path, qspec.refine.radius, qspec.qid.sequence_length);
-    index_query_loop(adb, &qspec);
+    index_query_loop(adb, &qspec, &qstate);
   }
   else{
     VERB_LOG(1, "Calling brute-force query on database %s\n", dbName);
-    if(query_loop(adb, &qspec)) {
+    if(query_loop(adb, &qspec, &qstate)) {
       error("query_loop failed");
     }
   }
 
-  adb_query_results_t *rs = accumulator->get_points();
+  adb_query_results_t *rs = qstate.accumulator->get_points();
+
+  delete qstate.accumulator;
+  delete qstate.allowed_keys;
+
+  /* End of audiodb_query_spec() function */
+
   for(unsigned int k = 0; k < rs->nresults; k++) {
     adb_result_t r = rs->results[k];
     reporter->add_point(audiodb_key_index(adb, r.key), r.qpos, r.ipos, r.dist);
@@ -606,13 +628,13 @@
 // Postconditions:
 // reporter contains the points and distances that meet the reporter constraints 
 
-void audioDB::query_loop_points(adb_t *adb, adb_query_spec_t *spec, double* query, adb_qpointers_internal_t *qpointers) {
+void audioDB::query_loop_points(adb_t *adb, adb_query_spec_t *spec, adb_qstate_internal_t *qstate, double *query, adb_qpointers_internal_t *qpointers) {
   adb_qpointers_internal_t dbpointers = {0};
 
   uint32_t sequence_length = spec->qid.sequence_length;
   bool power_refine = spec->refine.flags & (ADB_REFINE_ABSOLUTE_THRESHOLD|ADB_REFINE_RELATIVE_THRESHOLD);
 
-  if(exact_evaluation_queue->size() == 0) {
+  if(qstate->exact_evaluation_queue->size() == 0) {
     return;
   }
 
@@ -638,9 +660,9 @@
   Uns32T trackOffset = 0;
   Uns32T trackIndexOffset = 0;
   Uns32T currentTrack = 0x80000000; // Initialize with a value outside of track index range
-  Uns32T npairs = exact_evaluation_queue->size();
+  Uns32T npairs = qstate->exact_evaluation_queue->size();
   while(npairs--){
-    PointPair pp = exact_evaluation_queue->top();
+    PointPair pp = qstate->exact_evaluation_queue->top();
     // Large ADB track data must be loaded here for sPower
     if(adb->header->flags & O2_FLAG_LARGE_ADB) {
       trackOffset=0;
@@ -703,40 +725,25 @@
         r.dist = dist;
         r.qpos = pp.qpos;
         r.ipos = pp.spos;
-        accumulator->add_point(&r);
+        qstate->accumulator->add_point(&r);
       }
     }
-    exact_evaluation_queue->pop();
+    qstate->exact_evaluation_queue->pop();
   }
   // Cleanup
   SAFE_DELETE_ARRAY(dbpointers.l2norm_data);
   SAFE_DELETE_ARRAY(dbpointers.power_data);
   SAFE_DELETE_ARRAY(dbpointers.mean_duration);
+  delete qstate->exact_evaluation_queue;
 }
 
-int audioDB::query_loop(adb_t *adb, adb_query_spec_t *spec) {
+int audioDB::query_loop(adb_t *adb, adb_query_spec_t *spec, adb_qstate_internal_t *qstate) {
   
   double *query, *query_data;
   adb_qpointers_internal_t qpointers = {0}, dbpointers = {0};
 
   bool power_refine = spec->refine.flags & (ADB_REFINE_ABSOLUTE_THRESHOLD|ADB_REFINE_RELATIVE_THRESHOLD);
 
-  std::set<std::string> keys;
-  if(spec->refine.flags & ADB_REFINE_INCLUDE_KEYLIST) {
-    for(unsigned int k = 0; k < spec->refine.include.nkeys; k++) {
-      keys.insert(spec->refine.include.keys[k]);
-    }
-  } else {
-    for(unsigned int k = 0; k < adb->header->numFiles; k++) {
-      keys.insert((*adb->keys)[k]);
-    }
-  }
-  if(spec->refine.flags & ADB_REFINE_EXCLUDE_KEYLIST) {
-    for(unsigned int k = 0; k < spec->refine.exclude.nkeys; k++) {
-      keys.erase(spec->refine.exclude.keys[k]);
-    }
-  }
-
   if(adb->header->flags & O2_FLAG_LARGE_ADB) {
     /* FIXME: actually it would be nice to support this mode of
      * operation, but for now... */
@@ -766,10 +773,11 @@
   double *data_buffer = 0;
   lseek(adb->fd, adb->header->dataOffset, SEEK_SET);
 
+  std::set<std::string>::iterator keys_end = qstate->allowed_keys->end();
   for(track = 0; track < adb->header->numFiles; track++) {
     unsigned t = track;
-
-    while (keys.find((*adb->keys)[track]) == keys.end()) {
+    
+    while (qstate->allowed_keys->find((*adb->keys)[track]) == keys_end) {
       track++;
       if(track == adb->header->numFiles) {
         goto loop_finish;
@@ -822,7 +830,7 @@
                   r.qpos = spec->qid.sequence_start;
                 }
                 r.ipos = k;
-                accumulator->add_point(&r);
+                qstate->accumulator->add_point(&r);
               }
             }
           }