changeset 414:dc099cd34b5b

Small changes to add restrict list to indexed search. Removed redundant methods in indexed query code paths.

--- a/audioDB.cpp	Thu Nov 27 10:19:50 2008 +0000
+++ b/audioDB.cpp	Tue Dec 23 20:41:58 2008 +0000
@@ -326,6 +326,8 @@
     delete reporter;
   if(exact_evaluation_queue)
     delete exact_evaluation_queue;
+  if(allowed_keys)
+    delete allowed_keys;
   if(rng)
     gsl_rng_free(rng);
   if(vv)

--- a/audioDB.h	Thu Nov 27 10:19:50 2008 +0000
+++ b/audioDB.h	Tue Dec 23 20:41:58 2008 +0000
@@ -312,6 +312,7 @@
   
   ReporterBase* reporter;  // track/point reporter
   priority_queue<PointPair, std::vector<PointPair>, std::less<PointPair> >* exact_evaluation_queue;
+  set<Uns32T> * allowed_keys;    // search restrict list by key
 
   // Timers
   struct timeval tv1;
@@ -419,8 +420,7 @@
   int index_init_query(const char* dbName);
   int index_exists(const char* dbName, double radius, Uns32T sequenceLength);
   char* index_get_name(const char*dbName, double radius, Uns32T sequenceLength);
-  static void index_add_point_approximate(void* instance, Uns32T pointID, Uns32T qpos, float dist); // static point reporter callback method
-  static void index_add_point_exact(void* instance, Uns32T pointID, Uns32T qpos, float dist); // static point reporter callback method
+  static void index_add_point(void* instance, Uns32T pointID, Uns32T qpos, float dist); // static point reporter callback method
   static Uns32T index_to_trackID(Uns32T lshID, Uns32T nPntBits);  // Convert lsh point index to audioDB trackID
   static Uns32T index_to_trackPos(Uns32T lshID, Uns32T nPntBits); // Convert lsh point index to audioDB trackPos (spos)
   static Uns32T index_from_trackInfo(Uns32T trackID, Uns32T pntID, Uns32T nPntBits); // Convert audioDB trackID and trackPos to an lsh point index
@@ -428,6 +428,8 @@
   void index_insert_exact_evaluation_queue(Uns32T trackID, Uns32T qpos, Uns32T spos);
   LSH* index_allocate(char* indexName, bool load_hashTables);
   void init_track_aux_data(Uns32T trackID, double* fvp, double** sNormpp,double** snPtrp, double** sPowerp, double** spPtrp);
+  void initialize_allowed_keys(std::ifstream*); // implementation of restrict lists using STL "set" class
+  int is_in_allowed_keys(Uns32T trackID); // test method for allowed_keys used during search
   
   // Web Services
   void startServer();
@@ -508,6 +510,7 @@
     relative_threshold(0.0),			\
     reporter(0),                                \
     exact_evaluation_queue(0),                  \
+    allowed_keys(0),                            \
     lisztOffset(0),                             \
     lisztLength(0),                             \
     apierrortemp(0),                            \

--- a/index.cpp	Thu Nov 27 10:19:50 2008 +0000
+++ b/index.cpp	Tue Dec 23 20:41:58 2008 +0000
@@ -512,27 +512,30 @@
   return true;
 }
 
-// *Static* approximate NN point reporter callback method for lshlib
-void audioDB::index_add_point_approximate(void* instancePtr, Uns32T pointID, Uns32T qpos, float dist){
-  assert(instancePtr); // We need an instance for this callback
-  audioDB* myself = (audioDB*) instancePtr; // Use explicit cast to recover "this" instance
-  Uns32T trackID = index_to_trackID(pointID, myself->lsh_n_point_bits);
-  Uns32T spos = index_to_trackPos(pointID, myself->lsh_n_point_bits);
-  // Skip identity in query_from_key
-  if( !myself->query_from_key || (myself->query_from_key && ( trackID != myself->query_from_key_index )) )
-    myself->reporter->add_point(trackID, qpos, spos, dist);
-}
-
 // *Static* exact NN point reporter callback method for lshlib
-// Maintain a queue of points to pass to query_points() for exact evaluation
-void audioDB::index_add_point_exact(void* instancePtr, Uns32T pointID, Uns32T qpos, float dist){
+void audioDB::index_add_point(void* instancePtr, Uns32T pointID, Uns32T qpos, float dist){
   assert(instancePtr); // We need an instance for this callback
   audioDB* myself = (audioDB*) instancePtr; // Use explicit cast to recover "this" instance  
   Uns32T trackID = index_to_trackID(pointID, myself->lsh_n_point_bits);
   Uns32T spos = index_to_trackPos(pointID, myself->lsh_n_point_bits);
   // Skip identity in query_from_key
-  if( !myself->query_from_key || (myself->query_from_key && ( trackID != myself->query_from_key_index )) )
-    myself->index_insert_exact_evaluation_queue(trackID, qpos, spos);
+  if( (!myself->query_from_key || (myself->query_from_key && ( trackID != myself->query_from_key_index ))) 
+      && (!myself->trackFile || myself->is_in_allowed_keys(trackID)) )
+    if(myself->lsh_exact)
+      myself->index_insert_exact_evaluation_queue(trackID, qpos, spos);
+    else    
+      myself->reporter->add_point(trackID, qpos, spos, dist);
+}      
+
+int audioDB::is_in_allowed_keys(Uns32T trackID){
+  std::set<Uns32T>::iterator it;
+  if(!allowed_keys)
+    return 0;
+  it = allowed_keys->find(trackID);
+  if(it == allowed_keys->end())
+    return 0;
+  else
+    return 1;
 }
 
 void audioDB::initialize_exact_evalutation_queue(){
@@ -541,6 +544,21 @@
   exact_evaluation_queue = new priority_queue<PointPair, std::vector<PointPair>, std::less<PointPair> >;
 }
 
+void audioDB::initialize_allowed_keys(std::ifstream* trackFile){
+  Uns32T trackIndex;
+  char nextKey[MAXSTR];
+
+  allowed_keys = new std::set<Uns32T>;
+  // Read keys from file, look up the index for each and insert in allowed_keys set
+  do {
+    trackFile->getline(nextKey,MAXSTR);
+    if(!trackFile->eof()) {
+      trackIndex = getKeyPos(nextKey);
+      allowed_keys->insert(trackIndex);
+    }
+  } while(!trackFile->eof());
+} 
+
 void audioDB::index_insert_exact_evaluation_queue(Uns32T trackID, Uns32T qpos, Uns32T spos){
   PointPair p(trackID, qpos, spos);
   exact_evaluation_queue->push(p);
@@ -555,14 +573,11 @@
   double *qNorm = 0, *qnPtr = 0, *qPower = 0, *qpPtr = 0;
   double meanQdur = 0;
   void (*add_point_func)(void*,Uns32T,Uns32T,float);
-
+  
   // Set the point-reporter callback based on the value of lsh_exact
-  if(lsh_exact){
+  if(lsh_exact)
     initialize_exact_evalutation_queue();
-    add_point_func = &index_add_point_exact;
-  }
-  else
-    add_point_func = &index_add_point_approximate;  
+  add_point_func = &index_add_point;
 
   if(!index_init_query(dbName)) // sets-up LSH index structures for querying
     return 0;
@@ -584,6 +599,11 @@
   Uns32T Nq = (numVectors>O2_MAXTRACKLEN?O2_MAXTRACKLEN:numVectors) - sequenceLength + 1;
   vv = index_initialize_shingles(Nq); // allocate memory to copy query vectors to shingles
   VERB_LOG(1, "Nq=%d", Nq);
+
+  // restrictList initialization
+  if(trackFile)
+    initialize_allowed_keys(trackFile); // trackFile is list of valid keys to admit in search
+
   // Construct shingles from query features  
   for( Uns32T pointID = 0 ; pointID < Nq ; pointID++ ) 
     index_make_shingle(vv, pointID, query, dbH->dim, sequenceLength);

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/0046/run-test.sh	Tue Dec 23 20:41:58 2008 +0000
@@ -0,0 +1,48 @@
+#! /bin/bash
+
+. ../test-utils.sh
+
+if [ -f testdb ]; then rm -f testdb; fi
+
+${AUDIODB} -d testdb -N
+
+intstring 2 > testfeature
+floatstring 0 1 >> testfeature
+floatstring 1 0 >> testfeature
+floatstring 1 0 >> testfeature
+floatstring 0 1 >> testfeature
+
+intstring 1 > testpower
+floatstring -0.5 >> testpower
+floatstring -1 >> testpower
+floatstring -1 >> testpower
+floatstring -0.5 >> testpower
+
+${AUDIODB} -d testdb -L
+${AUDIODB} -d testdb -P
+
+# insert two instances of the same feature
+${AUDIODB} -d testdb -I -f testfeature -w testpower -k testfeatureA
+${AUDIODB} -d testdb -I -f testfeature -w testpower -k testfeatureB
+
+
+echo "query points (0.0,0.5),(0.0,0.5),(0.5,0.0)"
+intstring 2 > testquery
+floatstring 0 0.5 >> testquery
+floatstring 0 0.5 >> testquery
+floatstring 0.5 0 >> testquery
+
+# LSH Indexed query with restrict list test
+
+# Index with default LSH params
+${AUDIODB} -d testdb -X -l 1 -R 1
+
+# Query using the index
+
+echo testfeatureB > test-restrict-list
+${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -w testpower -K test-restrict-list -R 1 > testoutput
+echo testfeatureB 1 > test-expected-output
+
+cmp testoutput test-expected-output
+
+exit 104

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/0046/short-description	Tue Dec 23 20:41:58 2008 +0000
@@ -0,0 +1,1 @@
+indexed query with restrict list -K