changeset 529:e532666226bc multiprobeLSH

Fixed a problem with inserting into exact_evaluation_queue, duplicate pairs were being inserted when querying multiple hashtables. That is now fixed by adding a unique constraint using set<PointPair> inserts and testing. This should speed things up quite a bit (like, 10-fold with default num tables).

--- a/Makefile	Thu Jan 29 13:44:47 2009 +0000
+++ b/Makefile	Fri Jan 30 14:58:35 2009 +0000
@@ -17,15 +17,15 @@
 MINORVERSION=0
 LIBRARY=lib$(EXECUTABLE).so.$(SOVERSION).$(MINORVERSION)
 
-override CFLAGS+=-O3 -fPIC -pg
+override CFLAGS+=-O3 -fPIC
 
 # set to DUMP hashtables on QUERY load
 #override CFLAGS+=-DLSH_DUMP_CORE_TABLES
 
 # set to turn on debugging information for LSH hashtables
-#override CFLAGS+=-D_LSH_DEBUG_
+# override CFLAGS+=-D_LSH_DEBUG_
 
-# set to increase multiple probes in LSH QUERY (allowable range = 1 ... lsh_k*2)
+# set multiple probes in LSH QUERY (allowable range = 0 ... lsh_k*2) 0=multiprobe off
 #override CFLAGS+=-DLSH_MULTI_PROBE_COUNT=10
 
 ifeq ($(shell uname),Linux)

--- a/audioDB-internals.h	Thu Jan 29 13:44:47 2009 +0000
+++ b/audioDB-internals.h	Fri Jan 30 14:58:35 2009 +0000
@@ -51,6 +51,7 @@
 typedef struct adb_qstate_internal {
   Accumulator *accumulator;
   std::set<std::string> *allowed_keys;
+  std::set<PointPair> * set;
   std::priority_queue<PointPair> *exact_evaluation_queue;
   LSH *lsh;
 } adb_qstate_internal_t;

--- a/query-indexed.cpp	Thu Jan 29 13:44:47 2009 +0000
+++ b/query-indexed.cpp	Fri Jan 30 14:58:35 2009 +0000
@@ -70,7 +70,10 @@
   std::set<std::string>::iterator keys_end = qstate->allowed_keys->end();
   if(qstate->allowed_keys->find((*adb->keys)[trackID]) != keys_end) {
     PointPair p(trackID, qpos, spos);
-    qstate->exact_evaluation_queue->push(p);
+    if(qstate->set->find(p)==qstate->set->end()){
+      qstate->set->insert(p);
+      qstate->exact_evaluation_queue->push(p);
+    }
   }
 }
 
@@ -98,6 +101,7 @@
     add_point_func = &audiodb_index_add_point_approximate;  
   } else {
     qstate->exact_evaluation_queue = new std::priority_queue<PointPair>;
+    qstate->set = new std::set<PointPair, less<PointPair> >;
     add_point_func = &audiodb_index_add_point_exact;
   }
 

--- a/query.cpp	Thu Jan 29 13:44:47 2009 +0000
+++ b/query.cpp	Fri Jan 30 14:58:35 2009 +0000
@@ -293,17 +293,21 @@
   if(d->power)
     qpointers->power_data = new double[vector_size / d->dim];
 
-  uint32_t seq_len_dbl = sequence_length*d->dim*sizeof(double);
+  uint32_t seq_len_dim_dbl = sequence_length*d->dim*sizeof(double);
+  uint32_t seq_len_dbl = sequence_length*sizeof(double);
   PointPair pp = ppairs.top();
   uint32_t tid = pp.trackID;
 
   while( !ppairs.empty() && pp.trackID==tid){
     uint32_t spos = pp.spos;
     uint32_t spos_dim = spos*d->dim;
+#ifdef _LSH_DEBUG_
+    cout << "tid=" << pp.trackID << " qpos=" << pp.qpos << " spos=" << pp.spos << endl;
+    cout.flush();
+#endif
+    memcpy((void*)(*vector_data+spos_dim), (void*)(d->data+spos_dim), seq_len_dim_dbl); 
 
-    memcpy(*vector_data+spos_dim, d->data+spos_dim, seq_len_dbl); 
-
-    audiodb_l2norm_buffer(*vector_data+spos_dim, d->dim, 1, qpointers->l2norm_data+spos);
+    audiodb_l2norm_buffer(*vector_data+spos_dim, d->dim, sequence_length, qpointers->l2norm_data+spos);
     audiodb_sequence_sum(qpointers->l2norm_data+spos, sequence_length, sequence_length);
     audiodb_sequence_sqrt(qpointers->l2norm_data+spos, sequence_length, sequence_length);
 
@@ -516,10 +520,13 @@
       adb_datum_t d = {0};
       if(audiodb_track_id_datum(adb, pp.trackID, &d)) {
         delete qstate->exact_evaluation_queue;
+        delete qstate->set;
         return 1;
       }
-      if(audiodb_datum_qpointers_partial(&d, sequence_length, &dbdata, &dbdata_pointer, &dbpointers, qstate)) {
+      //      if(audiodb_datum_qpointers_partial(&d, sequence_length, &dbdata, &dbdata_pointer, &dbpointers, qstate)) {
+      if(audiodb_datum_qpointers(&d, sequence_length, &dbdata, &dbdata_pointer, &dbpointers)) {
         delete qstate->exact_evaluation_queue;
+        delete qstate->set;
         audiodb_free_datum(&d);
         return 1;
       }
@@ -561,6 +568,7 @@
   maybe_delete_array(dbpointers.power_data);
   maybe_delete_array(dbpointers.mean_duration);
   delete qstate->exact_evaluation_queue;
+  delete qstate->set;
   return 0;
 }