comparison index.cpp @ 459:fcc6f7c4856b api-inversion

No more global shingle vector of vectors.

Convert audioDB::index_initialize_shingles and
audioDB::index_norm_shingles to plain old functions.  In doing so, the
latter in particular acquires a silly argument list; we need that
complexity for now because it's called both from audioDB::query (which
we're currently inverting) and from audioDB::index (which is out of
scope for now).

The loss of the global vv thing made me check up on memory discipline
[hence the new API function audiodb_query_free_results() as well as the
internal audiodb_index_delete_shingles()].  It's not too bad, but there
are plenty of leaks for those with time to do
  AUDIODB="valgrind --leak-check=full ../../audioDB" sh ./run-test.sh
on their favourite test case.  For example, the Radius reporters leak
one triple per hit.

(Honestly, C++ memory management is teh suck.)

   }
   assert(audioDB::lsh);  
   return audioDB::lsh;
 }
 
-vector<vector<float> >* audioDB::index_initialize_shingles(Uns32T sz){
-  if(vv)
-    delete vv;
-  vv = new vector<vector<float> >(sz);
-  for(Uns32T i=0 ; i < sz ; i++)
-    (*vv)[i]=vector<float>(dbH->dim*sequenceLength);  // allocate shingle storage
+vector<vector<float> > *audiodb_index_initialize_shingles(Uns32T sz, Uns32T dim, Uns32T seqLen) {
+  std::vector<std::vector<float> > *vv = new vector<vector<float> >(sz);
+  for(Uns32T i=0 ; i < sz ; i++) {
+    (*vv)[i]=vector<float>(dim * seqLen);
+  }
   return vv;
+}
+
+void audiodb_index_delete_shingles(vector<vector<float> > *vv) {
+  delete vv;
 }
 
 /********************  LSH indexing audioDB database access forall s \in {S} ***********************/
 
 // Prepare the AudioDB database for read access and allocate auxillary memory

 }
 
 // norm shingles
 // in-place norming, no deletions
 // If using power, return number of shingles above power threshold
-int audioDB::index_norm_shingles(vector<vector<float> >* vv, double* snp, double* spp){  
+int audiodb_index_norm_shingles(vector<vector<float> >* vv, double* snp, double* spp, Uns32T dim, Uns32T seqLen, double radius, bool normed_vectors, bool use_pthreshold, float pthreshold) {
   int z = 0; // number of above-threshold shingles
   float l2norm;
   double power;
   float oneOverRadius = 1./(float)sqrt(radius); // Passed radius is really radius^2
   float oneOverSqrtl2NormDivRad = oneOverRadius;
-  if(!spp)
-    error("LSH indexing and query requires a power feature using -w or -W");
-  Uns32T shingleSize = sequenceLength*dbH->dim;
+  Uns32T shingleSize = seqLen * dim;
+
+  if(!spp) {
+    return -1;
+  }
   for(Uns32T a=0; a<(*vv).size(); a++){
     l2norm = (float)(*snp++);
-    if(audioDB::normalizedDistance)
+    if(normed_vectors)
       oneOverSqrtl2NormDivRad = (1./l2norm)*oneOverRadius;
     
     for(Uns32T b=0; b < shingleSize ; b++)
       (*vv)[a][b]*=oneOverSqrtl2NormDivRad;
 
     power = *spp++;
-    if(use_absolute_threshold){
-      if ( power >= absolute_threshold )
+    if(use_pthreshold){
+      if (power >= pthreshold)
 	z++;
     }
     else
       z++;	
   }

     }
   }
   
   Uns32T numVecsAboveThreshold = 0, collisionCount = 0; 
   if(numVecs){
-    vv = index_initialize_shingles(numVecs);
+    std::vector<std::vector<float> > *vv = audiodb_index_initialize_shingles(numVecs, dbH->dim, sequenceLength);
     
     for( Uns32T pointID = 0 ; pointID < numVecs; pointID++ )
       audiodb_index_make_shingle(vv, pointID, *fvpp, dbH->dim, sequenceLength);
-    
-    numVecsAboveThreshold = index_norm_shingles(vv, *snpp, *sppp);
+    int vcount = audiodb_index_norm_shingles(vv, *snpp, *sppp, dbH->dim, sequenceLength, radius, normalizedDistance, use_absolute_threshold, absolute_threshold);
+    if(vcount == -1) {
+      audiodb_index_delete_shingles(vv);
+      error("failed to norm shingles");
+    }
+    numVecsAboveThreshold = vcount;
     collisionCount = index_insert_shingles(vv, trackID, *sppp);
-  }
+    audiodb_index_delete_shingles(vv);
+  }
+
   float meanCollisionCount = numVecsAboveThreshold?(float)collisionCount/numVecsAboveThreshold:0;
 
-  /* index_norm_shingles() only goes as far as the end of the
+  /* audiodb_index_norm_shingles() only goes as far as the end of the
      sequence, which is right, but the space allocated is for the
      whole track.  */
 
   /* But numVecs will be <trackTable[track] if trackTable[track]>O2_MAXTRACKLEN
    * So let's be certain the pointers are in the correct place

     error("failed to set up qpointers");
   }
 
   // query vector index
   Uns32T Nq = (qpointers.nvectors>O2_MAXTRACKLEN?O2_MAXTRACKLEN:qpointers.nvectors) - sequenceLength + 1;
-  vv = index_initialize_shingles(Nq); // allocate memory to copy query vectors to shingles
+  std::vector<std::vector<float> > *vv = audiodb_index_initialize_shingles(Nq, adb->header->dim, sequenceLength); // allocate memory to copy query vectors to shingles
 
   // Construct shingles from query features  
   for( Uns32T pointID = 0 ; pointID < Nq ; pointID++ ) 
     audiodb_index_make_shingle(vv, pointID, query, dbH->dim, sequenceLength);
   
   // Normalize query vectors
-  Uns32T numVecsAboveThreshold = index_norm_shingles( vv, qpointers.l2norm, qpointers.power );
+  int vcount = audiodb_index_norm_shingles(vv, qpointers.l2norm, qpointers.power, dbH->dim, sequenceLength, radius, normalizedDistance, use_absolute_threshold, absolute_threshold);
+  if(vcount == -1) {
+    audiodb_index_delete_shingles(vv);
+    error("failed to norm shingles");
+  }
+  Uns32T numVecsAboveThreshold = vcount;
 
   // Nq contains number of inspected points in query file, 
   // numVecsAboveThreshold is number of points with power >= absolute_threshold
   double* qpp = qpointers.power; // Keep original qpPtr for possible exact evaluation
   if(usingQueryPoint && numVecsAboveThreshold){

 	  lsh->retrieve_point((*vv)[pointID], pointID, add_point_func, &callback_data);
         } else {
 	  lsh->serial_retrieve_point(database, (*vv)[pointID], pointID, add_point_func, &callback_data);   
         }
       }
+  audiodb_index_delete_shingles(vv);
 
   if(lsh_exact)
     // Perform exact distance computation on point pairs in exact_evaluation_queue
     query_loop_points(adb, spec, qstate, query, &qpointers);