changeset 589:9119f2fa3efe

Header file rearrangement

Make it so that lshlib.h is slightly more like a header file to include
to use the LSH ("class G") class, and slightly less a header file for
developing that class, by removing all the #includes and moving them to
the one-file lshlib.cpp instead.

Make audioDB-internals.h slightly more of a project header file, by
including there all the headers we actually need.

Remove some assert()s (which do nothing) and some Uns32Ts (-> uint32_t)

--- a/audioDB-internals.h	Tue Aug 11 21:42:07 2009 +0000
+++ b/audioDB-internals.h	Tue Aug 11 21:42:13 2009 +0000
@@ -1,16 +1,27 @@
+#include <sys/mman.h>
 #include <sys/types.h>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <math.h>
+#include <string.h>
 #include <unistd.h>
 
+#include <algorithm>
+#include <iostream>
+#include <map>
+#include <queue>
 #include <set>
-#include <queue>
-#include <map>
 #include <string>
-#include <algorithm>
+#include <vector>
 
+#include "accumulator.h"
 #include "pointpair.h"
-#include "accumulator.h"
 #include "lshlib.h"
 
+using namespace std;
+
 /* this struct is for writing polymorphic routines as puns.  When
  * inserting, we might have a "datum" (with actual numerical data) or
  * a "reference" (with strings denoting pathnames containing numerical

--- a/index-utils.cpp	Tue Aug 11 21:42:07 2009 +0000
+++ b/index-utils.cpp	Tue Aug 11 21:42:13 2009 +0000
@@ -2,6 +2,7 @@
 #include "audioDB_API.h"
 }
 #include "audioDB-internals.h"
+#include "lshlib.h"
 
 /*
  * Routines which are common to both indexed query and index creation:
@@ -15,7 +16,7 @@
  * information in a filename is going to lose: it's impossible to
  * maintain backwards-compatibility.  Instead we should probably store
  * the index metadata inside the audiodb instance. */
-char *audiodb_index_get_name(const char *dbName, double radius, Uns32T sequenceLength) {
+char *audiodb_index_get_name(const char *dbName, double radius, uint32_t sequenceLength) {
   char *indexName;
   if(strlen(dbName) > (ADB_MAXSTR - 32)) {
     return NULL;
@@ -26,7 +27,7 @@
   return indexName;
 }
 
-bool audiodb_index_exists(const char *dbName, double radius, Uns32T sequenceLength) {
+bool audiodb_index_exists(const char *dbName, double radius, uint32_t sequenceLength) {
   char *indexName = audiodb_index_get_name(dbName, radius, sequenceLength);
   if(!indexName) {
     return false;
@@ -68,9 +69,9 @@
   return lsh;
 }
 
-vector<vector<float> > *audiodb_index_initialize_shingles(Uns32T sz, Uns32T dim, Uns32T seqLen) {
+vector<vector<float> > *audiodb_index_initialize_shingles(uint32_t sz, uint32_t dim, uint32_t seqLen) {
   std::vector<std::vector<float> > *vv = new vector<vector<float> >(sz);
-  for(Uns32T i=0 ; i < sz ; i++) {
+  for(uint32_t i=0 ; i < sz ; i++) {
     (*vv)[i]=vector<float>(dim * seqLen);
   }
   return vv;
@@ -80,8 +81,8 @@
   delete vv;
 }
 
-void audiodb_index_make_shingle(vector<vector<float> >* vv, Uns32T idx, double* fvp, Uns32T dim, Uns32T seqLen){
-  assert(idx<(*vv).size());
+void audiodb_index_make_shingle(vector<vector<float> >* vv, uint32_t idx, double* fvp, uint32_t dim, uint32_t seqLen){
+
   vector<float>::iterator ve = (*vv)[idx].end();
   vector<float>::iterator vi = (*vv)[idx].begin();
   // First feature vector in shingle
@@ -107,23 +108,23 @@
 
 // in-place norming, no deletions.  If using power, return number of
 // shingles above power threshold.
-int audiodb_index_norm_shingles(vector<vector<float> >* vv, double* snp, double* spp, Uns32T dim, Uns32T seqLen, double radius, bool normed_vectors, bool use_pthreshold, float pthreshold) {
+int audiodb_index_norm_shingles(vector<vector<float> >* vv, double* snp, double* spp, uint32_t dim, uint32_t seqLen, double radius, bool normed_vectors, bool use_pthreshold, float pthreshold) {
   int z = 0; // number of above-threshold shingles
   float l2norm;
   double power;
   float oneOverRadius = 1./(float)sqrt(radius); // Passed radius is really radius^2
   float oneOverSqrtl2NormDivRad = oneOverRadius;
-  Uns32T shingleSize = seqLen * dim;
+  uint32_t shingleSize = seqLen * dim;
 
   if(!spp) {
     return -1;
   }
-  for(Uns32T a=0; a<(*vv).size(); a++){
+  for(uint32_t a=0; a<(*vv).size(); a++){
     l2norm = (float)(*snp++);
     if(normed_vectors)
       oneOverSqrtl2NormDivRad = (1./l2norm)*oneOverRadius;
     
-    for(Uns32T b=0; b < shingleSize ; b++)
+    for(uint32_t b=0; b < shingleSize ; b++)
       (*vv)[a][b]*=oneOverSqrtl2NormDivRad;
 
     power = *spp++;

--- a/lshlib.cpp	Tue Aug 11 21:42:07 2009 +0000
+++ b/lshlib.cpp	Tue Aug 11 21:42:13 2009 +0000
@@ -1,6 +1,28 @@
+#include <vector>
+#include <queue>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <fcntl.h>
+#include <string.h>
+#include <iostream>
+#include <fstream>
+#include <math.h>
+#include <sys/time.h>
+#include <assert.h>
+#include <float.h>
+#include <signal.h>
+#include <time.h>
+#include <limits.h>
+#include <errno.h>
+#ifdef MT19937
+#include "mt19937/mt19937ar.h"
+#endif
+
 #include "lshlib.h"
 
-
 void err(char*s){cout << s << endl;exit(2);}
 
 Uns32T get_page_logn(){
@@ -8,8 +30,6 @@
   return (Uns32T)log2((double)pagesz);  
 }
 
-unsigned align_up(unsigned x, unsigned w) { return (((x) + ((1<<w)-1)) & ~((1<<w)-1)); }
-
 void H::error(const char* a, const char* b, const char *sysFunc) {
   cerr << a << ": " << b << endl;
   if (sysFunc) {

--- a/lshlib.h	Tue Aug 11 21:42:07 2009 +0000
+++ b/lshlib.h	Tue Aug 11 21:42:13 2009 +0000
@@ -11,36 +11,13 @@
 #ifndef __LSHLIB_H
 #define __LSHLIB_H
 
-#include <vector>
-#include <queue>
-#include <stdio.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/mman.h>
-#include <fcntl.h>
-#include <string.h>
-#include <iostream>
-#include <fstream>
-#include <math.h>
-#include <sys/time.h>
-#include <assert.h>
-#include <float.h>
-#include <signal.h>
-#include <time.h>
-#include <limits.h>
-#include <errno.h>
-#ifdef MT19937
-#include "mt19937/mt19937ar.h"
-#endif
+using namespace std;
 
 #define IntT int
 #define LongUns64T long long unsigned
 #define Uns32T unsigned
 #define Int32T int
 #define BooleanT int
-#define TRUE 1
-#define FALSE 0
 
 // A big number (>> max #  of points)
 #define INDEX_START_EMPTY 1000000000U
@@ -78,7 +55,7 @@
 
 #define O2_INDEX_MAXSTR (256)
 
-unsigned align_up(unsigned x, unsigned w);
+#define align_up(x,w) (((x) + ((1<<w)-1)) & ~((1<<w)-1))
 
 #define O2_SERIAL_FUNCTIONS_SIZE (align_up(sizeof(float) * O2_SERIAL_MAX_TABLES * O2_SERIAL_MAX_FUNS * O2_SERIAL_MAX_DIM \
 + sizeof(float) * O2_SERIAL_MAX_TABLES * O2_SERIAL_MAX_FUNS + \
@@ -107,8 +84,6 @@
 
 #define LSH_CORE_ARRAY_BIT (0x80000000) //  LSH_CORE_ARRAY test bit for list head
 
-using namespace std;
-
 Uns32T get_page_logn();
 
 // Disk table entry

--- a/pointpair.cpp	Tue Aug 11 21:42:07 2009 +0000
+++ b/pointpair.cpp	Tue Aug 11 21:42:13 2009 +0000
@@ -3,7 +3,7 @@
 }
 #include "audioDB-internals.h"
 
-PointPair::PointPair(Uns32T a, Uns32T b, Uns32T c) :
+PointPair::PointPair(uint32_t a, uint32_t b, uint32_t c) :
   trackID(a), qpos(b), spos(c) {
 };
 

--- a/query-indexed.cpp	Tue Aug 11 21:42:07 2009 +0000
+++ b/query-indexed.cpp	Tue Aug 11 21:42:13 2009 +0000
@@ -2,6 +2,7 @@
 #include "audioDB_API.h"
 }
 #include "audioDB-internals.h"
+#include "lshlib.h"
 
 /*
  * Routines and datastructures which are specific to indexed queries.
@@ -40,7 +41,7 @@
   return true;
 }
 
-void audiodb_index_add_point_approximate(void *user_data, Uns32T pointID, Uns32T qpos, float dist) {
+void audiodb_index_add_point_approximate(void *user_data, uint32_t pointID, uint32_t qpos, float dist) {
   adb_qcallback_t *data = (adb_qcallback_t *) user_data;
   adb_t *adb = data->adb;
   adb_qstate_internal_t *qstate = data->qstate;
@@ -59,7 +60,7 @@
 
 // Maintain a queue of points to pass to audiodb_query_queue_loop()
 // for exact evaluation
-void audiodb_index_add_point_exact(void *user_data, Uns32T pointID, Uns32T qpos, float dist) {
+void audiodb_index_add_point_exact(void *user_data, uint32_t pointID, uint32_t qpos, float dist) {
   adb_qcallback_t *data = (adb_qcallback_t *) user_data;
   adb_t *adb = data->adb;
   adb_qstate_internal_t *qstate = data->qstate;

--- a/query.cpp	Tue Aug 11 21:42:07 2009 +0000
+++ b/query.cpp	Tue Aug 11 21:42:13 2009 +0000
@@ -104,6 +104,7 @@
   return 0;
 }
 
+/* FIXME: we should check the return values from allocation */
 static void audiodb_initialize_arrays(adb_t *adb, const adb_query_spec_t *spec, int track, unsigned int numVectors, double *query, double *data_buffer, double **D, double **DD) {
   unsigned int j, k, l, w;
   double *dp, *qp, *sp;
@@ -114,10 +115,8 @@
   for(j = 0; j < numVectors; j++) {
     // Sum products matrix
     D[j] = new double[(*adb->track_lengths)[track]]; 
-    assert(D[j]);
     // Matched filter matrix
     DD[j]=new double[(*adb->track_lengths)[track]];
-    assert(DD[j]);
   }
 
   // Dot product
@@ -447,8 +446,8 @@
    */
   double dist;
   double *dbdata = 0, *dbdata_pointer;
-  Uns32T currentTrack = 0x80000000; // KLUDGE: Initialize with a value outside of track index range
-  Uns32T npairs = qstate->exact_evaluation_queue->size();
+  uint32_t currentTrack = 0x80000000; // KLUDGE: Initialize with a value outside of track index range
+  uint32_t npairs = qstate->exact_evaluation_queue->size();
   while(npairs--) {
     PointPair pp = qstate->exact_evaluation_queue->top();
     if(currentTrack != pp.trackID) {
@@ -469,8 +468,8 @@
       }
       audiodb_really_free_datum(&d);
     }
-    Uns32T qPos = (spec->qid.flags & ADB_QID_FLAG_EXHAUSTIVE) ? pp.qpos : 0;
-    Uns32T sPos = pp.spos; // index into l2norm table
+    uint32_t qPos = (spec->qid.flags & ADB_QID_FLAG_EXHAUSTIVE) ? pp.qpos : 0;
+    uint32_t sPos = pp.spos; // index into l2norm table
     // Test power thresholds before computing distance
     if( ( (!power_refine) || audiodb_powers_acceptable(&spec->refine, qpointers->power[qPos], dbpointers.power[sPos])) &&
 	( qPos<qpointers->nvectors-sequence_length+1 && sPos<(*adb->track_lengths)[pp.trackID]-sequence_length+1 ) ){