Mercurial > hg > audiodb

changeset 319:b9eff6896943 large_adb

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added indexing support for O2_FLAG_LARGE_ADB. Tested on indexed query by features. No indexed query-by-key yet. No --lsh_exact yet.
author mas01mc
date Tue, 19 Aug 2008 20:27:15 +0000
parents c270d9e4659a
children a995e5ad999a
files audioDB.cpp audioDB.h common.cpp create.cpp dump.cpp index.cpp lshlib.h query.cpp sample.cpp soap.cpp
diffstat 10 files changed, 174 insertions(+), 60 deletions(-) [+]
line wrap: on
line diff
--- a/audioDB.cpp	Tue Aug 19 15:50:26 2008 +0000
+++ b/audioDB.cpp	Tue Aug 19 20:27:15 2008 +0000
@@ -597,7 +597,7 @@
 
   assert(l2normTable);
 
-  if( !append && (dbH->flags & O2_FLAG_L2NORM) )
+  if( !(dbH->flags & O2_FLAG_LARGE_ADB) && !append && (dbH->flags & O2_FLAG_L2NORM) )
     error("Database is already L2 normed", "automatic norm on insert is enabled");
 
   VERB_LOG(2, "norming %u vectors...", n);
--- a/audioDB.h	Tue Aug 19 15:50:26 2008 +0000
+++ b/audioDB.h	Tue Aug 19 20:27:15 2008 +0000
@@ -80,9 +80,13 @@
 #define O2_DEFAULTDBSIZE (2000000000) // 2GB table size
 
 // Bit masks for packing (trackID,pointID) into 32-bit unsigned int
-#define LSH_N_POINT_BITS 14
-#define LSH_TRACK_MASK 0xFFFFC000U // 2^18 = 262144 tracks
-#define LSH_POINT_MASK 0x00003FFFU // 2^14 = 16384 points per track
+// This can be controlled at compile time
+#define O2_DEFAULT_LSH_N_POINT_BITS 14
+
+// Override the default point bit width for large database support
+#ifndef LSH_N_POINT_BITS
+#define LSH_N_POINT_BITS 12
+#endif
 
 // LIMIT PARAMETERS
 #define O2_DEFAULT_DATASIZE (1355U) // in MB
@@ -99,7 +103,7 @@
 #define O2_MAXNN (1000000U)
 #define O2_MAXSEQLEN (8000U)            // maximum feature vectors in a sequence
 #define O2_MAXTRACKS (1000000U)           // maximum number of tracks
-#define O2_MAXTRACKLEN ((LSH_POINT_MASK+1)) // maximum shingles in a track
+#define O2_MAXTRACKLEN (1<<LSH_N_POINT_BITS) // maximum shingles in a track
 #define O2_MAXDOTPRODUCTMEMORY (sizeof(O2_REALTYPE)*O2_MAXSEQLEN*O2_MAXSEQLEN) // 512MB
 #define O2_DISTANCE_TOLERANCE (1e-6)
 #define O2_SERIAL_MAX_TRACKBATCH (1000000)
@@ -281,7 +285,7 @@
 
   void initialize_arrays(int track, unsigned int numVectors, double *query, double *data_buffer, double **D, double **DD);
   void delete_arrays(int track, unsigned int numVectors, double **D, double **DD);
-  void read_data(int track, double **data_buffer_p, size_t *data_buffer_size_p);
+  void read_data(int trkfid, int track, double **data_buffer_p, size_t *data_buffer_size_p);
   void set_up_query(double **qp, double **vqp, double **qnp, double **vqnp, double **qpp, double **vqpp, double *mqdp, unsigned int *nvp);
   void set_up_query_from_key(double **qp, double **vqp, double **qnp, double **vqnp, double **qpp, double **vqpp, double *mqdp, unsigned *nvp, Uns32T queryIndex);
   void set_up_db(double **snp, double **vsnp, double **spp, double **vspp, double **mddp, unsigned int *dvp);
@@ -335,6 +339,8 @@
   Uns32T lsh_param_N; // Number of rows per hash table
   Uns32T lsh_param_b; // Batch size, in number of tracks, per indexing iteration
   Uns32T lsh_param_ncols; // Maximum number of collision in a hash-table row
+  Uns32T lsh_n_point_bits; // How many bits to use to encode point ID within a track
+
 
   // LSH vector<> containers for one in-core copy of a set of feature vectors
   vector<float>::iterator vi; // feature vector iterator
@@ -355,13 +361,14 @@
   char* index_get_name(const char*dbName, double radius, Uns32T sequenceLength);
   static void index_add_point_approximate(void* instance, Uns32T pointID, Uns32T qpos, float dist); // static point reporter callback method
   static void index_add_point_exact(void* instance, Uns32T pointID, Uns32T qpos, float dist); // static point reporter callback method
-  static Uns32T index_to_trackID(Uns32T lshID);  // Convert lsh point index to audioDB trackID
-  static Uns32T index_to_trackPos(Uns32T lshID); // Convert lsh point index to audioDB trackPos (spos)
-  static Uns32T index_from_trackInfo(Uns32T, Uns32T); // Convert audioDB trackID and trackPos to an lsh point index
+  static Uns32T index_to_trackID(Uns32T lshID, Uns32T nPntBits);  // Convert lsh point index to audioDB trackID
+  static Uns32T index_to_trackPos(Uns32T lshID, Uns32T nPntBits); // Convert lsh point index to audioDB trackPos (spos)
+  static Uns32T index_from_trackInfo(Uns32T trackID, Uns32T pntID, Uns32T nPntBits); // Convert audioDB trackID and trackPos to an lsh point index
   void initialize_exact_evalutation_queue();
   void index_insert_exact_evaluation_queue(Uns32T trackID, Uns32T qpos, Uns32T spos);
   LSH* index_allocate(char* indexName, bool load_hashTables);
-
+  void init_track_aux_data(Uns32T trackID, double* fvp, double** sNormpp,double** snPtrp, double** sPowerp, double** spPtrp);
+  
   // Web Services
   void startServer();
   void ws_status(const char*dbName, char* hostport);
@@ -447,5 +454,6 @@
     lsh_param_N(0),				\
     lsh_param_b(0),				\
     lsh_param_ncols(0),                         \
+    lsh_n_point_bits(0),                        \
     vv(0)
 #endif
--- a/common.cpp	Tue Aug 19 15:50:26 2008 +0000
+++ b/common.cpp	Tue Aug 19 20:27:15 2008 +0000
@@ -171,7 +171,12 @@
   for(Uns32T k = 0; k < dbH->numFiles; k++){
     trackOffsetTable[k] = cumTrack;
     cumTrack += trackTable[k] * dbH->dim;
-  }  
+  }
+
+  // Assign correct number of point bits per track in LSH indexing / retrieval
+  lsh_n_point_bits = dbH->flags >> 28;
+  if( !lsh_n_point_bits )
+    lsh_n_point_bits = O2_DEFAULT_LSH_N_POINT_BITS;
 }
 
 void audioDB::initInputFile (const char *inFile, bool loadData) {
--- a/create.cpp	Tue Aug 19 15:50:26 2008 +0000
+++ b/create.cpp	Tue Aug 19 20:27:15 2008 +0000
@@ -52,6 +52,15 @@
   off_t databytes = ((off_t) datasize) * 1024 * 1024;
   off_t auxbytes = databytes / datadim;
 
+  // For backward-compatibility, Record the point-encoding parameter for LSH indexing in the adb header
+  // If this value is 0 then it will be set to 14
+
+#if O2_LSH_N_POINT_BITS > 15
+#error "AudioDB Compile ERROR: consistency check of O2_LSH_POINT_BITS failed (>15)"
+#endif
+  
+  dbH->flags |= LSH_N_POINT_BITS << 28;
+
   // If database will fit in a single file the vectors are copied into the AudioDB instance
   // Else all the vectors are left on the FileSystem and we use the dataOffset as storage
   // for the location of the features, powers and times files (assuming that arbitrary keys are used for the fileTable)
--- a/dump.cpp	Tue Aug 19 15:50:26 2008 +0000
+++ b/dump.cpp	Tue Aug 19 20:27:15 2008 +0000
@@ -5,6 +5,10 @@
     initTables(dbName, 0);
   }
 
+  if(dbH->flags & O2_FLAG_LARGE_ADB){
+    error("error: dump not supported for LARGE_ADB");
+  }
+
   if((mkdir(output, S_IRWXU|S_IRWXG|S_IRWXO)) < 0) {
     error("error making output directory", output, "mkdir");
   }
--- a/index.cpp	Tue Aug 19 15:50:26 2008 +0000
+++ b/index.cpp	Tue Aug 19 20:27:15 2008 +0000
@@ -8,22 +8,27 @@
 //
 // Author: Michael Casey
 //   Date: 23 June 2008
+//
+// 19th August 2008 - added O2_FLAG_LARGE_ADB support
 
 #include "audioDB.h"
 #include "ReporterBase.h"
 
 
 /************************* LSH point index to audioDB conversion  *****************/
-Uns32T audioDB::index_to_trackID(Uns32T lshID){
-  return lshID>>LSH_N_POINT_BITS;
+Uns32T audioDB::index_to_trackID(Uns32T lshID, Uns32T nPntBits){
+  assert(nPntBits);
+  return lshID>>nPntBits;
 }
 
-Uns32T audioDB::index_to_trackPos(Uns32T lshID){
-  return lshID&LSH_POINT_MASK;
+Uns32T audioDB::index_to_trackPos(Uns32T lshID, Uns32T nPntBits){
+  assert(nPntBits);
+  return lshID&((1<<nPntBits)-1);
 }
 
-Uns32T audioDB::index_from_trackInfo(Uns32T trackID, Uns32T spos){
-  return (trackID << LSH_N_POINT_BITS) | spos;
+Uns32T audioDB::index_from_trackInfo(Uns32T trackID, Uns32T spos, Uns32T nPntBits){
+  assert(nPntBits);
+  return (trackID << nPntBits) | spos;
 }
 
 /************************* LSH indexing and query initialization  *****************/
@@ -78,19 +83,20 @@
 
 // Prepare the AudioDB database for read access and allocate auxillary memory
 void audioDB::index_initialize(double **snp, double **vsnp, double **spp, double **vspp, Uns32T *dvp) {
+  if (!(dbH->flags & O2_FLAG_POWER)) {
+    error("INDEXed database must be power-enabled", dbName);
+  }
+
+  double *snpp = *snp, *sppp = 0;
+
   *dvp = dbH->length / (dbH->dim * sizeof(double)); // number of database vectors
   *snp = new double[*dvp];  // songs norm pointer: L2 norm table for each vector
-
-  double *snpp = *snp, *sppp = 0;
-  memcpy(*snp, l2normTable, *dvp * sizeof(double));
-
-  if (!(dbH->flags & O2_FLAG_POWER)) {
-    error("database not power-enabled", dbName);
-  }
   *spp = new double[*dvp]; // song powertable pointer
   sppp = *spp;
+  memcpy(*snp, l2normTable, *dvp * sizeof(double));
   memcpy(*spp, powerTable, *dvp * sizeof(double));
-
+  
+  
   for(Uns32T i = 0; i < dbH->numFiles; i++){
     if(trackTable[i] >= sequenceLength) {
       sequence_sum(snpp, trackTable[i], sequenceLength);
@@ -102,10 +108,10 @@
     snpp += trackTable[i];
     sppp += trackTable[i];
   }
-
+  
   *vsnp = *snp;
   *vspp = *spp;
-
+  
   // Move the feature vector read pointer to start of fetures in database
   lseek(dbfid, dbH->dataOffset, SEEK_SET);
 }
@@ -141,8 +147,6 @@
   fflush(stdout);
 
 
-  index_initialize(&sNorm, &snPtr, &sPower, &spPtr, &dbVectors);
-  
   if((lshfid = open(newIndexName,O_RDONLY))<0){
     printf("INDEX: constructing new LSH index\n");  
     printf("INDEX: making index file %s\n", newIndexName);
@@ -160,7 +164,12 @@
     if( endTrack > dbH->numFiles)
       endTrack = dbH->numFiles;
     // Insert up to lsh_param_b tracks
-    index_insert_tracks(0, endTrack, &fvp, &sNorm, &snPtr, &sPower, &spPtr);    
+    if( dbH->flags & O2_FLAG_LARGE_ADB ){
+    }
+    else{
+      index_initialize(&sNorm, &snPtr, &sPower, &spPtr, &dbVectors);  
+      index_insert_tracks(0, endTrack, &fvp, &sNorm, &snPtr, &sPower, &spPtr);
+    }
     lsh->serialize(newIndexName, lsh_in_core?O2_SERIAL_FILEFORMAT2:O2_SERIAL_FILEFORMAT1);
     
     // Clean up
@@ -177,7 +186,7 @@
     // Get the lsh header info and find how many tracks are inserted already
     lsh = new LSH(newIndexName, false); // lshInCore=false to avoid loading hashTables here
     assert(lsh);
-    Uns32T maxs = index_to_trackID(lsh->get_maxp())+1;
+    Uns32T maxs = index_to_trackID(lsh->get_maxp(), lsh_n_point_bits)+1;
     delete lsh;
     lsh = 0;
 
@@ -221,6 +230,57 @@
 
 }
 
+
+// initialize auxillary track data from filesystem
+// pre-conditions:
+// dbH->flags & O2_FLAG_LARGE_ADB
+// feature data allocated and copied (fvp)
+//
+// post-conditions:
+// allocated power data
+// allocated l2norm data
+//
+void audioDB::init_track_aux_data(Uns32T trackID, double* fvp, double** sNormpp,double** snPtrp, double** sPowerp, double** spPtrp){  
+  if( !(dbH->flags & O2_FLAG_LARGE_ADB) )
+    error("error: init_track_large_adb required O2_FLAG_LARGE_ADB");
+
+  // Allocate and read the power sequence
+  if(trackTable[trackID]>=sequenceLength){
+
+    // Open and check dimensions of power file
+    powerfd = open(powerFileNameTable+trackID*O2_FILETABLE_ENTRY_SIZE, O_RDONLY);
+    if (powerfd < 0) {
+      error("failed to open power file", powerFileNameTable+trackID*O2_FILETABLE_ENTRY_SIZE);
+    }
+    if (fstat(powerfd, &statbuf) < 0) {
+      error("fstat error finding size of power file", powerFileNameTable+trackID*O2_FILETABLE_ENTRY_SIZE, "fstat");
+    }
+    
+    if( (statbuf.st_size - sizeof(int)) / (sizeof(double)) != trackTable[trackID] )
+      error("Dimension mismatch: numPowers != numVectors", powerFileNameTable+trackID*O2_FILETABLE_ENTRY_SIZE);
+    
+    *sPowerp = new double[trackTable[trackID]]; // Allocate memory for power values
+    assert(*sPowerp);
+    *spPtrp = *sPowerp;
+    insertPowerData(trackTable[trackID], powerfd, *sPowerp);
+    if (0 < powerfd) {
+      close(powerfd);
+    }
+    
+    sequence_sum(*sPowerp, trackTable[trackID], sequenceLength);
+    sequence_average(*sPowerp, trackTable[trackID], sequenceLength);
+    powerTable = 0;
+
+    // Allocate and calculate the l2norm sequence
+    *sNormpp = new double[trackTable[trackID]];
+    assert(*sNormpp);
+    *snPtrp = *sNormpp;
+    unitNorm(fvp, dbH->dim, trackTable[trackID], *sNormpp);
+    sequence_sum(*sNormpp, trackTable[trackID], sequenceLength);
+    sequence_sqrt(*sNormpp, trackTable[trackID], sequenceLength);
+  }
+}
+
 void audioDB::index_insert_tracks(Uns32T start_track, Uns32T end_track,
 				  double** fvpp, double** sNormpp,double** snPtrp, 
 				  double** sPowerp, double** spPtrp){  
@@ -230,13 +290,29 @@
 
   VERB_LOG(1, "indexing tracks...");
 
-
+  int trackfd = dbfid;
   for(trackID = start_track ; trackID < end_track ; trackID++ ){
-    read_data(trackID, &fvp, &nfv); // over-writes fvp and nfv
+    if( dbH->flags & O2_FLAG_LARGE_ADB ){
+      // Open and check dimensions of feature file
+      initInputFile(featureFileNameTable+trackID*O2_FILETABLE_ENTRY_SIZE, false); // nommap, file pointer at correct position
+      trackfd = infid;
+    }
+    read_data(trackfd, trackID, &fvp, &nfv); // over-writes fvp and nfv
     *fvpp = fvp; // Protect memory allocation and free() for track data
+    
+    if( dbH->flags & O2_FLAG_LARGE_ADB )
+      // Load power and calculate power and l2norm sequence sums
+      init_track_aux_data(trackID, fvp, sNormpp, snPtrp, sPowerp, spPtrp);
+    
     if(!index_insert_track(trackID, fvpp, snPtrp, spPtrp))
       break;    
-  }
+    if ( dbH->flags & O2_FLAG_LARGE_ADB ){
+      close(infid);
+      delete *sNormpp;
+      delete *sPowerp;
+      *sNormpp = *sPowerp = *snPtrp = *snPtrp = 0;
+    }
+  } // end for(trackID = start_track ; ... )
   std::cout << "finished inserting." << endl;
 }
 
@@ -256,13 +332,17 @@
       numVecs = trackTable[trackID] - sequenceLength + 1;
     }
   }
-  vv = index_initialize_shingles(numVecs);
-
-  for( Uns32T pointID = 0 ; pointID < numVecs; pointID++ )
-    index_make_shingle(vv, pointID, *fvpp, dbH->dim, sequenceLength);
   
-  Uns32T numVecsAboveThreshold = index_norm_shingles(vv, *snpp, *sppp);
-  Uns32T collisionCount = index_insert_shingles(vv, trackID, *sppp);
+  Uns32T numVecsAboveThreshold = 0, collisionCount = 0; 
+  if(numVecs){
+    vv = index_initialize_shingles(numVecs);
+    
+    for( Uns32T pointID = 0 ; pointID < numVecs; pointID++ )
+      index_make_shingle(vv, pointID, *fvpp, dbH->dim, sequenceLength);
+    
+    numVecsAboveThreshold = index_norm_shingles(vv, *snpp, *sppp);
+    collisionCount = index_insert_shingles(vv, trackID, *sppp);
+  }
   float meanCollisionCount = numVecsAboveThreshold?(float)collisionCount/numVecsAboveThreshold:0;
 
   /* index_norm_shingles() only goes as far as the end of the
@@ -273,9 +353,11 @@
    * So let's be certain the pointers are in the correct place
    */
 
-  *snpp += trackTable[trackID];
-  *sppp += trackTable[trackID];
-  *fvpp += trackTable[trackID] * dbH->dim;
+  if( !(dbH->flags & O2_FLAG_LARGE_ADB) ){
+    *snpp += trackTable[trackID];
+    *sppp += trackTable[trackID];
+    *fvpp += trackTable[trackID] * dbH->dim;
+  }
 
   std::cout << " n=" << trackTable[trackID] << " n'=" << numVecsAboveThreshold << " E[#c]=" << lsh->get_mean_collision_rate() << " E[#p]=" << meanCollisionCount << endl;
   std::cout.flush();  
@@ -287,7 +369,7 @@
   cout << "[" << trackID << "]" << fileTable+trackID*O2_FILETABLE_ENTRY_SIZE;
   for( Uns32T pointID=0 ; pointID < (*vv).size(); pointID+=sequenceHop)
     if(!use_absolute_threshold || (use_absolute_threshold && (*spp >= absolute_threshold))){
-      collisionCount += lsh->insert_point((*vv)[pointID], index_from_trackInfo(trackID, pointID));
+      collisionCount += lsh->insert_point((*vv)[pointID], index_from_trackInfo(trackID, pointID, lsh_n_point_bits));
       spp+=sequenceHop;
     }
   return collisionCount;
@@ -393,7 +475,7 @@
     printf("INDEX: k %d\n", lsh->get_lshHeader()->get_numFuns());
     printf("INDEX: L (m*(m-1))/2 %d\n", lsh->get_lshHeader()->get_numTables());
     printf("INDEX: N %d\n", lsh->get_lshHeader()->get_numRows());
-    printf("INDEX: s %d\n", index_to_trackID(lsh->get_maxp()));
+    printf("INDEX: s %d\n", index_to_trackID(lsh->get_maxp(), lsh_n_point_bits));
     printf("INDEX: Opened LSH index file %s\n", indexName);
     fflush(stdout);
   }
@@ -415,8 +497,8 @@
 void audioDB::index_add_point_approximate(void* instancePtr, Uns32T pointID, Uns32T qpos, float dist){
   assert(instancePtr); // We need an instance for this callback
   audioDB* myself = (audioDB*) instancePtr; // Use explicit cast to recover "this" instance
-  Uns32T trackID = index_to_trackID(pointID);
-  Uns32T spos = index_to_trackPos(pointID);
+  Uns32T trackID = index_to_trackID(pointID, myself->lsh_n_point_bits);
+  Uns32T spos = index_to_trackPos(pointID, myself->lsh_n_point_bits);
   // Skip identity in query_from_key
   if( !myself->query_from_key || (myself->query_from_key && ( trackID != myself->query_from_key_index )) )
     myself->reporter->add_point(trackID, qpos, spos, dist);
@@ -427,8 +509,8 @@
 void audioDB::index_add_point_exact(void* instancePtr, Uns32T pointID, Uns32T qpos, float dist){
   assert(instancePtr); // We need an instance for this callback
   audioDB* myself = (audioDB*) instancePtr; // Use explicit cast to recover "this" instance  
-  Uns32T trackID = index_to_trackID(pointID);
-  Uns32T spos = index_to_trackPos(pointID);
+  Uns32T trackID = index_to_trackID(pointID, myself->lsh_n_point_bits);
+  Uns32T spos = index_to_trackPos(pointID, myself->lsh_n_point_bits);
   // Skip identity in query_from_key
   if( !myself->query_from_key || (myself->query_from_key && ( trackID != myself->query_from_key_index )) )
     myself->index_insert_exact_evaluation_queue(trackID, qpos, spos);
--- a/lshlib.h	Tue Aug 19 15:50:26 2008 +0000
+++ b/lshlib.h	Tue Aug 19 20:27:15 2008 +0000
@@ -58,8 +58,8 @@
 #define O2_SERIAL_HEADER_SIZE sizeof(SerialHeaderT)
 #define O2_SERIAL_ELEMENT_SIZE sizeof(SerialElementT)
 #define O2_SERIAL_MAX_TABLES (200)
-#define O2_SERIAL_MAX_ROWS (1000000)
-#define O2_SERIAL_MAX_COLS (100000)
+#define O2_SERIAL_MAX_ROWS (1000000000)
+#define O2_SERIAL_MAX_COLS (1000000)
 #define O2_SERIAL_MAX_DIM (2000)
 #define O2_SERIAL_MAX_FUNS (100)
 #define O2_SERIAL_MAX_BINWIDTH (200)
--- a/query.cpp	Tue Aug 19 15:50:26 2008 +0000
+++ b/query.cpp	Tue Aug 19 20:27:15 2008 +0000
@@ -46,7 +46,7 @@
       if(index_exists(dbName, radius, sequenceLength)){
 	char* indexName = index_get_name(dbName, radius, sequenceLength);
 	lsh = index_allocate(indexName, false);
-	reporter = new trackSequenceQueryRadReporter(trackNN, index_to_trackID(lsh->get_maxp())+1);
+	reporter = new trackSequenceQueryRadReporter(trackNN, index_to_trackID(lsh->get_maxp(), lsh_n_point_bits)+1);
 	delete[] indexName;
       }
       else
@@ -62,7 +62,7 @@
       if(index_exists(dbName, radius, sequenceLength)){
 	char* indexName = index_get_name(dbName, radius, sequenceLength);
 	lsh = index_allocate(indexName, false);
-	reporter = new trackSequenceQueryRadNNReporter(pointNN,trackNN, index_to_trackID(lsh->get_maxp())+1);
+	reporter = new trackSequenceQueryRadNNReporter(pointNN,trackNN, index_to_trackID(lsh->get_maxp(), lsh_n_point_bits)+1);
 	delete[] indexName;
       }
       else
@@ -220,7 +220,7 @@
   }
 }
 
-void audioDB::read_data(int track, double **data_buffer_p, size_t *data_buffer_size_p) {
+void audioDB::read_data(int trkfid, int track, double **data_buffer_p, size_t *data_buffer_size_p) {
   if (trackTable[track] * sizeof(double) * dbH->dim > *data_buffer_size_p) {
     if(*data_buffer_p) {
       free(*data_buffer_p);
@@ -235,7 +235,7 @@
     }
   }
 
-  read(dbfid, *data_buffer_p, trackTable[track] * sizeof(double) * dbH->dim);
+  read(trkfid, *data_buffer_p, trackTable[track] * sizeof(double) * dbH->dim);
 }
 
 // These names deserve some unpicking.  The names starting with a "q"
@@ -345,7 +345,7 @@
   *qp = NULL;
   lseek(dbfid, dbH->dataOffset + trackOffsetTable[queryIndex] * sizeof(double), SEEK_SET);
   size_t allocatedSize = 0;
-  read_data(queryIndex, qp, &allocatedSize);
+  read_data(dbfid, queryIndex, qp, &allocatedSize);
   // Consistency check on allocated memory and query feature size
   if(*nvp*sizeof(double)*dbH->dim != allocatedSize)
     error("Query memory allocation failed consitency check","set_up_query_from_key");
@@ -515,7 +515,7 @@
       if(currentTrack!=pp.trackID){
 	currentTrack=pp.trackID;
         lseek(dbfid, dbH->dataOffset + trackOffset * sizeof(double), SEEK_SET);
-	read_data(currentTrack, &data_buffer, &data_buffer_size);
+	read_data(dbfid, currentTrack, &data_buffer, &data_buffer_size);
       }
       dist = dot_product_points(query+(usingQueryPoint?0:pp.qpos*dbH->dim), data_buffer+pp.spos*dbH->dim, dbH->dim*sequenceLength);
       if(normalizedDistance) 
@@ -555,6 +555,9 @@
   double *qNorm, *qnPtr, *qPower = 0, *qpPtr = 0;
   double meanQdur;
 
+  if( dbH->flags & O2_FLAG_LARGE_ADB )
+    error("error: LARGE_ADB requires indexed query");
+
   if(query_from_key)
     set_up_query_from_key(&query_data, &query, &qNorm, &qnPtr, &qPower, &qpPtr, &meanQdur, &numVectors, queryIndex);
   else
@@ -618,7 +621,7 @@
 
     trackIndexOffset=trackOffset/dbH->dim; // numVectors offset
 
-    read_data(track, &data_buffer, &data_buffer_size);
+    read_data(dbfid, track, &data_buffer, &data_buffer_size);
     if(sequenceLength <= trackTable[track]) {  // test for short sequences
       
       VERB_LOG(7,"%u.%jd.%u | ", track, (intmax_t) trackIndexOffset, trackTable[track]);
--- a/sample.cpp	Tue Aug 19 15:50:26 2008 +0000
+++ b/sample.cpp	Tue Aug 19 20:27:15 2008 +0000
@@ -56,7 +56,10 @@
 
 void audioDB::sample(const char *dbName) {
   initTables(dbName, 0);
-
+  if(dbH->flags & O2_FLAG_LARGE_ADB){
+    error("error: sample not yet supported for LARGE_ADB");
+  }
+    
   // build track offset table (FIXME: cut'n'pasted from query.cpp)
   off_t *trackOffsetTable = new off_t[dbH->numFiles];
   unsigned cumTrack=0;
--- a/soap.cpp	Tue Aug 19 15:50:26 2008 +0000
+++ b/soap.cpp	Tue Aug 19 20:27:15 2008 +0000
@@ -18,7 +18,7 @@
     std::cout << "length = " << adbStatusResponse.result.length << std::endl;
     std::cout << "dudCount = " << adbStatusResponse.result.dudCount << std::endl;
     std::cout << "nullCount = " << adbStatusResponse.result.nullCount << std::endl;
-    std::cout << "flags = " << adbStatusResponse.result.flags << std::endl;
+    std::cout << "flags = " << (adbStatusResponse.result.flags & 0x00FFFFFF) << std::endl;
   } else {
     soap_print_fault(&soap,stderr);
   }