comparison audioDB.cpp @ 187:530154ca4cf5 no-big-mmap

Wow, two changes for the price of one

* remove dataBuf usage from trackPointQuery()

* make the restrict-list tests pass:
  ** lseek() to the right place if we're actually not doing tracks in
     sequential-order;
  ** deal with the off-by-one error in reading in lines from trackFile.

   unsigned insertoffset = dbH->length;// Store current state
 
   // Check times status and insert times from file
   unsigned timesoffset=insertoffset/(dbH->dim*sizeof(double));
   double* timesdata=timesTable+timesoffset;
-  assert(timesdata+numVectors<l2normTable);
+  /* FIXME: work out how to check against wandering off the end of the
+     times table.
+
+     assert(timesdata+numVectors<l2normTable);
+  */
   insertTimeStamps(numVectors, timesFile, timesdata);
 
   // Increment file count
   dbH->numFiles++;
 

       query=query+queryPoint*dbH->dim;
       numVectors=queryPoint+1;
     }
   
   // build track offset table
-  unsigned *trackOffsetTable = new unsigned[dbH->numFiles];
+  off_t *trackOffsetTable = new off_t[dbH->numFiles];
   unsigned cumTrack=0;
-  unsigned trackIndexOffset;
+  off_t trackIndexOffset;
   for(k=0; k<dbH->numFiles;k++){
     trackOffsetTable[k]=cumTrack;
     cumTrack+=trackTable[k]*dbH->dim;
   }
 
   char nextKey[MAXSTR];
 
   gettimeofday(&tv1, NULL); 
+
+  size_t data_buffer_size = 0;
+  double *data_buffer = 0;
+  lseek(dbfid, dbH->dataOffset, SEEK_SET);
         
   for(processedTracks=0, track=0 ; processedTracks < dbH->numFiles ; track++, processedTracks++){
-    if(trackFile){
-      if(!trackFile->eof()){
-	trackFile->getline(nextKey,MAXSTR);
-	track=getKeyPos(nextKey);
-      }
-      else
+
+    trackOffset = trackOffsetTable[track];     // numDoubles offset
+
+    // get trackID from file if using a control file
+    if(trackFile) {
+      trackFile->getline(nextKey,MAXSTR);
+      if(!trackFile->eof()) {
+	track = getKeyPos(nextKey);
+        trackOffset = trackOffsetTable[track];
+        lseek(dbfid, dbH->dataOffset + trackOffset * sizeof(double), SEEK_SET);
+      } else {
 	break;
-    }
-    trackOffset=trackOffsetTable[track];     // numDoubles offset
+      }
+    }
+
     trackIndexOffset=trackOffset/dbH->dim; // numVectors offset
+
     if(verbosity>7) {
       cerr << track << "." << trackOffset/(dbH->dim) << "." << trackTable[track] << " | ";cerr.flush();
     }
 
     if(dbH->flags & O2_FLAG_L2NORM)

       usingQueryPoint?query=(double*)(indata+sizeof(int))+queryPoint*dbH->dim:query=(double*)(indata+sizeof(int));
     if(usingQueryPoint)
       j=1;
     else
       j=numVectors;
+
+    if (trackTable[track] * sizeof(double) * dbH->dim > data_buffer_size) {
+      if(data_buffer) {
+        free(data_buffer);
+      }
+      { 
+        data_buffer_size = trackTable[track] * sizeof(double) * dbH->dim;
+        void *tmp = malloc(data_buffer_size);
+        if (tmp == NULL) {
+          error("error allocating data buffer");
+        }
+        data_buffer = (double *) tmp;
+      }
+    }
+
+    read(dbfid, data_buffer, trackTable[track] * sizeof(double) * dbH->dim);
+
     while(j--){
       k=trackTable[track];  // number of vectors in track
-      data=dataBuf+trackOffset; // data for track
+      data=data_buffer; // data for track
       while(k--){
 	thisDist=0;
 	l=dbH->dim;
 	double* q=query;
 	while(l--)

       distances[k]=-DBL_MAX;
       qIndexes[k]=~0;
       sIndexes[k]=~0;    
     }
   } // tracks
+
+  free(data_buffer);
+
   gettimeofday(&tv2, NULL); 
 
   if(verbosity>1) {
     cerr << endl << "processed tracks :" << processedTracks 
 	 << " elapsed time:" << ( tv2.tv_sec*1000 + tv2.tv_usec/1000 ) - ( tv1.tv_sec*1000+tv1.tv_usec/1000 ) << " msec" << endl;

   double* qp;
   double* sp;
   double* dp;
 
   // build track offset table
-  unsigned *trackOffsetTable = new unsigned[dbH->numFiles];
+  off_t *trackOffsetTable = new off_t[dbH->numFiles];
   unsigned cumTrack=0;
-  unsigned trackIndexOffset;
+  off_t trackIndexOffset;
   for(k=0; k<dbH->numFiles;k++){
     trackOffsetTable[k]=cumTrack;
     cumTrack+=trackTable[k]*dbH->dim;
   }
 

   double *data_buffer = 0;
   lseek(dbfid, dbH->dataOffset, SEEK_SET);
 
   for(processedTracks=0, track=0 ; processedTracks < dbH->numFiles ; track++, processedTracks++) {
 
+    trackOffset = trackOffsetTable[track];     // numDoubles offset
+
     // get trackID from file if using a control file
-    if(trackFile){
-      if(!trackFile->eof()){
-	trackFile->getline(nextKey,MAXSTR);
-	track=getKeyPos(nextKey);
-      }
-      else
+    if(trackFile) {
+      trackFile->getline(nextKey,MAXSTR);
+      if(!trackFile->eof()) {
+	track = getKeyPos(nextKey);
+        trackOffset = trackOffsetTable[track];
+        lseek(dbfid, dbH->dataOffset + trackOffset * sizeof(double), SEEK_SET);
+      } else {
 	break;
-    }
-
-    trackOffset=trackOffsetTable[track];     // numDoubles offset
+      }
+    }
+
     trackIndexOffset=trackOffset/dbH->dim; // numVectors offset
 
     if(sequenceLength<=trackTable[track]){  // test for short sequences
       
       if(verbosity>7) {

   double* qp;
   double* sp;
   double* dp;
 
   // build track offset table
-  unsigned *trackOffsetTable = new unsigned[dbH->numFiles];
+  off_t *trackOffsetTable = new off_t[dbH->numFiles];
   unsigned cumTrack=0;
-  unsigned trackIndexOffset;
+  off_t trackIndexOffset;
   for(k=0; k<dbH->numFiles;k++){
     trackOffsetTable[k]=cumTrack;
     cumTrack+=trackTable[k]*dbH->dim;
   }
 

   double *data_buffer = 0;
   lseek(dbfid, dbH->dataOffset, SEEK_SET);
 
   for(processedTracks=0, track=0 ; processedTracks < dbH->numFiles ; track++, processedTracks++){
 
+    trackOffset = trackOffsetTable[track];     // numDoubles offset
+
     // get trackID from file if using a control file
-    if(trackFile){
-      if(!trackFile->eof()){
-	trackFile->getline(nextKey,MAXSTR);
-	track=getKeyPos(nextKey);
-      }
-      else
+    if(trackFile) {
+      trackFile->getline(nextKey,MAXSTR);
+      if(!trackFile->eof()) {
+	track = getKeyPos(nextKey);
+        trackOffset = trackOffsetTable[track];
+        lseek(dbfid, dbH->dataOffset + trackOffset * sizeof(double), SEEK_SET);
+      } else {
 	break;
-    }
-
-    trackOffset=trackOffsetTable[track];     // numDoubles offset
+      }
+    }
+
     trackIndexOffset=trackOffset/dbH->dim; // numVectors offset
 
     if(sequenceLength<=trackTable[track]){  // test for short sequences
       
       if(verbosity>7) {