comparison audioDB.cpp @ 199:72a037f2b1e4

Alter timesTable handling.

Instead of inserting N boundary points per track, insert N (start,end)
pairs.  Since N boundaries define only N-1 intervals, we were losing
information on insert -- this means that dump/restore cycles of existing
times-enabled databases WILL NOT WORK.

The input format for times files is still the same, albeit a little
tighter: there must be exactly N+1 floats in the file.

- make times table default length twice as big (done!)
- ensure exactly n+1 times read in from timesFile (done!)
- store (beginning, end) in times table (done!)
- update file format version (done!)
- ensure dump exactly n+1 times (done!)
- adjust uses of timesTable to read at 2k and 2k+1 (done!)
- timesdata[x] in sequence query is duration of sequence of
  appropriate length from point [x] (done!)
- dbdurs calculation (done!)
- don't decrement j when using times in sequence query (done!)
- tests! (NOT DONE)

 
  retry:
   do {
     status = fcntl(fd, F_SETLKW, &lock);
   } while (status != 0 && errno == EINTR);
-
+  
   if (status) {
     if (errno == EAGAIN) {
       sleep(1);
       goto retry;
     } else {

    * a header (see dbTableHeader struct definition);
    * keyTable: list of keys of tracks;
    * trackTable: Maps implicit feature index to a feature vector
    matrix (sizes of tracks)
    * featureTable: Lots of doubles;
-   * timesTable: time points for each feature vector;
+   * timesTable: (start,end) time points for each feature vector;
+   * powerTable: associated power for each feature vector;
    * l2normTable: squared l2norms for each feature vector.
 */
 void audioDB::create(const char* dbName){
   if ((dbfid = open (dbName, O_RDWR|O_CREAT|O_EXCL, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)) < 0)
     error("Can't create database file", dbName, "open");

   dbH->fileTableOffset = ALIGN_PAGE_UP(O2_HEADERSIZE);
   dbH->trackTableOffset = ALIGN_PAGE_UP(dbH->fileTableOffset + O2_FILETABLESIZE*maxfiles);
   dbH->dataOffset = ALIGN_PAGE_UP(dbH->trackTableOffset + O2_TRACKTABLESIZE*maxfiles);
   dbH->l2normTableOffset = ALIGN_PAGE_DOWN(size - maxfiles*O2_MEANNUMVECTORS*sizeof(double));
   dbH->powerTableOffset = ALIGN_PAGE_DOWN(dbH->l2normTableOffset - maxfiles*O2_MEANNUMVECTORS*sizeof(double));
-  dbH->timesTableOffset = ALIGN_PAGE_DOWN(dbH->powerTableOffset - maxfiles*O2_MEANNUMVECTORS*sizeof(double));
+  dbH->timesTableOffset = ALIGN_PAGE_DOWN(dbH->powerTableOffset - 2*maxfiles*O2_MEANNUMVECTORS*sizeof(double));
   dbH->dbSize = size;
 
   write(dbfid, dbH, O2_HEADERSIZE);
 
   // go to the location corresponding to the last byte

       l2normTableLength = dbH->dbSize - dbH->l2normTableOffset;
     } else {
       fileTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLESIZE);
       trackTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_TRACKTABLESIZE);
       dataBufLength = ALIGN_PAGE_UP(dbH->length);
-      timesTableLength = ALIGN_PAGE_UP(dbH->length / dbH->dim);
+      timesTableLength = ALIGN_PAGE_UP(2*(dbH->length / dbH->dim));
       powerTableLength = ALIGN_PAGE_UP(dbH->length / dbH->dim);
       l2normTableLength = ALIGN_PAGE_UP(dbH->length / dbH->dim);
     }
     CHECKED_MMAP(char *, fileTable, dbH->fileTableOffset, fileTableLength);
     CHECKED_MMAP(unsigned *, trackTable, dbH->trackTableOffset, trackTableLength);

   strncpy(fileTable + dbH->numFiles*O2_FILETABLESIZE, key, strlen(key));
 
   off_t insertoffset = dbH->length;// Store current state
 
   // Check times status and insert times from file
-  unsigned timesoffset=insertoffset/(dbH->dim*sizeof(double));
-  double* timesdata=timesTable+timesoffset;
-
-  if(timesoffset + numVectors > timesTableLength) {
+  unsigned indexoffset = insertoffset/(dbH->dim*sizeof(double));
+  double *timesdata = timesTable + 2*indexoffset;
+
+  if(2*(indexoffset + numVectors) > timesTableLength) {
     error("out of space for times", key);
   }
-    
-  insertTimeStamps(numVectors, timesFile, timesdata);
-
-  double *powerdata = powerTable + timesoffset;
+  
+  if (usingTimes) {
+    insertTimeStamps(numVectors, timesFile, timesdata);
+  }
+
+  double *powerdata = powerTable + indexoffset;
   insertPowerData(numVectors, powerfd, powerdata);
 
   // Increment file count
   dbH->numFiles++;
 
   // Update Header information
   dbH->length+=(statbuf.st_size-sizeof(int));
 
   // Update track to file index map
-  memcpy (trackTable+dbH->numFiles-1, &numVectors, sizeof(unsigned));  
+  memcpy(trackTable + dbH->numFiles - 1, &numVectors, sizeof(unsigned));  
 
   insert_data_vectors(insertoffset, indata + sizeof(int), statbuf.st_size - sizeof(int));
   
   // Norm the vectors on input if the database is already L2 normed
   if(dbH->flags & O2_FLAG_L2NORM)

   // CLEAN UP
   munmap(indata,statbuf.st_size);
   close(infid);
 }
 
-void audioDB::insertTimeStamps(unsigned numVectors, ifstream* timesFile, double* timesdata){
-  unsigned numtimes=0;
- if(usingTimes){
-   if(!(dbH->flags & O2_FLAG_TIMES) && !dbH->numFiles)
-     dbH->flags=dbH->flags|O2_FLAG_TIMES;
-   else if(!(dbH->flags&O2_FLAG_TIMES)){
-     cerr << "Warning: timestamp file used with non time-stamped database: ignoring timestamps" << endl;
-     usingTimes=0;
-   }
+void audioDB::insertTimeStamps(unsigned numVectors, ifstream *timesFile, double *timesdata) {
+  assert(usingTimes);
+
+  unsigned numtimes = 0;
+
+  if(!(dbH->flags & O2_FLAG_TIMES) && !dbH->numFiles) {
+    dbH->flags=dbH->flags|O2_FLAG_TIMES;
+  } else if(!(dbH->flags & O2_FLAG_TIMES)) {
+    error("Timestamp file used with non-timestamped database", timesFileName);
+  }
    
-   if(!timesFile->is_open()){
-     if(dbH->flags & O2_FLAG_TIMES){
-       munmap(indata,statbuf.st_size);
-       munmap(db,dbH->dbSize);
-       error("problem opening times file on timestamped database",timesFileName);
-     }
-     else{
-       cerr << "Warning: problem opening times file. But non-timestamped database, so ignoring times file." << endl;
-       usingTimes=0;
-     }      
-   }
-
-    // Process time file
-   if(usingTimes){
-     do{
-       *timesFile>>*timesdata++;
-       if(timesFile->eof())
-	 break;
-       numtimes++;
-     }while(!timesFile->eof() && numtimes<numVectors);
-     if(!timesFile->eof()){
-	double dummy;
-	do{
-	  *timesFile>>dummy;
-	  if(timesFile->eof())
-	    break;
-	  numtimes++;
-	}while(!timesFile->eof());
-     }
-     if(numtimes<numVectors || numtimes>numVectors+2){
-       munmap(indata,statbuf.st_size);
-       munmap(db,dbH->dbSize);
-       close(infid);
-       cerr << "expected " << numVectors << " found " << numtimes << endl;
-       error("Times file is incorrect length for features file",inFile);
-     }
-     if(verbosity>2) {
-       cerr << "numtimes: " << numtimes << endl;
-     }
-   }
- }
+  if(!timesFile->is_open()) {
+    error("problem opening times file on timestamped database", timesFileName);
+  }
+
+  double timepoint, next;
+  *timesFile >> timepoint;
+  if (timesFile->eof()) {
+    error("no entries in times file", timesFileName);
+  }
+  numtimes++;
+  do {
+    *timesFile >> next;
+    if (timesFile->eof()) {
+      break;
+    }
+    numtimes++;
+    timesdata[0] = timepoint;
+    timepoint = (timesdata[1] = next);
+    timesdata += 2;
+  } while (numtimes < numVectors + 1);
+
+  if (numtimes < numVectors + 1) {
+    error("too few timepoints in times file", timesFileName);
+  }
+
+  *timesFile >> next;
+  if (!timesFile->eof()) {
+    error("too many timepoints in times file", timesFileName);
+  }
 }
 
 void audioDB::insertPowerData(unsigned numVectors, int powerfd, double *powerdata) {
   if (usingPower) {
     if (!(dbH->flags & O2_FLAG_POWER)) {

 	  cerr << "Warning: ignoring zero-length feature vector file:" << thisKey << endl;
         }
       }
       else{
 	if(usingTimes){
-	  if(timesFile->eof())
-	    error("not enough timestamp files in timesList");
-	  thisTimesFile=new ifstream(thisTimesFileName,ios::in);
-	  if(!thisTimesFile->is_open())
-	    error("Cannot open timestamp file",thisTimesFileName);
-	  off_t insertoffset=dbH->length;
-	  unsigned timesoffset=insertoffset/(dbH->dim*sizeof(double));
-	  double* timesdata=timesTable+timesoffset;
-          if(timesoffset + numVectors > timesTableLength) {
+	  if(timesFile->eof()) {
+	    error("not enough timestamp files in timesList", timesFileName);
+	  }
+	  thisTimesFile = new ifstream(thisTimesFileName,ios::in);
+	  if(!thisTimesFile->is_open()) {
+	    error("Cannot open timestamp file", thisTimesFileName);
+	  }
+	  off_t insertoffset = dbH->length;
+	  unsigned indexoffset = insertoffset / (dbH->dim*sizeof(double));
+	  double *timesdata = timesTable + 2*indexoffset;
+          if(2*(indexoffset + numVectors) > timesTableLength) {
             error("out of space for times", key);
           }
-	  insertTimeStamps(numVectors,thisTimesFile,timesdata);
+	  insertTimeStamps(numVectors, thisTimesFile, timesdata);
 	  if(thisTimesFile)
 	    delete thisTimesFile;
 	}
         
         if (usingPower) {

     free(data_buffer);
 
     fprintf(fLFile, "%s\n", fName);
     close(ffd);
 
-    if(times) {
+    if (times) {
       snprintf(fName, 256, "%05d.times", k);
       tFile = fopen(fName, "w");
       for(unsigned i = 0; i < trackTable[k]; i++) {
         // KLUDGE: specifying 16 digits of precision after the decimal
         // point is (but check this!) sufficient to uniquely identify
         // doubles; however, that will cause ugliness, as that's
         // vastly too many for most values of interest.  Moving to %a
         // here and scanf() in the timesFile reading might fix this.
         // -- CSR, 2007-10-19
-        fprintf(tFile, "%.16e\n", *(timesTable + pos + i));
-      }
+        fprintf(tFile, "%.16e\n", *(timesTable + 2*pos + 2*i));
+      }
+      fprintf(tFile, "%.16e\n", *(timesTable + 2*pos + 2*trackTable[k]-1));
+
       fprintf(tLFile, "%s\n", fName);
     }
 
     if (power) {
       uint32_t one = 1;

   unsigned k,l,n;
   double thisDist;
 
   unsigned totalVecs=dbH->length/(dbH->dim*sizeof(double));
   double meanQdur = 0;
-  double* timesdata = 0;
-  double* dbdurs = 0;
+  double *timesdata = 0;
+  double *querydurs = 0;
+  double *dbdurs = 0;
 
   if(usingTimes && !(dbH->flags & O2_FLAG_TIMES)){
     cerr << "warning: ignoring query timestamps for non-timestamped database" << endl;
     usingTimes=0;
   }
 
   else if(!usingTimes && (dbH->flags & O2_FLAG_TIMES))
     cerr << "warning: no timestamps given for query. Ignoring database timestamps." << endl;
   
   else if(usingTimes && (dbH->flags & O2_FLAG_TIMES)){
-    timesdata = new double[numVectors];
+    timesdata = new double[2*numVectors];
+    querydurs = new double[numVectors];
     insertTimeStamps(numVectors, timesFile, timesdata);
     // Calculate durations of points
     for(k=0; k<numVectors-1; k++){
-      timesdata[k]=timesdata[k+1]-timesdata[k];
-      meanQdur+=timesdata[k];
+      querydurs[k]=timesdata[2*k+1]-timesdata[2*k];
+      meanQdur+=querydurs[k];
     }
     meanQdur/=k;
     // Individual exhaustive timepoint durations
     dbdurs = new double[totalVecs];
-    for(k=0; k<totalVecs-1; k++)
-      dbdurs[k]=timesTable[k+1]-timesTable[k];
-    j--; // decrement vector counter by one
+    for(k=0; k<totalVecs-1; k++) {
+      dbdurs[k]=timesTable[2*k+1]-timesTable[2*k];
+    }
   }
 
   if(usingQueryPoint)
     if(queryPoint>numVectors-1)
       error("queryPoint > numVectors in query");

       double* q=query;
       while(l--)
 	thisDist+=*q++**data++;
       if(!usingTimes || 
 	 (usingTimes 
-	  && fabs(dbdurs[totalVecs-k-1]-timesdata[numVectors-j-1])<timesdata[numVectors-j-1]*timesTol)){
+	  && fabs(dbdurs[totalVecs-k-1]-querydurs[numVectors-j-1])<querydurs[numVectors-j-1]*timesTol)){
 	n=pointNN;
 	while(n--){
 	  if(thisDist>=distances[n]){
 	    if((n==0 || thisDist<=distances[n-1])){
 	      // Copy all values above up the queue

   if(queryCopy)
     delete queryCopy;
   if(qNorm)
     delete qNorm;
   if(timesdata)
-    delete timesdata;
+    delete[] timesdata;
+  if(querydurs)
+    delete[] querydurs;
   if(dbdurs)
     delete dbdurs;
 }
 
 // trackPointQuery  

     trackSIndexes[k]=~0;
     trackIDs[k]=~0;
   }
 
   double meanQdur = 0;
-  double* timesdata = 0;
-  double* meanDBdur = 0;
+  double *timesdata = 0;
+  double *querydurs = 0;
+  double *meanDBdur = 0;
   
   if(usingTimes && !(dbH->flags & O2_FLAG_TIMES)){
     cerr << "warning: ignoring query timestamps for non-timestamped database" << endl;
     usingTimes=0;
   }
   
   else if(!usingTimes && (dbH->flags & O2_FLAG_TIMES))
     cerr << "warning: no timestamps given for query. Ignoring database timestamps." << endl;
   
   else if(usingTimes && (dbH->flags & O2_FLAG_TIMES)){
-    timesdata = new double[numVectors];
+    timesdata = new double[2*numVectors];
+    querydurs = new double[numVectors];
     insertTimeStamps(numVectors, timesFile, timesdata);
     // Calculate durations of points
-    for(k=0; k<numVectors-1; k++){
-      timesdata[k]=timesdata[k+1]-timesdata[k];
-      meanQdur+=timesdata[k];
+    for(k=0; k<numVectors-1; k++) {
+      querydurs[k] = timesdata[2*k+1] - timesdata[2*k];
+      meanQdur += querydurs[k];
     }
     meanQdur/=k;
     meanDBdur = new double[dbH->numFiles];
     for(k=0; k<dbH->numFiles; k++){
       meanDBdur[k]=0.0;
-      for(j=0; j<trackTable[k]-1 ; j++)
-	meanDBdur[k]+=timesTable[j+1]-timesTable[j];
+      for(j=0; j<trackTable[k]-1 ; j++) {
+	meanDBdur[k]+=timesTable[2*j+1]-timesTable[2*j];
+      }
       meanDBdur[k]/=j;
     }
   }
 
   if(usingQueryPoint)

       adbQueryResponse->result.Spos[k]=trackSIndexes[k];
       sprintf(adbQueryResponse->result.Rlist[k], "%s", fileTable+trackIDs[k]*O2_FILETABLESIZE);
     }
   }
     
-
   // Clean up
   if(trackOffsetTable)
     delete trackOffsetTable;
   if(queryCopy)
     delete queryCopy;
   if(qNorm)
     delete qNorm;
   if(timesdata)
-    delete timesdata;
+    delete[] timesdata;
+  if(querydurs)
+    delete[] querydurs;
   if(meanDBdur)
     delete meanDBdur;
-
 }
 
 // This is a common pattern in sequence queries: what we are doing is
 // taking a window of length seqlen over a buffer of length length,
 // and placing the sum of the elements in that window in the first

     trackIDs[k]=~0;
   }
 
   // Timestamp and durations processing
   double meanQdur = 0;
-  double* timesdata = 0;
-  double* meanDBdur = 0;
+  double *timesdata = 0;
+  double *querydurs = 0;
+  double *meanDBdur = 0;
   
   if(usingTimes && !(dbH->flags & O2_FLAG_TIMES)){
     cerr << "warning: ignoring query timestamps for non-timestamped database" << endl;
     usingTimes=0;
   }
   
   else if(!usingTimes && (dbH->flags & O2_FLAG_TIMES))
     cerr << "warning: no timestamps given for query. Ignoring database timestamps." << endl;
   
   else if(usingTimes && (dbH->flags & O2_FLAG_TIMES)){
-    timesdata = new double[numVectors];
-    assert(timesdata);
+    timesdata = new double[2*numVectors];
+    querydurs = new double[numVectors];
+
     insertTimeStamps(numVectors, timesFile, timesdata);
     // Calculate durations of points
-    for(k=0; k<numVectors-1; k++){
-      timesdata[k]=timesdata[k+1]-timesdata[k];
-      meanQdur+=timesdata[k];
+    for(k=0; k<numVectors-1; k++) {
+      querydurs[k] = timesdata[2*k+1] - timesdata[2*k];
+      meanQdur += querydurs[k];
     }
     meanQdur/=k;
     if(verbosity>1) {
       cerr << "mean query file duration: " << meanQdur << endl;
     }
     meanDBdur = new double[dbH->numFiles];
     assert(meanDBdur);
     for(k=0; k<dbH->numFiles; k++){
       meanDBdur[k]=0.0;
-      for(j=0; j<trackTable[k]-1 ; j++)
-	meanDBdur[k]+=timesTable[j+1]-timesTable[j];
+      for(j=0; j<trackTable[k]-1 ; j++) {
+	meanDBdur[k]+=timesTable[2*j+1]-timesTable[2*j];
+      }
       meanDBdur[k]/=j;
     }
   }
 
   if(usingQueryPoint)

     delete[] D;
   if(DD)
     delete[] DD;
   if(timesdata)
     delete[] timesdata;
+  if(querydurs)
+    delete[] querydurs;
   if(meanDBdur)
     delete[] meanDBdur;
 }
 
 // Radius search between query and target tracks

     trackIDs[k]=~0;
   }
 
   // Timestamp and durations processing
   double meanQdur = 0;
-  double* timesdata = 0;
-  double* meanDBdur = 0;
+  double *timesdata = 0;
+  double *querydurs = 0;
+  double *meanDBdur = 0;
   
   if(usingTimes && !(dbH->flags & O2_FLAG_TIMES)){
     cerr << "warning: ignoring query timestamps for non-timestamped database" << endl;
     usingTimes=0;
   }
   
   else if(!usingTimes && (dbH->flags & O2_FLAG_TIMES))
     cerr << "warning: no timestamps given for query. Ignoring database timestamps." << endl;
   
   else if(usingTimes && (dbH->flags & O2_FLAG_TIMES)){
-    timesdata = new double[numVectors];
-    assert(timesdata);
+    timesdata = new double[2*numVectors];
+    querydurs = new double[numVectors];
+
     insertTimeStamps(numVectors, timesFile, timesdata);
     // Calculate durations of points
     for(k=0; k<numVectors-1; k++){
-      timesdata[k]=timesdata[k+1]-timesdata[k];
-      meanQdur+=timesdata[k];
+      querydurs[k] = timesdata[2*k+1] - timesdata[2*k];
+      meanQdur += querydurs[k];
     }
     meanQdur/=k;
     if(verbosity>1) {
       cerr << "mean query file duration: " << meanQdur << endl;
     }
     meanDBdur = new double[dbH->numFiles];
     assert(meanDBdur);
     for(k=0; k<dbH->numFiles; k++){
       meanDBdur[k]=0.0;
-      for(j=0; j<trackTable[k]-1 ; j++)
-	meanDBdur[k]+=timesTable[j+1]-timesTable[j];
+      for(j=0; j<trackTable[k]-1 ; j++) {
+	meanDBdur[k]+=timesTable[2*j+1]-timesTable[2*j];
+      }
       meanDBdur[k]/=j;
     }
   }
 
   if(usingQueryPoint)

     delete[] D;
   if(DD)
     delete[] DD;
   if(timesdata)
     delete[] timesdata;
+  if(querydurs)
+    delete[] querydurs;
   if(meanDBdur)
     delete[] meanDBdur;
 }
 
 // Unit norm block of features