diff common.cpp @ 498:342822c2d49a

Merge api-inversion branch (-r656:771, but I don't expect to return to
that branch) into the trunk.

I expect there to be minor performance regressions (e.g. in the SOAP
server index cacheing, which I have forcibly removed) and minor
unplugged memory leaks (e.g. in audioDB::query(), where I don't free up
the datum).  I hope that these leaks and performance regressions can be
plugged in short order.  I also expect that some (but maybe not all) of
the issues currently addressed in the memory-leaks branch are superseded
or fixed by this merge.

There remains much work to be done; go forth and do it.

--- a/common.cpp	Sat Jan 10 11:11:27 2009 +0000
+++ b/common.cpp	Sat Jan 10 16:47:57 2009 +0000
@@ -1,4 +1,8 @@
 #include "audioDB.h"
+extern "C" {
+#include "audioDB_API.h"
+#include "audioDB-internals.h"
+}
 
 #if defined(O2_DEBUG)
 void sigterm_action(int signal, siginfo_t *info, void *context) {
@@ -11,42 +15,15 @@
 #endif
 
 void audioDB::get_lock(int fd, bool exclusive) {
-  struct flock lock;
-  int status;
-  
-  lock.l_type = exclusive ? F_WRLCK : F_RDLCK;
-  lock.l_whence = SEEK_SET;
-  lock.l_start = 0;
-  lock.l_len = 0; /* "the whole file" */
-
- retry:
-  do {
-    status = fcntl(fd, F_SETLKW, &lock);
-  } while (status != 0 && errno == EINTR);
-  
-  if (status) {
-    if (errno == EAGAIN) {
-      sleep(1);
-      goto retry;
-    } else {
-      error("fcntl lock error", "", "fcntl");
-    }
+  if(acquire_lock(fd, exclusive)) {
+    error("fcntl lock error", "", "fcntl");
   }
 }
 
 void audioDB::release_lock(int fd) {
-  struct flock lock;
-  int status;
-
-  lock.l_type = F_UNLCK;
-  lock.l_whence = SEEK_SET;
-  lock.l_start = 0;
-  lock.l_len = 0;
-
-  status = fcntl(fd, F_SETLKW, &lock);
-
-  if (status)
+  if (divest_lock(fd)) {
     error("fcntl unlock error", "", "fcntl");
+  }
 }
 
 void audioDB::error(const char* a, const char* b, const char *sysFunc) {
@@ -63,9 +40,6 @@
            structured type, so that we can throw separate faultstring
            and details.  -- CSR, 2007-10-01 */
         throw(err);
-    } else if (UseApiError){
-        apierrortemp=-1;
-        throw(apierrortemp);
     } else {
         std::cerr << a << ": " << b << std::endl;
         if (sysFunc) {
@@ -73,7 +47,6 @@
         }
         exit(1);
     }
-
 }
 
 void audioDB::initRNG() {
@@ -86,46 +59,20 @@
 }
 
 void audioDB::initDBHeader(const char* dbName) {
-  if ((dbfid = open(dbName, forWrite ? O_RDWR : O_RDONLY)) < 0) {
-    error("Can't open database file", dbName, "open");
+  if(!adb) {
+    adb = audiodb_open(dbName, forWrite ? O_RDWR : O_RDONLY);
+    if(!adb) {
+      error("Failed to open database", dbName);
+    }
   }
-
-  get_lock(dbfid, forWrite);
-  // Get the database header info
-  dbH = new dbTableHeaderT();
-  assert(dbH);
-  
-  if(read(dbfid, (char *) dbH, O2_HEADERSIZE) != O2_HEADERSIZE) {
-    error("error reading db header", dbName, "read");
-  }
-
-  if(dbH->magic == O2_OLD_MAGIC) {
-    // FIXME: if anyone ever complains, write the program to convert
-    // from the old audioDB format to the new...
-    error("database file has old O2 header", dbName);
-  }
-
-  if(dbH->magic != O2_MAGIC) {
-    std::cerr << "expected: " << O2_MAGIC << ", got: " << dbH->magic << std::endl;
-    error("database file has incorrect header", dbName);
-  }
-
-  if(dbH->version != O2_FORMAT_VERSION) {
-    error("database file has incorrect version", dbName);
-  }
-
-  if(dbH->headerSize != O2_HEADERSIZE) {
-    error("sizeof(dbTableHeader) unexpected: platform ABI mismatch?", dbName);
-  }
-
-  CHECKED_MMAP(char *, db, 0, getpagesize());
+  dbfid = adb->fd;
+  dbH = adb->header;
 
   // Make some handy tables with correct types
   if(forWrite || (dbH->length > 0)) {
     if(forWrite) {
       fileTableLength = dbH->trackTableOffset - dbH->fileTableOffset;
       trackTableLength = dbH->dataOffset - dbH->trackTableOffset;
-      dataBufLength = dbH->timesTableOffset - dbH->dataOffset;
       timesTableLength = dbH->powerTableOffset - dbH->timesTableOffset;
       powerTableLength = dbH->l2normTableOffset - dbH->powerTableOffset;
       l2normTableLength = dbH->dbSize - dbH->l2normTableOffset;
@@ -133,13 +80,11 @@
       fileTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLE_ENTRY_SIZE);
       trackTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_TRACKTABLE_ENTRY_SIZE);
       if( dbH->flags & O2_FLAG_LARGE_ADB ){
-	dataBufLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLE_ENTRY_SIZE);
 	timesTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLE_ENTRY_SIZE);
 	powerTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLE_ENTRY_SIZE);
 	l2normTableLength = 0;
       }
       else{
-	dataBufLength = ALIGN_PAGE_UP(dbH->length);
 	timesTableLength = ALIGN_PAGE_UP(2*(dbH->length / dbH->dim));
 	powerTableLength = ALIGN_PAGE_UP(dbH->length / dbH->dim);
 	l2normTableLength = ALIGN_PAGE_UP(dbH->length / dbH->dim);
@@ -147,16 +92,6 @@
     }
     CHECKED_MMAP(char *, fileTable, dbH->fileTableOffset, fileTableLength);
     CHECKED_MMAP(unsigned *, trackTable, dbH->trackTableOffset, trackTableLength);
-    /*
-     * No more mmap() for dataBuf
-     *
-     * FIXME: Actually we do do the mmap() in the two cases where it's
-     * still "needed": in pointQuery and in l2norm if dbH->length is
-     * non-zero.  Removing those cases too (and deleting the dataBuf
-     * variable completely) would be cool.  -- CSR, 2007-11-19
-     *
-     * CHECKED_MMAP(double *, dataBuf, dbH->dataOffset, dataBufLength);
-     */
     if( dbH->flags & O2_FLAG_LARGE_ADB ){
       CHECKED_MMAP(char *, featureFileNameTable, dbH->dataOffset, fileTableLength);
       if( dbH->flags & O2_FLAG_TIMES )
@@ -170,22 +105,9 @@
       CHECKED_MMAP(double *, l2normTable, dbH->l2normTableOffset, l2normTableLength);
     }
   }
-
-  // build track offset table
-  trackOffsetTable = new off_t[dbH->numFiles];
-  Uns32T cumTrack=0;
-  for(Uns32T k = 0; k < dbH->numFiles; k++){
-    trackOffsetTable[k] = cumTrack;
-    cumTrack += trackTable[k] * dbH->dim;
-  }
-
-  // Assign correct number of point bits per track in LSH indexing / retrieval
-  lsh_n_point_bits = dbH->flags >> 28;
-  if( !lsh_n_point_bits )
-    lsh_n_point_bits = O2_DEFAULT_LSH_N_POINT_BITS;
 }
 
-void audioDB::initInputFile (const char *inFile, bool loadData) {
+void audioDB::initInputFile (const char *inFile) {
   if (inFile) {
     if ((infid = open(inFile, O_RDONLY)) < 0) {
       error("can't open input file for reading", inFile, "open");
@@ -216,10 +138,6 @@
 	error("feature dimensions do not match database table dimensions", inFile);
       }
     }
-    
-    if (loadData && ((indata = (char *) mmap(0, statbuf.st_size, PROT_READ, MAP_SHARED, infid, 0)) == (caddr_t) -1)) {
-      error("mmap error for input", inFile, "mmap");
-    }
   }
 }
 
@@ -254,3 +172,39 @@
   sprintf(prefixedName, "%s/%s", prefix, *name);
   *name = prefixedName; // side effect new name to old name
 }
+
+void audioDB::insertTimeStamps(unsigned numVectors, std::ifstream *timesFile, double *timesdata) {
+  assert(usingTimes);
+
+  unsigned numtimes = 0;
+
+  if(!timesFile->is_open()) {
+    error("problem opening times file on timestamped database", timesFileName);
+  }
+
+  double timepoint, next;
+  *timesFile >> timepoint;
+  if (timesFile->eof()) {
+    error("no entries in times file", timesFileName);
+  }
+  numtimes++;
+  do {
+    *timesFile >> next;
+    if (timesFile->eof()) {
+      break;
+    }
+    numtimes++;
+    timesdata[0] = timepoint;
+    timepoint = (timesdata[1] = next);
+    timesdata += 2;
+  } while (numtimes < numVectors + 1);
+
+  if (numtimes < numVectors + 1) {
+    error("too few timepoints in times file", timesFileName);
+  }
+
+  *timesFile >> next;
+  if (!timesFile->eof()) {
+    error("too many timepoints in times file", timesFileName);
+  }
+}