comparison audioDB.cpp @ 177:c32bf13c3978 no-big-mmap

forWrite audioDB flag (rather than function argument) so that we can
tell when it comes to munmap() how much we need to.

ALIGN_PAGE_FOO() takes one argument not two.

   dbH->version = O2_FORMAT_VERSION;
   dbH->numFiles = 0;
   dbH->dim = 0;
   dbH->flags = 0;
   dbH->length = 0;
-  dbH->fileTableOffset = ALIGN_PAGE_UP(O2_HEADERSIZE, 8);
-  dbH->trackTableOffset = ALIGN_PAGE_UP(dbH->fileTableOffset + O2_FILETABLESIZE*maxfiles, 8);
-  dbH->dataOffset = ALIGN_PAGE_UP(dbH->trackTableOffset + O2_TRACKTABLESIZE*maxfiles, 8);
-  dbH->l2normTableOffset = ALIGN_PAGE_DOWN(size - maxfiles*O2_MEANNUMVECTORS*sizeof(double), 8);
-  dbH->timesTableOffset = ALIGN_PAGE_DOWN(dbH->l2normTableOffset - maxfiles*O2_MEANNUMVECTORS*sizeof(double), 8);
+  dbH->fileTableOffset = ALIGN_PAGE_UP(O2_HEADERSIZE);
+  dbH->trackTableOffset = ALIGN_PAGE_UP(dbH->fileTableOffset + O2_FILETABLESIZE*maxfiles);
+  dbH->dataOffset = ALIGN_PAGE_UP(dbH->trackTableOffset + O2_TRACKTABLESIZE*maxfiles);
+  dbH->l2normTableOffset = ALIGN_PAGE_DOWN(size - maxfiles*O2_MEANNUMVECTORS*sizeof(double));
+  dbH->timesTableOffset = ALIGN_PAGE_DOWN(dbH->l2normTableOffset - maxfiles*O2_MEANNUMVECTORS*sizeof(double));
   dbH->dbSize = size;
 
   write(dbfid, dbH, O2_HEADERSIZE);
 
   // go to the location corresponding to the last byte

 
 void audioDB::drop(){
   // FIXME: drop something?  Should we even allow this?
 }
 
-void audioDB::initDBHeader(const char* dbName, bool forWrite) {
+void audioDB::initDBHeader(const char* dbName) {
   if ((dbfid = open(dbName, forWrite ? O_RDWR : O_RDONLY)) < 0) {
     error("Can't open database file", dbName, "open");
   }
 
   get_lock(dbfid, forWrite);

     var = (type) tmp; \
   }
 
   CHECKED_MMAP(char *, db, 0, getpagesize());
 
-  CHECKED_MMAP(char *, fileTable, dbH->fileTableOffset, dbH->trackTableOffset);
-  CHECKED_MMAP(unsigned *, trackTable, dbH->trackTableOffset, dbH->dataOffset);
-  CHECKED_MMAP(double *, dataBuf, dbH->dataOffset, dbH->timesTableOffset);
-  CHECKED_MMAP(double *, timesTable, dbH->timesTableOffset, dbH->l2normTableOffset);
-  CHECKED_MMAP(double *, l2normTable, dbH->l2normTableOffset, dbH->dbSize);
+  if(forWrite) {
+    CHECKED_MMAP(char *, fileTable, dbH->fileTableOffset, dbH->trackTableOffset);
+
+    CHECKED_MMAP(unsigned *, trackTable, dbH->trackTableOffset, dbH->dataOffset);
+    CHECKED_MMAP(double *, dataBuf, dbH->dataOffset, dbH->timesTableOffset);
+    CHECKED_MMAP(double *, timesTable, dbH->timesTableOffset, dbH->l2normTableOffset);
+    CHECKED_MMAP(double *, l2normTable, dbH->l2normTableOffset, dbH->dbSize);
+  } else {
+    if(dbH->length > 0) {
+      CHECKED_MMAP(char *, fileTable, dbH->fileTableOffset, dbH->fileTableOffset + ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLESIZE));
+      CHECKED_MMAP(unsigned *, trackTable, dbH->trackTableOffset, dbH->trackTableOffset + ALIGN_PAGE_UP(dbH->numFiles * O2_TRACKTABLESIZE));
+      CHECKED_MMAP(double *, dataBuf, dbH->dataOffset, dbH->dataOffset + ALIGN_PAGE_UP(dbH->length));
+      CHECKED_MMAP(double *, timesTable, dbH->timesTableOffset, dbH->timesTableOffset + ALIGN_PAGE_UP(dbH->length / dbH->dim));
+      CHECKED_MMAP(double *, l2normTable, dbH->l2normTableOffset, dbH->l2normTableOffset + ALIGN_PAGE_UP(dbH->length / dbH->dim));
+    }
+  }
 }
 
 void audioDB::initInputFile (const char *inFile) {
   if (inFile) {
     if ((infid = open(inFile, O_RDONLY)) < 0) {

       error("mmap error for input", inFile, "mmap");
     }
   }
 }
 
-void audioDB::initTables(const char* dbName, bool forWrite, const char* inFile = 0) {
-  
-  initDBHeader(dbName, forWrite);
+void audioDB::initTables(const char* dbName, const char* inFile = 0) {
+  initDBHeader(dbName);
   initInputFile(inFile);
 }
 
-void audioDB::insert(const char* dbName, const char* inFile){
-
-  initTables(dbName, 1, inFile);
+void audioDB::insert(const char* dbName, const char* inFile) {
+  forWrite = true;
+  initTables(dbName, inFile);
 
   if(!usingTimes && (dbH->flags & O2_FLAG_TIMES))
     error("Must use timestamps with timestamped database","use --times");
 
   // Check that there is room for at least 1 more file

  }
 }
 
 void audioDB::batchinsert(const char* dbName, const char* inFile) {
 
-  initDBHeader(dbName, true);
+  forWrite = true;
+  initDBHeader(dbName);
 
   if(!key)
     key=inFile;
   ifstream *filesIn = 0;
   ifstream *keysIn = 0;

 }
 
 
 void audioDB::status(const char* dbName, adb__statusResponse *adbStatusResponse){
   if(!dbH)
-    initTables(dbName, 0, 0);
+    initTables(dbName, 0);
 
   unsigned dudCount=0;
   unsigned nullCount=0;
   for(unsigned k=0; k<dbH->numFiles; k++){
     if(trackTable[k]<sequenceLength){

   }
 }
 
 void audioDB::dump(const char* dbName){
   if(!dbH) {
-    initTables(dbName, 0, 0);
+    initTables(dbName, 0);
   }
 
   if((mkdir(output, S_IRWXU|S_IRWXG|S_IRWXO)) < 0) {
     error("error making output directory", output, "mkdir");
   }

   delete[] fName;
     
   status(dbName);
 }
 
-void audioDB::l2norm(const char* dbName){
-  initTables(dbName, true, 0);
+void audioDB::l2norm(const char* dbName) {
+  forWrite = true;
+  initTables(dbName, 0);
   if(dbH->length>0){
     unsigned numVectors = dbH->length/(sizeof(double)*dbH->dim);
     unitNormAndInsertL2(dataBuf, dbH->dim, numVectors, 0); // No append
   }
   // Update database flags

   error("Key not found",key);
   return O2_ERR_KEYNOTFOUND;
 }
 
 // Basic point query engine
-void audioDB::pointQuery(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse){
-  
-  initTables(dbName, 0, inFile);
+void audioDB::pointQuery(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse) {
+  
+  initTables(dbName, inFile);
   
   // For each input vector, find the closest pointNN matching output vectors and report
   // we use stdout in this stub version
   unsigned numVectors = (statbuf.st_size-sizeof(int))/(sizeof(double)*dbH->dim);
     

 }
 
 // trackPointQuery  
 // return the trackNN closest tracks to the query track
 // uses average of pointNN points per track 
-void audioDB::trackPointQuery(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse){  
-  initTables(dbName, 0, inFile);
+void audioDB::trackPointQuery(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse) {
+  initTables(dbName, inFile);
   
   // For each input vector, find the closest pointNN matching output vectors and report
   unsigned numVectors = (statbuf.st_size-sizeof(int))/(sizeof(double)*dbH->dim);
   double* query = (double*)(indata+sizeof(int));
   double* data = dataBuf;

 // efficient implementation based on matched filter
 // assumes normed shingles
 // outputs distances of retrieved shingles, max retreived = pointNN shingles per per track
 void audioDB::trackSequenceQueryNN(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse){
   
-  initTables(dbName, 0, inFile);
+  initTables(dbName, inFile);
   
   // For each input vector, find the closest pointNN matching output vectors and report
   // we use stdout in this stub version
   unsigned numVectors = (statbuf.st_size-sizeof(int))/(sizeof(double)*dbH->dim);
   double* query = (double*)(indata+sizeof(int));

 // efficient implementation based on matched filter
 // assumes normed shingles
 // outputs count of retrieved shingles, max retreived = one shingle per query shingle per track
 void audioDB::trackSequenceQueryRad(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse){
   
-  initTables(dbName, 0, inFile);
+  initTables(dbName, inFile);
   
   // For each input vector, find the closest pointNN matching output vectors and report
   // we use stdout in this stub version
   unsigned numVectors = (statbuf.st_size-sizeof(int))/(sizeof(double)*dbH->dim);
   double* query = (double*)(indata+sizeof(int));