comparison audioDB.cpp @ 120:fce73e4afa15

Make a start on refactoring: work the DB header initialization into its
own function, used from both initTables and batchinsert.

This actually removes a mmap()/munmap() pair from inside the batchinsert
loop over files.  Not really sure why that was there in the first place;
maybe to sync the header to disk more forcefully?  In any case, no
change in behaviour should happen as a result of this code change.

     return 0;
   }
   return -1; // no command found
 }
 
-/* Make a new database
-
-   The database consists of:
-
-   header
-   ---------------------------------------------------------------------------------
-   | magic 4 bytes| numFiles 4 bytes | dim 4 bytes | length 4 bytes |flags 4 bytes |
-   ---------------------------------------------------------------------------------
-   
-
-   keyTable : list of keys of tracks
-   --------------------------------------------------------------------------
-   | key 256 bytes                                                          |
-   --------------------------------------------------------------------------
-   O2_MAXFILES*O2_FILENAMELENGTH
-
-   trackTable : Maps implicit feature index to a feature vector matrix
-   --------------------------------------------------------------------------
-   | numVectors (4 bytes)                                                   |
-   --------------------------------------------------------------------------
-   O2_MAXFILES * O2_MEANNUMFEATURES * sizeof(INT)
-
-   featureTable
-   --------------------------------------------------------------------------
-   | v1 v2 v3 ... vd (double)                                               |
-   --------------------------------------------------------------------------
-   O2_MAXFILES * O2_MEANNUMFEATURES * DIM * sizeof(DOUBLE)
-
-   timesTable
-   --------------------------------------------------------------------------
-   | timestamp (double)                                                     |
-   --------------------------------------------------------------------------
-   O2_MAXFILES * O2_MEANNUMFEATURES * sizeof(DOUBLE)
-
-   l2normTable
-   --------------------------------------------------------------------------
-   | nm (double)                                                            |
-   --------------------------------------------------------------------------
-   O2_MAXFILES * O2_MEANNUMFEATURES * sizeof(DOUBLE)
-
-*/
-
 void audioDB::get_lock(int fd, bool exclusive) {
   struct flock lock;
   int status;
   
   lock.l_type = exclusive ? F_WRLCK : F_RDLCK;

 
   if (status)
     error("fcntl unlock error", "", "fcntl");
 }
 
+/* Make a new database.
+
+   The database consists of:
+
+   * a header (see dbTableHeader struct definition);
+   * keyTable: list of keys of tracks;
+   * trackTable: Maps implicit feature index to a feature vector
+   matrix (sizes of tracks)
+   * featureTable: Lots of doubles;
+   * timesTable: time points for each feature vector;
+   * l2normTable: squared l2norms for each feature vector.
+*/
 void audioDB::create(const char* dbName){
   if ((dbfid = open (dbName, O_RDWR|O_CREAT|O_EXCL, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)) < 0)
     error("Can't create database file", dbName, "open");
   get_lock(dbfid, 1);
 

 
 void audioDB::drop(){
   // FIXME: drop something?  Should we even allow this?
 }
 
-// initTables - memory map files passed as arguments
-// Precondition: database has already been created
-void audioDB::initTables(const char* dbName, bool forWrite, const char* inFile = 0) {
+void audioDB::initDBHeader(const char* dbName, bool forWrite) {
   if ((dbfid = open(dbName, forWrite ? O_RDWR : O_RDONLY)) < 0) {
     error("Can't open database file", dbName, "open");
   }
+
   get_lock(dbfid, forWrite);
-
-  // open the input file
-  if (inFile && (infid = open(inFile, O_RDONLY)) < 0) {
-    error("can't open input file for reading", inFile, "open");
-  }
-  // find size of input file
-  if (inFile && fstat(infid, &statbuf) < 0) {
-    error("fstat error finding size of input", inFile, "fstat");
-  }
   // Get the database header info
   dbH = new dbTableHeaderT();
   assert(dbH);
   
   if(read(dbfid, (char *) dbH, O2_HEADERSIZE) != O2_HEADERSIZE) {

 
   if(dbH->version != O2_FORMAT_VERSION) {
     error("database file has incorect version", dbName);
   }
 
+  // mmap the database file
+  if ((db = (char*) mmap(0, O2_DEFAULTDBSIZE, PROT_READ | (forWrite ? PROT_WRITE : 0),
+			 MAP_SHARED, dbfid, 0)) == (caddr_t) -1)
+    error("mmap error for initting tables of database", "", "mmap");
+
+  // Make some handy tables with correct types
+  fileTable = (char *) (db + dbH->fileTableOffset);
+  trackTable = (unsigned *) (db + dbH->trackTableOffset);
+  dataBuf = (double *) (db + dbH->dataOffset);
+  l2normTable = (double *) (db + dbH->l2normTableOffset);
+  timesTable = (double *) (db + dbH->timesTableOffset);
+}
+
+void audioDB::initTables(const char* dbName, bool forWrite, const char* inFile = 0) {
+  
+  initDBHeader(dbName, forWrite);
+
+  // open the input file
+  if (inFile && (infid = open(inFile, O_RDONLY)) < 0) {
+    error("can't open input file for reading", inFile, "open");
+  }
+  // find size of input file
+  if (inFile && fstat(infid, &statbuf) < 0) {
+    error("fstat error finding size of input", inFile, "fstat");
+  }
   if(inFile)
     if(dbH->dim == 0 && dbH->length == 0) // empty database
       // initialize with input dimensionality
       read(infid, &dbH->dim, sizeof(unsigned)); 
     else {

   
   // mmap the input file 
   if (inFile && (indata = (char*)mmap (0, statbuf.st_size, PROT_READ, MAP_SHARED, infid, 0))
       == (caddr_t) -1)
     error("mmap error for input", inFile, "mmap");
-
-  // mmap the database file
-  if ((db = (char*) mmap(0, O2_DEFAULTDBSIZE, PROT_READ | (forWrite ? PROT_WRITE : 0),
-			 MAP_SHARED, dbfid, 0)) == (caddr_t) -1)
-    error("mmap error for initting tables of database", "", "mmap");
-
-  // Make some handy tables with correct types
-  fileTable= (char*)(db+dbH->fileTableOffset);
-  trackTable = (unsigned*)(db+dbH->trackTableOffset);
-  dataBuf  = (double*)(db+dbH->dataOffset);
-  l2normTable = (double*)(db+dbH->l2normTableOffset);
-  timesTable = (double*)(db+dbH->timesTableOffset);
 }
 
 void audioDB::insert(const char* dbName, const char* inFile){
 
   initTables(dbName, 1, inFile);

      }
    }
  }
 }
 
-void audioDB::batchinsert(const char* dbName, const char* inFile){
-
-  if ((dbfid = open (dbName, O_RDWR)) < 0)
-    error("Can't open database file", dbName, "open");
-  get_lock(dbfid, 1);
+void audioDB::batchinsert(const char* dbName, const char* inFile) {
+
+  initDBHeader(dbName, true);
 
   if(!key)
     key=inFile;
   ifstream *filesIn = 0;
   ifstream *keysIn = 0;

     error("Could not open batch in file", inFile);
   if(key && key!=inFile)
     if(!(keysIn = new ifstream(key)))
       error("Could not open batch key file",key);
   
-  // Get the database header info
-  dbH = new dbTableHeaderT();
-  assert(dbH);
-  
-  if(read(dbfid,(char*)dbH,sizeof(dbTableHeaderT))!=sizeof(dbTableHeaderT))
-    error("error reading db header");
-
   if(!usingTimes && (dbH->flags & O2_FLAG_TIMES))
     error("Must use timestamps with timestamped database","use --times");
 
-  if(dbH->magic!=O2_MAGIC){
-    cerr << "expected:" << O2_MAGIC << ", got:" << dbH->magic << endl;
-    error("database file has incorrect header",dbName);
-  }
-  
   unsigned totalVectors=0;
   char *thisKey = new char[MAXSTR];
   char *thisFile = new char[MAXSTR];
   char *thisTimesFileName = new char[MAXSTR];
-    
+  
   do{
     filesIn->getline(thisFile,MAXSTR);
     if(key && key!=inFile)
       keysIn->getline(thisKey,MAXSTR);
     else

       error("can't open feature file for reading", thisFile, "open");
   
     // find size of input file
     if (thisFile && fstat (infid,&statbuf) < 0)
       error("fstat error finding size of input", "", "fstat");
-
-    // mmap the database file
-    if ((db = (char*) mmap(0, O2_DEFAULTDBSIZE, PROT_READ | PROT_WRITE,
-			   MAP_SHARED, dbfid, 0)) == (caddr_t) -1)
-      error("mmap error for batchinsert into database", "", "mmap");
-    
-    // Make some handy tables with correct types
-    fileTable= (char*)(db+dbH->fileTableOffset);
-    trackTable = (unsigned*)(db+dbH->trackTableOffset);
-    dataBuf  = (double*)(db+dbH->dataOffset);
-    l2normTable = (double*)(db+dbH->l2normTableOffset);
-    timesTable = (double*)(db+dbH->timesTableOffset);
 
     // Check that there is room for at least 1 more file
     if((char*)timesTable<((char*)dataBuf+(dbH->length+statbuf.st_size-sizeof(int))))
       error("No more room in database","insert failed: reason database is full.");
     

       }
     }
     // CLEAN UP
     munmap(indata,statbuf.st_size);
     close(infid);
-    munmap(db,O2_DEFAULTDBSIZE);
   }while(!filesIn->eof());
 
-  // mmap the database file
-  if ((db = (char*) mmap(0, O2_DEFAULTDBSIZE, PROT_READ | PROT_WRITE,
-			 MAP_SHARED, dbfid, 0)) == (caddr_t) -1)
-    error("mmap error for creating database", "", "mmap");
-  
   if(verbosity) {
     cerr << COM_BATCHINSERT << " " << dbName << " " << totalVectors << " vectors " 
 	 << totalVectors*dbH->dim*sizeof(double) << " bytes." << endl;
   }
   
   // Report status
   status(dbName);
-  
-  munmap(db,O2_DEFAULTDBSIZE);
 }
 
 // FIXME: this can't propagate the sequence length argument (used for
 // dudCount).  See adb__status() definition for the other half of
 // this.  -- CSR, 2007-10-01