Mercurial > hg > audiodb

--- a/Makefile	Fri Oct 05 15:42:50 2007 +0000
+++ b/Makefile	Wed Oct 17 14:14:02 2007 +0000
@@ -19,7 +19,7 @@
 soapServer.cpp soapClient.cpp soapC.cpp: audioDBws.h
 	soapcpp2 audioDBws.h

-${EXECUTABLE}: audioDB.cpp soapServer.cpp soapClient.cpp soapC.cpp cmdline.c cmdline.h
+${EXECUTABLE}: audioDB.cpp audioDB.h soapServer.cpp soapClient.cpp soapC.cpp cmdline.c cmdline.h
 	g++ -c ${CFLAGS} -Wall -Werror audioDB.cpp
 	g++ -o ${EXECUTABLE} ${CFLAGS} audioDB.o soapServer.cpp soapClient.cpp soapC.cpp cmdline.c ${LIBS}
--- a/audioDB.cpp	Fri Oct 05 15:42:50 2007 +0000
+++ b/audioDB.cpp	Wed Oct 17 14:14:02 2007 +0000
@@ -106,6 +106,7 @@
 }

 void audioDB::cleanup() {
+  cmdline_parser_free(&args_info);
   if(indata)
     munmap(indata,statbuf.st_size);
   if(db)
@@ -314,48 +315,6 @@
   return -1; // no command found
 }

-/* Make a new database
-
-   The database consists of:
-
-   header
-   ---------------------------------------------------------------------------------
-   | magic 4 bytes| numFiles 4 bytes | dim 4 bytes | length 4 bytes |flags 4 bytes |
-   ---------------------------------------------------------------------------------
-
-
-   keyTable : list of keys of tracks
-   --------------------------------------------------------------------------
-   | key 256 bytes                                                          |
-   --------------------------------------------------------------------------
-   O2_MAXFILES*O2_FILENAMELENGTH
-
-   trackTable : Maps implicit feature index to a feature vector matrix
-   --------------------------------------------------------------------------
-   | numVectors (4 bytes)                                                   |
-   --------------------------------------------------------------------------
-   O2_MAXFILES * O2_MEANNUMFEATURES * sizeof(INT)
-
-   featureTable
-   --------------------------------------------------------------------------
-   | v1 v2 v3 ... vd (double)                                               |
-   --------------------------------------------------------------------------
-   O2_MAXFILES * O2_MEANNUMFEATURES * DIM * sizeof(DOUBLE)
-
-   timesTable
-   --------------------------------------------------------------------------
-   | timestamp (double)                                                     |
-   --------------------------------------------------------------------------
-   O2_MAXFILES * O2_MEANNUMFEATURES * sizeof(DOUBLE)
-
-   l2normTable
-   --------------------------------------------------------------------------
-   | nm (double)                                                            |
-   --------------------------------------------------------------------------
-   O2_MAXFILES * O2_MEANNUMFEATURES * sizeof(DOUBLE)
-
-*/
-
 void audioDB::get_lock(int fd, bool exclusive) {
   struct flock lock;
   int status;
@@ -395,6 +354,18 @@
     error("fcntl unlock error", "", "fcntl");
 }

+/* Make a new database.
+
+   The database consists of:
+
+   * a header (see dbTableHeader struct definition);
+   * keyTable: list of keys of tracks;
+   * trackTable: Maps implicit feature index to a feature vector
+   matrix (sizes of tracks)
+   * featureTable: Lots of doubles;
+   * timesTable: time points for each feature vector;
+   * l2normTable: squared l2norms for each feature vector.
+*/
 void audioDB::create(const char* dbName){
   if ((dbfid = open (dbName, O_RDWR|O_CREAT|O_EXCL, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)) < 0)
     error("Can't create database file", dbName, "open");
@@ -443,22 +414,12 @@
   // FIXME: drop something?  Should we even allow this?
 }

-// initTables - memory map files passed as arguments
-// Precondition: database has already been created
-void audioDB::initTables(const char* dbName, bool forWrite, const char* inFile = 0) {
+void audioDB::initDBHeader(const char* dbName, bool forWrite) {
   if ((dbfid = open(dbName, forWrite ? O_RDWR : O_RDONLY)) < 0) {
     error("Can't open database file", dbName, "open");
   }
+
   get_lock(dbfid, forWrite);
-
-  // open the input file
-  if (inFile && (infid = open(inFile, O_RDONLY)) < 0) {
-    error("can't open input file for reading", inFile, "open");
-  }
-  // find size of input file
-  if (inFile && fstat(infid, &statbuf) < 0) {
-    error("fstat error finding size of input", inFile, "fstat");
-  }
   // Get the database header info
   dbH = new dbTableHeaderT();
   assert(dbH);
@@ -482,6 +443,31 @@
     error("database file has incorect version", dbName);
   }

+  // mmap the database file
+  if ((db = (char*) mmap(0, O2_DEFAULTDBSIZE, PROT_READ | (forWrite ? PROT_WRITE : 0),
+			 MAP_SHARED, dbfid, 0)) == (caddr_t) -1)
+    error("mmap error for initting tables of database", "", "mmap");
+
+  // Make some handy tables with correct types
+  fileTable = (char *) (db + dbH->fileTableOffset);
+  trackTable = (unsigned *) (db + dbH->trackTableOffset);
+  dataBuf = (double *) (db + dbH->dataOffset);
+  l2normTable = (double *) (db + dbH->l2normTableOffset);
+  timesTable = (double *) (db + dbH->timesTableOffset);
+}
+
+void audioDB::initTables(const char* dbName, bool forWrite, const char* inFile = 0) {
+
+  initDBHeader(dbName, forWrite);
+
+  // open the input file
+  if (inFile && (infid = open(inFile, O_RDONLY)) < 0) {
+    error("can't open input file for reading", inFile, "open");
+  }
+  // find size of input file
+  if (inFile && fstat(infid, &statbuf) < 0) {
+    error("fstat error finding size of input", inFile, "fstat");
+  }
   if(inFile)
     if(dbH->dim == 0 && dbH->length == 0) // empty database
       // initialize with input dimensionality
@@ -499,18 +485,6 @@
   if (inFile && (indata = (char*)mmap (0, statbuf.st_size, PROT_READ, MAP_SHARED, infid, 0))
       == (caddr_t) -1)
     error("mmap error for input", inFile, "mmap");
-
-  // mmap the database file
-  if ((db = (char*) mmap(0, O2_DEFAULTDBSIZE, PROT_READ | (forWrite ? PROT_WRITE : 0),
-			 MAP_SHARED, dbfid, 0)) == (caddr_t) -1)
-    error("mmap error for initting tables of database", "", "mmap");
-
-  // Make some handy tables with correct types
-  fileTable= (char*)(db+dbH->fileTableOffset);
-  trackTable = (unsigned*)(db+dbH->trackTableOffset);
-  dataBuf  = (double*)(db+dbH->dataOffset);
-  l2normTable = (double*)(db+dbH->l2normTableOffset);
-  timesTable = (double*)(db+dbH->timesTableOffset);
 }

 void audioDB::insert(const char* dbName, const char* inFile){
@@ -649,11 +623,9 @@
  }
 }

-void audioDB::batchinsert(const char* dbName, const char* inFile){
+void audioDB::batchinsert(const char* dbName, const char* inFile) {

-  if ((dbfid = open (dbName, O_RDWR)) < 0)
-    error("Can't open database file", dbName, "open");
-  get_lock(dbfid, 1);
+  initDBHeader(dbName, true);

   if(!key)
     key=inFile;
@@ -667,26 +639,14 @@
     if(!(keysIn = new ifstream(key)))
       error("Could not open batch key file",key);

-  // Get the database header info
-  dbH = new dbTableHeaderT();
-  assert(dbH);
-
-  if(read(dbfid,(char*)dbH,sizeof(dbTableHeaderT))!=sizeof(dbTableHeaderT))
-    error("error reading db header");
-
   if(!usingTimes && (dbH->flags & O2_FLAG_TIMES))
     error("Must use timestamps with timestamped database","use --times");

-  if(dbH->magic!=O2_MAGIC){
-    cerr << "expected:" << O2_MAGIC << ", got:" << dbH->magic << endl;
-    error("database file has incorrect header",dbName);
-  }
-
   unsigned totalVectors=0;
   char *thisKey = new char[MAXSTR];
   char *thisFile = new char[MAXSTR];
   char *thisTimesFileName = new char[MAXSTR];
-
+
   do{
     filesIn->getline(thisFile,MAXSTR);
     if(key && key!=inFile)
@@ -707,18 +667,6 @@
     if (thisFile && fstat (infid,&statbuf) < 0)
       error("fstat error finding size of input", "", "fstat");

-    // mmap the database file
-    if ((db = (char*) mmap(0, O2_DEFAULTDBSIZE, PROT_READ | PROT_WRITE,
-			   MAP_SHARED, dbfid, 0)) == (caddr_t) -1)
-      error("mmap error for batchinsert into database", "", "mmap");
-
-    // Make some handy tables with correct types
-    fileTable= (char*)(db+dbH->fileTableOffset);
-    trackTable = (unsigned*)(db+dbH->trackTableOffset);
-    dataBuf  = (double*)(db+dbH->dataOffset);
-    l2normTable = (double*)(db+dbH->l2normTableOffset);
-    timesTable = (double*)(db+dbH->timesTableOffset);
-
     // Check that there is room for at least 1 more file
     if((char*)timesTable<((char*)dataBuf+(dbH->length+statbuf.st_size-sizeof(int))))
       error("No more room in database","insert failed: reason database is full.");
@@ -809,14 +757,8 @@
     // CLEAN UP
     munmap(indata,statbuf.st_size);
     close(infid);
-    munmap(db,O2_DEFAULTDBSIZE);
   }while(!filesIn->eof());

-  // mmap the database file
-  if ((db = (char*) mmap(0, O2_DEFAULTDBSIZE, PROT_READ | PROT_WRITE,
-			 MAP_SHARED, dbfid, 0)) == (caddr_t) -1)
-    error("mmap error for creating database", "", "mmap");
-
   if(verbosity) {
     cerr << COM_BATCHINSERT << " " << dbName << " " << totalVectors << " vectors "
 	 << totalVectors*dbH->dim*sizeof(double) << " bytes." << endl;
@@ -824,8 +766,6 @@

   // Report status
   status(dbName);
-
-  munmap(db,O2_DEFAULTDBSIZE);
 }

 // FIXME: this can't propagate the sequence length argument (used for
@@ -1891,8 +1831,12 @@
     delete[] trackOffsetTable;
   if(queryCopy)
     delete[] queryCopy;
-  //if(qNorm)
-  //delete qNorm;
+  if(qNorm)
+    delete[] qNorm;
+  if(sNorm)
+    delete[] sNorm;
+  if(sMeanL2)
+    delete[] sMeanL2;
   if(D)
     delete[] D;
   if(DD)
@@ -2361,14 +2305,17 @@
     }
   }

-
   // Clean up
   if(trackOffsetTable)
     delete[] trackOffsetTable;
   if(queryCopy)
     delete[] queryCopy;
-  //if(qNorm)
-  //delete qNorm;
+  if(qNorm)
+    delete[] qNorm;
+  if(sNorm)
+    delete[] sNorm;
+  if(sMeanL2)
+    delete[] sMeanL2;
   if(D)
     delete[] D;
   if(DD)
--- a/audioDB.h	Fri Oct 05 15:42:50 2007 +0000
+++ b/audioDB.h	Wed Oct 17 14:14:02 2007 +0000
@@ -86,22 +86,23 @@
 using namespace std;

 typedef struct dbTableHeader{
-  unsigned magic;
-  unsigned version;
-  unsigned numFiles;
-  unsigned dim;
-  unsigned flags;
+  uint32_t magic;
+  uint32_t version;
+  uint32_t numFiles;
+  uint32_t dim;
+  uint32_t flags;
   // FIXME: these lengths and offsets should be size_t or off_t, but
   // that causes this header (and hence audioDB files) to be
   // unportable between 32 and 64-bit architectures.  Making them
-  // unsigned isn't the real answer, but it works around the problem.
-  // -- CSR, 2007-10-05
-  unsigned length;
-  unsigned fileTableOffset;
-  unsigned trackTableOffset;
-  unsigned dataOffset;
-  unsigned l2normTableOffset;
-  unsigned timesTableOffset;
+  // uint32_t isn't the real answer, as it means we won't be able to
+  // scale to really large collections easily but it works around the
+  // problem.  -- CSR, 2007-10-05
+  uint32_t length;
+  uint32_t fileTableOffset;
+  uint32_t trackTableOffset;
+  uint32_t dataOffset;
+  uint32_t l2normTableOffset;
+  uint32_t timesTableOffset;
 } dbTableHeaderT, *dbTableHeaderPtr;


@@ -163,6 +164,7 @@
   void trackSequenceQueryNN(const char* dbName, const char* inFile, adb__queryResult *adbQueryResult=0);
   void trackSequenceQueryRad(const char* dbName, const char* inFile, adb__queryResult *adbQueryResult=0);

+  void initDBHeader(const char *dbName, bool forWrite);
   void initTables(const char* dbName, bool forWrite, const char* inFile);
   void unitNorm(double* X, unsigned d, unsigned n, double* qNorm);
   void unitNormAndInsertL2(double* X, unsigned dim, unsigned n, unsigned append);
--- a/debian/changelog	Fri Oct 05 15:42:50 2007 +0000
+++ b/debian/changelog	Wed Oct 17 14:14:02 2007 +0000
@@ -1,3 +1,10 @@
+audiodb (1.0-11) unstable; urgency=low
+
+  * Updated to svn version #142
+  * (fixes some memory leaks in the server process)
+
+ -- Christophe Rhodes <c.rhodes@gold.ac.uk>  Wed, 17 Oct 2007 15:10:42 +0100
+
 audiodb (1.0-10) unstable; urgency=low

   * Updated to svn version #128