Mercurial > hg > audiodb

changeset 177:c32bf13c3978 no-big-mmap

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
forWrite audioDB flag (rather than function argument) so that we can tell when it comes to munmap() how much we need to. ALIGN_PAGE_FOO() takes one argument not two.
author mas01cr
date Thu, 15 Nov 2007 14:58:16 +0000
parents 8193dbd66e34
children 7bbe5d48a7ef
files audioDB.cpp audioDB.h
diffstat 2 files changed, 46 insertions(+), 32 deletions(-) [+]
line wrap: on
line diff
--- a/audioDB.cpp	Wed Nov 14 17:58:57 2007 +0000
+++ b/audioDB.cpp	Thu Nov 15 14:58:16 2007 +0000
@@ -406,11 +406,11 @@
   dbH->dim = 0;
   dbH->flags = 0;
   dbH->length = 0;
-  dbH->fileTableOffset = ALIGN_PAGE_UP(O2_HEADERSIZE, 8);
-  dbH->trackTableOffset = ALIGN_PAGE_UP(dbH->fileTableOffset + O2_FILETABLESIZE*maxfiles, 8);
-  dbH->dataOffset = ALIGN_PAGE_UP(dbH->trackTableOffset + O2_TRACKTABLESIZE*maxfiles, 8);
-  dbH->l2normTableOffset = ALIGN_PAGE_DOWN(size - maxfiles*O2_MEANNUMVECTORS*sizeof(double), 8);
-  dbH->timesTableOffset = ALIGN_PAGE_DOWN(dbH->l2normTableOffset - maxfiles*O2_MEANNUMVECTORS*sizeof(double), 8);
+  dbH->fileTableOffset = ALIGN_PAGE_UP(O2_HEADERSIZE);
+  dbH->trackTableOffset = ALIGN_PAGE_UP(dbH->fileTableOffset + O2_FILETABLESIZE*maxfiles);
+  dbH->dataOffset = ALIGN_PAGE_UP(dbH->trackTableOffset + O2_TRACKTABLESIZE*maxfiles);
+  dbH->l2normTableOffset = ALIGN_PAGE_DOWN(size - maxfiles*O2_MEANNUMVECTORS*sizeof(double));
+  dbH->timesTableOffset = ALIGN_PAGE_DOWN(dbH->l2normTableOffset - maxfiles*O2_MEANNUMVECTORS*sizeof(double));
   dbH->dbSize = size;
 
   write(dbfid, dbH, O2_HEADERSIZE);
@@ -432,7 +432,7 @@
   // FIXME: drop something?  Should we even allow this?
 }
 
-void audioDB::initDBHeader(const char* dbName, bool forWrite) {
+void audioDB::initDBHeader(const char* dbName) {
   if ((dbfid = open(dbName, forWrite ? O_RDWR : O_RDONLY)) < 0) {
     error("Can't open database file", dbName, "open");
   }
@@ -477,11 +477,22 @@
 
   CHECKED_MMAP(char *, db, 0, getpagesize());
 
-  CHECKED_MMAP(char *, fileTable, dbH->fileTableOffset, dbH->trackTableOffset);
-  CHECKED_MMAP(unsigned *, trackTable, dbH->trackTableOffset, dbH->dataOffset);
-  CHECKED_MMAP(double *, dataBuf, dbH->dataOffset, dbH->timesTableOffset);
-  CHECKED_MMAP(double *, timesTable, dbH->timesTableOffset, dbH->l2normTableOffset);
-  CHECKED_MMAP(double *, l2normTable, dbH->l2normTableOffset, dbH->dbSize);
+  if(forWrite) {
+    CHECKED_MMAP(char *, fileTable, dbH->fileTableOffset, dbH->trackTableOffset);
+
+    CHECKED_MMAP(unsigned *, trackTable, dbH->trackTableOffset, dbH->dataOffset);
+    CHECKED_MMAP(double *, dataBuf, dbH->dataOffset, dbH->timesTableOffset);
+    CHECKED_MMAP(double *, timesTable, dbH->timesTableOffset, dbH->l2normTableOffset);
+    CHECKED_MMAP(double *, l2normTable, dbH->l2normTableOffset, dbH->dbSize);
+  } else {
+    if(dbH->length > 0) {
+      CHECKED_MMAP(char *, fileTable, dbH->fileTableOffset, dbH->fileTableOffset + ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLESIZE));
+      CHECKED_MMAP(unsigned *, trackTable, dbH->trackTableOffset, dbH->trackTableOffset + ALIGN_PAGE_UP(dbH->numFiles * O2_TRACKTABLESIZE));
+      CHECKED_MMAP(double *, dataBuf, dbH->dataOffset, dbH->dataOffset + ALIGN_PAGE_UP(dbH->length));
+      CHECKED_MMAP(double *, timesTable, dbH->timesTableOffset, dbH->timesTableOffset + ALIGN_PAGE_UP(dbH->length / dbH->dim));
+      CHECKED_MMAP(double *, l2normTable, dbH->l2normTableOffset, dbH->l2normTableOffset + ALIGN_PAGE_UP(dbH->length / dbH->dim));
+    }
+  }
 }
 
 void audioDB::initInputFile (const char *inFile) {
@@ -522,15 +533,14 @@
   }
 }
 
-void audioDB::initTables(const char* dbName, bool forWrite, const char* inFile = 0) {
-  
-  initDBHeader(dbName, forWrite);
+void audioDB::initTables(const char* dbName, const char* inFile = 0) {
+  initDBHeader(dbName);
   initInputFile(inFile);
 }
 
-void audioDB::insert(const char* dbName, const char* inFile){
-
-  initTables(dbName, 1, inFile);
+void audioDB::insert(const char* dbName, const char* inFile) {
+  forWrite = true;
+  initTables(dbName, inFile);
 
   if(!usingTimes && (dbH->flags & O2_FLAG_TIMES))
     error("Must use timestamps with timestamped database","use --times");
@@ -665,7 +675,8 @@
 
 void audioDB::batchinsert(const char* dbName, const char* inFile) {
 
-  initDBHeader(dbName, true);
+  forWrite = true;
+  initDBHeader(dbName);
 
   if(!key)
     key=inFile;
@@ -834,7 +845,7 @@
 
 void audioDB::status(const char* dbName, adb__statusResponse *adbStatusResponse){
   if(!dbH)
-    initTables(dbName, 0, 0);
+    initTables(dbName, 0);
 
   unsigned dudCount=0;
   unsigned nullCount=0;
@@ -873,7 +884,7 @@
 
 void audioDB::dump(const char* dbName){
   if(!dbH) {
-    initTables(dbName, 0, 0);
+    initTables(dbName, 0);
   }
 
   if((mkdir(output, S_IRWXU|S_IRWXG|S_IRWXO)) < 0) {
@@ -986,8 +997,9 @@
   status(dbName);
 }
 
-void audioDB::l2norm(const char* dbName){
-  initTables(dbName, true, 0);
+void audioDB::l2norm(const char* dbName) {
+  forWrite = true;
+  initTables(dbName, 0);
   if(dbH->length>0){
     unsigned numVectors = dbH->length/(sizeof(double)*dbH->dim);
     unitNormAndInsertL2(dataBuf, dbH->dim, numVectors, 0); // No append
@@ -1029,9 +1041,9 @@
 }
 
 // Basic point query engine
-void audioDB::pointQuery(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse){
+void audioDB::pointQuery(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse) {
   
-  initTables(dbName, 0, inFile);
+  initTables(dbName, inFile);
   
   // For each input vector, find the closest pointNN matching output vectors and report
   // we use stdout in this stub version
@@ -1212,8 +1224,8 @@
 // trackPointQuery  
 // return the trackNN closest tracks to the query track
 // uses average of pointNN points per track 
-void audioDB::trackPointQuery(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse){  
-  initTables(dbName, 0, inFile);
+void audioDB::trackPointQuery(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse) {
+  initTables(dbName, inFile);
   
   // For each input vector, find the closest pointNN matching output vectors and report
   unsigned numVectors = (statbuf.st_size-sizeof(int))/(sizeof(double)*dbH->dim);
@@ -1468,7 +1480,7 @@
 // outputs distances of retrieved shingles, max retreived = pointNN shingles per per track
 void audioDB::trackSequenceQueryNN(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse){
   
-  initTables(dbName, 0, inFile);
+  initTables(dbName, inFile);
   
   // For each input vector, find the closest pointNN matching output vectors and report
   // we use stdout in this stub version
@@ -1975,7 +1987,7 @@
 // outputs count of retrieved shingles, max retreived = one shingle per query shingle per track
 void audioDB::trackSequenceQueryRad(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse){
   
-  initTables(dbName, 0, inFile);
+  initTables(dbName, inFile);
   
   // For each input vector, find the closest pointNN matching output vectors and report
   // we use stdout in this stub version
--- a/audioDB.h	Wed Nov 14 17:58:57 2007 +0000
+++ b/audioDB.h	Thu Nov 15 14:58:16 2007 +0000
@@ -83,8 +83,8 @@
 #define ALIGN_UP(x,w) ((x) + ((1<<w)-1) & ~((1<<w)-1))
 #define ALIGN_DOWN(x,w) ((x) & ~((1<<w)-1))
 
-#define ALIGN_PAGE_UP(x,w) ((x) + (getpagesize()-1) & ~(getpagesize()-1))
-#define ALIGN_PAGE_DOWN(x,w) ((x) & ~(getpagesize()-1))
+#define ALIGN_PAGE_UP(x) ((x) + (getpagesize()-1) & ~(getpagesize()-1))
+#define ALIGN_PAGE_DOWN(x) ((x) & ~(getpagesize()-1))
 
 #define ENSURE_STRING(x) ((x) ? (x) : "")
 
@@ -130,6 +130,7 @@
   ifstream *timesFile;
 
   int dbfid;
+  bool forWrite;
   int infid;
   char* db;
   char* indata;
@@ -173,9 +174,9 @@
   void trackSequenceQueryNN(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse=0);
   void trackSequenceQueryRad(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse=0);
 
-  void initDBHeader(const char *dbName, bool forWrite);
+  void initDBHeader(const char *dbName);
   void initInputFile(const char *inFile);
-  void initTables(const char* dbName, bool forWrite, const char* inFile);
+  void initTables(const char* dbName, const char* inFile);
   void unitNorm(double* X, unsigned d, unsigned n, double* qNorm);
   void unitNormAndInsertL2(double* X, unsigned dim, unsigned n, unsigned append);
   void insertTimeStamps(unsigned n, ifstream* timesFile, double* timesdata);
@@ -218,6 +219,7 @@
   timesFileName(0), \
   timesFile(0), \
   dbfid(0), \
+  forWrite(false), \
   infid(0), \
   db(0), \
   indata(0), \