Mercurial > hg > audiodb

changeset 318:c270d9e4659a large_adb

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Ammended batchinsert and status to memory map featureFilenameTables, powerFileNameTable and timesFileNameTable for O2_FLAG_LARGE_ADB
author mas01mc
date Tue, 19 Aug 2008 15:50:26 +0000
parents ab411674dad4
children b9eff6896943
files audioDB.cpp audioDB.h common.cpp insert.cpp
diffstat 4 files changed, 44 insertions(+), 14 deletions(-) [+]
line wrap: on
line diff
--- a/audioDB.cpp	Tue Aug 19 14:38:39 2008 +0000
+++ b/audioDB.cpp	Tue Aug 19 15:50:26 2008 +0000
@@ -125,6 +125,12 @@
     munmap(powerTable, powerTableLength);
   if(l2normTable)
     munmap(l2normTable, l2normTableLength);
+  if(featureFileNameTable)
+    munmap(featureFileNameTable, fileTableLength);
+  if(timesFileNameTable)
+    munmap(timesFileNameTable, fileTableLength);
+  if(powerFileNameTable)
+    munmap(powerFileNameTable, fileTableLength);
   if(trackOffsetTable)
     delete trackOffsetTable;
   if(reporter)
@@ -542,7 +548,7 @@
 	      << "] power[" << DISPLAY_FLAG(dbH->flags&O2_FLAG_POWER)
 	      << "] times[" << DISPLAY_FLAG(dbH->flags&O2_FLAG_TIMES) 
 	      << "] largeADB[" << DISPLAY_FLAG(dbH->flags&O2_FLAG_LARGE_ADB)
-	      << endl;    
+	      << "]" << endl;    
               
     std::cout << "null count: " << nullCount << " small sequence count " << dudCount-nullCount << std::endl;    
   } else {
@@ -558,7 +564,7 @@
 void audioDB::l2norm(const char* dbName) {
   forWrite = true;
   initTables(dbName, 0);
-  if(dbH->length>0){
+  if( !(dbH->flags & O2_FLAG_LARGE_ADB ) && (dbH->length>0) ){
     /* FIXME: should probably be uint64_t */
     unsigned numVectors = dbH->length/(sizeof(double)*dbH->dim);
     CHECKED_MMAP(double *, dataBuf, dbH->dataOffset, dataBufLength);
@@ -571,8 +577,8 @@
 
 void audioDB::power_flag(const char *dbName) {
   forWrite = true;
-  initTables(dbName, 0);
-  if (dbH->length > 0) {
+  initTables(dbName, 0);  
+  if( !(dbH->flags & O2_FLAG_LARGE_ADB ) && (dbH->length>0) ){
     error("cannot turn on power storage for non-empty database", dbName);
   }
   dbH->flags |= O2_FLAG_POWER;
--- a/audioDB.h	Tue Aug 19 14:38:39 2008 +0000
+++ b/audioDB.h	Tue Aug 19 15:50:26 2008 +0000
@@ -222,6 +222,10 @@
   double* timesTable;
   double* powerTable;
 
+  char* featureFileNameTable;
+  char* timesFileNameTable;
+  char* powerFileNameTable;
+
   size_t fileTableLength;
   size_t trackTableLength;
   off_t dataBufLength;
@@ -395,6 +399,9 @@
     l2normTable(0),				\
     timesTable(0),				\
     powerTable(0),                              \
+    featureFileNameTable(0),                    \
+    timesFileNameTable(0),                      \
+    powerFileNameTable(0),                      \
     fileTableLength(0),				\
     trackTableLength(0),			\
     dataBufLength(0),				\
--- a/common.cpp	Tue Aug 19 14:38:39 2008 +0000
+++ b/common.cpp	Tue Aug 19 15:50:26 2008 +0000
@@ -126,10 +126,18 @@
     } else {
       fileTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLE_ENTRY_SIZE);
       trackTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_TRACKTABLE_ENTRY_SIZE);
-      dataBufLength = ALIGN_PAGE_UP(dbH->length);
-      timesTableLength = ALIGN_PAGE_UP(2*(dbH->length / dbH->dim));
-      powerTableLength = ALIGN_PAGE_UP(dbH->length / dbH->dim);
-      l2normTableLength = ALIGN_PAGE_UP(dbH->length / dbH->dim);
+      if( dbH->flags & O2_FLAG_LARGE_ADB ){
+	dataBufLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLE_ENTRY_SIZE);
+	timesTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLE_ENTRY_SIZE);
+	powerTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLE_ENTRY_SIZE);
+	l2normTableLength = 0;
+      }
+      else{
+	dataBufLength = ALIGN_PAGE_UP(dbH->length);
+	timesTableLength = ALIGN_PAGE_UP(2*(dbH->length / dbH->dim));
+	powerTableLength = ALIGN_PAGE_UP(dbH->length / dbH->dim);
+	l2normTableLength = ALIGN_PAGE_UP(dbH->length / dbH->dim);
+      }
     }
     CHECKED_MMAP(char *, fileTable, dbH->fileTableOffset, fileTableLength);
     CHECKED_MMAP(unsigned *, trackTable, dbH->trackTableOffset, trackTableLength);
@@ -143,9 +151,18 @@
      *
      * CHECKED_MMAP(double *, dataBuf, dbH->dataOffset, dataBufLength);
      */
-    CHECKED_MMAP(double *, timesTable, dbH->timesTableOffset, timesTableLength);
-    CHECKED_MMAP(double *, powerTable, dbH->powerTableOffset, powerTableLength);
-    CHECKED_MMAP(double *, l2normTable, dbH->l2normTableOffset, l2normTableLength);
+    if( dbH->flags & O2_FLAG_LARGE_ADB ){
+      CHECKED_MMAP(char *, featureFileNameTable, dbH->dataOffset, fileTableLength);
+      if( dbH->flags & O2_FLAG_TIMES )
+	CHECKED_MMAP(char *, timesFileNameTable, dbH->timesTableOffset, fileTableLength);
+      if( dbH->flags & O2_FLAG_POWER )
+	CHECKED_MMAP(char *, powerFileNameTable, dbH->powerTableOffset, fileTableLength);
+    }
+    else{
+      CHECKED_MMAP(double *, timesTable, dbH->timesTableOffset, timesTableLength);
+      CHECKED_MMAP(double *, powerTable, dbH->powerTableOffset, powerTableLength);
+      CHECKED_MMAP(double *, l2normTable, dbH->l2normTableOffset, l2normTableLength);
+    }
   }
 
   // build track offset table
--- a/insert.cpp	Tue Aug 19 14:38:39 2008 +0000
+++ b/insert.cpp	Tue Aug 19 15:50:26 2008 +0000
@@ -465,16 +465,16 @@
 	INSERT_FILETABLE_STRING(fileTable, thisKey);
 	
 	// Feature Vector fileNames
-	INSERT_FILETABLE_STRING(dbH->dataOffset, thisFile);
+	INSERT_FILETABLE_STRING(featureFileNameTable, thisFile);
 	
 	// Time Stamp fileNames
 	if(usingTimes)
-	  INSERT_FILETABLE_STRING(dbH->timesTableOffset, thisTimesFileName);
+	  INSERT_FILETABLE_STRING(timesFileNameTable, thisTimesFileName);
 
 
 	// Power fileNames
 	if(usingPower)
-	  INSERT_FILETABLE_STRING(dbH->powerTableOffset, thisPowerFileName);
+	  INSERT_FILETABLE_STRING(powerFileNameTable, thisPowerFileName);
 
 	// Increment file count
 	dbH->numFiles++;