# HG changeset patch
# User mas01cr
# Date 1191598970 0
# Node ID ae045842d29fd0b9eb9b94d133f206fddf3bfd73
# Parent  1521d46bc1ac046f404536c9254bc378f5780e8e
Merge trunk changes -r123:128 to audiodb-debian branch

(+ new debian/changelog version)

diff -r 1521d46bc1ac -r ae045842d29f audioDB.cpp
--- a/audioDB.cpp	Fri Oct 05 11:45:03 2007 +0000
+++ b/audioDB.cpp	Fri Oct 05 15:42:50 2007 +0000
@@ -420,11 +420,17 @@
   assert(dbH);
 
   // Initialize header
-  dbH->magic=O2_MAGIC;
-  dbH->numFiles=0;
-  dbH->length=0;
-  dbH->dim=0;
-  dbH->flags=0; //O2_FLAG_L2NORM;
+  dbH->magic = O2_MAGIC;
+  dbH->version = O2_FORMAT_VERSION;
+  dbH->numFiles = 0;
+  dbH->dim = 0;
+  dbH->flags = 0;
+  dbH->length = 0;
+  dbH->fileTableOffset = ALIGN_UP(O2_HEADERSIZE, 8);
+  dbH->trackTableOffset = ALIGN_UP(dbH->fileTableOffset + O2_FILETABLESIZE*O2_MAXFILES, 8);
+  dbH->dataOffset = ALIGN_UP(dbH->trackTableOffset + O2_TRACKTABLESIZE*O2_MAXFILES, 8);
+  dbH->l2normTableOffset = ALIGN_DOWN(O2_DEFAULTDBSIZE - O2_MAXFILES*O2_MEANNUMVECTORS*sizeof(double), 8);
+  dbH->timesTableOffset = ALIGN_DOWN(dbH->l2normTableOffset - O2_MAXFILES*O2_MEANNUMVECTORS*sizeof(double), 8);
 
   memcpy (db, dbH, O2_HEADERSIZE);
   if(verbosity) {
@@ -434,8 +440,7 @@
 
 
 void audioDB::drop(){
-    
-    
+  // FIXME: drop something?  Should we even allow this?
 }
 
 // initTables - memory map files passed as arguments
@@ -462,15 +467,19 @@
     error("error reading db header", dbName, "read");
   }
 
-  fileTableOffset = O2_HEADERSIZE;
-  trackTableOffset = fileTableOffset + O2_FILETABLESIZE*O2_MAXFILES;
-  dataoffset = trackTableOffset + O2_TRACKTABLESIZE*O2_MAXFILES;
-  l2normTableOffset = O2_DEFAULTDBSIZE - O2_MAXFILES*O2_MEANNUMVECTORS*sizeof(double);
-  timesTableOffset = l2normTableOffset - O2_MAXFILES*O2_MEANNUMVECTORS*sizeof(double);
+  if(dbH->magic == O2_OLD_MAGIC) {
+    // FIXME: if anyone ever complains, write the program to convert
+    // from the old audioDB format to the new...
+    error("database file has old O2 header", dbName);
+  }
 
   if(dbH->magic != O2_MAGIC) {
     cerr << "expected: " << O2_MAGIC << ", got: " << dbH->magic << endl;
-    error("database file has incorrect header",dbName);
+    error("database file has incorrect header", dbName);
+  }
+
+  if(dbH->version != O2_FORMAT_VERSION) {
+    error("database file has incorect version", dbName);
   }
 
   if(inFile)
@@ -497,11 +506,11 @@
     error("mmap error for initting tables of database", "", "mmap");
 
   // Make some handy tables with correct types
-  fileTable= (char*)(db+fileTableOffset);
-  trackTable = (unsigned*)(db+trackTableOffset);
-  dataBuf  = (double*)(db+dataoffset);
-  l2normTable = (double*)(db+l2normTableOffset);
-  timesTable = (double*)(db+timesTableOffset);
+  fileTable= (char*)(db+dbH->fileTableOffset);
+  trackTable = (unsigned*)(db+dbH->trackTableOffset);
+  dataBuf  = (double*)(db+dbH->dataOffset);
+  l2normTable = (double*)(db+dbH->l2normTableOffset);
+  timesTable = (double*)(db+dbH->timesTableOffset);
 }
 
 void audioDB::insert(const char* dbName, const char* inFile){
@@ -569,11 +578,11 @@
   memcpy (trackTable+dbH->numFiles-1, &numVectors, sizeof(unsigned));  
 
   // Update the feature database
-  memcpy (db+dataoffset+insertoffset, indata+sizeof(int), statbuf.st_size-sizeof(int));
+  memcpy (db+dbH->dataOffset+insertoffset, indata+sizeof(int), statbuf.st_size-sizeof(int));
   
   // Norm the vectors on input if the database is already L2 normed
   if(dbH->flags & O2_FLAG_L2NORM)
-    unitNormAndInsertL2((double*)(db+dataoffset+insertoffset), dbH->dim, numVectors, 1); // append
+    unitNormAndInsertL2((double*)(db+dbH->dataOffset+insertoffset), dbH->dim, numVectors, 1); // append
 
   // Report status
   status(dbName);
@@ -668,17 +677,10 @@
   if(!usingTimes && (dbH->flags & O2_FLAG_TIMES))
     error("Must use timestamps with timestamped database","use --times");
 
-  fileTableOffset = O2_HEADERSIZE;
-  trackTableOffset = fileTableOffset + O2_FILETABLESIZE*O2_MAXFILES;
-  dataoffset = trackTableOffset + O2_TRACKTABLESIZE*O2_MAXFILES;
-  l2normTableOffset = O2_DEFAULTDBSIZE - O2_MAXFILES*O2_MEANNUMVECTORS*sizeof(double);
-  timesTableOffset = l2normTableOffset - O2_MAXFILES*O2_MEANNUMVECTORS*sizeof(double);
-
   if(dbH->magic!=O2_MAGIC){
     cerr << "expected:" << O2_MAGIC << ", got:" << dbH->magic << endl;
     error("database file has incorrect header",dbName);
   }
-
   
   unsigned totalVectors=0;
   char *thisKey = new char[MAXSTR];
@@ -711,11 +713,11 @@
       error("mmap error for batchinsert into database", "", "mmap");
     
     // Make some handy tables with correct types
-    fileTable= (char*)(db+fileTableOffset);
-    trackTable = (unsigned*)(db+trackTableOffset);
-    dataBuf  = (double*)(db+dataoffset);
-    l2normTable = (double*)(db+l2normTableOffset);
-    timesTable = (double*)(db+timesTableOffset);
+    fileTable= (char*)(db+dbH->fileTableOffset);
+    trackTable = (unsigned*)(db+dbH->trackTableOffset);
+    dataBuf  = (double*)(db+dbH->dataOffset);
+    l2normTable = (double*)(db+dbH->l2normTableOffset);
+    timesTable = (double*)(db+dbH->timesTableOffset);
 
     // Check that there is room for at least 1 more file
     if((char*)timesTable<((char*)dataBuf+(dbH->length+statbuf.st_size-sizeof(int))))
@@ -795,11 +797,11 @@
 	memcpy (trackTable+dbH->numFiles-1, &numVectors, sizeof(unsigned));  
 	
 	// Update the feature database
-	memcpy (db+dataoffset+insertoffset, indata+sizeof(int), statbuf.st_size-sizeof(int));
+	memcpy (db+dbH->dataOffset+insertoffset, indata+sizeof(int), statbuf.st_size-sizeof(int));
 	
 	// Norm the vectors on input if the database is already L2 normed
 	if(dbH->flags & O2_FLAG_L2NORM)
-	  unitNormAndInsertL2((double*)(db+dataoffset+insertoffset), dbH->dim, numVectors, 1); // append
+	  unitNormAndInsertL2((double*)(db+dbH->dataOffset+insertoffset), dbH->dim, numVectors, 1); // append
 	
 	totalVectors+=numVectors;
       }
@@ -895,11 +897,11 @@
     cout << "data dim:" << dbH->dim <<endl;
     if(dbH->dim>0){
       cout << "total vectors:" << dbH->length/(sizeof(double)*dbH->dim)<<endl;
-      cout << "vectors available:" << (timesTableOffset-(dataoffset+dbH->length))/(sizeof(double)*dbH->dim) << endl;
+      cout << "vectors available:" << (dbH->timesTableOffset-(dbH->dataOffset+dbH->length))/(sizeof(double)*dbH->dim) << endl;
     }
-    cout << "total bytes:" << dbH->length << " (" << (100.0*dbH->length)/(timesTableOffset-dataoffset) << "%)" << endl;
-    cout << "bytes available:" << timesTableOffset-(dataoffset+dbH->length) << " (" <<
-      (100.0*(timesTableOffset-(dataoffset+dbH->length)))/(timesTableOffset-dataoffset) << "%)" << endl;
+    cout << "total bytes:" << dbH->length << " (" << (100.0*dbH->length)/(dbH->timesTableOffset-dbH->dataOffset) << "%)" << endl;
+    cout << "bytes available:" << dbH->timesTableOffset-(dbH->dataOffset+dbH->length) << " (" <<
+      (100.0*(dbH->timesTableOffset-(dbH->dataOffset+dbH->length)))/(dbH->timesTableOffset-dbH->dataOffset) << "%)" << endl;
     cout << "flags:" << dbH->flags << endl;
     
     cout << "null count: " << nullCount << " small sequence count " << dudCount-nullCount << endl;    
diff -r 1521d46bc1ac -r ae045842d29f audioDB.h
--- a/audioDB.h	Fri Oct 05 11:45:03 2007 +0000
+++ b/audioDB.h	Fri Oct 05 15:42:50 2007 +0000
@@ -44,7 +44,9 @@
 #define COM_KEYLIST "--keyList"
 #define COM_TIMES "--times"
 
-#define O2_MAGIC 1111765583 // 'B'<<24|'D'<<16|'2'<<8|'O' reads O2DB in little endian order
+#define O2_OLD_MAGIC ('O'|'2'<<8|'D'<<16|'B'<<24)
+#define O2_MAGIC ('o'|'2'<<8|'d'<<16|'b'<<24)
+#define O2_FORMAT_VERSION (0U)
 
 #define O2_DEFAULT_POINTNN (10U)
 #define O2_DEFAULT_TRACKNN  (10U)
@@ -78,15 +80,28 @@
 // Macros
 #define O2_ACTION(a) (strcmp(command,a)==0)
 
+#define ALIGN_UP(x,w) ((x) + ((1<<w)-1) & ~((1<<w)-1))
+#define ALIGN_DOWN(x,w) ((x) & ~((1<<w)-1))
+
 using namespace std;
 
-// 64 byte header
 typedef struct dbTableHeader{
   unsigned magic;
+  unsigned version;
   unsigned numFiles;
   unsigned dim;
+  unsigned flags;
+  // FIXME: these lengths and offsets should be size_t or off_t, but
+  // that causes this header (and hence audioDB files) to be
+  // unportable between 32 and 64-bit architectures.  Making them
+  // unsigned isn't the real answer, but it works around the problem.
+  // -- CSR, 2007-10-05
   unsigned length;
-  unsigned flags;
+  unsigned fileTableOffset;
+  unsigned trackTableOffset;
+  unsigned dataOffset;
+  unsigned l2normTableOffset;
+  unsigned timesTableOffset;
 } dbTableHeaderT, *dbTableHeaderPtr;
 
 
@@ -111,11 +126,6 @@
   char* indata;
   struct stat statbuf;  
   dbTableHeaderPtr dbH;
-  size_t fileTableOffset;
-  size_t trackTableOffset;
-  size_t dataoffset;
-  size_t l2normTableOffset;
-  size_t timesTableOffset;
   
   char *fileTable;
   unsigned* trackTable;
diff -r 1521d46bc1ac -r ae045842d29f debian/changelog
--- a/debian/changelog	Fri Oct 05 11:45:03 2007 +0000
+++ b/debian/changelog	Fri Oct 05 15:42:50 2007 +0000
@@ -1,3 +1,9 @@
+audiodb (1.0-10) unstable; urgency=low
+
+  * Updated to svn version #128
+
+ -- Christophe Rhodes <c.rhodes@gold.ac.uk>  Fri,  5 Oct 2007 16:39:12 +0100
+
 audiodb (1.0-9) unstable; urgency=low
 
   * Updated to svn version #122
diff -r 1521d46bc1ac -r ae045842d29f tests/0021/run-test.sh
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/0021/run-test.sh	Fri Oct 05 15:42:50 2007 +0000
@@ -0,0 +1,46 @@
+#! /bin/sh
+
+. ../test-utils.sh
+
+if [ -f testdb ]; then rm -f testdb; fi
+
+${AUDIODB} -d testdb -N
+
+intstring 2 > testfeature
+floatstring 0 0.5 >> testfeature
+floatstring 0.5 0 >> testfeature
+
+${AUDIODB} -d testdb -I -f testfeature
+
+# sequence queries require L2NORM
+${AUDIODB} -d testdb -L
+
+start_server ${AUDIODB} 10021
+
+echo "query point (0.0,0.5)"
+intstring 2 > testquery
+floatstring 0 0.5 >> testquery
+
+${AUDIODB} -c localhost:10021 -d testdb -Q sequence -l 1 -f testquery > testoutput
+echo testfeature 1 0 0 > test-expected-output
+cmp testoutput test-expected-output
+${AUDIODB} -c localhost:10021 -d testdb -Q sequence -l 1 -f testquery -n 1 > testoutput
+echo testfeature 0 0 0 > test-expected-output
+cmp testoutput test-expected-output
+
+check_server $!
+
+echo "query point (0.5,0.0)"
+intstring 2 > testquery
+floatstring 0.5 0 >> testquery
+
+${AUDIODB} -c localhost:10021 -d testdb -Q sequence -l 1 -f testquery > testoutput
+echo testfeature 1 0 1 > test-expected-output
+cmp testoutput test-expected-output
+${AUDIODB} -c localhost:10021 -d testdb -Q sequence -l 1 -f testquery -n 1 > testoutput
+echo testfeature 0 0 1 > test-expected-output
+cmp testoutput test-expected-output
+
+stop_server $!
+
+exit 104
diff -r 1521d46bc1ac -r ae045842d29f tests/0021/short-description
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/0021/short-description	Fri Oct 05 15:42:50 2007 +0000
@@ -0,0 +1,1 @@
+WS version of 0011
\ No newline at end of file
diff -r 1521d46bc1ac -r ae045842d29f tests/0022/run-test.sh
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/0022/run-test.sh	Fri Oct 05 15:42:50 2007 +0000
@@ -0,0 +1,48 @@
+#! /bin/sh
+
+. ../test-utils.sh
+
+if [ -f testdb ]; then rm -f testdb; fi
+
+${AUDIODB} -d testdb -N
+
+intstring 2 > testfeature01
+floatstring 0 1 >> testfeature01
+intstring 2 > testfeature10
+floatstring 1 0 >> testfeature10
+
+cat > testfeaturefiles <<EOF
+testfeature01
+testfeature10
+EOF
+
+${AUDIODB} -d testdb -B -F testfeaturefiles
+
+# sequence queries require L2NORM
+${AUDIODB} -d testdb -L
+
+echo "query point (0.0,0.5)"
+intstring 2 > testquery
+floatstring 0 0.5 >> testquery
+
+${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput
+echo testfeature01 0 0 0 > test-expected-output
+echo testfeature10 2 0 0 >> test-expected-output
+cmp testoutput test-expected-output
+${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -r 1 > testoutput
+echo testfeature01 0 0 0 > test-expected-output
+cmp testoutput test-expected-output
+
+echo "query point (0.5,0.0)"
+intstring 2 > testquery
+floatstring 0.5 0 >> testquery
+
+${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput
+echo testfeature10 0 0 0 > test-expected-output
+echo testfeature01 2 0 0 >> test-expected-output
+cmp testoutput test-expected-output
+${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -r 1 > testoutput
+echo testfeature10 0 0 0 > test-expected-output
+cmp testoutput test-expected-output
+
+exit 104
diff -r 1521d46bc1ac -r ae045842d29f tests/0022/short-description
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/0022/short-description	Fri Oct 05 15:42:50 2007 +0000
@@ -0,0 +1,1 @@
+Batchinsert / sequence search
\ No newline at end of file