Mercurial > hg > audiodb
changeset 112:ae045842d29f audiodb-debian
Merge trunk changes -r123:128 to audiodb-debian branch
(+ new debian/changelog version)
| author | mas01cr |
|---|---|
| date | Fri, 05 Oct 2007 15:42:50 +0000 |
| parents | 1521d46bc1ac |
| children | 8d0942525fab |
| files | audioDB.cpp audioDB.h debian/changelog tests/0021/run-test.sh tests/0021/short-description tests/0022/run-test.sh tests/0022/short-description |
| diffstat | 7 files changed, 160 insertions(+), 46 deletions(-) [+] |
line wrap: on
line diff
--- a/audioDB.cpp Fri Oct 05 11:45:03 2007 +0000 +++ b/audioDB.cpp Fri Oct 05 15:42:50 2007 +0000 @@ -420,11 +420,17 @@ assert(dbH); // Initialize header - dbH->magic=O2_MAGIC; - dbH->numFiles=0; - dbH->length=0; - dbH->dim=0; - dbH->flags=0; //O2_FLAG_L2NORM; + dbH->magic = O2_MAGIC; + dbH->version = O2_FORMAT_VERSION; + dbH->numFiles = 0; + dbH->dim = 0; + dbH->flags = 0; + dbH->length = 0; + dbH->fileTableOffset = ALIGN_UP(O2_HEADERSIZE, 8); + dbH->trackTableOffset = ALIGN_UP(dbH->fileTableOffset + O2_FILETABLESIZE*O2_MAXFILES, 8); + dbH->dataOffset = ALIGN_UP(dbH->trackTableOffset + O2_TRACKTABLESIZE*O2_MAXFILES, 8); + dbH->l2normTableOffset = ALIGN_DOWN(O2_DEFAULTDBSIZE - O2_MAXFILES*O2_MEANNUMVECTORS*sizeof(double), 8); + dbH->timesTableOffset = ALIGN_DOWN(dbH->l2normTableOffset - O2_MAXFILES*O2_MEANNUMVECTORS*sizeof(double), 8); memcpy (db, dbH, O2_HEADERSIZE); if(verbosity) { @@ -434,8 +440,7 @@ void audioDB::drop(){ - - + // FIXME: drop something? Should we even allow this? } // initTables - memory map files passed as arguments @@ -462,15 +467,19 @@ error("error reading db header", dbName, "read"); } - fileTableOffset = O2_HEADERSIZE; - trackTableOffset = fileTableOffset + O2_FILETABLESIZE*O2_MAXFILES; - dataoffset = trackTableOffset + O2_TRACKTABLESIZE*O2_MAXFILES; - l2normTableOffset = O2_DEFAULTDBSIZE - O2_MAXFILES*O2_MEANNUMVECTORS*sizeof(double); - timesTableOffset = l2normTableOffset - O2_MAXFILES*O2_MEANNUMVECTORS*sizeof(double); + if(dbH->magic == O2_OLD_MAGIC) { + // FIXME: if anyone ever complains, write the program to convert + // from the old audioDB format to the new... + error("database file has old O2 header", dbName); + } if(dbH->magic != O2_MAGIC) { cerr << "expected: " << O2_MAGIC << ", got: " << dbH->magic << endl; - error("database file has incorrect header",dbName); + error("database file has incorrect header", dbName); + } + + if(dbH->version != O2_FORMAT_VERSION) { + error("database file has incorect version", dbName); } if(inFile) @@ -497,11 +506,11 @@ error("mmap error for initting tables of database", "", "mmap"); // Make some handy tables with correct types - fileTable= (char*)(db+fileTableOffset); - trackTable = (unsigned*)(db+trackTableOffset); - dataBuf = (double*)(db+dataoffset); - l2normTable = (double*)(db+l2normTableOffset); - timesTable = (double*)(db+timesTableOffset); + fileTable= (char*)(db+dbH->fileTableOffset); + trackTable = (unsigned*)(db+dbH->trackTableOffset); + dataBuf = (double*)(db+dbH->dataOffset); + l2normTable = (double*)(db+dbH->l2normTableOffset); + timesTable = (double*)(db+dbH->timesTableOffset); } void audioDB::insert(const char* dbName, const char* inFile){ @@ -569,11 +578,11 @@ memcpy (trackTable+dbH->numFiles-1, &numVectors, sizeof(unsigned)); // Update the feature database - memcpy (db+dataoffset+insertoffset, indata+sizeof(int), statbuf.st_size-sizeof(int)); + memcpy (db+dbH->dataOffset+insertoffset, indata+sizeof(int), statbuf.st_size-sizeof(int)); // Norm the vectors on input if the database is already L2 normed if(dbH->flags & O2_FLAG_L2NORM) - unitNormAndInsertL2((double*)(db+dataoffset+insertoffset), dbH->dim, numVectors, 1); // append + unitNormAndInsertL2((double*)(db+dbH->dataOffset+insertoffset), dbH->dim, numVectors, 1); // append // Report status status(dbName); @@ -668,17 +677,10 @@ if(!usingTimes && (dbH->flags & O2_FLAG_TIMES)) error("Must use timestamps with timestamped database","use --times"); - fileTableOffset = O2_HEADERSIZE; - trackTableOffset = fileTableOffset + O2_FILETABLESIZE*O2_MAXFILES; - dataoffset = trackTableOffset + O2_TRACKTABLESIZE*O2_MAXFILES; - l2normTableOffset = O2_DEFAULTDBSIZE - O2_MAXFILES*O2_MEANNUMVECTORS*sizeof(double); - timesTableOffset = l2normTableOffset - O2_MAXFILES*O2_MEANNUMVECTORS*sizeof(double); - if(dbH->magic!=O2_MAGIC){ cerr << "expected:" << O2_MAGIC << ", got:" << dbH->magic << endl; error("database file has incorrect header",dbName); } - unsigned totalVectors=0; char *thisKey = new char[MAXSTR]; @@ -711,11 +713,11 @@ error("mmap error for batchinsert into database", "", "mmap"); // Make some handy tables with correct types - fileTable= (char*)(db+fileTableOffset); - trackTable = (unsigned*)(db+trackTableOffset); - dataBuf = (double*)(db+dataoffset); - l2normTable = (double*)(db+l2normTableOffset); - timesTable = (double*)(db+timesTableOffset); + fileTable= (char*)(db+dbH->fileTableOffset); + trackTable = (unsigned*)(db+dbH->trackTableOffset); + dataBuf = (double*)(db+dbH->dataOffset); + l2normTable = (double*)(db+dbH->l2normTableOffset); + timesTable = (double*)(db+dbH->timesTableOffset); // Check that there is room for at least 1 more file if((char*)timesTable<((char*)dataBuf+(dbH->length+statbuf.st_size-sizeof(int)))) @@ -795,11 +797,11 @@ memcpy (trackTable+dbH->numFiles-1, &numVectors, sizeof(unsigned)); // Update the feature database - memcpy (db+dataoffset+insertoffset, indata+sizeof(int), statbuf.st_size-sizeof(int)); + memcpy (db+dbH->dataOffset+insertoffset, indata+sizeof(int), statbuf.st_size-sizeof(int)); // Norm the vectors on input if the database is already L2 normed if(dbH->flags & O2_FLAG_L2NORM) - unitNormAndInsertL2((double*)(db+dataoffset+insertoffset), dbH->dim, numVectors, 1); // append + unitNormAndInsertL2((double*)(db+dbH->dataOffset+insertoffset), dbH->dim, numVectors, 1); // append totalVectors+=numVectors; } @@ -895,11 +897,11 @@ cout << "data dim:" << dbH->dim <<endl; if(dbH->dim>0){ cout << "total vectors:" << dbH->length/(sizeof(double)*dbH->dim)<<endl; - cout << "vectors available:" << (timesTableOffset-(dataoffset+dbH->length))/(sizeof(double)*dbH->dim) << endl; + cout << "vectors available:" << (dbH->timesTableOffset-(dbH->dataOffset+dbH->length))/(sizeof(double)*dbH->dim) << endl; } - cout << "total bytes:" << dbH->length << " (" << (100.0*dbH->length)/(timesTableOffset-dataoffset) << "%)" << endl; - cout << "bytes available:" << timesTableOffset-(dataoffset+dbH->length) << " (" << - (100.0*(timesTableOffset-(dataoffset+dbH->length)))/(timesTableOffset-dataoffset) << "%)" << endl; + cout << "total bytes:" << dbH->length << " (" << (100.0*dbH->length)/(dbH->timesTableOffset-dbH->dataOffset) << "%)" << endl; + cout << "bytes available:" << dbH->timesTableOffset-(dbH->dataOffset+dbH->length) << " (" << + (100.0*(dbH->timesTableOffset-(dbH->dataOffset+dbH->length)))/(dbH->timesTableOffset-dbH->dataOffset) << "%)" << endl; cout << "flags:" << dbH->flags << endl; cout << "null count: " << nullCount << " small sequence count " << dudCount-nullCount << endl;
--- a/audioDB.h Fri Oct 05 11:45:03 2007 +0000 +++ b/audioDB.h Fri Oct 05 15:42:50 2007 +0000 @@ -44,7 +44,9 @@ #define COM_KEYLIST "--keyList" #define COM_TIMES "--times" -#define O2_MAGIC 1111765583 // 'B'<<24|'D'<<16|'2'<<8|'O' reads O2DB in little endian order +#define O2_OLD_MAGIC ('O'|'2'<<8|'D'<<16|'B'<<24) +#define O2_MAGIC ('o'|'2'<<8|'d'<<16|'b'<<24) +#define O2_FORMAT_VERSION (0U) #define O2_DEFAULT_POINTNN (10U) #define O2_DEFAULT_TRACKNN (10U) @@ -78,15 +80,28 @@ // Macros #define O2_ACTION(a) (strcmp(command,a)==0) +#define ALIGN_UP(x,w) ((x) + ((1<<w)-1) & ~((1<<w)-1)) +#define ALIGN_DOWN(x,w) ((x) & ~((1<<w)-1)) + using namespace std; -// 64 byte header typedef struct dbTableHeader{ unsigned magic; + unsigned version; unsigned numFiles; unsigned dim; + unsigned flags; + // FIXME: these lengths and offsets should be size_t or off_t, but + // that causes this header (and hence audioDB files) to be + // unportable between 32 and 64-bit architectures. Making them + // unsigned isn't the real answer, but it works around the problem. + // -- CSR, 2007-10-05 unsigned length; - unsigned flags; + unsigned fileTableOffset; + unsigned trackTableOffset; + unsigned dataOffset; + unsigned l2normTableOffset; + unsigned timesTableOffset; } dbTableHeaderT, *dbTableHeaderPtr; @@ -111,11 +126,6 @@ char* indata; struct stat statbuf; dbTableHeaderPtr dbH; - size_t fileTableOffset; - size_t trackTableOffset; - size_t dataoffset; - size_t l2normTableOffset; - size_t timesTableOffset; char *fileTable; unsigned* trackTable;
--- a/debian/changelog Fri Oct 05 11:45:03 2007 +0000 +++ b/debian/changelog Fri Oct 05 15:42:50 2007 +0000 @@ -1,3 +1,9 @@ +audiodb (1.0-10) unstable; urgency=low + + * Updated to svn version #128 + + -- Christophe Rhodes <c.rhodes@gold.ac.uk> Fri, 5 Oct 2007 16:39:12 +0100 + audiodb (1.0-9) unstable; urgency=low * Updated to svn version #122
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/0021/run-test.sh Fri Oct 05 15:42:50 2007 +0000 @@ -0,0 +1,46 @@ +#! /bin/sh + +. ../test-utils.sh + +if [ -f testdb ]; then rm -f testdb; fi + +${AUDIODB} -d testdb -N + +intstring 2 > testfeature +floatstring 0 0.5 >> testfeature +floatstring 0.5 0 >> testfeature + +${AUDIODB} -d testdb -I -f testfeature + +# sequence queries require L2NORM +${AUDIODB} -d testdb -L + +start_server ${AUDIODB} 10021 + +echo "query point (0.0,0.5)" +intstring 2 > testquery +floatstring 0 0.5 >> testquery + +${AUDIODB} -c localhost:10021 -d testdb -Q sequence -l 1 -f testquery > testoutput +echo testfeature 1 0 0 > test-expected-output +cmp testoutput test-expected-output +${AUDIODB} -c localhost:10021 -d testdb -Q sequence -l 1 -f testquery -n 1 > testoutput +echo testfeature 0 0 0 > test-expected-output +cmp testoutput test-expected-output + +check_server $! + +echo "query point (0.5,0.0)" +intstring 2 > testquery +floatstring 0.5 0 >> testquery + +${AUDIODB} -c localhost:10021 -d testdb -Q sequence -l 1 -f testquery > testoutput +echo testfeature 1 0 1 > test-expected-output +cmp testoutput test-expected-output +${AUDIODB} -c localhost:10021 -d testdb -Q sequence -l 1 -f testquery -n 1 > testoutput +echo testfeature 0 0 1 > test-expected-output +cmp testoutput test-expected-output + +stop_server $! + +exit 104
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/0021/short-description Fri Oct 05 15:42:50 2007 +0000 @@ -0,0 +1,1 @@ +WS version of 0011 \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/0022/run-test.sh Fri Oct 05 15:42:50 2007 +0000 @@ -0,0 +1,48 @@ +#! /bin/sh + +. ../test-utils.sh + +if [ -f testdb ]; then rm -f testdb; fi + +${AUDIODB} -d testdb -N + +intstring 2 > testfeature01 +floatstring 0 1 >> testfeature01 +intstring 2 > testfeature10 +floatstring 1 0 >> testfeature10 + +cat > testfeaturefiles <<EOF +testfeature01 +testfeature10 +EOF + +${AUDIODB} -d testdb -B -F testfeaturefiles + +# sequence queries require L2NORM +${AUDIODB} -d testdb -L + +echo "query point (0.0,0.5)" +intstring 2 > testquery +floatstring 0 0.5 >> testquery + +${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput +echo testfeature01 0 0 0 > test-expected-output +echo testfeature10 2 0 0 >> test-expected-output +cmp testoutput test-expected-output +${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -r 1 > testoutput +echo testfeature01 0 0 0 > test-expected-output +cmp testoutput test-expected-output + +echo "query point (0.5,0.0)" +intstring 2 > testquery +floatstring 0.5 0 >> testquery + +${AUDIODB} -d testdb -Q sequence -l 1 -f testquery > testoutput +echo testfeature10 0 0 0 > test-expected-output +echo testfeature01 2 0 0 >> test-expected-output +cmp testoutput test-expected-output +${AUDIODB} -d testdb -Q sequence -l 1 -f testquery -r 1 > testoutput +echo testfeature10 0 0 0 > test-expected-output +cmp testoutput test-expected-output + +exit 104