changeset 548:e18843dc0aea

Implement a rudimentary API for audioDB::liszt

The API is rudimentary because we've dropped support for the incremental
retrieval of tracks and their number of vectors (at the API level; the
SOAP and command-line support is still there -- no changes should be
visible).  This is potentially bad for the large-scale databases, of
course; one million tracks will take of the order of 16MB of RAM, more
if I'm unlucky about how std::string.c_str() is implemented.

Both this liszt operation and querying (and sampling, forthcoming...)
would benefit from a `cursor-like' interface to retrieval results: for
an API like that, instead of getting a struct with the data there, you
get a cookie with which you can ask the database for successive results.
This would be neat for all sorts of reasons.  In the meantime, at least
this change fixes SOAP memory leaks related to liszt.

Make liszt.o part of LIBOBJS rather than ordinary OBJS, so that the
liszt functionality is actually compiled into the library.

Add a test for this library functionality; also modify the command-line
test file to run the SOAP server on its own port.

--- a/Makefile	Wed Feb 11 10:42:23 2009 +0000
+++ b/Makefile	Wed Feb 11 12:38:03 2009 +0000
@@ -8,8 +8,8 @@
 
 SHARED_LIB_FLAGS=-shared -Wl,-soname,
 
-LIBOBJS=lock.o pointpair.o create.o open.o power.o l2norm.o insert.o status.o query.o dump.o close.o lshlib.o index-utils.o query-indexed.o
-OBJS=$(LIBOBJS) index.o soap.o liszt.o sample.o cmdline.o audioDB.o common.o
+LIBOBJS=lock.o pointpair.o create.o open.o power.o l2norm.o insert.o status.o query.o dump.o close.o lshlib.o index-utils.o query-indexed.o liszt.o
+OBJS=$(LIBOBJS) index.o soap.o sample.o cmdline.o audioDB.o common.o
 
 EXECUTABLE=audioDB
 

--- a/audioDB.cpp	Wed Feb 11 10:42:23 2009 +0000
+++ b/audioDB.cpp	Wed Feb 11 12:38:03 2009 +0000
@@ -111,7 +111,7 @@
   }
 }
 
-audioDB::audioDB(const unsigned argc, const char *argv[], adb__lisztResponse *adbLisztResponse): O2_AUDIODB_INITIALIZERS
+audioDB::audioDB(const unsigned argc, const char *argv[], struct soap *soap, adb__lisztResponse *adbLisztResponse): O2_AUDIODB_INITIALIZERS
 {
   try {
     isServer = 1; // Set to make errors report over SOAP
@@ -120,7 +120,7 @@
     if(dbName && adb_root)
       prefix_name((char** const)&dbName, adb_root);
     assert(O2_ACTION(COM_LISZT));
-    liszt(dbName, lisztOffset, lisztLength, adbLisztResponse);
+    liszt(dbName, lisztOffset, lisztLength, soap, adbLisztResponse);
   } catch(char *err) {
     cleanup();
     throw(err);
@@ -950,6 +950,44 @@
   reporter->report(adb, soap, adbQueryResponse);
 }
 
+void audioDB::liszt(const char* dbName, unsigned offset, unsigned numLines, struct soap *soap, adb__lisztResponse* adbLisztResponse) {
+  if(!adb) {
+    if(!(adb = audiodb_open(dbName, O_RDONLY))) {
+      error("failed to open database", dbName);
+    }
+  }
+
+  adb_liszt_results_t *results = audiodb_liszt(adb);
+  if(!results) {
+    error("audiodb_liszt() failed");
+  }
+
+  if(offset > results->nresults) {
+    audiodb_liszt_free_results(adb, results);
+    error("listKeys offset out of range");
+  }
+
+  if(!adbLisztResponse){
+    for(uint32_t k = 0; k < numLines && offset + k < results->nresults; k++) {
+      uint32_t index = offset + k;
+      printf("[%d] %s (%d)\n", index, results->entries[index].key, results->entries[index].nvectors);
+    }
+  } else {
+    adbLisztResponse->result.Rkey = (char **) soap_malloc(soap, numLines * sizeof(char *));
+    adbLisztResponse->result.Rlen = (unsigned int *) soap_malloc(soap, numLines * sizeof(unsigned int));
+    uint32_t k;
+    for(k = 0; k < numLines && offset + k < results->nresults; k++) {
+      uint32_t index = offset + k;
+      adbLisztResponse->result.Rkey[k] = (char *) soap_malloc(soap, O2_MAXFILESTR);
+      snprintf(adbLisztResponse->result.Rkey[k], O2_MAXFILESTR, "%s", results->entries[index].key);
+      adbLisztResponse->result.Rlen[k] = results->entries[index].nvectors;
+    }
+    adbLisztResponse->result.__sizeRkey = k;
+    adbLisztResponse->result.__sizeRlen = k;
+  }
+  audiodb_liszt_free_results(adb, results);  
+}
+
 // This entry point is visited once per instance
 // so it is a good place to set any global state variables
 int main(const int argc, const char* argv[]){

--- a/audioDB.h	Wed Feb 11 10:42:23 2009 +0000
+++ b/audioDB.h	Wed Feb 11 12:38:03 2009 +0000
@@ -273,7 +273,7 @@
   audioDB(const unsigned argc, const char *argv[]);
   audioDB(const unsigned argc, const char *argv[], struct soap *soap, adb__queryResponse *adbQueryResponse);
   audioDB(const unsigned argc, const char *argv[], adb__statusResponse *adbStatusResponse);
-  audioDB(const unsigned argc, const char *argv[], adb__lisztResponse *adbLisztResponse);
+  audioDB(const unsigned argc, const char *argv[], struct soap *soap, adb__lisztResponse *adbLisztResponse);
 
   void cleanup();
   ~audioDB();
@@ -291,7 +291,7 @@
   void l2norm(const char* dbName);
   void power_flag(const char *dbName);
   void dump(const char* dbName);
-  void liszt(const char* dbName, unsigned offset, unsigned numLines, adb__lisztResponse* adbLisztResponse=0);
+  void liszt(const char* dbName, unsigned offset, unsigned numLines, struct soap *soap=0, adb__lisztResponse* adbLisztResponse=0);
 
   // LSH indexing parameters and data structures
   LSH* lsh;

--- a/audioDB_API.h	Wed Feb 11 10:42:23 2009 +0000
+++ b/audioDB_API.h	Wed Feb 11 12:38:03 2009 +0000
@@ -163,6 +163,16 @@
   adb_query_refine_t refine;
 } adb_query_spec_t;
 
+typedef struct adbtrackentry {
+  uint32_t nvectors;
+  const char *key;
+} adb_track_entry_t;
+
+typedef struct adblisztresults {
+  uint32_t nresults;
+  adb_track_entry_t *entries;
+} adb_liszt_results_t;
+
 /*******************************************************************/
 /* Function prototypes for API */
 
@@ -198,4 +208,8 @@
 /* varoius dump formats */
 int audiodb_dump(adb_ptr mydb, const char *outputdir);
 
+/* liszt */
+adb_liszt_results_t *audiodb_liszt(adb_t *);
+int audiodb_liszt_free_results(adb_t *, adb_liszt_results_t *);
+
 #endif

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/libtests/0039/prog1.c	Wed Feb 11 12:38:03 2009 +0000
@@ -0,0 +1,28 @@
+#include "audioDB_API.h"
+#include "test_utils_lib.h"
+
+int main(int argc, char *argv[]) {
+  adb_t *adb;
+  adb_liszt_results_t *liszt;
+
+  clean_remove_db(TESTDB);
+  if(!(adb = audiodb_create(TESTDB, 0, 0, 0)))
+    return 1;
+  adb_datum_t datum1 = {2, 2, "testfeature01", (double[4]) {0, 1, 1, 0}};
+  adb_datum_t datum2 = {2, 2, "testfeature10", (double[4]) {1, 0, 0, 1}};
+  if(audiodb_insert_datum(adb, &datum1))
+    return 1;
+  if(audiodb_insert_datum(adb, &datum2))
+    return 1;
+
+  liszt = audiodb_liszt(adb);
+  if(!liszt || liszt->nresults != 2)
+    return 1;
+  entry_present_or_fail(liszt, "testfeature01", 2);
+  entry_present_or_fail(liszt, "testfeature10", 2);
+  audiodb_liszt_free_results(adb, liszt);
+
+  audiodb_close(adb);
+
+  return 104;
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/libtests/0039/short-description	Wed Feb 11 12:38:03 2009 +0000
@@ -0,0 +1,1 @@
+liszt
\ No newline at end of file

--- a/libtests/test_utils_lib.h	Wed Feb 11 10:42:23 2009 +0000
+++ b/libtests/test_utils_lib.h	Wed Feb 11 12:38:03 2009 +0000
@@ -40,3 +40,16 @@
 
 #define result_present_or_fail(r, k, d, q, i) \
   if(result_position(r, k, d, q, i) < 0) return 1;
+
+int entry_position(adb_liszt_results_t *l, const char *key, uint32_t nvectors) {
+  for(uint32_t k = 0; k < l->nresults; k++) {
+    adb_track_entry_t entry = l->entries[k];
+    if((nvectors == entry.nvectors) && !strcmp(key, entry.key)) {
+      return k;
+    }
+  }
+  return -1;
+}
+
+#define entry_present_or_fail(l, k, n) \
+  if(entry_position(l, k, n) < 0) return 1;

--- a/liszt.cpp	Wed Feb 11 10:42:23 2009 +0000
+++ b/liszt.cpp	Wed Feb 11 12:38:03 2009 +0000
@@ -1,34 +1,25 @@
-#include "audioDB.h"
+extern "C" {
+#include "audioDB_API.h"
+}
+#include "audioDB-internals.h"
 
-void audioDB::liszt(const char* dbName, unsigned offset, unsigned numLines, adb__lisztResponse* adbLisztResponse){
-  if(!dbH) {
-    initTables(dbName, 0);
+adb_liszt_results_t *audiodb_liszt(adb_t *adb) {
+  uint32_t nfiles = adb->header->numFiles;
+  adb_liszt_results_t *results;
+  results = (adb_liszt_results_t *) calloc(sizeof(adb_liszt_results_t),1);
+  results->nresults = nfiles;
+  if(nfiles > 0) {
+    results->entries = (adb_track_entry_t *) malloc(nfiles * sizeof(adb_track_entry_t));
   }
+  for(uint32_t k = 0; k < nfiles; k++) {
+    results->entries[k].nvectors = (*adb->track_lengths)[k];
+    results->entries[k].key = audiodb_index_key(adb, k);
+  }
+  return results;
+}
 
-  assert(trackTable && fileTable);
-
-  if(offset>dbH->numFiles){
-    char tmpStr[MAXSTR];
-    sprintf(tmpStr, "numFiles=%u, lisztOffset=%u", dbH->numFiles, offset);
-    error("listKeys offset out of range", tmpStr);
-  }
-
-  if(!adbLisztResponse){
-    for(Uns32T k=0; k<numLines && offset+k<dbH->numFiles; k++){
-      fprintf(stdout, "[%d] %s (%d)\n", offset+k, fileTable+(offset+k)*O2_FILETABLE_ENTRY_SIZE, trackTable[offset+k]);
-    }
-  }
-  else{
-    adbLisztResponse->result.Rkey = new char*[numLines];
-    adbLisztResponse->result.Rlen = new unsigned int[numLines];
-    Uns32T k = 0;
-    for( ; k<numLines && offset+k<dbH->numFiles; k++){    
-      adbLisztResponse->result.Rkey[k] = new char[MAXSTR];
-      snprintf(adbLisztResponse->result.Rkey[k], O2_MAXFILESTR, "%s", fileTable+(offset+k)*O2_FILETABLE_ENTRY_SIZE);
-      adbLisztResponse->result.Rlen[k] = trackTable[offset+k];
-    }
-    adbLisztResponse->result.__sizeRkey = k;
-    adbLisztResponse->result.__sizeRlen = k;
-  }
-  
+int audiodb_liszt_free_results(adb_t *adb, adb_liszt_results_t *results) {
+  free(results->entries);
+  free(results);
+  return 0;
 }

--- a/soap.cpp	Wed Feb 11 10:42:23 2009 +0000
+++ b/soap.cpp	Wed Feb 11 12:38:03 2009 +0000
@@ -164,7 +164,7 @@
   const char *argv[] = {"./audioDB", COM_LISZT, "-d",dbName, "--lisztOffset", lisztOffsetStr, "--lisztLength", lisztLengthStr};
   const unsigned argc = 8;
   try{
-    audioDB(argc, argv, &adbLisztResponse);
+    audioDB(argc, argv, soap, &adbLisztResponse);
     return SOAP_OK;
   } catch(char *err) {
     soap_receiver_fault(soap, err, "");

--- a/tests/0039/run-test.sh	Wed Feb 11 10:42:23 2009 +0000
+++ b/tests/0039/run-test.sh	Wed Feb 11 12:38:03 2009 +0000
@@ -46,7 +46,7 @@
 echo "[1] testkey02 (2)" >> test-expected-output
 cmp testoutput test-expected-output
 
-WSPORT=10020
+WSPORT=10039
 start_server ${AUDIODB} ${WSPORT}
 
 expect_clean_error_exit ${AUDIODB} -d testdb -c localhost:${WSPORT} --LISZT --lisztOffset -1