Mercurial > hg > audiodb

changeset 316:25572f1bd37f large_adb

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Adding large_adb support (up to 1M tracks)
author mas01mc
date Tue, 19 Aug 2008 14:27:21 +0000
parents d2c56d4f841e
children ab411674dad4
files audioDB.h common.cpp create.cpp insert.cpp lshlib.cpp lshlib.h reporter.h soap.cpp
diffstat 8 files changed, 332 insertions(+), 59 deletions(-) [+]
line wrap: on
line diff
--- a/audioDB.h	Tue Aug 12 14:25:51 2008 +0000
+++ b/audioDB.h	Tue Aug 19 14:27:21 2008 +0000
@@ -98,17 +98,19 @@
 #define O2_MAXDIM (2000U)
 #define O2_MAXNN (1000000U)
 #define O2_MAXSEQLEN (8000U)            // maximum feature vectors in a sequence
-#define O2_MAXTRACKS (10000U)           // maximum number of tracks
+#define O2_MAXTRACKS (1000000U)           // maximum number of tracks
 #define O2_MAXTRACKLEN ((LSH_POINT_MASK+1)) // maximum shingles in a track
 #define O2_MAXDOTPRODUCTMEMORY (sizeof(O2_REALTYPE)*O2_MAXSEQLEN*O2_MAXSEQLEN) // 512MB
 #define O2_DISTANCE_TOLERANCE (1e-6)
 #define O2_SERIAL_MAX_TRACKBATCH (10000)
+#define O2_LARGE_ADB_SIZE (3500) // datasize at which features are kept externally (in Mbytes)
 
 // Flags
 #define O2_FLAG_L2NORM (0x1U)
 #define O2_FLAG_MINMAX (0x2U)
 #define O2_FLAG_POWER (0x4U)
 #define O2_FLAG_TIMES (0x20U)
+#define O2_FLAG_LARGE_ADB (0x40U)
 #define DISPLAY_FLAG(x) (x?"on":"off")
 
 // Query types
@@ -146,6 +148,11 @@
     fflush(stderr); \
   }
 
+// We will only use this in a 32-bit address space
+// So map the off_t down to 32-bits first
+#define INSERT_FILETABLE_STRING(OFFSET, STR) \
+    strncpy((char*)((Uns32T)OFFSET) + dbH->numFiles*O2_FILETABLE_ENTRY_SIZE, STR, strlen(STR));
+
 extern LSH* SERVER_LSH_INDEX_SINGLETON;
 
 typedef struct dbTableHeader {
@@ -205,9 +212,9 @@
 
   gsl_rng *rng;
   
-  char *fileTable;
+  char* fileTable;
   unsigned* trackTable;
-  off_t *trackOffsetTable;
+  off_t* trackOffsetTable;
   double* dataBuf;
   double* inBuf;
   double* l2normTable;
@@ -278,7 +285,7 @@
   double dot_product_points(double* q, double* p, Uns32T  L);
   void initRNG();
   void initDBHeader(const char *dbName);
-  void initInputFile(const char *inFile);
+  void initInputFile(const char *inFile, bool loadData = true);
   void initTables(const char* dbName, const char* inFile = 0);
   void initTablesFromKey(const char* dbName, const Uns32T queryIndex);
   void unitNorm(double* X, unsigned d, unsigned n, double* qNorm);
@@ -301,6 +308,7 @@
   void insert_data_vectors(off_t offset, void *buffer, size_t size);
   void insert(const char* dbName, const char* inFile);
   void batchinsert(const char* dbName, const char* inFile);
+  void batchinsert_large_adb(const char* dbName, const char* inFile);
   void query(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse=0);
   void status(const char* dbName, adb__statusResponse *adbStatusResponse=0);
   unsigned random_track(unsigned *propTable, unsigned total);
--- a/common.cpp	Tue Aug 12 14:25:51 2008 +0000
+++ b/common.cpp	Tue Aug 19 14:27:21 2008 +0000
@@ -157,7 +157,7 @@
   }  
 }
 
-void audioDB::initInputFile (const char *inFile) {
+void audioDB::initInputFile (const char *inFile, bool loadData) {
   if (inFile) {
     if ((infid = open(inFile, O_RDONLY)) < 0) {
       error("can't open input file for reading", inFile, "open");
@@ -189,7 +189,7 @@
       }
     }
     
-    if ((indata = (char *) mmap(0, statbuf.st_size, PROT_READ, MAP_SHARED, infid, 0)) == (caddr_t) -1) {
+    if (loadData && ((indata = (char *) mmap(0, statbuf.st_size, PROT_READ, MAP_SHARED, infid, 0)) == (caddr_t) -1)) {
       error("mmap error for input", inFile, "mmap");
     }
   }
--- a/create.cpp	Tue Aug 12 14:25:51 2008 +0000
+++ b/create.cpp	Tue Aug 19 14:27:21 2008 +0000
@@ -2,6 +2,7 @@
 
 /* Make a new database.
 
+IF size(featuredata) < O2_LARGE_ADB_SIZE 
    The database consists of:
 
    * a header (see dbTableHeader struct definition);
@@ -12,6 +13,16 @@
    * timesTable: (start,end) time points for each feature vector;
    * powerTable: associated power for each feature vector;
    * l2normTable: squared l2norms for each feature vector.
+   
+ELSE the database consists of:
+   
+   * a header (see dbTableHeader struct definition);
+   * keyTable: list of keys of tracks
+   * trackTable: sizes of tracks
+   * featureTable: list of feature file names
+   * timesTable: list of times file names
+   * powerTable: list of power file names
+
 */
 
 void audioDB::create(const char* dbName){
@@ -41,10 +52,22 @@
   off_t databytes = ((off_t) datasize) * 1024 * 1024;
   off_t auxbytes = databytes / datadim;
 
-  dbH->timesTableOffset = ALIGN_PAGE_UP(dbH->dataOffset + databytes);
-  dbH->powerTableOffset = ALIGN_PAGE_UP(dbH->timesTableOffset + 2*auxbytes);
-  dbH->l2normTableOffset = ALIGN_PAGE_UP(dbH->powerTableOffset + auxbytes);
-  dbH->dbSize = ALIGN_PAGE_UP(dbH->l2normTableOffset + auxbytes);
+  // If database will fit in a single file the vectors are copied into the AudioDB instance
+  // Else all the vectors are left on the FileSystem and we use the dataOffset as storage
+  // for the location of the features, powers and times files (assuming that arbitrary keys are used for the fileTable)
+  if(datasize<O2_LARGE_ADB_SIZE){
+    dbH->timesTableOffset = ALIGN_PAGE_UP(dbH->dataOffset + databytes);
+    dbH->powerTableOffset = ALIGN_PAGE_UP(dbH->timesTableOffset + 2*auxbytes);
+    dbH->l2normTableOffset = ALIGN_PAGE_UP(dbH->powerTableOffset + auxbytes);
+    dbH->dbSize = ALIGN_PAGE_UP(dbH->l2normTableOffset + auxbytes);
+  }
+  else{
+    dbH->flags |= O2_FLAG_LARGE_ADB;
+    dbH->timesTableOffset = ALIGN_PAGE_UP(dbH->dataOffset + O2_FILETABLE_ENTRY_SIZE*ntracks);
+    dbH->powerTableOffset = ALIGN_PAGE_UP(dbH->timesTableOffset + O2_FILETABLE_ENTRY_SIZE*ntracks);
+    dbH->l2normTableOffset = ALIGN_PAGE_UP(dbH->powerTableOffset + O2_FILETABLE_ENTRY_SIZE*ntracks);
+    dbH->dbSize = dbH->l2normTableOffset;
+  } 
 
   write(dbfid, dbH, O2_HEADERSIZE);
 
--- a/insert.cpp	Tue Aug 12 14:25:51 2008 +0000
+++ b/insert.cpp	Tue Aug 19 14:27:21 2008 +0000
@@ -11,7 +11,7 @@
 }
 
 bool audioDB::enough_data_space_free(off_t size) {
-  return(dbH->timesTableOffset > dbH->dataOffset + dbH->length + size);
+    return(dbH->timesTableOffset > dbH->dataOffset + dbH->length + size);
 }
 
 void audioDB::insert_data_vectors(off_t offset, void *buffer, size_t size) {
@@ -23,6 +23,9 @@
   forWrite = true;
   initTables(dbName, inFile);
 
+  if(dbH->flags & O2_FLAG_LARGE_ADB)
+    error("Single-feature inserts not allowed with LARGE audioDB instances");
+
   if(!usingTimes && (dbH->flags & O2_FLAG_TIMES))
     error("Must use timestamps with timestamped database","use --times");
 
@@ -49,6 +52,7 @@
 
   if(alreadyInserted) {
     VERB_LOG(0, "key already exists in database; ignoring: %s\n", inFile);
+    // FIXME: Do we need to munmap here (see below) ? MKC 18/08/08
     return;
   }
   
@@ -64,7 +68,7 @@
     return;
   }
 
-  strncpy(fileTable + dbH->numFiles*O2_FILETABLE_ENTRY_SIZE, key, strlen(key));
+  INSERT_FILETABLE_STRING(fileTable, key);
 
   off_t insertoffset = dbH->length;// Store current state
 
@@ -183,6 +187,12 @@
   forWrite = true;
   initDBHeader(dbName);
 
+  // Treat large ADB instances differently
+  if( dbH->flags & O2_FLAG_LARGE_ADB ){
+    batchinsert_large_adb(dbName, inFile) ;
+    return;
+  }
+    
   if(!key)
     key=inFile;
   std::ifstream *filesIn = 0;
@@ -289,8 +299,9 @@
             close(thispowerfd);
           }
         }
-	strncpy(fileTable + dbH->numFiles*O2_FILETABLE_ENTRY_SIZE, thisKey, strlen(thisKey));
-  
+
+	INSERT_FILETABLE_STRING(fileTable, thisKey);
+
 	off_t insertoffset = dbH->length;// Store current state
 
 	// Increment file count
@@ -301,7 +312,7 @@
   
 	// Update track to file index map
 	memcpy (trackTable+dbH->numFiles-1, &numVectors, sizeof(unsigned));  
-	
+
 	insert_data_vectors(insertoffset, indata + sizeof(int), statbuf.st_size - sizeof(int));
 	
 	// Norm the vectors on input if the database is already L2 normed
@@ -334,3 +345,169 @@
   // Report status
   status(dbName);
 }
+
+
+// BATCHINSERT_LARGE_ADB
+//
+// This method inserts file pointers into the ADB instance rather than the actual feature data
+//
+// This method is intended for databases that are large enough to only support indexed query
+// So exhaustive searching across all feature vectors will not be performed
+//
+// We insert featureFileName, [powerFileName], [timesFileName]
+//
+// l2norms and power sequence sums are calculated on-the-fly at INDEX and --lsh_exact QUERY time
+//
+// LIMITS:
+//
+// We impose an upper limit of 1M keys, 1M featureFiles, 1M powerFiles and 1M timesFiles
+//
+void audioDB::batchinsert_large_adb(const char* dbName, const char* inFile) {
+
+  if(!key)
+    key=inFile;
+  std::ifstream *filesIn = 0;
+  std::ifstream *keysIn = 0;
+  std::ifstream* thisTimesFile = 0;
+  int thispowerfd = 0;
+
+  if(!(filesIn = new std::ifstream(inFile)))
+    error("Could not open batch in file", inFile);
+  if(key && key!=inFile)
+    if(!(keysIn = new std::ifstream(key)))
+      error("Could not open batch key file",key);
+  
+  if(!usingTimes && (dbH->flags & O2_FLAG_TIMES))
+    error("Must use timestamps with timestamped database","use --times");
+
+  if(!usingPower && (dbH->flags & O2_FLAG_POWER))
+    error("Must use power with power-enabled database", dbName);
+
+  unsigned totalVectors=0;
+  char *thisFile = new char[MAXSTR];
+  char *thisKey = 0;
+  if (key && (key != inFile)) {
+    thisKey = new char[MAXSTR];
+  }
+  char *thisTimesFileName = new char[MAXSTR];
+  char *thisPowerFileName = new char[MAXSTR];
+
+  std::set<std::string> s;
+
+  for (unsigned k = 0; k < dbH->numFiles; k++) {
+    s.insert(fileTable + k*O2_FILETABLE_ENTRY_SIZE);
+  }
+
+  do {
+    filesIn->getline(thisFile,MAXSTR);
+    if(key && key!=inFile) {
+      keysIn->getline(thisKey,MAXSTR);
+    } else {
+      thisKey = thisFile;
+    }
+    if(usingTimes) {
+      timesFile->getline(thisTimesFileName,MAXSTR);
+    }
+    if(usingPower) {
+      powerFile->getline(thisPowerFileName, MAXSTR);
+    }
+    
+    if(filesIn->eof()) {
+      break;
+    }
+    
+    initInputFile(thisFile, false);
+
+    if(!enough_per_file_space_free()) {
+      error("batchinsert failed: no more room for metadata", thisFile);
+    }
+
+    if(s.count(thisKey)) {
+      VERB_LOG(0, "key already exists in database: %s\n", thisKey);
+    } else {
+      s.insert(thisKey);
+      // Make a track index table of features to file indexes
+      unsigned numVectors = (statbuf.st_size-sizeof(int))/(sizeof(double)*dbH->dim);
+      if(!numVectors) {
+        VERB_LOG(0, "ignoring zero-length feature vector file: %s\n", thisKey);
+      }
+      else{
+	// Check that time-stamp file exists
+	if(usingTimes){
+	  if(timesFile->eof()) {
+	    error("not enough timestamp files in timesList", timesFileName);
+	  }
+	  thisTimesFile = new std::ifstream(thisTimesFileName,std::ios::in);
+	  if(!thisTimesFile->is_open()) {
+	    error("Cannot open timestamp file", thisTimesFileName);
+	  }
+	  if(thisTimesFile)
+	    delete thisTimesFile;
+	}
+
+	// Check that power file exists        
+        if (usingPower) {
+          if(powerFile->eof()) {
+            error("not enough power files in powerList", powerFileName);
+          }
+          thispowerfd = open(thisPowerFileName, O_RDONLY);
+          if (thispowerfd < 0) {
+            error("failed to open power file", thisPowerFileName);
+          }
+          if (0 < thispowerfd) {
+            close(thispowerfd);
+          }
+        }
+
+	// persist links to the feature files for reading from filesystem later
+	
+	// Primary Keys
+	INSERT_FILETABLE_STRING(fileTable, thisKey);
+	
+	// Feature Vector fileNames
+	INSERT_FILETABLE_STRING(dbH->dataOffset, thisFile);
+	
+	// Time Stamp fileNames
+	if(usingTimes)
+	  INSERT_FILETABLE_STRING(dbH->timesTableOffset, thisTimesFileName);
+
+
+	// Power fileNames
+	if(usingPower)
+	  INSERT_FILETABLE_STRING(dbH->powerTableOffset, thisPowerFileName);
+
+	// Increment file count
+	dbH->numFiles++;  
+  
+	// Update Header information
+	dbH->length+=(statbuf.st_size-sizeof(int));
+  
+	// Update track to file index map
+	memcpy (trackTable+dbH->numFiles-1, &numVectors, sizeof(unsigned));  
+
+	totalVectors+=numVectors;
+
+	// Copy the header back to the database
+	memcpy (db, dbH, sizeof(dbTableHeaderT));  
+      }
+    }
+    // CLEAN UP
+    munmap(indata,statbuf.st_size);
+    close(infid);
+  } while(!filesIn->eof());
+
+  VERB_LOG(0, "%s %s %u vectors %ju bytes.\n", COM_BATCHINSERT, dbName, totalVectors, (intmax_t) (totalVectors * dbH->dim * sizeof(double)));
+
+  delete [] thisPowerFileName;
+  if(key && (key != inFile)) {
+    delete [] thisKey;
+  }
+  delete [] thisFile;
+  delete [] thisTimesFileName;
+  
+  delete filesIn;
+  delete keysIn;
+
+  // Report status
+  status(dbName);
+}
--- a/lshlib.cpp	Tue Aug 12 14:25:51 2008 +0000
+++ b/lshlib.cpp	Tue Aug 19 14:27:21 2008 +0000
@@ -771,9 +771,12 @@
     // Align each hash table to page boundary
     char* dbtable = serial_mmap(fid, hashTableSize, 1, 
 				align_up(get_serial_hashtable_offset()+x*hashTableSize, get_page_logn()));
+#ifdef __CYGWIN__
+    // No madvise in CYGWIN
+#else
     if(madvise(dbtable, hashTableSize, MADV_SEQUENTIAL)<0)
       error("could not advise hashtable memory","","madvise");
-    
+#endif
     maxColCount=0;
     minColCount=O2_SERIAL_MAX_COLS;
     meanColCount=0;
@@ -1161,8 +1164,12 @@
     // Align each hash table to page boundary
     char* dbtable = serial_mmap(fid, hashTableSize, 0, 
 				align_up(get_serial_hashtable_offset()+x*hashTableSize, get_page_logn()));
+#ifdef __CYGWIN__
+    // No madvise in CYGWIN
+#else
     if(madvise(dbtable, hashTableSize, MADV_SEQUENTIAL)<0)
       error("could not advise hashtable memory","","madvise");    
+#endif
     pt=(SerialElementT*)dbtable;
     for( y = 0 ; y < H::N ; y++ ){
       // Move disk pointer to beginning of row
@@ -1331,8 +1338,12 @@
     // memory map a single hash table for random access
     char* db = serial_mmap(dbfid, hashTableSize, 0, 
 			   align_up(get_serial_hashtable_offset()+j*hashTableSize,get_page_logn()));
+#ifdef __CYGWIN__
+    // No madvise in CYGWIN
+#else
     if(madvise(db, hashTableSize, MADV_RANDOM)<0)
       error("could not advise local hashtable memory","","madvise");
+#endif
     SerialElementT* pe = (SerialElementT*)db ;
     for(Uns32T qpos=0; qpos<vv.size(); qpos++){
       H::compute_hash_functions(vv[qpos]);
@@ -1364,8 +1375,12 @@
     // memory map a single hash table for random access
     char* db = serial_mmap(dbfid, hashTableSize, 0, 
 			   align_up(get_serial_hashtable_offset()+j*hashTableSize,get_page_logn()));
+#ifdef __CYGWIN__
+    // No madvise in CYGWIN
+#else
     if(madvise(db, hashTableSize, MADV_RANDOM)<0)
       error("could not advise local hashtable memory","","madvise");
+#endif
     SerialElementT* pe = (SerialElementT*)db ;
     H::generate_hash_keys(*(g+j),*(r1+j),*(r2+j)); 
     serial_bucket_chain_point(pe+t1*lshHeader->numCols, qpos); // Point to correct row
@@ -1384,8 +1399,12 @@
     // memory map a single hash table for random access
     char* db = serial_mmap(dbfid, hashTableSize, 0, 
 			   align_up(get_serial_hashtable_offset()+j*hashTableSize,get_page_logn()));
+#ifdef __CYGWIN__
+    // No madvise in CYGWIN
+#else
     if(madvise(db, hashTableSize, MADV_SEQUENTIAL)<0)
       error("could not advise local hashtable memory","","madvise");
+#endif
     SerialElementT* pe = (SerialElementT*)db ;
     printf("*********** TABLE %d ***************\n", j);
     fflush(stdout);
--- a/lshlib.h	Tue Aug 12 14:25:51 2008 +0000
+++ b/lshlib.h	Tue Aug 19 14:27:21 2008 +0000
@@ -66,15 +66,17 @@
 #define O2_SERIAL_MAXFILESIZE (4000000000UL)
 
 // Flags for Serial Header
-#define O2_SERIAL_FILEFORMAT1 (0x1U)       // Optimize for on-disk search
-#define O2_SERIAL_FILEFORMAT2 (0x2U)       // Optimize for in-core search
+#define O2_SERIAL_FILEFORMAT1 (0x1U)       // Optimize disk format for on-disk search
+#define O2_SERIAL_FILEFORMAT2 (0x2U)       // Optimize disk format for in-core search
+#define O2_SERIAL_COREFORMAT1 (0x4U)
+#define O2_SERIAL_COREFORMAT2 (0x8U)
 
 // Flags for serialization fileformat2: use high 3 bits of Uns32T
-#define O2_SERIAL_TOKEN_T1 (0xFFFFFFFC)
+#define O2_SERIAL_TOKEN_T1 (0xFFFFFFFCU)
 #define O2_SERIAL_TOKEN_T2 (0xFFFFFFFDU)
 #define O2_SERIAL_TOKEN_ENDTABLE (0xFFFFFFFEU)
 
-#define O2_INDEX_MAXSTR (512)
+#define O2_INDEX_MAXSTR (256)
 
 unsigned align_up(unsigned x, unsigned w);
 
@@ -320,7 +322,7 @@
 
   // Callback Function for point reporting
   void* calling_instance; // store calling object instance for member-function callback
-  void (*add_point_callback)(void*, Uns32T, Uns32T, float); // The callback
+  ReporterCallbackPtr add_point_callback; // Pointer to the callback function
 
  public:
   G(char* lshFile, bool lshInCore = false); // unserialize constructor
--- a/reporter.h	Tue Aug 12 14:25:51 2008 +0000
+++ b/reporter.h	Tue Aug 19 14:27:21 2008 +0000
@@ -292,6 +292,7 @@
   }
   std::vector<NNresult>::reverse_iterator rit;
   std::priority_queue< NNresult, std::vector< NNresult>, std::greater<NNresult> > point_queue;      
+  NNresult rk;
 
   if(adbQueryResponse==0) {
     for(rit = v.rbegin(); rit < v.rend(); rit++) {
@@ -309,31 +310,57 @@
       }
       
       for(unsigned int k = 0; k < qsize; k++) {
-	NNresult rk = point_queue.top();
+	rk = point_queue.top();
 	std::cout << rk.dist << " " << rk.qpos << " " << rk.spos << std::endl;
 	point_queue.pop();
       }
     }
   } else {
-    ((adb__queryResponse*)adbQueryResponse)->result.__sizeRlist=size;
-    ((adb__queryResponse*)adbQueryResponse)->result.__sizeDist=size;
-    ((adb__queryResponse*)adbQueryResponse)->result.__sizeQpos=size;
-    ((adb__queryResponse*)adbQueryResponse)->result.__sizeSpos=size;
-    ((adb__queryResponse*)adbQueryResponse)->result.Rlist= new char*[size];
-    ((adb__queryResponse*)adbQueryResponse)->result.Dist = new double[size];
-    ((adb__queryResponse*)adbQueryResponse)->result.Qpos = new unsigned int[size];
-    ((adb__queryResponse*)adbQueryResponse)->result.Spos = new unsigned int[size];
+   ((adb__queryResponse*)adbQueryResponse)->result.__sizeRlist=size*pointNN;
+    ((adb__queryResponse*)adbQueryResponse)->result.__sizeDist=size*pointNN;
+    ((adb__queryResponse*)adbQueryResponse)->result.__sizeQpos=size*pointNN;
+    ((adb__queryResponse*)adbQueryResponse)->result.__sizeSpos=size*pointNN;
+    ((adb__queryResponse*)adbQueryResponse)->result.Rlist= new char*[size*pointNN];
+    ((adb__queryResponse*)adbQueryResponse)->result.Dist = new double[size*pointNN];
+    ((adb__queryResponse*)adbQueryResponse)->result.Qpos = new unsigned int[size*pointNN];
+    ((adb__queryResponse*)adbQueryResponse)->result.Spos = new unsigned int[size*pointNN];
     unsigned int k = 0;
-    for(rit = v.rbegin(); rit < v.rend(); rit++, k++) {
+    // Loop over returned tracks
+    for(rit = v.rbegin(); rit < v.rend(); rit++) {
       r = *rit;
-      ((adb__queryResponse*)adbQueryResponse)->result.Rlist[k] = new char[O2_MAXFILESTR];
-      ((adb__queryResponse*)adbQueryResponse)->result.Dist[k] = r.dist;
-      ((adb__queryResponse*)adbQueryResponse)->result.Qpos[k] = r.qpos;
-      ((adb__queryResponse*)adbQueryResponse)->result.Spos[k] = r.spos;
-      if(fileTable)
-	snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "%s", fileTable+r.trackID*O2_FILETABLE_ENTRY_SIZE);
-      else
-	snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "%d", r.trackID);
+      // Reverse the order of the points stored in point_queues
+      unsigned int qsize=point_queues[r.trackID].size();
+      while(qsize--){
+	point_queue.push(point_queues[r.trackID].top());
+	point_queues[r.trackID].pop();
+      }
+      qsize=point_queue.size();
+      unsigned int numReports = pointNN;
+      while(numReports--){ // pop the rest of the points
+	if(qsize)
+	  rk = point_queue.top(); // Take one point from the top of the queue
+	else{
+	  rk.dist = 1000000000.0;
+	  rk.qpos = 0xFFFFFFFF;
+	  rk.spos = 0xFFFFFFFF;
+	}
+	  
+	((adb__queryResponse*)adbQueryResponse)->result.Rlist[k] = new char[O2_MAXFILESTR];
+	((adb__queryResponse*)adbQueryResponse)->result.Dist[k] = rk.dist;
+	((adb__queryResponse*)adbQueryResponse)->result.Qpos[k] = rk.qpos;
+	((adb__queryResponse*)adbQueryResponse)->result.Spos[k] = rk.spos;
+	if(qsize){
+	  if(fileTable)
+	    snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "%s", fileTable+r.trackID*O2_FILETABLE_ENTRY_SIZE);
+	  else
+	    snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "%d", r.trackID);	
+	  point_queue.pop();
+	  qsize--;
+	}
+	else
+	  snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "NULL");		  
+	k++;
+      }
     }
   }
   // clean up
@@ -641,17 +668,17 @@
     }
   }
  else {
-   ((adb__queryResponse*)adbQueryResponse)->result.__sizeRlist=size;
-    ((adb__queryResponse*)adbQueryResponse)->result.__sizeDist=size;
-    ((adb__queryResponse*)adbQueryResponse)->result.__sizeQpos=size;
-    ((adb__queryResponse*)adbQueryResponse)->result.__sizeSpos=size;
-    ((adb__queryResponse*)adbQueryResponse)->result.Rlist= new char*[size];
-    ((adb__queryResponse*)adbQueryResponse)->result.Dist = new double[size];
-    ((adb__queryResponse*)adbQueryResponse)->result.Qpos = new unsigned int[size];
-    ((adb__queryResponse*)adbQueryResponse)->result.Spos = new unsigned int[size];
+   ((adb__queryResponse*)adbQueryResponse)->result.__sizeRlist=size*pointNN;
+    ((adb__queryResponse*)adbQueryResponse)->result.__sizeDist=size*pointNN;
+    ((adb__queryResponse*)adbQueryResponse)->result.__sizeQpos=size*pointNN;
+    ((adb__queryResponse*)adbQueryResponse)->result.__sizeSpos=size*pointNN;
+    ((adb__queryResponse*)adbQueryResponse)->result.Rlist= new char*[size*pointNN];
+    ((adb__queryResponse*)adbQueryResponse)->result.Dist = new double[size*pointNN];
+    ((adb__queryResponse*)adbQueryResponse)->result.Qpos = new unsigned int[size*pointNN];
+    ((adb__queryResponse*)adbQueryResponse)->result.Spos = new unsigned int[size*pointNN];
     unsigned int k = 0;
     // Loop over returned tracks
-    for(rit = v.rbegin(); rit < v.rend(); rit++, k++) {
+    for(rit = v.rbegin(); rit < v.rend(); rit++) {
       r = *rit;
       // Reverse the order of the points stored in point_queues
       unsigned int qsize=point_queues[r.trackID].size();
@@ -660,17 +687,32 @@
 	point_queues[r.trackID].pop();
       }
       qsize=point_queue.size();
-      rk = point_queue.top(); // Take one point from the top of the queue
-      ((adb__queryResponse*)adbQueryResponse)->result.Rlist[k] = new char[O2_MAXFILESTR];
-      ((adb__queryResponse*)adbQueryResponse)->result.Dist[k] = rk.dist;
-      ((adb__queryResponse*)adbQueryResponse)->result.Qpos[k] = rk.qpos;
-      ((adb__queryResponse*)adbQueryResponse)->result.Spos[k] = rk.spos;
-      if(fileTable)
-	snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "%s", fileTable+r.trackID*O2_FILETABLE_ENTRY_SIZE);
-      else
-	snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "%d", r.trackID);
-      while(qsize--) // pop the rest of the points
-	point_queue.pop();
+      unsigned int numReports = pointNN;
+      while(numReports--){ // pop the rest of the points
+	if(qsize)
+	  rk = point_queue.top(); // Take one point from the top of the queue
+	else{
+	  rk.dist = 1000000000.0;
+	  rk.qpos = 0xFFFFFFFF;
+	  rk.spos = 0xFFFFFFFF;
+	}
+	  
+	((adb__queryResponse*)adbQueryResponse)->result.Rlist[k] = new char[O2_MAXFILESTR];
+	((adb__queryResponse*)adbQueryResponse)->result.Dist[k] = rk.dist;
+	((adb__queryResponse*)adbQueryResponse)->result.Qpos[k] = rk.qpos;
+	((adb__queryResponse*)adbQueryResponse)->result.Spos[k] = rk.spos;
+	if(qsize){
+	  if(fileTable)
+	    snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "%s", fileTable+r.trackID*O2_FILETABLE_ENTRY_SIZE);
+	  else
+	    snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "%d", r.trackID);	
+	  point_queue.pop();
+	  qsize--;
+	}
+	else
+	  snprintf(((adb__queryResponse*)adbQueryResponse)->result.Rlist[k], O2_MAXFILESTR, "NULL");
+	k++;
+      }
     }
  }
   delete[] point_queues;
--- a/soap.cpp	Tue Aug 12 14:25:51 2008 +0000
+++ b/soap.cpp	Tue Aug 19 14:27:21 2008 +0000
@@ -126,6 +126,8 @@
     strncpy(queryType, "sequence", strlen("sequence"));
   else if(qType == O2_TRACK_QUERY)
     strncpy(queryType,"track", strlen("track"));
+  else if(qType == O2_N_SEQUENCE_QUERY)
+    strncpy(queryType,"nsequence", strlen("nsequence"));
   else
     strncpy(queryType, "", strlen(""));