Mercurial > hg > audiodb

changeset 474:f9d86b1db21c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Fixed memory leaks, added WS --no_unit_norming, and removed capping of LSH_N_POINT_BITS to 15 bits, instead allow any number of bits to encode points, remaining bits encode tracks
author mas01mc
date Fri, 09 Jan 2009 18:05:32 +0000
parents add65705e655
children fd890d2b38da
files QueryADB.py audioDB.cpp audioDB.h create.cpp lshlib.cpp lshlib.h query.cpp
diffstat 7 files changed, 112 insertions(+), 19 deletions(-) [+]
line wrap: on
line diff
--- a/QueryADB.py	Tue Jan 06 07:02:11 2009 +0000
+++ b/QueryADB.py	Fri Jan 09 18:05:32 2009 +0000
@@ -15,7 +15,7 @@
 # From: http://www.informit.com/articles/article.aspx?p=686162&seqNum=2
 #serverHost = 'research-hm3.corp.sk1.yahoo.com'
 serverHost = 'localhost'
-serverPort = 14475
+serverPort = 14476
 
 # Start the server on serverHost with
 #	./audioDB -s 14475
@@ -117,6 +117,39 @@
 </SOAP-ENV:Envelope>
 """
 
+
+
+FEATURE_QUERY_TEMPLATE = """
+<?xml version="1.0" encoding="UTF-8"?>
+<SOAP-ENV:Envelope
+ xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/"
+ xmlns:SOAP-ENC="http://schemas.xmlsoap.org/soap/encoding/"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:xsd="http://www.w3.org/2001/XMLSchema"
+ xmlns:adb="http://tempuri.org/adb.xsd">
+ <SOAP-ENV:Body>
+  <adb:query>
+   <dbName>%s</dbName>
+   <qKey>%s</qKey>
+   <keyList>%s</keyList>
+   <timesFileName>%s</timesFileName>
+   <powerFileName>%s</powerFileName>
+   <qType>%s</qType>
+   <qPos>%s</qPos>
+   <pointNN>%s</pointNN>
+   <segNN>%s</segNN>
+   <segLen>%s</segLen>
+   <radius>%s</radius>
+   <absolute-threshold>%s</absolute-threshold>
+   <relative-threshold>%s</relative-threshold>
+   <exhaustive>%s</exhaustive>
+   <lsh-exact>%s</lsh-exact>
+   <no-unit-norming>%s</no-unit-norming>
+  </adb:query>
+ </SOAP-ENV:Body>
+</SOAP-ENV:Envelope>
+"""
+
 ###############  List Query - Show the files in the database ###########
 # Return a list of (key identifier, frame length) pairs.
 def RunListQuery():
@@ -210,7 +243,7 @@
 		pointNN = '10'
 		trackNN = '5'
 		seqLen = argv[4]
-		queryRadius = '0.5'
+		queryRadius = '2'
 	else:
 		dbKey = 'tmp/3.chr'
 		qType = '32'			# nSequence
@@ -226,6 +259,51 @@
 	response = SendXMLCommand(message)
 	ParseShingleXML(response)
 
+###############  Sequence Query - Show the data closest to one query ###########
+def RunQuery(argv):
+#   <dbName>%s</dbName>
+#   <qKey>%s</qKey>
+#   <keyList>%s</keyList>
+#   <timesFileName>%s</timesFileName>
+#   <powerFileName>%s</powerFileName>
+#   <qType>%s</qType>
+#   <qPos>%s</qPos>
+#   <pointNN>%s</pointNN>
+#   <segNN>%s</segNN>
+#   <segLen>%s</segLen>
+#   <radius>%s</radius>
+#   <absolute-threshold>%s</absolute-threshold>
+#   <relative-threshold>%s</relative-threshold>
+#   <exhaustive>%s</exhaustive>
+#   <lsh-exact>%s</lsh-exact>
+#   <no-unit-norming>%s</no-unit-norming>
+	global debug, dbName
+	if len(argv) > 2:
+		featureFile = argv[2]
+		powerFile = argv[3]
+		qType = '32'			# nSequence
+		qPos = argv[4]
+		pointNN = '20'
+		trackNN = '5'
+		seqLen = argv[5]
+		queryRadius = '0.2'
+	else:
+		featureFile = 'foo.chr12'
+		powerFile = 'foo.power'
+		qType = '32'			# nSequence
+		qPos = '0'
+		pointNN = '3'
+		trackNN = '5'
+		seqLen = '10'
+		queryRadius = '0.2'
+
+	message = FEATURE_QUERY_TEMPLATE
+	message = FEATURE_QUERY_TEMPLATE%(dbName, featureFile, "", "", powerFile, qType, qPos, pointNN, trackNN, seqLen, queryRadius, '0.0', '0.0', '0', '1','0')
+
+	print message
+	response = SendXMLCommand(message)
+	ParseShingleXML(response)
+
 def ParseShingleXML(response):
 	# Grab all the responses
 	#	See http://diveintopython.org/xml_processing/parsing_xml.html
@@ -301,7 +379,7 @@
 if __name__=="__main__":
 	cmdname = sys.argv[0]
 	if len(sys.argv) == 1:
-		print "Syntax: " + sys.argv[0] + " -q feature_file pos len"
+		print "Syntax: " + sys.argv[0] + " -{s,q,f,l} feature_file [power_file] pos len"
 		sys.exit(1)
 
 	queryType = sys.argv[1]
@@ -311,6 +389,8 @@
 			print k, v
 	elif queryType == '-q' or queryType == 'query':
 		RunSequenceQuery(sys.argv)
+	elif queryType == '-f' or queryType == 'feature':
+		RunQuery(sys.argv)
 	elif queryType == '-l' or queryType == 'list':
 		response = RunListQuery()
 		# print response
--- a/audioDB.cpp	Tue Jan 06 07:02:11 2009 +0000
+++ b/audioDB.cpp	Fri Jan 09 18:05:32 2009 +0000
@@ -40,13 +40,15 @@
   if(dbName && adb_root)
     prefix_name((char** const)&dbName, adb_root);
 
-  if(O2_ACTION(COM_SERVER))
+  if(O2_ACTION(COM_SERVER)){
 #ifdef LIBRARY
     ;
 #else
     startServer();
+    if(SERVER_LSH_INDEX_SINGLETON)
+      delete lsh;
 #endif
-
+  }
   else  if(O2_ACTION(COM_CREATE))
     create(dbName);
 
--- a/audioDB.h	Tue Jan 06 07:02:11 2009 +0000
+++ b/audioDB.h	Fri Jan 09 18:05:32 2009 +0000
@@ -69,8 +69,6 @@
 // Because we work in Radius^2 units,
 // The sqrt of this number is the multiplier on the radius
 
-#define O2_LSH_EXACT_MULT 1
-
 #define O2_OLD_MAGIC ('O'|'2'<<8|'D'<<16|'B'<<24)
 #define O2_MAGIC ('o'|'2'<<8|'d'<<16|'b'<<24)
 #define O2_FORMAT_VERSION (4U)
--- a/create.cpp	Tue Jan 06 07:02:11 2009 +0000
+++ b/create.cpp	Fri Jan 09 18:05:32 2009 +0000
@@ -55,9 +55,9 @@
   // For backward-compatibility, Record the point-encoding parameter for LSH indexing in the adb header
   // If this value is 0 then it will be set to 14
 
-#if O2_LSH_N_POINT_BITS > 15
-#error "AudioDB Compile ERROR: consistency check of O2_LSH_POINT_BITS failed (>15)"
-#endif
+  //#if O2_LSH_N_POINT_BITS > 28
+  //#error "AudioDB Compile ERROR: consistency check of O2_LSH_POINT_BITS failed (>15)"
+  //#endif
   
   dbH->flags |= LSH_N_POINT_BITS << 28;
 
--- a/lshlib.cpp	Tue Jan 06 07:02:11 2009 +0000
+++ b/lshlib.cpp	Fri Jan 09 18:05:32 2009 +0000
@@ -252,9 +252,9 @@
       }
       delete[] H::h[ j ];
   }
-  delete[] H::r1;
-  delete[] H::r2;
-  delete[] H::h;
+    delete[] H::r1;
+    delete[] H::r2;
+    delete[] H::h;
 }
 
 
@@ -472,6 +472,7 @@
   calling_instance(0),
   add_point_callback(0)
 {
+  FILE* dbFile = 0;
   int dbfid = unserialize_lsh_header(filename);
   indexName = new char[O2_INDEX_MAXSTR];
   strncpy(indexName, filename, O2_INDEX_MAXSTR); // COPY THE CONTENTS TO THE NEW POINTER
@@ -485,16 +486,21 @@
 
   // Format2 always needs unserializing
   if(lshHeader->flags&O2_SERIAL_FILEFORMAT2 && lshInCoreFlag){
-    FILE* dbFile = fdopen(dbfid, "rb");
+    dbFile = fdopen(dbfid, "rb");
     if(!dbFile)
       error("Cannot open LSH file for reading", filename);
     unserialize_lsh_hashtables_format2(dbFile);
   }
   serial_close(dbfid);
+  if(dbFile){
+    fclose(dbFile);
+    dbFile = 0;
+  }
 }
 
 G::~G(){
   delete lshHeader;
+  delete[] indexName;
 }
 
 // single point insertion; inserted values are hash value and pointID
@@ -680,7 +686,7 @@
   int dbfid;
   char* db;
   int dbIsNew=0;
-
+  FILE* dbFile = 0;
   // Check requested serialFormat
   if(!(serialFormat==O2_SERIAL_FILEFORMAT1 || serialFormat==O2_SERIAL_FILEFORMAT2))
     error("Unrecognized serial file format request: ", "serialize()");
@@ -717,7 +723,7 @@
   if(serialFormat == O2_SERIAL_FILEFORMAT1)
     serialize_lsh_hashtables_format1(dbfid, !dbIsNew);
   else{
-    FILE* dbFile = fdopen(dbfid, "r+b");
+    dbFile = fdopen(dbfid, "r+b");
     if(!dbFile)
       error("Cannot open LSH file for writing",filename);
     serialize_lsh_hashtables_format2(dbFile, !dbIsNew);
@@ -736,6 +742,10 @@
     serial_munmap(db, O2_SERIAL_HEADER_SIZE); // drop mmap
   }  
     serial_close(dbfid);
+    if(dbFile){
+      fclose(dbFile);
+      dbFile = 0;
+    }
 }
 
 // Test to see if core structure and requested format is
--- a/lshlib.h	Tue Jan 06 07:02:11 2009 +0000
+++ b/lshlib.h	Fri Jan 09 18:05:32 2009 +0000
@@ -244,7 +244,7 @@
 
   H();
   H(Uns32T k, Uns32T m, Uns32T d, Uns32T N, Uns32T C, float w, float r);
-  ~H();
+  virtual ~H();
 
   float get_w(){return w;}
   float get_radius(){return radius;}  
@@ -341,7 +341,7 @@
  public:
   G(char* lshFile, bool lshInCore = false); // unserialize constructor
   G(float w, Uns32T k,Uns32T m, Uns32T d, Uns32T N, Uns32T C, float r); // core constructor
-  ~G();
+  virtual ~G();
 
   Uns32T insert_point(vector<float>&, Uns32T pointID);
   void insert_point_set(vector<vector<float> >& vv, Uns32T basePointID);
--- a/query.cpp	Tue Jan 06 07:02:11 2009 +0000
+++ b/query.cpp	Fri Jan 09 18:05:32 2009 +0000
@@ -589,12 +589,15 @@
 	  dist = qn*qn + sn*sn - 2*dist;
       // else
       // dist = dist;      
-      if((!radius) || dist <= (O2_LSH_EXACT_MULT*radius+O2_DISTANCE_TOLERANCE)) 
+      if((!radius) || dist <= (radius+O2_DISTANCE_TOLERANCE)) 
 	reporter->add_point(pp.trackID, pp.qpos, pp.spos, dist);    
     }
     exact_evaluation_queue->pop();
   }
+
+  
   // Cleanup
+  free(data_buffer);
   SAFE_DELETE_ARRAY(sNorm);
   SAFE_DELETE_ARRAY(sPower);
   SAFE_DELETE_ARRAY(meanDBdur);