comparison lshlib.cpp @ 336:fe4d5b763086

converted read/write into fread/fwrite for LSH hashtable serialize and unserialize. INDEXING is now faster.

     unserialize_lsh_hashtables_format1(dbfid);
   }
 
   // Format2 always needs unserializing
   if(lshHeader->flags&O2_SERIAL_FILEFORMAT2 && lshInCoreFlag){
-    unserialize_lsh_hashtables_format2(dbfid);
-  }
-
-  close(dbfid);}
+    FILE* dbFile = fdopen(dbfid, "rb");
+    if(!dbFile)
+      error("Cannot open LSH file for reading", filename);
+    unserialize_lsh_hashtables_format2(dbFile);
+  }
+  serial_close(dbfid);
+}
 
 G::~G(){
   delete lshHeader;
 }
 

   if(dbIsNew)
     serialize_lsh_hashfunctions(dbfid);
   // Write the hashtables in the requested format
   if(serialFormat == O2_SERIAL_FILEFORMAT1)
     serialize_lsh_hashtables_format1(dbfid, !dbIsNew);
-  else
-    serialize_lsh_hashtables_format2(dbfid, !dbIsNew);
-
-  if(!dbIsNew){
+  else{
+    FILE* dbFile = fdopen(dbfid, "r+b");
+    if(!dbFile)
+      error("Cannot open LSH file for writing",filename);
+    serialize_lsh_hashtables_format2(dbFile, !dbIsNew);
+    fflush(dbFile);
+  }
+
+  if(!dbIsNew) { 
     db = serial_mmap(dbfid, O2_SERIAL_HEADER_SIZE, 1);// get database pointer
     //serial_get_header(db);           // read header
     cout << "maxp = " << H::maxp << endl;
     lshHeader->maxp=H::maxp;
     // Default to FILEFORMAT1
     if(!(lshHeader->flags&O2_SERIAL_FILEFORMAT2))
       lshHeader->flags|=O2_SERIAL_FILEFORMAT1;
     memcpy((char*)db, (char*)lshHeader, sizeof(SerialHeaderT));
     serial_munmap(db, O2_SERIAL_HEADER_SIZE); // drop mmap
-  }
-
-  serial_close(dbfid);
+  }  
+    serial_close(dbfid);
 }
 
 // Test to see if core structure and requested format is
 // compatible with currently opened database
 int G::serial_can_merge(Uns32T format){

   }
   pe->hashValue=IFLAG;
   return;
 }
 
-int G::serialize_lsh_hashtables_format2(int fid, int merge){
+int G::serialize_lsh_hashtables_format2(FILE* dbFile, int merge){
   Uns32T x,y;
 
   if( merge && !serial_can_merge(O2_SERIAL_FILEFORMAT2) )
     error("Cannot merge core and serial LSH, data structure dimensions mismatch.");
 
   // We must pereform FORMAT2 merges in core
   if(merge)
-    unserialize_lsh_hashtables_format2(fid);
+    unserialize_lsh_hashtables_format2(dbFile);
 
   Uns32T colCount, meanColCount, colCountN, maxColCount, minColCount, t1;
-  lseek(fid, get_serial_hashtable_offset(), SEEK_SET);
+  if(fseek(dbFile, get_serial_hashtable_offset(), SEEK_SET)){
+    fclose(dbFile);
+    error("fSeek error in serialize_lsh_hashtables_format2");
+  }
 
   // Write the hash tables
   for( x = 0 ; x < H::L ; x++ ){
     std::cout << (merge ? "merging":"writing") << " hash table " << x << " FORMAT2...";
     std::cout.flush();

       colCount=0;
       if(bucket* bPtr = h[x][y]){
 	// Check for empty row (even though row was allocated)
 #ifdef LSH_BLOCK_FULL_ROWS
 	if(bPtr->next->t2==IFLAG){
-	  close(fid);
+	  fclose(dbFile);
 	  error("b->next->t2==IFLAG","serialize_lsh_hashtables_format2()");
 	}
 #else
 	if(bPtr->t2==IFLAG){
-	  close(fid);
+	  fclose(dbFile);
 	  error("b->t2==IFLAG","serialize_lsh_hashtables_format2()");
 	}
 #endif
 	t1 = O2_SERIAL_TOKEN_T1;
-	if( write(fid, &t1, sizeof(Uns32T)) != sizeof(Uns32T) ){
-	  close(fid);
+	if( fwrite(&t1, sizeof(Uns32T), 1, dbFile) != 1 ){
+	  fclose(dbFile);
 	  error("write error in serial_write_hashtable_format2() [T1]");
 	}
 	t1 = y;
-	if( write(fid, &t1, sizeof(Uns32T)) != sizeof(Uns32T) ){
-	  close(fid);
+	if( fwrite(&t1, sizeof(Uns32T), 1, dbFile) != 1 ){
+	  fclose(dbFile);
 	  error("write error in serial_write_hashtable_format2() [t1]");
 	}
 #ifdef LSH_BLOCK_FULL_ROWS
-	serial_write_hashtable_row_format2(fid, bPtr->next, colCount); // skip collision counter bucket
-#else
-      serial_write_hashtable_row_format2(fid, bPtr, colCount);
+	serial_write_hashtable_row_format2(dbFile, bPtr->next, colCount); // skip collision counter bucket
+#else
+      serial_write_hashtable_row_format2(dbFile, bPtr, colCount);
 #endif
       }
       if(colCount){
 	if(colCount<minColCount)
 	  minColCount=colCount;

 	colCountN++;
       }
     }
     // Write END of table marker
     t1 = O2_SERIAL_TOKEN_ENDTABLE;
-    if( write(fid, &t1, sizeof(Uns32T)) != sizeof(Uns32T) ){
-      close(fid);
+    if( fwrite(&t1, sizeof(Uns32T), 1, dbFile ) != 1 ){
+      fclose(dbFile);
       error("write error in serial_write_hashtable_format2() [end]");
     }
 
     if(colCountN)
       std::cout << "#rows with collisions =" << colCountN << ", mean = " << meanColCount/(float)colCountN 

   
   // We're done writing
   return 1;
 }
 
-void G::serial_write_hashtable_row_format2(int fid, bucket* b, Uns32T& colCount){
+void G::serial_write_hashtable_row_format2(FILE* dbFile, bucket* b, Uns32T& colCount){
   while(b && b->t2!=IFLAG){
     if(!b->snext){
-      close(fid);
+      fclose(dbFile);
       error("Empty collision chain in serial_write_hashtable_row_format2()");
     }
     t2 = O2_SERIAL_TOKEN_T2;
-    if( write(fid, &t2, sizeof(Uns32T)) != sizeof(Uns32T) ){
-      close(fid);
+    if( fwrite(&t2, sizeof(Uns32T), 1, dbFile) != 1 ){
+      fclose(dbFile);
       error("write error in serial_write_hashtable_row_format2()");
     }
     t2 = b->t2;
-    if( write(fid, &t2, sizeof(Uns32T)) != sizeof(Uns32T) ){
-      close(fid);
+    if( fwrite(&t2, sizeof(Uns32T), 1, dbFile) != 1 ){
+      fclose(dbFile);
       error("write error in serial_write_hashtable_row_format2()");
     }
-    serial_write_element_format2(fid, b->snext, colCount);
+    serial_write_element_format2(dbFile, b->snext, colCount);
     b=b->next;
   }
 }
 
-void G::serial_write_element_format2(int fid, sbucket* sb, Uns32T& colCount){
+void G::serial_write_element_format2(FILE* dbFile, sbucket* sb, Uns32T& colCount){
   while(sb){
-    if(write(fid, &sb->pointID, sizeof(Uns32T))!=sizeof(Uns32T)){
-      close(fid);
+    if(fwrite(&sb->pointID, sizeof(Uns32T), 1, dbFile) != 1){
+      fclose(dbFile);
       error("Write error in serial_write_element_format2()");
     }
     colCount++;
     sb=sb->snext;  
   }

     pe++;
     colCount++;
   }
 }
  
-void G::unserialize_lsh_hashtables_format2(int fid){
+void G::unserialize_lsh_hashtables_format2(FILE* dbFile){
   Uns32T x=0,y=0;
 
   // Seek to hashtable base offset
-  if(lseek(fid, get_serial_hashtable_offset(), SEEK_SET)!=get_serial_hashtable_offset()){
-    close(fid);
-    error("Seek error in unserialize_lsh_hashtables_format2");
+  if(fseek(dbFile, get_serial_hashtable_offset(), SEEK_SET)){
+    fclose(dbFile);
+    error("fSeek error in unserialize_lsh_hashtables_format2");
   }
 
   // Read the hash tables into core (structure is given in header) 
   while( x < H::L){
-    if(read(fid, &(H::t1), sizeof(Uns32T))!=sizeof(Uns32T)){
-      close(fid);
+    if(fread(&(H::t1), sizeof(Uns32T), 1, dbFile) != 1){
+      fclose(dbFile);
       error("Read error","unserialize_lsh_hashtables_format2()");
     }
     if(H::t1==O2_SERIAL_TOKEN_ENDTABLE)
       x++; // End of table
     else
       while(y < H::N){
 	// Read a row and move file pointer to beginning of next row or _bittable
 	if(!(H::t1==O2_SERIAL_TOKEN_T1)){
-	  close(fid);
+	  fclose(dbFile);
 	  error("State matchine error T1","unserialize_lsh_hashtables_format2()");
 	}
-	if(read(fid, &(H::t1), sizeof(Uns32T))!=sizeof(Uns32T)){
-	  close(fid);
+	if(fread(&(H::t1), sizeof(Uns32T), 1, dbFile) != 1){
+	  fclose(dbFile);
 	  error("Read error: t1","unserialize_lsh_hashtables_format2()");
 	}
 	y = H::t1;
 	if(y>=H::N){
-	  close(fid);
+	  fclose(dbFile);
 	  error("Unserialized hashtable row pointer out of range","unserialize_lsh_hashtables_format2()");
 	}
-	Uns32T token = unserialize_hashtable_row_format2(fid, h[x]+y);
+	Uns32T token = unserialize_hashtable_row_format2(dbFile, h[x]+y);
 	
 #ifdef __LSH_DUMP_CORE_TABLES__
 	printf("C[%d,%d]", x, y);
 	dump_hashtable_row(h[x][y]);
 #endif
 	// Check that token is valid
 	if( !(token==O2_SERIAL_TOKEN_T1 || token==O2_SERIAL_TOKEN_ENDTABLE) ){
-	  close(fid);
+	  fclose(dbFile);
 	  error("State machine error end of row/table", "unserialize_lsh_hashtables_format2()");
 	}
 	// Check for end of table flag
 	if(token==O2_SERIAL_TOKEN_ENDTABLE){
 	  x++;

 	  H::t1 = token;
       }
   }
 }
  
-Uns32T G::unserialize_hashtable_row_format2(int fid, bucket** b){
+Uns32T G::unserialize_hashtable_row_format2(FILE* dbFile, bucket** b){
   bool pointFound = false;
-  if(read(fid, &(H::t2), sizeof(Uns32T)) != sizeof(Uns32T)){
-    close(fid);
+  if(fread(&(H::t2), sizeof(Uns32T), 1, dbFile) != 1){
+    fclose(dbFile);
     error("Read error T2 token","unserialize_hashtable_row_format2");
   }
   if( !(H::t2==O2_SERIAL_TOKEN_ENDTABLE || H::t2==O2_SERIAL_TOKEN_T2)){
-    close(fid);
+    fclose(dbFile);
     error("State machine error: expected E or T2");
   }
   while(!(H::t2==O2_SERIAL_TOKEN_ENDTABLE || H::t2==O2_SERIAL_TOKEN_T1)){
     pointFound=false;
     // Check for T2 token
     if(H::t2!=O2_SERIAL_TOKEN_T2){
-      close(fid);
+      fclose(dbFile);
       error("State machine error T2 token", "unserialize_hashtable_row_format2()");
     }
     // Read t2 value
-    if(read(fid, &(H::t2), sizeof(Uns32T)) != sizeof(Uns32T)){
-      close(fid);
+    if(fread(&(H::t2), sizeof(Uns32T), 1, dbFile) != 1){
+      fclose(dbFile);
       error("Read error t2","unserialize_hashtable_row_format2");
     }
-    if(read(fid, &(H::p), sizeof(Uns32T)) != sizeof(Uns32T)){
-      close(fid);
+    if(fread(&(H::p), sizeof(Uns32T), 1, dbFile) != 1){
+      fclose(dbFile);
       error("Read error H::p","unserialize_hashtable_row_format2");
     }
     while(!(H::p==O2_SERIAL_TOKEN_ENDTABLE || H::p==O2_SERIAL_TOKEN_T1 || H::p==O2_SERIAL_TOKEN_T2 )){
       pointFound=true;
       bucket_insert_point(b);
-      if(read(fid, &(H::p), sizeof(Uns32T)) != sizeof(Uns32T)){
-	close(fid);
+      if(fread(&(H::p), sizeof(Uns32T), 1, dbFile) != 1){
+	fclose(dbFile);
 	error("Read error H::p","unserialize_hashtable_row_format2");
       }
     }
     if(!pointFound)
       error("State machine error: point", "unserialize_hashtable_row_format2()");    

   char* dbheader = serial_mmap(dbfid, O2_SERIAL_HEADER_SIZE, 0);// get database pointer
   serial_get_header(dbheader);           // read header
   serial_munmap(dbheader, O2_SERIAL_HEADER_SIZE); // drop header mmap
 
   if((lshHeader->flags & O2_SERIAL_FILEFORMAT2)){
-    close(dbfid);
+    serial_close(dbfid);
     error("serial_retrieve_point_set is for SERIAL_FILEFORMAT1 only");
   }
 
   // size of each hash table
   Uns32T hashTableSize=sizeof(SerialElementT)*lshHeader->numRows*lshHeader->numCols;

   char* dbheader = serial_mmap(dbfid, O2_SERIAL_HEADER_SIZE, 0);// get database pointer
   serial_get_header(dbheader);           // read header
   serial_munmap(dbheader, O2_SERIAL_HEADER_SIZE); // drop header mmap
 
   if((lshHeader->flags & O2_SERIAL_FILEFORMAT2)){
-    close(dbfid);
+    serial_close(dbfid);
     error("serial_retrieve_point is for SERIAL_FILEFORMAT1 only");
   }
 
   // size of each hash table
   Uns32T hashTableSize=sizeof(SerialElementT)*lshHeader->numRows*lshHeader->numCols;