comparison audioDB.h @ 196:8c81cacf5aab

Merge -r228:254 from no-big-mmap branch.

Although the last log message from that branch only mentioned working
create and status (-N and -S), it turned out that I seemed to have done
everything right for dump and search on huge DBs to work too.

Additionally:
* bump the DB format version;
* CHECKED_MMAP() for the powerTable;
* move the powerTable above the timesTable, so that all the code
  everywhere which computes the length of the data buffer assuming
  that the timesTable is the next thing on the disk still works.

 #define COM_RELATIVE_THRESH "--relative-threshold"
 #define COM_ABSOLUTE_THRESH "--absolute-threshold"
 
 #define O2_OLD_MAGIC ('O'|'2'<<8|'D'<<16|'B'<<24)
 #define O2_MAGIC ('o'|'2'<<8|'d'<<16|'b'<<24)
-#define O2_FORMAT_VERSION (1U)
+#define O2_FORMAT_VERSION (2U)
 
 #define O2_DEFAULT_POINTNN (10U)
 #define O2_DEFAULT_TRACKNN  (10U)
 
 #define O2_DEFAULTDBSIZE (2000000000) // 2GB table size

 #define O2_ACTION(a) (strcmp(command,a)==0)
 
 #define ALIGN_UP(x,w) ((x) + ((1<<w)-1) & ~((1<<w)-1))
 #define ALIGN_DOWN(x,w) ((x) & ~((1<<w)-1))
 
+#define ALIGN_PAGE_UP(x) ((x) + (getpagesize()-1) & ~(getpagesize()-1))
+#define ALIGN_PAGE_DOWN(x) ((x) & ~(getpagesize()-1))
+
 #define ENSURE_STRING(x) ((x) ? (x) : "")
 
 using namespace std;
 
 typedef struct dbTableHeader{
   uint32_t magic;
   uint32_t version;
   uint32_t numFiles;
   uint32_t dim;
   uint32_t flags;
-  // FIXME: these lengths and offsets should be size_t or off_t, but
-  // that causes this header (and hence audioDB files) to be
-  // unportable between 32 and 64-bit architectures.  Making them
-  // uint32_t isn't the real answer, as it means we won't be able to
-  // scale to really large collections easily but it works around the
-  // problem.  Expanding to 64 bits will of course need a change in
-  // file format version.  -- CSR, 2007-10-05
-  uint32_t length;
-  uint32_t fileTableOffset;
-  uint32_t trackTableOffset;
-  uint32_t dataOffset;
-  uint32_t l2normTableOffset;
-  uint32_t timesTableOffset;
-  uint32_t powerTableOffset;
-  uint32_t dbSize;
+  off_t length;
+  off_t fileTableOffset;
+  off_t trackTableOffset;
+  off_t dataOffset;
+  off_t l2normTableOffset;
+  off_t timesTableOffset;
+  off_t powerTableOffset;
+  off_t dbSize;
 } dbTableHeaderT, *dbTableHeaderPtr;
 
 
 class audioDB{
   

   const char *powerFileName;
   ifstream *powerFile;
   int powerfd;
 
   int dbfid;
+  bool forWrite;
   int infid;
   char* db;
   char* indata;
   struct stat statbuf;  
   dbTableHeaderPtr dbH;

   double* dataBuf;
   double* inBuf;
   double* l2normTable;
   double* qNorm;
   double* sNorm;
-  double* timesTable;  
+  double* timesTable;
   double* powerTable;
+
+  size_t fileTableLength;
+  size_t trackTableLength;
+  off_t dataBufLength;
+  size_t timesTableLength;
+  size_t powerTableLength;
+  size_t l2normTableLength;
 
   // Flags and parameters
   unsigned verbosity;   // how much do we want to know?
-  unsigned size; // given size (for creation)
+  off_t size; // given size (for creation)
   unsigned queryType; // point queries default
   unsigned pointNN;   // how many point NNs ?
   unsigned trackNN;   // how many track NNs ?
   unsigned sequenceLength;
   unsigned sequenceHop;

   void sequence_average(double *buffer, int length, int seqlen);
 
   void trackSequenceQueryNN(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse=0);
   void trackSequenceQueryRad(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse=0);
 
-  void initDBHeader(const char *dbName, bool forWrite);
+  void initDBHeader(const char *dbName);
   void initInputFile(const char *inFile);
-  void initTables(const char* dbName, bool forWrite, const char* inFile);
+  void initTables(const char* dbName, const char* inFile);
   void unitNorm(double* X, unsigned d, unsigned n, double* qNorm);
   void unitNormAndInsertL2(double* X, unsigned dim, unsigned n, unsigned append);
   void insertTimeStamps(unsigned n, ifstream* timesFile, double* timesdata);
   void insertPowerData(unsigned n, int powerfd, double *powerdata);
   unsigned getKeyPos(char* key);

   int processArgs(const unsigned argc, char* const argv[]);
   void get_lock(int fd, bool exclusive);
   void release_lock(int fd);
   void create(const char* dbName);
   void drop();
+  bool enough_data_space_free(off_t size);
+  void insert_data_vectors(off_t offset, void *buffer, size_t size);
   void insert(const char* dbName, const char* inFile);
   void batchinsert(const char* dbName, const char* inFile);
   void query(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse=0);
   void status(const char* dbName, adb__statusResponse *adbStatusResponse=0);
   void ws_status(const char*dbName, char* hostport);

   timesFile(0), \
   powerFileName(0), \
   powerFile(0), \
   powerfd(0), \
   dbfid(0), \
+  forWrite(false), \
   infid(0), \
   db(0), \
   indata(0), \
   dbH(0), \
   fileTable(0), \
   trackTable(0), \
   dataBuf(0), \
   l2normTable(0), \
   qNorm(0), \
   timesTable(0), \
+  fileTableLength(0), \
+  trackTableLength(0), \
+  dataBufLength(0), \
+  timesTableLength(0), \
+  powerTableLength(0), \
+  l2normTableLength(0), \
   verbosity(1), \
   size(O2_DEFAULTDBSIZE), \
   queryType(O2_POINT_QUERY), \
   pointNN(O2_DEFAULT_POINTNN), \
   trackNN(O2_DEFAULT_TRACKNN), \