comparison audioDB-internals.h @ 509:cc2b97d020b1

Code rearrangements to tease apart library code from C++ audioDB code.

There should be precisely no functional changes in this commit.
Instead, the only thing that has happened is that all the abstraction
violation and other horribleness is concentrated in one place: the
include of "audioDB-internals.h" in audioDB.h -- the separation will be
complete once that include can be removed.

This include is necessary because the command-line binary / SOAP server
still does some things directly rather than through an API: not least of
which the operations that have not yet been integrated into the API yet,
but also some messing around with constants, flags and nominally
internal functions.  The intent is to remove as many of these as
possible and think quite hard about the rest.

In the meantime, the library is now much more self-contained: the only
things it uses are in the audioDB_API.h and audioDB-internals.h headers;
thus there are fewer nasty surprises lurking for readers of the code.

The Makefile has been adjusted to take advantage of this rearrangement
in the dependencies.

+#include <set>
+#include <queue>
+#include <map>
+#include <string>
+
+#include "pointpair.h"
 #include "accumulator.h"
+#include "lshlib.h"
 
 /* this struct is for writing polymorphic routines as puns.  When
  * inserting, we might have a "datum" (with actual numerical data) or
  * a "reference" (with strings denoting pathnames containing numerical
  * data), but most of the operations are the same.  This struct, used

   std::set<std::string> *allowed_keys;
   std::priority_queue<PointPair> *exact_evaluation_queue;
   LSH *lsh;
 } adb_qstate_internal_t;
 
+/* this struct is the in-memory representation of the binary
+ * information stored at the head of each adb file */
+typedef struct adbheader {
+  uint32_t magic;
+  uint32_t version;
+  uint32_t numFiles;
+  uint32_t dim;
+  uint32_t flags;
+  uint32_t headerSize;
+  off_t length;
+  off_t fileTableOffset;
+  off_t trackTableOffset;
+  off_t dataOffset;
+  off_t l2normTableOffset;
+  off_t timesTableOffset;
+  off_t powerTableOffset;
+  off_t dbSize;
+} adb_header_t;
+
+#define ADB_HEADER_SIZE (sizeof(struct adbheader))
+
+#define ADB_HEADER_FLAG_L2NORM		(0x1U)
+#define ADB_HEADER_FLAG_POWER		(0x4U)
+#define ADB_HEADER_FLAG_TIMES		(0x20U)
+#define ADB_HEADER_FLAG_REFERENCES	(0x40U)
+
 /* the transparent version of the opaque (forward-declared) adb_t. */
 struct adb {
   char *path;
   int fd;
   int flags;

   { if(table) { \
       munmap(table, length); \
     } \
   }
 
+#define maybe_delete_array(pointer) \
+  { if(pointer) { \
+      delete [] pointer; \
+      pointer = NULL; \
+    } \
+  }
+
 #define write_or_goto_error(fd, buffer, size) \
   { ssize_t tmp = size; \
     if(write(fd, buffer, size) != tmp) { \
       goto error; \
     } \

     goto error;
   }
   if(lseek(adb->fd, (off_t) 0, SEEK_SET) == (off_t) -1) {
     goto error;
   }
-  if(write(adb->fd, adb->header, O2_HEADERSIZE) != O2_HEADERSIZE) {
+  if(write(adb->fd, adb->header, ADB_HEADER_SIZE) != ADB_HEADER_SIZE) {
     goto error;
   }
 
   /* can be fsync() if fdatasync() is racily exciting and new */
   fdatasync(adb->fd);

 
 static inline uint32_t audiodb_index_from_trackinfo(uint32_t track_id, uint32_t track_pos, uint32_t n_point_bits) {
   return ((track_id << n_point_bits) | track_pos);
 }
 
+#define ADB_FIXME_DEFAULT_LSH_N_POINT_BITS 14
+#ifndef ADB_FIXME_LSH_N_POINT_BITS
+#define ADB_FIXME_LSH_N_POINT_BITS ADB_FIXME_DEFAULT_LSH_N_POINT_BITS
+#endif
+
 static inline uint32_t audiodb_lsh_n_point_bits(adb_t *adb) {
   uint32_t nbits = adb->header->flags >> 28;
-  return (nbits ? nbits : O2_DEFAULT_LSH_N_POINT_BITS);
+  return (nbits ? nbits : ADB_FIXME_LSH_N_POINT_BITS);
 }
 
 int audiodb_read_data(adb_t *, int, int, double **, size_t *);
 int audiodb_insert_create_datum(adb_insert_t *, adb_datum_t *);
 int audiodb_track_id_datum(adb_t *, uint32_t, adb_datum_t *);

 int audiodb_query_queue_loop(adb_t *, const adb_query_spec_t *, adb_qstate_internal_t *, double *, adb_qpointers_internal_t *);
 int audiodb_query_loop(adb_t *, const adb_query_spec_t *, adb_qstate_internal_t *);
 char *audiodb_index_get_name(const char *, double, uint32_t);
 bool audiodb_index_exists(const char *, double, uint32_t);
 int audiodb_index_query_loop(adb_t *, const adb_query_spec_t *, adb_qstate_internal_t *);
+LSH *audiodb_index_allocate(adb_t *, char *, bool);
+vector<vector<float> > *audiodb_index_initialize_shingles(uint32_t, uint32_t, uint32_t);
+void audiodb_index_delete_shingles(vector<vector<float> > *);
+void audiodb_index_make_shingle(vector<vector<float> > *, uint32_t, double *, uint32_t, uint32_t);
+int audiodb_index_norm_shingles(vector<vector<float> > *, double *, double *, uint32_t, uint32_t, double, bool, bool, float);
+
+#define ADB_MAXSTR (512U)
+#define ADB_FILETABLE_ENTRY_SIZE (256U)
+#define ADB_TRACKTABLE_ENTRY_SIZE (sizeof(uint32_t))
+#define ADB_DISTANCE_TOLERANCE (1e-6)
+
+#define ADB_DEFAULT_DATASIZE (1355U) /* in MB */
+#define ADB_DEFAULT_NTRACKS (20000U)
+#define ADB_DEFAULT_DATADIM (9U)
+
+#define ADB_FIXME_LARGE_ADB_SIZE (ADB_DEFAULT_DATASIZE+1)
+#define ADB_FIXME_LARGE_ADB_NTRACKS (ADB_DEFAULT_NTRACKS+1)
+
+#define ADB_OLD_MAGIC ('O'|'2'<<8|'D'<<16|'B'<<24)
+#define ADB_MAGIC ('o'|'2'<<8|'d'<<16|'b'<<24)
+#define ADB_FORMAT_VERSION (4U)
+
+#define ADB_LSH_MAXTRACKLEN (1 << ADB_FIXME_LSH_N_POINT_BITS)
+
+#define align_up(x,w) (((x) + ((1<<w)-1)) & ~((1<<w)-1))
+#define align_down(x,w) ((x) & ~((1<<w)-1))
+
+#define align_page_up(x) (((x) + (getpagesize()-1)) & ~(getpagesize()-1))
+#define align_page_down(x) ((x) & ~(getpagesize()-1))
+