mas01cr@0: #include <stdio.h>
mas01cr@0: #include <stdlib.h>
mas01cr@0: #include <sys/types.h>
mas01cr@0: #include <sys/stat.h>
mas01cr@0: #include <sys/mman.h>
mas01cr@0: #include <fcntl.h>
mas01cr@0: #include <string.h>
mas01cr@0: #include <iostream>
mas01cr@0: #include <fstream>
mas01cr@0: #include <math.h>
mas01cr@0: #include <sys/time.h>
mas01cr@0: #include <assert.h>
mas01cr@62: #include <float.h>
mas01cr@104: #include <signal.h>
mas01cr@278: #include <gsl/gsl_rng.h>
mas01cr@0: 
mas01cr@0: // includes for web services
mas01cr@0: #include "soapH.h"
mas01cr@0: #include "cmdline.h"
mas01cr@0: 
mas01cr@0: #define MAXSTR 512
mas01cr@0: 
mas01cr@0: // Databse PRIMARY commands
mas01cr@0: #define COM_CREATE "--NEW"
mas01cr@0: #define COM_INSERT "--INSERT"
mas01cr@0: #define COM_BATCHINSERT "--BATCHINSERT"
mas01cr@0: #define COM_QUERY "--QUERY"
mas01cr@0: #define COM_STATUS "--STATUS"
mas01cr@0: #define COM_L2NORM "--L2NORM"
mas01cr@193: #define COM_POWER "--POWER"
mas01cr@0: #define COM_DUMP "--DUMP"
mas01cr@0: #define COM_SERVER "--SERVER"
mas01cr@266: #define COM_SAMPLE "--SAMPLE"
mas01cr@0: 
mas01cr@0: // parameters
mas01cr@0: #define COM_CLIENT "--client"
mas01cr@0: #define COM_DATABASE "--database"
mas01cr@0: #define COM_QTYPE "--qtype"
mas01cr@0: #define COM_SEQLEN "--sequencelength"
mas01cr@0: #define COM_SEQHOP "--sequencehop"
mas01cr@0: #define COM_POINTNN "--pointnn"
mas01mc@18: #define COM_TRACKNN "--resultlength"
mas01cr@0: #define COM_QPOINT "--qpoint"
mas01cr@0: #define COM_FEATURES "--features"
mas01cr@0: #define COM_QUERYKEY "--key"
mas01cr@0: #define COM_KEYLIST "--keyList"
mas01cr@0: #define COM_TIMES "--times"
mas01cr@193: #define COM_QUERYPOWER "--power"
mas01cr@193: #define COM_RELATIVE_THRESH "--relative-threshold"
mas01cr@193: #define COM_ABSOLUTE_THRESH "--absolute-threshold"
mas01cr@0: 
mas01cr@108: #define O2_OLD_MAGIC ('O'|'2'<<8|'D'<<16|'B'<<24)
mas01cr@108: #define O2_MAGIC ('o'|'2'<<8|'d'<<16|'b'<<24)
mas01cr@210: #define O2_FORMAT_VERSION (4U)
mas01cr@0: 
mas01cr@0: #define O2_DEFAULT_POINTNN (10U)
mas01mc@18: #define O2_DEFAULT_TRACKNN  (10U)
mas01cr@0: 
mas01mc@248: //#define O2_DEFAULTDBSIZE (4000000000) // 4GB table size
mas01mc@7: #define O2_DEFAULTDBSIZE (2000000000) // 2GB table size
mas01cr@0: 
mas01cr@256: #define O2_DEFAULT_DATASIZE (1355U) // in MB
mas01cr@256: #define O2_DEFAULT_NTRACKS (20000U)
mas01cr@256: #define O2_DEFAULT_DATADIM (9U)
mas01cr@256: 
mas01cr@239: #define O2_MAXFILES (20000U)
mas01cr@0: #define O2_MAXFILESTR (256U)
mas01cr@256: #define O2_FILETABLE_ENTRY_SIZE (O2_MAXFILESTR)
mas01cr@256: #define O2_TRACKTABLE_ENTRY_SIZE (sizeof(unsigned))
mas01cr@0: #define O2_HEADERSIZE (sizeof(dbTableHeaderT))
mas01cr@0: #define O2_MEANNUMVECTORS (1000U)
mas01cr@0: #define O2_MAXDIM (1000U)
mas01mc@263: #define O2_MAXNN (1000000U)
mas01cr@0: 
mas01cr@0: // Flags
mas01cr@0: #define O2_FLAG_L2NORM (0x1U)
mas01cr@0: #define O2_FLAG_MINMAX (0x2U)
mas01cr@193: #define O2_FLAG_POWER (0x4U)
mas01cr@0: #define O2_FLAG_TIMES (0x20U)
mas01cr@0: 
mas01cr@105: // Query types
mas01cr@105: #define O2_POINT_QUERY (0x4U)
mas01cr@105: #define O2_SEQUENCE_QUERY (0x8U)
mas01cr@105: #define O2_TRACK_QUERY (0x10U)
mas01mc@248: #define O2_N_SEQUENCE_QUERY (0x20U)
mas01mc@263: #define O2_ONE_TO_ONE_N_SEQUENCE_QUERY (0x40U)
mas01mc@248: 
mas01cr@105: 
mas01cr@0: // Error Codes
mas01cr@0: #define O2_ERR_KEYNOTFOUND (0xFFFFFF00)
mas01cr@0: 
mas01cr@0: // Macros
mas01cr@0: #define O2_ACTION(a) (strcmp(command,a)==0)
mas01cr@0: 
mas01cr@108: #define ALIGN_UP(x,w) ((x) + ((1<<w)-1) & ~((1<<w)-1))
mas01cr@108: #define ALIGN_DOWN(x,w) ((x) & ~((1<<w)-1))
mas01cr@108: 
mas01cr@196: #define ALIGN_PAGE_UP(x) ((x) + (getpagesize()-1) & ~(getpagesize()-1))
mas01cr@196: #define ALIGN_PAGE_DOWN(x) ((x) & ~(getpagesize()-1))
mas01cr@196: 
mas01cr@166: #define ENSURE_STRING(x) ((x) ? (x) : "")
mas01cr@166: 
mas01cr@239: #define CHECKED_MMAP(type, var, start, length) \
mas01cr@239:   { void *tmp = mmap(0, length, (PROT_READ | (forWrite ? PROT_WRITE : 0)), MAP_SHARED, dbfid, (start)); \
mas01cr@239:     if(tmp == (void *) -1) { \
mas01cr@239:       error("mmap error for db table", #var, "mmap"); \
mas01cr@239:     } \
mas01cr@239:     var = (type) tmp; \
mas01cr@239:   }
mas01cr@239: 
mas01cr@239: #define VERB_LOG(vv, ...) \
mas01cr@239:   if(verbosity > vv) { \
mas01cr@239:     fprintf(stderr, __VA_ARGS__); \
mas01cr@239:     fflush(stderr); \
mas01cr@239:   }
mas01cr@0: 
mas01cr@210: typedef struct dbTableHeader {
mas01cr@114:   uint32_t magic;
mas01cr@114:   uint32_t version;
mas01cr@114:   uint32_t numFiles;
mas01cr@114:   uint32_t dim;
mas01cr@114:   uint32_t flags;
mas01cr@210:   uint32_t headerSize;
mas01cr@196:   off_t length;
mas01cr@196:   off_t fileTableOffset;
mas01cr@196:   off_t trackTableOffset;
mas01cr@196:   off_t dataOffset;
mas01cr@196:   off_t l2normTableOffset;
mas01cr@196:   off_t timesTableOffset;
mas01cr@196:   off_t powerTableOffset;
mas01cr@196:   off_t dbSize;
mas01cr@0: } dbTableHeaderT, *dbTableHeaderPtr;
mas01cr@0: 
mas01cr@239: class Reporter;
mas01cr@0: 
mas01cr@0: class audioDB{
mas01cr@0:   
mas01cr@0:  private:
mas01cr@0:   gengetopt_args_info args_info;
mas01cr@0:   unsigned dim;
mas01cr@0:   const char *dbName;
mas01cr@0:   const char *inFile;
mas01cr@0:   const char *hostport;
mas01cr@0:   const char *key;
mas01mc@18:   const char* trackFileName;
mas01cr@239:   std::ifstream *trackFile;
mas01cr@0:   const char *command;
mas01cr@131:   const char *output;
mas01cr@0:   const char *timesFileName;
mas01cr@239:   std::ifstream *timesFile;
mas01cr@193:   const char *powerFileName;
mas01cr@239:   std::ifstream *powerFile;
mas01cr@193:   int powerfd;
mas01cr@0: 
mas01cr@0:   int dbfid;
mas01cr@196:   bool forWrite;
mas01cr@0:   int infid;
mas01cr@0:   char* db;
mas01cr@0:   char* indata;
mas01cr@0:   struct stat statbuf;  
mas01cr@0:   dbTableHeaderPtr dbH;
mas01cr@279: 
mas01cr@279:   gsl_rng *rng;
mas01cr@0:   
mas01cr@0:   char *fileTable;
mas01mc@18:   unsigned* trackTable;
mas01cr@0:   double* dataBuf;
mas01cr@0:   double* inBuf;
mas01cr@0:   double* l2normTable;
mas01cr@196:   double* timesTable;
mas01cr@193:   double* powerTable;
mas01cr@0: 
mas01cr@196:   size_t fileTableLength;
mas01cr@196:   size_t trackTableLength;
mas01cr@196:   off_t dataBufLength;
mas01cr@196:   size_t timesTableLength;
mas01cr@196:   size_t powerTableLength;
mas01cr@196:   size_t l2normTableLength;
mas01cr@196: 
mas01cr@0:   // Flags and parameters
mas01cr@0:   unsigned verbosity;   // how much do we want to know?
mas01cr@256: 
mas01cr@274:   unsigned nsamples;
mas01cr@274: 
mas01cr@256:   //off_t size; // given size (for creation)
mas01cr@256:   unsigned datasize; // size in MB
mas01cr@256:   unsigned ntracks;
mas01cr@256:   unsigned datadim;
mas01cr@256: 
mas01cr@0:   unsigned queryType; // point queries default
mas01cr@0:   unsigned pointNN;   // how many point NNs ?
mas01mc@18:   unsigned trackNN;   // how many track NNs ?
mas01cr@0:   unsigned sequenceLength;
mas01cr@0:   unsigned sequenceHop;
mas01cr@239:   bool normalizedDistance;
mas01cr@0:   unsigned queryPoint;
mas01cr@0:   unsigned usingQueryPoint;
mas01cr@0:   unsigned usingTimes;
mas01cr@193:   unsigned usingPower;
mas01cr@0:   unsigned isClient;
mas01cr@0:   unsigned isServer;
mas01cr@0:   unsigned port;
mas01cr@0:   double timesTol;
mas01mc@17:   double radius;
mas01cr@193: 
mas01cr@193:   bool use_absolute_threshold;
mas01cr@193:   double absolute_threshold;
mas01cr@193:   bool use_relative_threshold;
mas01cr@193:   double relative_threshold;
mas01cr@193: 
mas01mc@17:   
mas01cr@0:   // Timers
mas01cr@0:   struct timeval tv1;
mas01cr@0:   struct timeval tv2;
mas01cr@0:     
mas01cr@0:   // private methods
mas01cr@32:   void error(const char* a, const char* b = "", const char *sysFunc = 0);
mas01cr@193:   void sequence_sum(double *buffer, int length, int seqlen);
mas01cr@193:   void sequence_sqrt(double *buffer, int length, int seqlen);
mas01cr@193:   void sequence_average(double *buffer, int length, int seqlen);
mas01cr@193: 
mas01cr@239:   void initialize_arrays(int track, unsigned int numVectors, double *query, double *data_buffer, double **D, double **DD);
mas01cr@239:   void delete_arrays(int track, unsigned int numVectors, double **D, double **DD);
mas01cr@239:   void read_data(int track, double **data_buffer_p, size_t *data_buffer_size_p);
mas01cr@239:   void set_up_query(double **qp, double **vqp, double **qnp, double **vqnp, double **qpp, double **vqpp, double *mqdp, unsigned int *nvp);
mas01cr@239:   void set_up_db(double **snp, double **vsnp, double **spp, double **vspp, double **mddp, unsigned int *dvp);
mas01cr@240:   void query_loop(const char* dbName, const char* inFile, Reporter *reporter);
mas01cr@0: 
mas01cr@279:   void initRNG();
mas01cr@196:   void initDBHeader(const char *dbName);
mas01cr@169:   void initInputFile(const char *inFile);
mas01cr@196:   void initTables(const char* dbName, const char* inFile);
mas01cr@0:   void unitNorm(double* X, unsigned d, unsigned n, double* qNorm);
mas01cr@0:   void unitNormAndInsertL2(double* X, unsigned dim, unsigned n, unsigned append);
mas01cr@239:   void insertTimeStamps(unsigned n, std::ifstream* timesFile, double* timesdata);
mas01cr@193:   void insertPowerData(unsigned n, int powerfd, double *powerdata);
mas01cr@0:   unsigned getKeyPos(char* key);
mas01cr@0:  public:
mas01cr@0: 
mas01cr@76:   audioDB(const unsigned argc, char* const argv[]);
mas01cr@133:   audioDB(const unsigned argc, char* const argv[], adb__queryResponse *adbQueryResponse);
mas01cr@133:   audioDB(const unsigned argc, char* const argv[], adb__statusResponse *adbStatusResponse);
mas01cr@97:   void cleanup();
mas01cr@0:   ~audioDB();
mas01cr@0:   int processArgs(const unsigned argc, char* const argv[]);
mas01cr@30:   void get_lock(int fd, bool exclusive);
mas01cr@30:   void release_lock(int fd);
mas01cr@0:   void create(const char* dbName);
mas01cr@0:   void drop();
mas01cr@251:   bool enough_per_file_space_free();
mas01cr@196:   bool enough_data_space_free(off_t size);
mas01cr@196:   void insert_data_vectors(off_t offset, void *buffer, size_t size);
mas01cr@0:   void insert(const char* dbName, const char* inFile);
mas01cr@0:   void batchinsert(const char* dbName, const char* inFile);
mas01cr@133:   void query(const char* dbName, const char* inFile, adb__queryResponse *adbQueryResponse=0);
mas01cr@133:   void status(const char* dbName, adb__statusResponse *adbStatusResponse=0);
mas01cr@279:   unsigned random_track(unsigned *propTable, unsigned total);
mas01cr@266:   void sample(const char *dbName);
mas01cr@0:   void ws_status(const char*dbName, char* hostport);
mas01mc@18:   void ws_query(const char*dbName, const char *trackKey, const char* hostport);
mas01cr@0:   void l2norm(const char* dbName);
mas01cr@193:   void power_flag(const char *dbName);
mas01cr@193:   bool powers_acceptable(double p1, double p2);
mas01cr@0:   void dump(const char* dbName);
mas01cr@0: 
mas01cr@0:   // web services
mas01cr@0:   void startServer();
mas01cr@0:   
mas01cr@0: };
mas01mc@17: 
mas01cr@105: #define O2_AUDIODB_INITIALIZERS \
mas01cr@105:   dim(0), \
mas01cr@105:   dbName(0), \
mas01cr@105:   inFile(0), \
mas01cr@105:   key(0), \
mas01cr@105:   trackFileName(0), \
mas01cr@105:   trackFile(0), \
mas01cr@105:   command(0), \
mas01cr@131:   output(0), \
mas01cr@105:   timesFileName(0), \
mas01cr@105:   timesFile(0), \
mas01cr@193:   powerFileName(0), \
mas01cr@193:   powerFile(0), \
mas01cr@193:   powerfd(0), \
mas01cr@105:   dbfid(0), \
mas01cr@196:   forWrite(false), \
mas01cr@105:   infid(0), \
mas01cr@105:   db(0), \
mas01cr@105:   indata(0), \
mas01cr@105:   dbH(0), \
mas01cr@279:   rng(0), \
mas01cr@105:   fileTable(0), \
mas01cr@105:   trackTable(0), \
mas01cr@105:   dataBuf(0), \
mas01cr@105:   l2normTable(0), \
mas01cr@105:   timesTable(0), \
mas01cr@196:   fileTableLength(0), \
mas01cr@196:   trackTableLength(0), \
mas01cr@196:   dataBufLength(0), \
mas01cr@196:   timesTableLength(0), \
mas01cr@196:   powerTableLength(0), \
mas01cr@196:   l2normTableLength(0), \
mas01cr@105:   verbosity(1), \
mas01cr@274:   nsamples(2000), \
mas01cr@256:   datasize(O2_DEFAULT_DATASIZE), \
mas01cr@256:   ntracks(O2_DEFAULT_NTRACKS), \
mas01cr@256:   datadim(O2_DEFAULT_DATADIM), \
mas01cr@105:   queryType(O2_POINT_QUERY), \
mas01cr@105:   pointNN(O2_DEFAULT_POINTNN), \
mas01cr@105:   trackNN(O2_DEFAULT_TRACKNN), \
mas01cr@105:   sequenceLength(16), \
mas01cr@105:   sequenceHop(1), \
mas01cr@239:   normalizedDistance(true), \
mas01cr@105:   queryPoint(0), \
mas01cr@105:   usingQueryPoint(0), \
mas01cr@105:   usingTimes(0), \
mas01cr@193:   usingPower(0), \
mas01cr@105:   isClient(0), \
mas01cr@105:   isServer(0), \
mas01cr@105:   port(0), \
mas01cr@105:   timesTol(0.1), \
mas01cr@193:   radius(0), \
mas01cr@193:   use_absolute_threshold(false), \
mas01cr@193:   absolute_threshold(0.0), \
mas01cr@193:   use_relative_threshold(false), \
mas01cr@193:   relative_threshold(0.0)