mas01cr@239: #include "audioDB.h"
mas01cr@239: 
mas01cr@239: #if defined(O2_DEBUG)
mas01cr@239: void sigterm_action(int signal, siginfo_t *info, void *context) {
mas01cr@239:   exit(128+signal);
mas01cr@239: }
mas01cr@239: 
mas01cr@239: void sighup_action(int signal, siginfo_t *info, void *context) {
mas01cr@239:   // FIXME: reread any configuration files
mas01cr@239: }
mas01cr@239: #endif
mas01cr@239: 
mas01cr@239: void audioDB::get_lock(int fd, bool exclusive) {
mas01cr@239:   struct flock lock;
mas01cr@239:   int status;
mas01cr@239:   
mas01cr@239:   lock.l_type = exclusive ? F_WRLCK : F_RDLCK;
mas01cr@239:   lock.l_whence = SEEK_SET;
mas01cr@239:   lock.l_start = 0;
mas01cr@239:   lock.l_len = 0; /* "the whole file" */
mas01cr@239: 
mas01cr@239:  retry:
mas01cr@239:   do {
mas01cr@239:     status = fcntl(fd, F_SETLKW, &lock);
mas01cr@239:   } while (status != 0 && errno == EINTR);
mas01cr@239:   
mas01cr@239:   if (status) {
mas01cr@239:     if (errno == EAGAIN) {
mas01cr@239:       sleep(1);
mas01cr@239:       goto retry;
mas01cr@239:     } else {
mas01cr@239:       error("fcntl lock error", "", "fcntl");
mas01cr@239:     }
mas01cr@239:   }
mas01cr@239: }
mas01cr@239: 
mas01cr@239: void audioDB::release_lock(int fd) {
mas01cr@239:   struct flock lock;
mas01cr@239:   int status;
mas01cr@239: 
mas01cr@239:   lock.l_type = F_UNLCK;
mas01cr@239:   lock.l_whence = SEEK_SET;
mas01cr@239:   lock.l_start = 0;
mas01cr@239:   lock.l_len = 0;
mas01cr@239: 
mas01cr@239:   status = fcntl(fd, F_SETLKW, &lock);
mas01cr@239: 
mas01cr@239:   if (status)
mas01cr@239:     error("fcntl unlock error", "", "fcntl");
mas01cr@239: }
mas01cr@239: 
mas01cr@239: void audioDB::error(const char* a, const char* b, const char *sysFunc) {
mas01cr@239:   if(isServer) {
mas01cr@239:     /* FIXME: I think this is leaky -- we never delete err.  actually
mas01cr@239:        deleting it is tricky, though; it gets placed into some
mas01cr@239:        soap-internal struct with uncertain extent... -- CSR,
mas01cr@239:        2007-10-01 */
mas01cr@239:     char *err = new char[256]; /* FIXME: overflows */
mas01cr@239:     snprintf(err, 255, "%s: %s\n%s", a, b, sysFunc ? strerror(errno) : "");
mas01cr@239:     /* FIXME: actually we could usefully do with a properly structured
mas01cr@239:        type, so that we can throw separate faultstring and details.
mas01cr@239:        -- CSR, 2007-10-01 */
mas01cr@239:     throw(err);
mas01cr@239:   } else {
mas01cr@239:     std::cerr << a << ": " << b << std::endl;
mas01cr@239:     if (sysFunc) {
mas01cr@239:       perror(sysFunc);
mas01cr@239:     }
mas01cr@239:     exit(1);
mas01cr@239:   }
mas01cr@239: }
mas01cr@239: 
mas01cr@284: void audioDB::initRNG() {
mas01cr@284:   rng = gsl_rng_alloc(gsl_rng_mt19937);
mas01cr@284:   if(!rng) {
mas01cr@284:     error("could not allocate Random Number Generator");
mas01cr@284:   }
mas01cr@284:   /* FIXME: maybe we should use a real source of entropy? */
mas01cr@284:   gsl_rng_set(rng, time(NULL));
mas01cr@284: }
mas01cr@284: 
mas01cr@239: void audioDB::initDBHeader(const char* dbName) {
mas01cr@239:   if ((dbfid = open(dbName, forWrite ? O_RDWR : O_RDONLY)) < 0) {
mas01cr@239:     error("Can't open database file", dbName, "open");
mas01cr@239:   }
mas01cr@239: 
mas01cr@239:   get_lock(dbfid, forWrite);
mas01cr@239:   // Get the database header info
mas01cr@239:   dbH = new dbTableHeaderT();
mas01cr@239:   assert(dbH);
mas01cr@239:   
mas01cr@239:   if(read(dbfid, (char *) dbH, O2_HEADERSIZE) != O2_HEADERSIZE) {
mas01cr@239:     error("error reading db header", dbName, "read");
mas01cr@239:   }
mas01cr@239: 
mas01cr@239:   if(dbH->magic == O2_OLD_MAGIC) {
mas01cr@239:     // FIXME: if anyone ever complains, write the program to convert
mas01cr@239:     // from the old audioDB format to the new...
mas01cr@239:     error("database file has old O2 header", dbName);
mas01cr@239:   }
mas01cr@239: 
mas01cr@239:   if(dbH->magic != O2_MAGIC) {
mas01cr@239:     std::cerr << "expected: " << O2_MAGIC << ", got: " << dbH->magic << std::endl;
mas01cr@239:     error("database file has incorrect header", dbName);
mas01cr@239:   }
mas01cr@239: 
mas01cr@239:   if(dbH->version != O2_FORMAT_VERSION) {
mas01cr@239:     error("database file has incorrect version", dbName);
mas01cr@239:   }
mas01cr@239: 
mas01cr@239:   if(dbH->headerSize != O2_HEADERSIZE) {
mas01cr@239:     error("sizeof(dbTableHeader) unexpected: platform ABI mismatch?", dbName);
mas01cr@239:   }
mas01cr@239: 
mas01cr@239:   CHECKED_MMAP(char *, db, 0, getpagesize());
mas01cr@239: 
mas01cr@239:   // Make some handy tables with correct types
mas01cr@239:   if(forWrite || (dbH->length > 0)) {
mas01cr@239:     if(forWrite) {
mas01cr@239:       fileTableLength = dbH->trackTableOffset - dbH->fileTableOffset;
mas01cr@239:       trackTableLength = dbH->dataOffset - dbH->trackTableOffset;
mas01cr@239:       dataBufLength = dbH->timesTableOffset - dbH->dataOffset;
mas01cr@239:       timesTableLength = dbH->powerTableOffset - dbH->timesTableOffset;
mas01cr@239:       powerTableLength = dbH->l2normTableOffset - dbH->powerTableOffset;
mas01cr@239:       l2normTableLength = dbH->dbSize - dbH->l2normTableOffset;
mas01cr@239:     } else {
mas01cr@256:       fileTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLE_ENTRY_SIZE);
mas01cr@256:       trackTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_TRACKTABLE_ENTRY_SIZE);
mas01mc@318:       if( dbH->flags & O2_FLAG_LARGE_ADB ){
mas01mc@318: 	dataBufLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLE_ENTRY_SIZE);
mas01mc@318: 	timesTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLE_ENTRY_SIZE);
mas01mc@318: 	powerTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLE_ENTRY_SIZE);
mas01mc@318: 	l2normTableLength = 0;
mas01mc@318:       }
mas01mc@318:       else{
mas01mc@318: 	dataBufLength = ALIGN_PAGE_UP(dbH->length);
mas01mc@318: 	timesTableLength = ALIGN_PAGE_UP(2*(dbH->length / dbH->dim));
mas01mc@318: 	powerTableLength = ALIGN_PAGE_UP(dbH->length / dbH->dim);
mas01mc@318: 	l2normTableLength = ALIGN_PAGE_UP(dbH->length / dbH->dim);
mas01mc@318:       }
mas01cr@239:     }
mas01cr@239:     CHECKED_MMAP(char *, fileTable, dbH->fileTableOffset, fileTableLength);
mas01cr@239:     CHECKED_MMAP(unsigned *, trackTable, dbH->trackTableOffset, trackTableLength);
mas01cr@239:     /*
mas01cr@239:      * No more mmap() for dataBuf
mas01cr@239:      *
mas01cr@239:      * FIXME: Actually we do do the mmap() in the two cases where it's
mas01cr@239:      * still "needed": in pointQuery and in l2norm if dbH->length is
mas01cr@239:      * non-zero.  Removing those cases too (and deleting the dataBuf
mas01cr@239:      * variable completely) would be cool.  -- CSR, 2007-11-19
mas01cr@239:      *
mas01cr@239:      * CHECKED_MMAP(double *, dataBuf, dbH->dataOffset, dataBufLength);
mas01cr@239:      */
mas01mc@318:     if( dbH->flags & O2_FLAG_LARGE_ADB ){
mas01mc@318:       CHECKED_MMAP(char *, featureFileNameTable, dbH->dataOffset, fileTableLength);
mas01mc@318:       if( dbH->flags & O2_FLAG_TIMES )
mas01mc@318: 	CHECKED_MMAP(char *, timesFileNameTable, dbH->timesTableOffset, fileTableLength);
mas01mc@318:       if( dbH->flags & O2_FLAG_POWER )
mas01mc@318: 	CHECKED_MMAP(char *, powerFileNameTable, dbH->powerTableOffset, fileTableLength);
mas01mc@318:     }
mas01mc@318:     else{
mas01mc@318:       CHECKED_MMAP(double *, timesTable, dbH->timesTableOffset, timesTableLength);
mas01mc@318:       CHECKED_MMAP(double *, powerTable, dbH->powerTableOffset, powerTableLength);
mas01mc@318:       CHECKED_MMAP(double *, l2normTable, dbH->l2normTableOffset, l2normTableLength);
mas01mc@318:     }
mas01cr@239:   }
mas01mc@292: 
mas01mc@292:   // build track offset table
mas01mc@292:   trackOffsetTable = new off_t[dbH->numFiles];
mas01mc@292:   Uns32T cumTrack=0;
mas01mc@292:   for(Uns32T k = 0; k < dbH->numFiles; k++){
mas01mc@292:     trackOffsetTable[k] = cumTrack;
mas01mc@292:     cumTrack += trackTable[k] * dbH->dim;
mas01mc@319:   }
mas01mc@319: 
mas01mc@319:   // Assign correct number of point bits per track in LSH indexing / retrieval
mas01mc@319:   lsh_n_point_bits = dbH->flags >> 28;
mas01mc@319:   if( !lsh_n_point_bits )
mas01mc@319:     lsh_n_point_bits = O2_DEFAULT_LSH_N_POINT_BITS;
mas01cr@239: }
mas01cr@239: 
mas01mc@316: void audioDB::initInputFile (const char *inFile, bool loadData) {
mas01cr@239:   if (inFile) {
mas01cr@239:     if ((infid = open(inFile, O_RDONLY)) < 0) {
mas01cr@239:       error("can't open input file for reading", inFile, "open");
mas01cr@239:     }
mas01cr@239: 
mas01cr@239:     if (fstat(infid, &statbuf) < 0) {
mas01cr@239:       error("fstat error finding size of input", inFile, "fstat");
mas01cr@239:     }
mas01cr@239: 
mas01cr@239:     if(dbH->dim == 0 && dbH->length == 0) { // empty database
mas01cr@239:       // initialize with input dimensionality
mas01cr@239:       if(read(infid, &dbH->dim, sizeof(unsigned)) != sizeof(unsigned)) {
mas01cr@239:         error("short read of input file", inFile);
mas01cr@239:       }
mas01cr@239:       if(dbH->dim == 0) {
mas01cr@239:         error("dimensionality of zero in input file", inFile);
mas01cr@239:       }
mas01cr@239:     } else {
mas01cr@239:       unsigned test;
mas01cr@239:       if(read(infid, &test, sizeof(unsigned)) != sizeof(unsigned)) {
mas01cr@239:         error("short read of input file", inFile);
mas01cr@239:       }
mas01cr@239:       if(dbH->dim == 0) {
mas01cr@239:         error("dimensionality of zero in input file", inFile);
mas01cr@239:       }
mas01cr@239:       if(dbH->dim != test) {      
mas01cr@239: 	std::cerr << "error: expected dimension: " << dbH->dim << ", got : " << test <<std::endl;
mas01cr@239: 	error("feature dimensions do not match database table dimensions", inFile);
mas01cr@239:       }
mas01cr@239:     }
mas01cr@239:     
mas01mc@316:     if (loadData && ((indata = (char *) mmap(0, statbuf.st_size, PROT_READ, MAP_SHARED, infid, 0)) == (caddr_t) -1)) {
mas01cr@239:       error("mmap error for input", inFile, "mmap");
mas01cr@239:     }
mas01cr@239:   }
mas01cr@239: }
mas01cr@239: 
mas01mc@292: void audioDB::initTables(const char* dbName, const char* inFile) {
mas01cr@284:   /* FIXME: initRNG() really logically belongs in the audioDB
mas01cr@284:      contructor.  However, there are of the order of four constructors
mas01cr@284:      at the moment, and more to come from API implementation.  Given
mas01cr@284:      that duplication, I think this is the least worst place to put
mas01cr@284:      it; the assumption is that nothing which doesn't look at a
mas01cr@284:      database will need an RNG.  -- CSR, 2008-07-02 */
mas01cr@284:   initRNG();
mas01cr@239:   initDBHeader(dbName);
mas01mc@292:   if(inFile)
mas01mc@292:     initInputFile(inFile);
mas01cr@239: }
mas01mc@292: 
mas01mc@321: // If name is relative path, side effect name with prefix/name
mas01mc@321: // Do not free original pointer
mas01mc@321: void audioDB::prefix_name(char** const name, const char* prefix){
mas01mc@321:   // No prefix if prefix is empty
mas01mc@321:   if(!prefix)
mas01mc@321:     return;
mas01mc@321:   // Allocate new memory, keep old memory
mas01mc@321:   assert(name && *name);
mas01mc@321:   if (strlen(*name) + strlen(prefix) + 1 > O2_MAXFILESTR)
mas01mc@321:     error("error: path prefix + filename too long",prefix);
mas01mc@321:   // Do not prefix absolute path+filename
mas01mc@321:   if(**name=='/')
mas01mc@321:     return;
mas01mc@321:   // OK to prefix relative path+filename
mas01mc@321:   char* prefixedName = (char*) malloc(O2_MAXFILESTR);
mas01mc@321:   sprintf(prefixedName, "%s/%s", prefix, *name);
mas01mc@321:   *name = prefixedName; // side effect new name to old name
mas01mc@321: }