view common.cpp @ 408:f0a69693eaef api-inversion

The lesser of two evils, part 1.

Most of the body of audiodb_insert_datum() will apply to
"LARGE_ADB"-type insertions: checking for the right flags, checking for
enough space free, synchronizing the header.  Wouldn't it be nice if we
could reuse all that code (or at least the bits that apply) without one
horrible almost-identical cut-and-paste job (see
batchinsert_large_adb(), or if that's not compelling enough, the four
almost-identical query loops from before the Great Refactoring).

Well, yes, it would.  Sadly C makes it mildly difficult, because its
functions are explicitly typed (so we can't pass arbitrary arguments of
other types, even if they're ABI-compatible), while its macros are
textual (which makes writing and maintaining them horrible).  The
thought of a union argument was briefly entertained and then discarded
as being just Too Weird.

So, instead, (ab)use the oldest trick in the book: void *.  Define an
adb_datum_internal_t which has void * instead of double *; the intention
is that this internal data type can be constructed both from an
adb_datum_t and some notional adb_reference_t (which looks very much
like an adb_insert_t at the time of writing, with char * structure
entries representing filenames).  This adb_datum_internal_t structure is
very much an internals-only thing, so put its definition in the
internals header.

Call what was previously audiodb_insert_datum() a new function
audiodb_insert_datum_internal(), made static so that really no-one is
tempted to call it other than ourselves.  audiodb_insert_datum() is then
trivial in terms of this new function, if stupidly tedious.  (If we were
playing dangerously, we could just perform a cast, but relying on the
fact that sizeof(double *) = sizeof(void *) would almost certainly end
up biting when we least expect.

Incidental inclusion in this patch, since I noticed it at the time:
actually check for the O2_FLAG_L2NORM before scribbling all over the
l2norm table.  Somewhat unsurprisingly, there are as yet no tests to
defend against this (harmless, as it turns out) erroneous behaviour.

#include "audioDB.h"
extern "C" {
#include "audioDB_API.h"
#include "audioDB-internals.h"
}

#if defined(O2_DEBUG)
void sigterm_action(int signal, siginfo_t *info, void *context) {
  exit(128+signal);
}

void sighup_action(int signal, siginfo_t *info, void *context) {
  // FIXME: reread any configuration files
}
#endif

int acquire_lock(int fd, bool exclusive) {
  struct flock lock;
  int status;
  
  lock.l_type = exclusive ? F_WRLCK : F_RDLCK;
  lock.l_whence = SEEK_SET;
  lock.l_start = 0;
  lock.l_len = 0; /* "the whole file" */

 retry:
  do {
    status = fcntl(fd, F_SETLKW, &lock);
  } while (status != 0 && errno == EINTR);
  
  if (status) {
    if (errno == EAGAIN) {
      sleep(1);
      goto retry;
    } else {
      return status;
    }
  }
  return 0;
}

int divest_lock(int fd) {
  struct flock lock;

  lock.l_type = F_UNLCK;
  lock.l_whence = SEEK_SET;
  lock.l_start = 0;
  lock.l_len = 0;

  return fcntl(fd, F_SETLKW, &lock);
}

void audioDB::get_lock(int fd, bool exclusive) {
  if(acquire_lock(fd, exclusive)) {
    error("fcntl lock error", "", "fcntl");
  }
}

void audioDB::release_lock(int fd) {
  if (divest_lock(fd)) {
    error("fcntl unlock error", "", "fcntl");
  }
}

void audioDB::error(const char* a, const char* b, const char *sysFunc) {
 

    if(isServer) {
        /* FIXME: I think this is leaky -- we never delete err.
           actually deleting it is tricky, though; it gets placed into
           some soap-internal struct with uncertain extent... -- CSR,
           2007-10-01 */
        char *err = new char[256]; /* FIXME: overflows */
        snprintf(err, 255, "%s: %s\n%s", a, b, sysFunc ? strerror(errno) : "");
        /* FIXME: actually we could usefully do with a properly
           structured type, so that we can throw separate faultstring
           and details.  -- CSR, 2007-10-01 */
        throw(err);
    } else if (UseApiError){
        apierrortemp=-1;
        throw(apierrortemp);
    } else {
        std::cerr << a << ": " << b << std::endl;
        if (sysFunc) {
            perror(sysFunc);
        }
        exit(1);
    }

}

void audioDB::initRNG() {
  rng = gsl_rng_alloc(gsl_rng_mt19937);
  if(!rng) {
    error("could not allocate Random Number Generator");
  }
  /* FIXME: maybe we should use a real source of entropy? */
  gsl_rng_set(rng, time(NULL));
}

void audioDB::initDBHeader(const char* dbName) {
  if(!adb) {
    adb = audiodb_open(dbName, forWrite ? O_RDWR : O_RDONLY);
    if(!adb) {
      error("Failed to open database", dbName);
    }
  }
  dbfid = adb->fd;
  dbH = adb->header;

  CHECKED_MMAP(char *, db, 0, getpagesize());

  // Make some handy tables with correct types
  if(forWrite || (dbH->length > 0)) {
    if(forWrite) {
      fileTableLength = dbH->trackTableOffset - dbH->fileTableOffset;
      trackTableLength = dbH->dataOffset - dbH->trackTableOffset;
      dataBufLength = dbH->timesTableOffset - dbH->dataOffset;
      timesTableLength = dbH->powerTableOffset - dbH->timesTableOffset;
      powerTableLength = dbH->l2normTableOffset - dbH->powerTableOffset;
      l2normTableLength = dbH->dbSize - dbH->l2normTableOffset;
    } else {
      fileTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLE_ENTRY_SIZE);
      trackTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_TRACKTABLE_ENTRY_SIZE);
      if( dbH->flags & O2_FLAG_LARGE_ADB ){
	dataBufLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLE_ENTRY_SIZE);
	timesTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLE_ENTRY_SIZE);
	powerTableLength = ALIGN_PAGE_UP(dbH->numFiles * O2_FILETABLE_ENTRY_SIZE);
	l2normTableLength = 0;
      }
      else{
	dataBufLength = ALIGN_PAGE_UP(dbH->length);
	timesTableLength = ALIGN_PAGE_UP(2*(dbH->length / dbH->dim));
	powerTableLength = ALIGN_PAGE_UP(dbH->length / dbH->dim);
	l2normTableLength = ALIGN_PAGE_UP(dbH->length / dbH->dim);
      }
    }
    CHECKED_MMAP(char *, fileTable, dbH->fileTableOffset, fileTableLength);
    CHECKED_MMAP(unsigned *, trackTable, dbH->trackTableOffset, trackTableLength);
    /*
     * No more mmap() for dataBuf
     *
     * FIXME: Actually we do do the mmap() in the two cases where it's
     * still "needed": in pointQuery and in l2norm if dbH->length is
     * non-zero.  Removing those cases too (and deleting the dataBuf
     * variable completely) would be cool.  -- CSR, 2007-11-19
     *
     * CHECKED_MMAP(double *, dataBuf, dbH->dataOffset, dataBufLength);
     */
    if( dbH->flags & O2_FLAG_LARGE_ADB ){
      CHECKED_MMAP(char *, featureFileNameTable, dbH->dataOffset, fileTableLength);
      if( dbH->flags & O2_FLAG_TIMES )
	CHECKED_MMAP(char *, timesFileNameTable, dbH->timesTableOffset, fileTableLength);
      if( dbH->flags & O2_FLAG_POWER )
	CHECKED_MMAP(char *, powerFileNameTable, dbH->powerTableOffset, fileTableLength);
    }
    else{
      CHECKED_MMAP(double *, timesTable, dbH->timesTableOffset, timesTableLength);
      CHECKED_MMAP(double *, powerTable, dbH->powerTableOffset, powerTableLength);
      CHECKED_MMAP(double *, l2normTable, dbH->l2normTableOffset, l2normTableLength);
    }
  }

  // build track offset table
  trackOffsetTable = new off_t[dbH->numFiles];
  Uns32T cumTrack=0;
  for(Uns32T k = 0; k < dbH->numFiles; k++){
    trackOffsetTable[k] = cumTrack;
    cumTrack += trackTable[k] * dbH->dim;
  }

  // Assign correct number of point bits per track in LSH indexing / retrieval
  lsh_n_point_bits = dbH->flags >> 28;
  if( !lsh_n_point_bits )
    lsh_n_point_bits = O2_DEFAULT_LSH_N_POINT_BITS;
}

void audioDB::initInputFile (const char *inFile, bool loadData) {
  if (inFile) {
    if ((infid = open(inFile, O_RDONLY)) < 0) {
      error("can't open input file for reading", inFile, "open");
    }

    if (fstat(infid, &statbuf) < 0) {
      error("fstat error finding size of input", inFile, "fstat");
    }

    if(dbH->dim == 0 && dbH->length == 0) { // empty database
      // initialize with input dimensionality
      if(read(infid, &dbH->dim, sizeof(unsigned)) != sizeof(unsigned)) {
        error("short read of input file", inFile);
      }
      if(dbH->dim == 0) {
        error("dimensionality of zero in input file", inFile);
      }
    } else {
      unsigned test;
      if(read(infid, &test, sizeof(unsigned)) != sizeof(unsigned)) {
        error("short read of input file", inFile);
      }
      if(dbH->dim == 0) {
        error("dimensionality of zero in input file", inFile);
      }
      if(dbH->dim != test) {      
	std::cerr << "error: expected dimension: " << dbH->dim << ", got : " << test <<std::endl;
	error("feature dimensions do not match database table dimensions", inFile);
      }
    }
    
    if (loadData && ((indata = (char *) mmap(0, statbuf.st_size, PROT_READ, MAP_SHARED, infid, 0)) == (caddr_t) -1)) {
      error("mmap error for input", inFile, "mmap");
    }
  }
}

void audioDB::initTables(const char* dbName, const char* inFile) {
  /* FIXME: initRNG() really logically belongs in the audioDB
     contructor.  However, there are of the order of four constructors
     at the moment, and more to come from API implementation.  Given
     that duplication, I think this is the least worst place to put
     it; the assumption is that nothing which doesn't look at a
     database will need an RNG.  -- CSR, 2008-07-02 */
  initRNG();
  initDBHeader(dbName);
  if(inFile)
    initInputFile(inFile);
}

// If name is relative path, side effect name with prefix/name
// Do not free original pointer
void audioDB::prefix_name(char** const name, const char* prefix){
  // No prefix if prefix is empty
  if(!prefix)
    return;
  // Allocate new memory, keep old memory
  assert(name && *name);
  if (strlen(*name) + strlen(prefix) + 1 > O2_MAXFILESTR)
    error("error: path prefix + filename too long",prefix);
  // Do not prefix absolute path+filename
  if(**name=='/')
    return;
  // OK to prefix relative path+filename
  char* prefixedName = (char*) malloc(O2_MAXFILESTR);
  sprintf(prefixedName, "%s/%s", prefix, *name);
  *name = prefixedName; // side effect new name to old name
}