view create.cpp @ 395:bc7a821004bb api-inversion

Invert audioDB::status / audiodb_status().

To do that without breaking abstractions, we actually need a new field
in the status structure, storing the size of the data region.
Previously, this was computed in the audioDB::status request from the
database header, but I'm assuming that "user" code doesn't have access
to such internals.  While we're at it, name some intermediate values in
audioDB::status() so that I don't get confused.

Here's the thing, though: we need to make sure that the adb_t * that we
have from audiodb_open() or audiodb_create() is propagated all the way
through into the C++ routines that implement library functions -- in
particular those which actually write to the database; otherwise we
won't have a consistent view in memory of the header on-disk (as the adb
header that will have been written to disk won't be the same as the one
in memory).

We can do that, by altering the "API" audioDB constructors to take the
adb_t * argument, and setting the adb field in the audioDB object that
we've already introduced to that.  But now we need to be careful a
couple of times: if we have one, then audioDB::initTables() mustn't
stomp on it; also, if we're only constructing an audioDB instance to
fulfil an API request, we mustn't audiodb_close() the one we have when
we destroy the audioDB object, because the adb_t * is the one we have
passed in and are going to reuse in later calls to the API.

The good news is that we can be careful in just these ways with minimal
code.  The really good news is that once the inversion is complete, all
of this horribleness will automatically go away (as there will be no
code which constructs audioDB objects to fulfil API functions).  Hooray!
It's almost like it was all planned this way.

#include "audioDB.h"
extern "C" {
#include "audioDB_API.h"
}
/* Make a new database.

IF size(featuredata) < O2_LARGE_ADB_SIZE 
   The database consists of:

   * a header (see dbTableHeader struct definition);
   * keyTable: list of keys of tracks;
   * trackTable: Maps implicit feature index to a feature vector
     matrix (sizes of tracks)
   * featureTable: Lots of doubles;
   * timesTable: (start,end) time points for each feature vector;
   * powerTable: associated power for each feature vector;
   * l2normTable: squared l2norms for each feature vector.
   
ELSE the database consists of:
   
   * a header (see dbTableHeader struct definition);
   * keyTable: list of keys of tracks
   * trackTable: sizes of tracks
   * featureTable: list of feature file names
   * timesTable: list of times file names
   * powerTable: list of power file names

*/

extern "C" {
  adb_t *audiodb_create(const char *path, unsigned datasize, unsigned ntracks, unsigned datadim) {
    int fd;
    adb_header_t *header = 0;
    off_t databytes, auxbytes;
    if(datasize == 0) {
      datasize = O2_DEFAULT_DATASIZE;
    }
    if(ntracks == 0) {
      ntracks = O2_DEFAULT_NTRACKS;
    }
    if(datadim == 0) {
      datadim = O2_DEFAULT_DATADIM;
    }

    if ((fd = open(path, O_RDWR|O_CREAT|O_EXCL, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)) < 0) {
      goto error;
    }
    if (acquire_lock(fd, true)) {
      goto error;
    }

    header = (adb_header_t *) malloc(sizeof(adb_header_t));
    if(!header) {
      goto error;
    }
    
    // Initialize header
    header->magic = O2_MAGIC;
    header->version = O2_FORMAT_VERSION;
    header->numFiles = 0;
    header->dim = 0;
    header->flags = 0;
    header->headerSize = O2_HEADERSIZE;
    header->length = 0;
    header->fileTableOffset = ALIGN_PAGE_UP(O2_HEADERSIZE);
    header->trackTableOffset = ALIGN_PAGE_UP(header->fileTableOffset + O2_FILETABLE_ENTRY_SIZE*ntracks);
    header->dataOffset = ALIGN_PAGE_UP(header->trackTableOffset + O2_TRACKTABLE_ENTRY_SIZE*ntracks);
    
    databytes = ((off_t) datasize) * 1024 * 1024;
    auxbytes = databytes / datadim;

    /* FIXME: what's going on here?  There are two distinct
       preprocessor constants (O2_LSH_N_POINT_BITS, LSH_N_POINT_BITS);
       a third is presumably some default
       (O2_DEFAULT_LSH_N_POINT_BITS), and then there's this magic 28
       bits.  Should this really be part of the flags structure at
       all?  Putting it elsewhere will of course break backwards
       compatibility, unless 14 is the only value that's been used
       anywhere... */
    
    // For backward-compatibility, Record the point-encoding parameter for LSH indexing in the adb header
    // If this value is 0 then it will be set to 14
    
#if O2_LSH_N_POINT_BITS > 15
#error "AudioDB Compile ERROR: consistency check of O2_LSH_POINT_BITS failed (>15)"
#endif
    
    header->flags |= LSH_N_POINT_BITS << 28;
    
    // If database will fit in a single file the vectors are copied into the AudioDB instance
    // Else all the vectors are left on the FileSystem and we use the dataOffset as storage
    // for the location of the features, powers and times files (assuming that arbitrary keys are used for the fileTable)
    if(ntracks<O2_LARGE_ADB_NTRACKS && datasize<O2_LARGE_ADB_SIZE){
      header->timesTableOffset = ALIGN_PAGE_UP(header->dataOffset + databytes);
      header->powerTableOffset = ALIGN_PAGE_UP(header->timesTableOffset + 2*auxbytes);
      header->l2normTableOffset = ALIGN_PAGE_UP(header->powerTableOffset + auxbytes);
      header->dbSize = ALIGN_PAGE_UP(header->l2normTableOffset + auxbytes);
    } else { // Create LARGE_ADB, features and powers kept on filesystem 
      header->flags |= O2_FLAG_LARGE_ADB;
      header->timesTableOffset = ALIGN_PAGE_UP(header->dataOffset + O2_FILETABLE_ENTRY_SIZE*ntracks);
      header->powerTableOffset = ALIGN_PAGE_UP(header->timesTableOffset + O2_FILETABLE_ENTRY_SIZE*ntracks);
      header->l2normTableOffset = ALIGN_PAGE_UP(header->powerTableOffset + O2_FILETABLE_ENTRY_SIZE*ntracks);
      header->dbSize = header->l2normTableOffset;
    } 
    
    if (write(fd, header, O2_HEADERSIZE) != O2_HEADERSIZE) {
      goto error;
    }
    
    // go to the location corresponding to the last byte
    if (lseek (fd, header->dbSize - 1, SEEK_SET) == -1) {
      goto error;
    }

    // write a dummy byte at the last location
    if (write (fd, "", 1) != 1) {
      goto error;
    }

    free(header);
    return audiodb_open(path, O_RDWR);

  error:
    if(header) {
      free(header);
    }
    return NULL;
  }
}