view open.cpp @ 548:e18843dc0aea

Implement a rudimentary API for audioDB::liszt

The API is rudimentary because we've dropped support for the incremental
retrieval of tracks and their number of vectors (at the API level; the
SOAP and command-line support is still there -- no changes should be
visible).  This is potentially bad for the large-scale databases, of
course; one million tracks will take of the order of 16MB of RAM, more
if I'm unlucky about how std::string.c_str() is implemented.

Both this liszt operation and querying (and sampling, forthcoming...)
would benefit from a `cursor-like' interface to retrieval results: for
an API like that, instead of getting a struct with the data there, you
get a cookie with which you can ask the database for successive results.
This would be neat for all sorts of reasons.  In the meantime, at least
this change fixes SOAP memory leaks related to liszt.

Make liszt.o part of LIBOBJS rather than ordinary OBJS, so that the
liszt functionality is actually compiled into the library.

Add a test for this library functionality; also modify the command-line
test file to run the SOAP server on its own port.

extern "C" {
#include "audioDB_API.h"
}
#include "audioDB-internals.h"

static bool audiodb_check_header(adb_header_t *header) {
  /* FIXME: use syslog() or write to stderr or something to give the
     poor user some diagnostics. */
  if(header->magic == ADB_OLD_MAGIC) {
    return false;
  }
  if(header->magic != ADB_MAGIC) {
    return false;
  }
  if(header->version != ADB_FORMAT_VERSION) {
    return false;
  }
  if(header->headerSize != ADB_HEADER_SIZE) {
    return false;
  }
  return true;
}

static int audiodb_collect_keys(adb_t *adb) {
  char *key_table = 0;
  size_t key_table_length = 0;

  if(adb->header->length > 0) {
    unsigned nfiles = adb->header->numFiles;
    key_table_length = align_page_up(nfiles * ADB_FILETABLE_ENTRY_SIZE);
    mmap_or_goto_error(char *, key_table, adb->header->fileTableOffset, key_table_length);
    for (unsigned int k = 0; k < nfiles; k++) {
      adb->keys->push_back(key_table + k*ADB_FILETABLE_ENTRY_SIZE);
      (*adb->keymap)[(key_table + k*ADB_FILETABLE_ENTRY_SIZE)] = k;
    }
    munmap(key_table, key_table_length);
  }

  return 0;

 error:
  maybe_munmap(key_table, key_table_length);
  return 1;
}

static int audiodb_collect_track_lengths(adb_t *adb) {
  uint32_t *track_table = 0;
  size_t track_table_length = 0;
  if(adb->header->length > 0) {
    unsigned nfiles = adb->header->numFiles;
    track_table_length = align_page_up(nfiles * ADB_TRACKTABLE_ENTRY_SIZE);
    mmap_or_goto_error(uint32_t *, track_table, adb->header->trackTableOffset, track_table_length);
    off_t offset = 0;
    for (unsigned int k = 0; k < nfiles; k++) {
      uint32_t track_length = track_table[k];
      adb->track_lengths->push_back(track_length);
      adb->track_offsets->push_back(offset);
      offset += track_length * adb->header->dim * sizeof(double);
    }
    munmap(track_table, track_table_length);
  }

  return 0;

 error:
  maybe_munmap(track_table, track_table_length);
  return 1;
}

adb_t *audiodb_open(const char *path, int flags) {
  adb_t *adb = 0;
  int fd = -1;

  flags &= (O_RDONLY|O_RDWR);
  fd = open(path, flags);
  if(fd == -1) {
    goto error;
  }
  if(acquire_lock(fd, flags == O_RDWR)) {
    goto error;
  }

  adb = (adb_t *) calloc(1, sizeof(adb_t));
  if(!adb) {
    goto error;
  }
  adb->fd = fd;
  adb->flags = flags;
  adb->path = (char *) malloc(1+strlen(path));
  if(!(adb->path)) {
    goto error;
  }
  strcpy(adb->path, path);

  adb->header = (adb_header_t *) malloc(sizeof(adb_header_t));
  if(!(adb->header)) {
    goto error;
  }
  if(read(fd, (char *) adb->header, ADB_HEADER_SIZE) != ADB_HEADER_SIZE) {
    goto error;
  }
  if(!audiodb_check_header(adb->header)) {
    goto error;
  }

  adb->keys = new std::vector<std::string>;
  if(!adb->keys) {
    goto error;
  }
  adb->keymap = new std::map<std::string,uint32_t>;
  if(!adb->keymap) {
    goto error;
  }
  if(audiodb_collect_keys(adb)) {
    goto error;
  }
  adb->track_lengths = new std::vector<uint32_t>;
  if(!adb->track_lengths) {
    goto error;
  }
  adb->track_lengths->reserve(adb->header->numFiles);
  adb->track_offsets = new std::vector<off_t>;
  if(!adb->track_offsets) {
    goto error;
  }
  adb->track_offsets->reserve(adb->header->numFiles);
  if(audiodb_collect_track_lengths(adb)) {
    goto error;
  }
  adb->cached_lsh = 0;
  return adb;

 error:
  if(adb) {
    if(adb->header) {
      free(adb->header);
    }
    if(adb->path) {
      free(adb->path);
    }
    if(adb->keys) {
      delete adb->keys;
    }
    if(adb->keymap) {
      delete adb->keymap;
    }
    if(adb->track_lengths) {
      delete adb->track_lengths;
    }
    free(adb);
  }
  if(fd != -1) {
    close(fd);
  }
  return NULL;
}