view l2norm.cpp @ 465:1030664df98c api-inversion

No more audioDB::index_allocate and audioDB::index_init_query

No more SERVER_LSH_INDEX_SINGLETON, either; instead each adb_t contains
a single cache of the last used in-core index.  At the moment, this
cache is unused by the server (and the previous cache code has been
replaced by a comment), but I think that this way everyone can be
allowed to benefit without anyone having to explicitly manage indexes
themselves.

I'm not going to say how long I wandered in a maze of valgrind before
giving up and keeping the hacky workaround for loading the lsh tables
[see the FIXME comment in audiodb_index_init_query()]; let's just say
that it was long enough to find the extra bonus crashy close(lshfid) in
audioDB::index_index_db.

Also, delete the abstraction-inverting LSH stuff from query.cpp where we
are making our reporters; the fix for that, which is presumably when
creating small indexes for large datasets, is to implement
space-efficient reporters.  (The accumulator code, which is my second
attempt, is more space-efficient than the reporters; inspiration may
wish to be drawn...)

#include "audioDB.h"
extern "C" {
#include "audioDB_API.h"
#include "audioDB-internals.h"
}

static int audiodb_l2norm_existing(adb_t *adb) {
  double *data_buffer, *l2norm_buffer;
  adb_header_t *header = adb->header;
  size_t data_buffer_size = ALIGN_PAGE_UP(header->length);
  size_t nvectors = header->length / (sizeof(double) * header->dim);
  /* FIXME: this map of the vector data will lose if we ever turn the
   * l2norm flag on when we have already inserted a large number of
   * vectors, as the mmap() will fail.  "Don't do that, then" is one
   * possible answer. */
  mmap_or_goto_error(double *, data_buffer, header->dataOffset, data_buffer_size);
  l2norm_buffer = (double *) malloc(nvectors * sizeof(double));
  if(!l2norm_buffer) {
    goto error;
  }
  audiodb_l2norm_buffer(data_buffer, header->dim, nvectors, l2norm_buffer);
  if(lseek(adb->fd, adb->header->l2normTableOffset, SEEK_SET) == (off_t) -1) {
    goto error;
  }
  write_or_goto_error(adb->fd, l2norm_buffer, nvectors * sizeof(double));

  munmap(data_buffer, data_buffer_size);
  free(l2norm_buffer);

  return 0;

 error:
  maybe_munmap(data_buffer, data_buffer_size);
  if(l2norm_buffer) {
    free(l2norm_buffer);
  }
  return 1;
}

int audiodb_l2norm(adb_t *adb) {
  adb_header_t *header = adb->header;
  if(!(adb->flags & O_RDWR)) {
    return 1;
  }
  if(header->flags & O2_FLAG_L2NORM) {
    /* non-error code for forthcoming backwards-compatibility
     * reasons */
    return 0;
  }
  if((!(header->flags & O2_FLAG_LARGE_ADB)) && (header->length > 0)) {
    if(audiodb_l2norm_existing(adb)) {
      goto error;
    }
  }
  adb->header->flags |= O2_FLAG_L2NORM;
  return audiodb_sync_header(adb);

 error:
  return 1;
}