mas01cr@509: extern "C" {
mas01cr@509: #include "audioDB_API.h"
mas01cr@509: }
mas01cr@498: #include "audioDB-internals.h"
mas01cr@498: #include "accumulators.h"
mas01cr@239: 
mas01cr@498: bool audiodb_powers_acceptable(const adb_query_refine_t *r, double p1, double p2) {
mas01cr@498:   if (r->flags & ADB_REFINE_ABSOLUTE_THRESHOLD) {
mas01cr@498:     if ((p1 < r->absolute_threshold) || (p2 < r->absolute_threshold)) {
mas01cr@239:       return false;
mas01cr@239:     }
mas01cr@239:   }
mas01cr@498:   if (r->flags & ADB_REFINE_RELATIVE_THRESHOLD) {
mas01cr@498:     if (fabs(p1-p2) > fabs(r->relative_threshold)) {
mas01cr@239:       return false;
mas01cr@239:     }
mas01cr@239:   }
mas01cr@239:   return true;
mas01cr@239: }
mas01cr@239: 
mas01cr@498: adb_query_results_t *audiodb_query_spec(adb_t *adb, const adb_query_spec_t *qspec) {
mas01cr@498:   adb_qstate_internal_t qstate = {0};
mas01cr@498:   qstate.allowed_keys = new std::set<std::string>;
mas01cr@498:   adb_query_results_t *results;
mas01cr@498:   if(qspec->refine.flags & ADB_REFINE_INCLUDE_KEYLIST) {
mas01cr@498:     for(unsigned int k = 0; k < qspec->refine.include.nkeys; k++) {
mas01cr@498:       qstate.allowed_keys->insert(qspec->refine.include.keys[k]);
mas01cr@498:     }
mas01cr@498:   } else {
mas01cr@498:     for(unsigned int k = 0; k < adb->header->numFiles; k++) {
mas01cr@498:       qstate.allowed_keys->insert((*adb->keys)[k]);
mas01cr@498:     }
mas01cr@498:   }
mas01cr@498:   if(qspec->refine.flags & ADB_REFINE_EXCLUDE_KEYLIST) {
mas01cr@498:     for(unsigned int k = 0; k < qspec->refine.exclude.nkeys; k++) {
mas01cr@498:       qstate.allowed_keys->erase(qspec->refine.exclude.keys[k]);
mas01cr@498:     }
mas01cr@498:   }
mas01mc@292: 
mas01cr@498:   switch(qspec->params.distance) {
mas01cr@498:   case ADB_DISTANCE_DOT_PRODUCT:
mas01cr@498:     switch(qspec->params.accumulation) {
mas01cr@498:     case ADB_ACCUMULATION_DB:
mas01cr@498:       qstate.accumulator = new DBAccumulator<adb_result_dist_gt>(qspec->params.npoints);
mas01cr@498:       break;
mas01cr@498:     case ADB_ACCUMULATION_PER_TRACK:
mas01cr@498:       qstate.accumulator = new PerTrackAccumulator<adb_result_dist_gt>(qspec->params.npoints, qspec->params.ntracks);
mas01cr@498:       break;
mas01cr@498:     case ADB_ACCUMULATION_ONE_TO_ONE:
mas01cr@498:       qstate.accumulator = new NearestAccumulator<adb_result_dist_gt>();
mas01cr@498:       break;
mas01cr@498:     default:
mas01cr@498:       goto error;
mas01cr@239:     }
mas01cr@239:     break;
mas01cr@498:   case ADB_DISTANCE_EUCLIDEAN_NORMED:
mas01cr@498:   case ADB_DISTANCE_EUCLIDEAN:
mas01cr@498:     switch(qspec->params.accumulation) {
mas01cr@498:     case ADB_ACCUMULATION_DB:
mas01cr@498:       qstate.accumulator = new DBAccumulator<adb_result_dist_lt>(qspec->params.npoints);
mas01cr@498:       break;
mas01cr@498:     case ADB_ACCUMULATION_PER_TRACK:
mas01cr@498:       qstate.accumulator = new PerTrackAccumulator<adb_result_dist_lt>(qspec->params.npoints, qspec->params.ntracks);
mas01cr@498:       break;
mas01cr@498:     case ADB_ACCUMULATION_ONE_TO_ONE:
mas01cr@498:       qstate.accumulator = new NearestAccumulator<adb_result_dist_lt>();
mas01cr@498:       break;
mas01cr@498:     default:
mas01cr@498:       goto error;
mas01mc@263:     }
mas01mc@263:     break;
mas01cr@239:   default:
mas01cr@498:     goto error;
mas01mc@329:   }
mas01cr@498:   
mas01cr@498:   if((qspec->refine.flags & ADB_REFINE_RADIUS) && audiodb_index_exists(adb->path, qspec->refine.radius, qspec->qid.sequence_length)) {
mas01cr@498:     if(audiodb_index_query_loop(adb, qspec, &qstate) < 0) {
mas01cr@498:       goto error;
mas01cr@498:     }
mas01cr@498:   } else {
mas01cr@498:     if(audiodb_query_loop(adb, qspec, &qstate)) {
mas01cr@498:       goto error;
mas01cr@498:     }
mas01mc@329:   }
mas01mc@292: 
mas01cr@498:   results = qstate.accumulator->get_points();
mas01cr@498: 
mas01cr@498:   delete qstate.accumulator;
mas01cr@498:   delete qstate.allowed_keys;
mas01cr@498: 
mas01cr@498:   return results;
mas01cr@498: 
mas01cr@498:  error:
mas01cr@498:   if(qstate.accumulator)
mas01cr@498:     delete qstate.accumulator;
mas01cr@498:   if(qstate.allowed_keys)
mas01cr@498:     delete qstate.allowed_keys;
mas01cr@498:   return NULL;
mas01cr@239: }
mas01cr@239: 
mas01cr@498: int audiodb_query_free_results(adb_t *adb, const adb_query_spec_t *spec, adb_query_results_t *rs) {
mas01cr@498:   free(rs->results);
mas01cr@498:   free(rs);
mas01cr@498:   return 0;
mas01cr@239: }
mas01cr@239: 
mas01cr@498: static void audiodb_initialize_arrays(adb_t *adb, const adb_query_spec_t *spec, int track, unsigned int numVectors, double *query, double *data_buffer, double **D, double **DD) {
mas01cr@239:   unsigned int j, k, l, w;
mas01cr@239:   double *dp, *qp, *sp;
mas01cr@239: 
mas01cr@498:   const unsigned HOP_SIZE = spec->refine.hopsize;
mas01cr@498:   const unsigned wL = spec->qid.sequence_length;
mas01cr@239: 
mas01cr@239:   for(j = 0; j < numVectors; j++) {
mas01cr@239:     // Sum products matrix
mas01cr@498:     D[j] = new double[(*adb->track_lengths)[track]]; 
mas01cr@239:     assert(D[j]);
mas01cr@239:     // Matched filter matrix
mas01cr@498:     DD[j]=new double[(*adb->track_lengths)[track]];
mas01cr@239:     assert(DD[j]);
mas01cr@239:   }
mas01cr@239: 
mas01cr@239:   // Dot product
mas01cr@239:   for(j = 0; j < numVectors; j++)
mas01cr@498:     for(k = 0; k < (*adb->track_lengths)[track]; k++){
mas01cr@498:       qp = query + j * adb->header->dim;
mas01cr@498:       sp = data_buffer + k * adb->header->dim;
mas01cr@239:       DD[j][k] = 0.0; // Initialize matched filter array
mas01cr@239:       dp = &D[j][k];  // point to correlation cell j,k
mas01cr@239:       *dp = 0.0;      // initialize correlation cell
mas01cr@498:       l = adb->header->dim;         // size of vectors
mas01cr@239:       while(l--)
mas01cr@239:         *dp += *qp++ * *sp++;
mas01cr@239:     }
mas01cr@239:   
mas01cr@239:   // Matched Filter
mas01cr@239:   // HOP SIZE == 1
mas01cr@239:   double* spd;
mas01cr@239:   if(HOP_SIZE == 1) { // HOP_SIZE = shingleHop
mas01cr@239:     for(w = 0; w < wL; w++) {
mas01cr@239:       for(j = 0; j < numVectors - w; j++) { 
mas01cr@239:         sp = DD[j];
mas01cr@239:         spd = D[j+w] + w;
mas01cr@498:         k = (*adb->track_lengths)[track] - w;
mas01mc@292: 	while(k--)
mas01mc@292: 	  *sp++ += *spd++;
mas01cr@239:       }
mas01cr@239:     }
mas01cr@239:   } else { // HOP_SIZE != 1
mas01cr@239:     for(w = 0; w < wL; w++) {
mas01cr@239:       for(j = 0; j < numVectors - w; j += HOP_SIZE) {
mas01cr@239:         sp = DD[j];
mas01cr@239:         spd = D[j+w]+w;
mas01cr@498:         for(k = 0; k < (*adb->track_lengths)[track] - w; k += HOP_SIZE) {
mas01cr@239:           *sp += *spd;
mas01cr@239:           sp += HOP_SIZE;
mas01cr@239:           spd += HOP_SIZE;
mas01cr@239:         }
mas01cr@239:       }
mas01cr@239:     }
mas01cr@239:   }
mas01cr@239: }
mas01cr@239: 
mas01cr@498: static void audiodb_delete_arrays(int track, unsigned int numVectors, double **D, double **DD) {
mas01cr@239:   if(D != NULL) {
mas01cr@239:     for(unsigned int j = 0; j < numVectors; j++) {
mas01cr@239:       delete[] D[j];
mas01cr@239:     }
mas01cr@239:   }
mas01cr@239:   if(DD != NULL) {
mas01cr@239:     for(unsigned int j = 0; j < numVectors; j++) {
mas01cr@239:       delete[] DD[j];
mas01cr@239:     }
mas01cr@239:   }
mas01cr@239: }
mas01cr@239: 
mas01cr@498: int audiodb_read_data(adb_t *adb, int trkfid, int track, double **data_buffer_p, size_t *data_buffer_size_p) {
mas01cr@498:   uint32_t track_length = (*adb->track_lengths)[track];
mas01cr@498:   size_t track_size = track_length * sizeof(double) * adb->header->dim;
mas01cr@498:   if (track_size > *data_buffer_size_p) {
mas01cr@239:     if(*data_buffer_p) {
mas01cr@239:       free(*data_buffer_p);
mas01cr@239:     }
mas01cr@239:     { 
mas01cr@498:       *data_buffer_size_p = track_size;
mas01cr@498:       void *tmp = malloc(track_size);
mas01cr@239:       if (tmp == NULL) {
mas01cr@498:         goto error;
mas01cr@239:       }
mas01cr@239:       *data_buffer_p = (double *) tmp;
mas01cr@239:     }
mas01cr@239:   }
mas01cr@239: 
mas01cr@498:   read_or_goto_error(trkfid, *data_buffer_p, track_size);
mas01cr@498:   return 0;
mas01cr@498: 
mas01cr@498:  error:
mas01cr@498:   return 1;
mas01cr@239: }
mas01cr@239: 
mas01mc@545: int audiodb_track_id_datum(adb_t * adb, uint32_t track_id, adb_datum_t *datum){
mas01mc@545:   return audiodb_track_id_datum_offset(adb, track_id , datum, 0, 0, 0);
mas01mc@545: }
mas01mc@545: 
mas01mc@545: int audiodb_track_id_datum_offset(adb_t *adb, uint32_t track_id, adb_datum_t *d, off_t vector_offset, size_t num_vectors, adb_fd_cache_t* cache){
mas01cr@509:   if(adb->header->flags & ADB_HEADER_FLAG_REFERENCES) {
mas01cr@498:     /* create a reference/insert, then use adb_insert_create_datum() */
mas01mc@541:     adb_reference_t *reference = NULL;
mas01mc@541:     if(! (cache && cache->reference) ){
mas01mc@541:       reference = (adb_reference_t *) malloc(sizeof(adb_reference_t));
mas01mc@541:       reference->features = (char*) malloc(ADB_MAXSTR*sizeof(char));
mas01mc@546:       if(adb->header->flags & ADB_HEADER_FLAG_POWER) {
mas01mc@541: 	reference->power = (char*) malloc(ADB_MAXSTR*sizeof(char));
mas01mc@546:       }
mas01mc@546:       else{
mas01mc@543: 	reference->power = NULL;
mas01mc@546:       }
mas01mc@546:       if(adb->header->flags & ADB_HEADER_FLAG_TIMES){
mas01mc@541: 	reference->times = (char*)malloc(ADB_MAXSTR*sizeof(char));
mas01mc@546:       }
mas01mc@546:       else{
mas01mc@543: 	reference->times = NULL;
mas01mc@546:       }
mas01mc@546:       if(cache){
mas01mc@541: 	cache->reference = reference;
mas01mc@546:       }
mas01cr@498:     }
mas01mc@546:     else{
mas01mc@541:       reference = cache->reference;
mas01mc@546:     }
mas01mc@541:     if(! (cache && cache->track_id==track_id) ){
mas01mc@546:       if(cache){
mas01mc@541: 	cache->track_id = track_id;
mas01mc@546:       }
mas01mc@541:       lseek(adb->fd, adb->header->dataOffset + track_id * ADB_FILETABLE_ENTRY_SIZE, SEEK_SET);
mas01mc@541:       read_or_goto_error(adb->fd, (void *)reference->features, ADB_MAXSTR);
mas01mc@541:       if(adb->header->flags & ADB_HEADER_FLAG_POWER) {
mas01mc@541: 	lseek(adb->fd, adb->header->powerTableOffset + track_id * ADB_FILETABLE_ENTRY_SIZE, SEEK_SET);
mas01mc@541: 	read_or_goto_error(adb->fd, (void *)reference->power, ADB_MAXSTR);
mas01mc@541:       }
mas01mc@541:       if(adb->header->flags & ADB_HEADER_FLAG_TIMES) {
mas01mc@541: 	lseek(adb->fd, adb->header->timesTableOffset + track_id * ADB_FILETABLE_ENTRY_SIZE, SEEK_SET);
mas01mc@541: 	read_or_goto_error(adb->fd, (void *)reference->times, ADB_MAXSTR);
mas01mc@541:       }
mas01cr@498:     }
mas01mc@546:     int retval = audiodb_insert_create_datum_offset(reference, d, vector_offset, num_vectors, cache);
mas01mc@541:     if(!cache){
mas01mc@541:       audiodb_free_datum_reference(reference);
mas01mc@541:       free(reference);      
mas01mc@541:     }
mas01mc@541:     return retval;
mas01mc@546:   } 
mas01mc@546:   else {
mas01mc@546:     off_t track_offset = (*adb->track_offsets)[track_id];
mas01cr@498:     /* initialize from sources of data that we already have */
mas01mc@539:     if(num_vectors)
mas01mc@539:       d->nvectors = num_vectors;
mas01mc@539:     else
mas01mc@539:       d->nvectors = (*adb->track_lengths)[track_id];
mas01cr@498:     d->dim = adb->header->dim;
mas01cr@498:     d->key = (*adb->keys)[track_id].c_str();
mas01cr@498:     /* read out stuff from the database tables */
mas01cr@498:     d->data = (double *) malloc(d->nvectors * d->dim * sizeof(double));
mas01mc@539:     lseek(adb->fd, adb->header->dataOffset + track_offset + vector_offset*d->dim*sizeof(double), SEEK_SET);
mas01cr@498:     read_or_goto_error(adb->fd, d->data, d->nvectors * d->dim * sizeof(double));
mas01cr@509:     if(adb->header->flags & ADB_HEADER_FLAG_POWER) {
mas01cr@498:       d->power = (double *) malloc(d->nvectors * sizeof(double));
mas01mc@539:       lseek(adb->fd, adb->header->powerTableOffset + track_offset / d->dim + vector_offset*sizeof(double), SEEK_SET);
mas01cr@498:       read_or_goto_error(adb->fd, d->power, d->nvectors * sizeof(double));
mas01cr@498:     }
mas01cr@509:     if(adb->header->flags & ADB_HEADER_FLAG_TIMES) {
mas01cr@498:       d->times = (double *) malloc(2 * d->nvectors * sizeof(double));
mas01mc@539:       lseek(adb->fd, adb->header->timesTableOffset + track_offset / d->dim + 2 * vector_offset*sizeof(double), SEEK_SET);
mas01cr@498:       read_or_goto_error(adb->fd, d->times, 2 * d->nvectors * sizeof(double));
mas01cr@498:     }
mas01cr@498:     return 0;
mas01cr@498:   }
mas01cr@498:  error:
mas01cr@498:   audiodb_free_datum(d);
mas01cr@498:   return 1;
mas01cr@498: }
mas01mc@292: 
mas01cr@498: int audiodb_datum_qpointers(adb_datum_t *d, uint32_t sequence_length, double **vector_data, double **vector, adb_qpointers_internal_t *qpointers) {
mas01cr@498:   uint32_t nvectors = d->nvectors;
mas01cr@498: 
mas01cr@498:   qpointers->nvectors = nvectors;
mas01cr@498: 
mas01cr@498:   size_t vector_size = nvectors * sizeof(double) * d->dim;
mas01cr@498:   *vector_data = new double[vector_size];
mas01cr@498:   memcpy(*vector_data, d->data, vector_size);
mas01cr@498: 
mas01cr@498:   qpointers->l2norm_data = new double[vector_size / d->dim];
mas01cr@498:   audiodb_l2norm_buffer(*vector_data, d->dim, nvectors, qpointers->l2norm_data);
mas01cr@498:   audiodb_sequence_sum(qpointers->l2norm_data, nvectors, sequence_length);
mas01cr@498:   audiodb_sequence_sqrt(qpointers->l2norm_data, nvectors, sequence_length);
mas01cr@498: 
mas01cr@498:   if(d->power) {
mas01cr@498:     qpointers->power_data = new double[vector_size / d->dim];
mas01cr@498:     memcpy(qpointers->power_data, d->power, vector_size / d->dim);
mas01cr@498:     audiodb_sequence_sum(qpointers->power_data, nvectors, sequence_length);
mas01cr@498:     audiodb_sequence_average(qpointers->power_data, nvectors, sequence_length);
mas01cr@239:   }
mas01cr@239: 
mas01cr@498:   if(d->times) {
mas01cr@498:     qpointers->mean_duration = new double[1];
mas01cr@498:     *qpointers->mean_duration = 0;
mas01cr@498:     for(unsigned int k = 0; k < nvectors; k++) {
mas01cr@498:       *qpointers->mean_duration += d->times[2*k+1] - d->times[2*k];
mas01cr@239:     }
mas01cr@498:     *qpointers->mean_duration /= nvectors;
mas01cr@239:   }
mas01cr@239: 
mas01cr@498:   *vector = *vector_data;
mas01cr@498:   qpointers->l2norm = qpointers->l2norm_data;
mas01cr@498:   qpointers->power = qpointers->power_data;
mas01cr@498:   return 0;
mas01cr@498: }
mas01cr@498: 
mas01mc@528: int audiodb_datum_qpointers_partial(adb_datum_t *d, uint32_t sequence_length, double **vector_data, 
mas01mc@528: 				    double **vector, adb_qpointers_internal_t *qpointers, 
mas01mc@528: 				    adb_qstate_internal_t *qstate){
mas01mc@528:   uint32_t nvectors = d->nvectors;
mas01mc@528:   qpointers->nvectors = nvectors;
mas01mc@528:   
mas01mc@539:   PointPair pp = (*qstate->exact_evaluation_queue).top();
mas01mc@529: #ifdef _LSH_DEBUG_
mas01mc@539:   cout << "tid=" << pp.trackID << " qpos=" << pp.qpos << " spos=" << pp.spos << endl;
mas01mc@539:   cout.flush();
mas01mc@529: #endif
mas01mc@539:   
mas01mc@539:   if(d->power) {
mas01mc@539:     //memcpy(qpointers->power_data, d->power, seq_len_dbl);
mas01mc@539:     audiodb_sequence_sum(d->power, sequence_length, sequence_length);
mas01mc@539:     audiodb_sequence_average(d->power, sequence_length, sequence_length);
mas01mc@528:   }
mas01mc@539:   
mas01mc@528:   if(d->times) {
mas01mc@528:     qpointers->mean_duration = new double[1];
mas01mc@528:     *qpointers->mean_duration = 0;
mas01mc@528:     for(unsigned int k = 0; k < nvectors; k++) {
mas01mc@528:       *qpointers->mean_duration += d->times[2*k+1] - d->times[2*k];
mas01mc@528:     }
mas01mc@528:     *qpointers->mean_duration /= nvectors;
mas01mc@528:   }  
mas01mc@539:   
mas01mc@531:   *vector = d->data;
mas01mc@531:   *vector_data = d->data;
mas01mc@531:   qpointers->l2norm = 0 ;
mas01mc@539:   qpointers->power = d->power;
mas01mc@528:   return 0;
mas01mc@528: }
mas01mc@528: 
mas01cr@498: int audiodb_query_spec_qpointers(adb_t *adb, const adb_query_spec_t *spec, double **vector_data, double **vector, adb_qpointers_internal_t *qpointers) {
mas01cr@498:   adb_datum_t *datum;
mas01cr@498:   adb_datum_t d = {0};
mas01cr@498:   uint32_t sequence_length;
mas01cr@498:   uint32_t sequence_start;
mas01cr@498: 
mas01cr@498:   datum = spec->qid.datum;
mas01cr@498:   sequence_length = spec->qid.sequence_length;
mas01cr@498:   sequence_start = spec->qid.sequence_start;
mas01cr@498:     
mas01cr@498:   if(datum->data) {
mas01cr@498:     if(datum->dim != adb->header->dim) {
mas01cr@498:       return 1;
mas01cr@239:     }
mas01cr@498:     /* initialize d, and mark that nothing needs freeing later. */
mas01cr@498:     d = *datum;
mas01cr@498:     datum = &d;
mas01cr@498:   } else if (datum->key) {
mas01cr@498:     uint32_t track_id;
mas01cr@498:     if((track_id = audiodb_key_index(adb, datum->key)) == (uint32_t) -1) {
mas01cr@498:       return 1;
mas01cr@498:     }
mas01cr@498:     audiodb_track_id_datum(adb, track_id, &d);
mas01cr@498:   } else {
mas01cr@498:     return 1;
mas01cr@239:   }
mas01cr@239: 
mas01cr@498:   /* FIXME: check the overflow logic here */
mas01cr@498:   if(sequence_start + sequence_length > d.nvectors) {
mas01cr@498:     if(datum != &d) {
mas01cr@498:       audiodb_free_datum(&d);
mas01cr@498:     }
mas01cr@498:     return 1;
mas01cr@498:   }
mas01cr@239: 
mas01cr@498:   audiodb_datum_qpointers(&d, sequence_length, vector_data, vector, qpointers);
mas01cr@498: 
mas01cr@498:   /* Finally, if applicable, set up the moving qpointers. */
mas01cr@498:   if(spec->qid.flags & ADB_QID_FLAG_EXHAUSTIVE) {
mas01cr@498:     /* the qpointers are already at the start, and so correct. */
mas01cr@498:   } else {
mas01cr@498:     /* adjust the qpointers to point to the correct place in the sequence */
mas01cr@498:     *vector = *vector_data + spec->qid.sequence_start * d.dim;
mas01cr@498:     qpointers->l2norm = qpointers->l2norm_data + spec->qid.sequence_start;
mas01cr@498:     if(d.power) {
mas01cr@498:       qpointers->power = qpointers->power_data + spec->qid.sequence_start;
mas01cr@239:     }
mas01cr@498:     qpointers->nvectors = sequence_length;
mas01cr@239:   }
mas01cr@498: 
mas01cr@498:   /* Clean up: free any bits of datum that we have ourselves
mas01cr@498:    * allocated. */
mas01cr@498:   if(datum != &d) {
mas01cr@498:     audiodb_free_datum(&d);
mas01cr@498:   }
mas01cr@498: 
mas01cr@498:   return 0;
mas01cr@239: }
mas01cr@239: 
mas01cr@498: static int audiodb_set_up_dbpointers(adb_t *adb, const adb_query_spec_t *spec, adb_qpointers_internal_t *dbpointers) {
mas01cr@498:   uint32_t nvectors = adb->header->length / (adb->header->dim * sizeof(double));
mas01cr@498:   uint32_t sequence_length = spec->qid.sequence_length;
mas01mc@292: 
mas01cr@498:   bool using_power = spec->refine.flags & (ADB_REFINE_ABSOLUTE_THRESHOLD|ADB_REFINE_RELATIVE_THRESHOLD);
mas01cr@498:   bool using_times = spec->refine.flags & ADB_REFINE_DURATION_RATIO;
mas01cr@498:   double *times_table = NULL;
mas01cr@498: 
mas01cr@498: 
mas01cr@498:   dbpointers->nvectors = nvectors;
mas01cr@498:   dbpointers->l2norm_data = new double[nvectors];
mas01cr@498: 
mas01cr@498:   double *snpp = dbpointers->l2norm_data, *sppp = 0;
mas01cr@498:   lseek(adb->fd, adb->header->l2normTableOffset, SEEK_SET);
mas01cr@498:   read_or_goto_error(adb->fd, dbpointers->l2norm_data, nvectors * sizeof(double));
mas01cr@498: 
mas01cr@498:   if (using_power) {
mas01cr@509:     if (!(adb->header->flags & ADB_HEADER_FLAG_POWER)) {
mas01cr@498:       goto error;
mas01cr@498:     }
mas01cr@498:     dbpointers->power_data = new double[nvectors];
mas01cr@498:     sppp = dbpointers->power_data;
mas01cr@498:     lseek(adb->fd, adb->header->powerTableOffset, SEEK_SET);
mas01cr@498:     read_or_goto_error(adb->fd, dbpointers->power_data, nvectors * sizeof(double));
mas01mc@292:   }
mas01mc@292: 
mas01cr@498:   for(unsigned int i = 0; i < adb->header->numFiles; i++){
mas01cr@498:     size_t track_length = (*adb->track_lengths)[i];
mas01cr@498:     if(track_length >= sequence_length) {
mas01cr@498:       audiodb_sequence_sum(snpp, track_length, sequence_length);
mas01cr@498:       audiodb_sequence_sqrt(snpp, track_length, sequence_length);
mas01cr@498:       if (using_power) {
mas01cr@498: 	audiodb_sequence_sum(sppp, track_length, sequence_length);
mas01cr@498:         audiodb_sequence_average(sppp, track_length, sequence_length);
mas01cr@498:       }
mas01mc@324:     }
mas01cr@498:     snpp += track_length;
mas01cr@498:     if (using_power) {
mas01cr@498:       sppp += track_length;
mas01mc@324:     }
mas01mc@292:   }
mas01mc@292: 
mas01cr@498:   if (using_times) {
mas01cr@509:     if(!(adb->header->flags & ADB_HEADER_FLAG_TIMES)) {
mas01cr@498:       goto error;
mas01cr@498:     }
mas01mc@292: 
mas01cr@498:     dbpointers->mean_duration = new double[adb->header->numFiles];
mas01cr@498: 
mas01cr@498:     times_table = (double *) malloc(2 * nvectors * sizeof(double));
mas01cr@498:     if(!times_table) {
mas01cr@498:       goto error;
mas01cr@498:     }
mas01cr@498:     lseek(adb->fd, adb->header->timesTableOffset, SEEK_SET);
mas01cr@498:     read_or_goto_error(adb->fd, times_table, 2 * nvectors * sizeof(double));
mas01cr@498:     for(unsigned int k = 0; k < adb->header->numFiles; k++) {
mas01cr@498:       size_t track_length = (*adb->track_lengths)[k];
mas01cr@498:       unsigned int j;
mas01cr@498:       dbpointers->mean_duration[k] = 0.0;
mas01cr@498:       for(j = 0; j < track_length; j++) {
mas01cr@498: 	dbpointers->mean_duration[k] += times_table[2*j+1] - times_table[2*j];
mas01mc@292:       }
mas01cr@498:       dbpointers->mean_duration[k] /= j;
mas01mc@292:     }
mas01cr@498: 
mas01cr@498:     free(times_table);
mas01cr@498:     times_table = NULL;
mas01mc@292:   }
mas01cr@498: 
mas01cr@498:   dbpointers->l2norm = dbpointers->l2norm_data;
mas01cr@498:   dbpointers->power = dbpointers->power_data;
mas01cr@498:   return 0;
mas01cr@498: 
mas01cr@498:  error:
mas01cr@498:   if(dbpointers->l2norm_data) {
mas01cr@498:     delete [] dbpointers->l2norm_data;
mas01cr@498:   }
mas01cr@498:   if(dbpointers->power_data) {
mas01cr@498:     delete [] dbpointers->power_data;
mas01cr@498:   }
mas01cr@498:   if(dbpointers->mean_duration) {
mas01cr@498:     delete [] dbpointers->mean_duration;
mas01cr@498:   }
mas01cr@498:   if(times_table) {
mas01cr@498:     free(times_table);
mas01cr@498:   }
mas01cr@498:   return 1;
mas01cr@498:   
mas01mc@292: }
mas01mc@292: 
mas01cr@498: int audiodb_query_queue_loop(adb_t *adb, const adb_query_spec_t *spec, adb_qstate_internal_t *qstate, double *query, adb_qpointers_internal_t *qpointers) {
mas01cr@498:   adb_qpointers_internal_t dbpointers = {0};
mas01mc@292: 
mas01cr@498:   uint32_t sequence_length = spec->qid.sequence_length;
mas01cr@498:   bool power_refine = spec->refine.flags & (ADB_REFINE_ABSOLUTE_THRESHOLD|ADB_REFINE_RELATIVE_THRESHOLD);
mas01cr@239: 
mas01cr@498:   if(qstate->exact_evaluation_queue->size() == 0) {
mas01cr@498:     return 0;
mas01cr@239:   }
mas01cr@239: 
mas01cr@498:   /* We are guaranteed that the order of points is sorted by:
mas01cr@498:    * {trackID, spos, qpos} so we can be relatively efficient in
mas01cr@498:    * initialization of track data.  We assume that points usually
mas01cr@498:    * don't overlap, so we will use exhaustive dot product evaluation
mas01cr@498:    * (instead of memoization of partial sums, as in query_loop()).
mas01cr@498:    */
mas01cr@498:   double dist;
mas01cr@498:   double *dbdata = 0, *dbdata_pointer;
mas01cr@498:   Uns32T npairs = qstate->exact_evaluation_queue->size();
mas01mc@541:   Uns32T currentTrack = qstate->exact_evaluation_queue->top().trackID+1; // i.e. not first track
mas01mc@527: #ifdef _LSH_DEBUG_
mas01mc@527:   cout << "Num vector pairs to evaluate: " << npairs << "..." << endl;
mas01mc@527:   cout.flush();
mas01mc@527: #endif  
mas01mc@531:   adb_datum_t d = {0};
mas01mc@541:   adb_fd_cache_t c = {0};
mas01mc@541:   c.track_id = currentTrack;
mas01cr@498:   while(npairs--) {
mas01cr@498:     PointPair pp = qstate->exact_evaluation_queue->top();
mas01mc@541:     if(pp.trackID != currentTrack){
mas01mc@541:           audiodb_free_datum(&d);
mas01mc@541: 	  currentTrack = pp.trackID;
mas01mc@541:     }
mas01mc@539:     maybe_delete_array(dbpointers.mean_duration);
mas01mc@545:     if(audiodb_track_id_datum_offset(adb, pp.trackID, &d, pp.spos, sequence_length, &c)) {
mas01mc@539:       delete qstate->exact_evaluation_queue;
mas01mc@539:       delete qstate->set;
mas01mc@541:       audiodb_free_datum(&d);
mas01mc@541:       audiodb_free_datum_cache(&c);
mas01mc@539:       return 1;
mas01mc@539:     }
mas01mc@539:       
mas01mc@539:     if(audiodb_datum_qpointers_partial(&d, sequence_length, &dbdata, &dbdata_pointer, &dbpointers, qstate)) {
mas01mc@539:       delete qstate->exact_evaluation_queue;
mas01mc@539:       delete qstate->set;
mas01mc@531:       audiodb_free_datum(&d);
mas01mc@541:       audiodb_free_datum_cache(&c);
mas01mc@539:       return 1;
mas01mc@539:     }
mas01mc@531:       
mas01cr@498:     Uns32T qPos = (spec->qid.flags & ADB_QID_FLAG_EXHAUSTIVE) ? pp.qpos : 0;
mas01cr@498:     // Test power thresholds before computing distance
mas01mc@539:     if( ( (!power_refine) || audiodb_powers_acceptable(&spec->refine, qpointers->power[qPos], dbpointers.power[0])) &&
mas01mc@539: 	( qPos<qpointers->nvectors-sequence_length+1 && pp.spos<(*adb->track_lengths)[pp.trackID]-sequence_length+1 ) ){
mas01mc@531:       // Compute distance    
mas01mc@539:       dist = audiodb_dot_product(query + qPos*adb->header->dim, dbdata, adb->header->dim*sequence_length);
mas01mc@531:       double qn = audiodb_dot_product(query + qPos*adb->header->dim, query + qPos*adb->header->dim, adb->header->dim*sequence_length);
mas01mc@539:       double sn = audiodb_dot_product(dbdata, dbdata, adb->header->dim*sequence_length);
mas01mc@531:       qn = sqrt(qn);
mas01mc@531:       sn = sqrt(sn);
mas01cr@498:       switch(spec->params.distance) {
mas01cr@498:       case ADB_DISTANCE_EUCLIDEAN_NORMED:
mas01cr@498: 	dist = 2 - (2/(qn*sn))*dist;
mas01cr@498:         break;
mas01cr@498:       case ADB_DISTANCE_EUCLIDEAN:
mas01cr@498:         dist = qn*qn + sn*sn - 2*dist;
mas01cr@498:         break;
mas01cr@498:       }
mas01cr@498:       if((!(spec->refine.flags & ADB_REFINE_RADIUS)) || 
mas01cr@509:          dist <= (spec->refine.radius + ADB_DISTANCE_TOLERANCE)) {
mas01cr@498:         adb_result_t r;
mas01cr@498:         r.key = (*adb->keys)[pp.trackID].c_str();
mas01cr@498:         r.dist = dist;
mas01cr@498:         r.qpos = pp.qpos;
mas01cr@498:         r.ipos = pp.spos;
mas01cr@498:         qstate->accumulator->add_point(&r);
mas01cr@239:       }
mas01cr@239:     }
mas01cr@498:     qstate->exact_evaluation_queue->pop();
mas01mc@292:   }
mas01mc@474: 
mas01mc@315:   // Cleanup
mas01mc@541:   audiodb_free_datum(&d);
mas01mc@541:   audiodb_free_datum_cache(&c);
mas01cr@509:   maybe_delete_array(dbpointers.mean_duration);
mas01cr@498:   delete qstate->exact_evaluation_queue;
mas01mc@529:   delete qstate->set;
mas01cr@498:   return 0;
mas01mc@292: }
mas01mc@292: 
mas01cr@498: int audiodb_query_loop(adb_t *adb, const adb_query_spec_t *spec, adb_qstate_internal_t *qstate) {
mas01cr@498:   
mas01cr@498:   double *query, *query_data;
mas01cr@498:   adb_qpointers_internal_t qpointers = {0}, dbpointers = {0};
mas01mc@292: 
mas01cr@498:   bool power_refine = spec->refine.flags & (ADB_REFINE_ABSOLUTE_THRESHOLD|ADB_REFINE_RELATIVE_THRESHOLD);
mas01cr@239: 
mas01cr@509:   if(adb->header->flags & ADB_HEADER_FLAG_REFERENCES) {
mas01cr@498:     /* FIXME: actually it would be nice to support this mode of
mas01cr@498:      * operation, but for now... */
mas01cr@498:     return 1;
mas01cr@498:   }
mas01mc@324: 
mas01cr@498:   if(audiodb_query_spec_qpointers(adb, spec, &query_data, &query, &qpointers)) {
mas01cr@498:     return 1;
mas01cr@498:   }
mas01cr@239: 
mas01cr@498:   if(audiodb_set_up_dbpointers(adb, spec, &dbpointers)) {
mas01cr@498:     return 1;
mas01cr@498:   }
mas01cr@239: 
mas01cr@498:   unsigned j,k,track,trackOffset=0, HOP_SIZE = spec->refine.hopsize;
mas01cr@498:   unsigned wL = spec->qid.sequence_length;
mas01cr@239:   double **D = 0;    // Differences query and target 
mas01cr@239:   double **DD = 0;   // Matched filter distance
mas01cr@239: 
mas01cr@498:   D = new double*[qpointers.nvectors]; // pre-allocate 
mas01cr@498:   DD = new double*[qpointers.nvectors];
mas01cr@239: 
mas01cr@239:   off_t trackIndexOffset;
mas01cr@239: 
mas01cr@239:   // Track loop 
mas01cr@239:   size_t data_buffer_size = 0;
mas01cr@239:   double *data_buffer = 0;
mas01cr@498:   lseek(adb->fd, adb->header->dataOffset, SEEK_SET);
mas01cr@239: 
mas01cr@498:   std::set<std::string>::iterator keys_end = qstate->allowed_keys->end();
mas01cr@498:   for(track = 0; track < adb->header->numFiles; track++) {
mas01cr@498:     unsigned t = track;
mas01cr@498:     
mas01cr@498:     while (qstate->allowed_keys->find((*adb->keys)[track]) == keys_end) {
mas01cr@498:       track++;
mas01cr@498:       if(track == adb->header->numFiles) {
mas01cr@498:         goto loop_finish;
mas01cr@239:       }
mas01cr@239:     }
mas01cr@498:     trackOffset = (*adb->track_offsets)[track];
mas01cr@498:     if(track != t) {
mas01cr@498:       lseek(adb->fd, adb->header->dataOffset + trackOffset, SEEK_SET);
mas01cr@498:     }
mas01cr@498:     trackIndexOffset = trackOffset / (adb->header->dim * sizeof(double)); // dbpointers.nvectors offset
mas01cr@239: 
mas01cr@498:     if(audiodb_read_data(adb, adb->fd, track, &data_buffer, &data_buffer_size)) {
mas01cr@498:       return 1;
mas01mc@292:     }
mas01cr@498:     if(wL <= (*adb->track_lengths)[track]) {  // test for short sequences
mas01cr@498:       
mas01cr@498:       audiodb_initialize_arrays(adb, spec, track, qpointers.nvectors, query, data_buffer, D, DD);
mas01mc@292: 
mas01cr@498:       if((!(spec->refine.flags & ADB_REFINE_DURATION_RATIO)) || 
mas01cr@498:          fabs(dbpointers.mean_duration[track]-qpointers.mean_duration[0]) < qpointers.mean_duration[0]*spec->refine.duration_ratio) {
mas01cr@239: 
mas01cr@239: 	// Search for minimum distance by shingles (concatenated vectors)
mas01cr@498: 	for(j = 0; j <= qpointers.nvectors - wL; j += HOP_SIZE) {
mas01cr@498: 	  for(k = 0; k <= (*adb->track_lengths)[track] - wL; k += HOP_SIZE) {
mas01cr@498:             double thisDist = 0;
mas01cr@498:             double qn = qpointers.l2norm[j];
mas01cr@498:             double sn = dbpointers.l2norm[trackIndexOffset + k];
mas01cr@498:             switch(spec->params.distance) {
mas01cr@498:             case ADB_DISTANCE_EUCLIDEAN_NORMED:
mas01cr@498:               thisDist = 2-(2/(qn*sn))*DD[j][k];
mas01cr@498:               break;
mas01cr@498:             case ADB_DISTANCE_EUCLIDEAN:
mas01cr@498:               thisDist = qn*qn + sn*sn - 2*DD[j][k];
mas01cr@498:               break;
mas01cr@498:             case ADB_DISTANCE_DOT_PRODUCT:
mas01cr@498:               thisDist = DD[j][k];
mas01cr@498:               break;
mas01cr@498:             }
mas01cr@239: 	    // Power test
mas01cr@498: 	    if ((!power_refine) || audiodb_powers_acceptable(&spec->refine, qpointers.power[j], dbpointers.power[trackIndexOffset + k])) {
mas01cr@239:               // radius test
mas01cr@498:               if((!(spec->refine.flags & ADB_REFINE_RADIUS)) || 
mas01cr@509:                  thisDist <= (spec->refine.radius + ADB_DISTANCE_TOLERANCE)) {
mas01cr@498:                 adb_result_t r;
mas01cr@498:                 r.key = (*adb->keys)[track].c_str();
mas01cr@498:                 r.dist = thisDist;
mas01cr@498:                 if(spec->qid.flags & ADB_QID_FLAG_EXHAUSTIVE) {
mas01cr@498:                   r.qpos = j;
mas01cr@498:                 } else {
mas01cr@498:                   r.qpos = spec->qid.sequence_start;
mas01cr@498:                 }
mas01cr@498:                 r.ipos = k;
mas01cr@498:                 qstate->accumulator->add_point(&r);
mas01cr@239:               }
mas01cr@239:             }
mas01cr@239:           }
mas01cr@239:         }
mas01cr@239:       } // Duration match            
mas01cr@498:       audiodb_delete_arrays(track, qpointers.nvectors, D, DD);
mas01cr@239:     }
mas01cr@239:   }
mas01cr@239: 
mas01cr@498:  loop_finish:
mas01cr@498: 
mas01cr@239:   free(data_buffer);
mas01cr@239: 
mas01cr@239:   // Clean up
mas01cr@239:   if(query_data)
mas01cr@239:     delete[] query_data;
mas01cr@498:   if(qpointers.l2norm_data)
mas01cr@498:     delete[] qpointers.l2norm_data;
mas01cr@498:   if(qpointers.power_data)
mas01cr@498:     delete[] qpointers.power_data;
mas01cr@498:   if(qpointers.mean_duration)
mas01cr@498:     delete[] qpointers.mean_duration;
mas01cr@498:   if(dbpointers.power_data)
mas01cr@498:     delete[] dbpointers.power_data;
mas01cr@498:   if(dbpointers.l2norm_data)
mas01cr@498:     delete[] dbpointers.l2norm_data;
mas01cr@239:   if(D)
mas01cr@239:     delete[] D;
mas01cr@239:   if(DD)
mas01cr@239:     delete[] DD;
mas01cr@498:   if(dbpointers.mean_duration)
mas01cr@498:     delete[] dbpointers.mean_duration;
mas01cr@498: 
mas01cr@498:   return 0;
mas01cr@239: }