Chris@1: /******************************************************************** Chris@1: * * Chris@1: * THIS FILE IS PART OF THE OggVorbis SOFTWARE CODEC SOURCE CODE. * Chris@1: * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS * Chris@1: * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE * Chris@1: * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING. * Chris@1: * * Chris@1: * THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2001 * Chris@1: * by the Xiph.Org Foundation http://www.xiph.org/ * Chris@1: * * Chris@1: ******************************************************************** Chris@1: Chris@1: function: train a VQ codebook Chris@1: last mod: $Id: vqgen.c 16037 2009-05-26 21:10:58Z xiphmont $ Chris@1: Chris@1: ********************************************************************/ Chris@1: Chris@1: /* This code is *not* part of libvorbis. It is used to generate Chris@1: trained codebooks offline and then spit the results into a Chris@1: pregenerated codebook that is compiled into libvorbis. It is an Chris@1: expensive (but good) algorithm. Run it on big iron. */ Chris@1: Chris@1: /* There are so many optimizations to explore in *both* stages that Chris@1: considering the undertaking is almost withering. For now, we brute Chris@1: force it all */ Chris@1: Chris@1: #include Chris@1: #include Chris@1: #include Chris@1: #include Chris@1: Chris@1: #include "vqgen.h" Chris@1: #include "bookutil.h" Chris@1: Chris@1: /* Codebook generation happens in two steps: Chris@1: Chris@1: 1) Train the codebook with data collected from the encoder: We use Chris@1: one of a few error metrics (which represent the distance between a Chris@1: given data point and a candidate point in the training set) to Chris@1: divide the training set up into cells representing roughly equal Chris@1: probability of occurring. Chris@1: Chris@1: 2) Generate the codebook and auxiliary data from the trained data set Chris@1: */ Chris@1: Chris@1: /* Codebook training **************************************************** Chris@1: * Chris@1: * The basic idea here is that a VQ codebook is like an m-dimensional Chris@1: * foam with n bubbles. The bubbles compete for space/volume and are Chris@1: * 'pressurized' [biased] according to some metric. The basic alg Chris@1: * iterates through allowing the bubbles to compete for space until Chris@1: * they converge (if the damping is dome properly) on a steady-state Chris@1: * solution. Individual input points, collected from libvorbis, are Chris@1: * used to train the algorithm monte-carlo style. */ Chris@1: Chris@1: /* internal helpers *****************************************************/ Chris@1: #define vN(data,i) (data+v->elements*i) Chris@1: Chris@1: /* default metric; squared 'distance' from desired value. */ Chris@1: float _dist(vqgen *v,float *a, float *b){ Chris@1: int i; Chris@1: int el=v->elements; Chris@1: float acc=0.f; Chris@1: for(i=0;ientries;i++) Chris@1: memcpy(_now(v,i),_point(v,i),sizeof(float)*v->elements); Chris@1: v->seeded=1; Chris@1: } Chris@1: Chris@1: int directdsort(const void *a, const void *b){ Chris@1: float av=*((float *)a); Chris@1: float bv=*((float *)b); Chris@1: return (avbv); Chris@1: } Chris@1: Chris@1: void vqgen_cellmetric(vqgen *v){ Chris@1: int j,k; Chris@1: float min=-1.f,max=-1.f,mean=0.f,acc=0.f; Chris@1: long dup=0,unused=0; Chris@1: #ifdef NOISY Chris@1: int i; Chris@1: char buff[80]; Chris@1: float spacings[v->entries]; Chris@1: int count=0; Chris@1: FILE *cells; Chris@1: sprintf(buff,"cellspace%d.m",v->it); Chris@1: cells=fopen(buff,"w"); Chris@1: #endif Chris@1: Chris@1: /* minimum, maximum, cell spacing */ Chris@1: for(j=0;jentries;j++){ Chris@1: float localmin=-1.; Chris@1: Chris@1: for(k=0;kentries;k++){ Chris@1: if(j!=k){ Chris@1: float this=_dist(v,_now(v,j),_now(v,k)); Chris@1: if(this>0){ Chris@1: if(v->assigned[k] && (localmin==-1 || thisentries)continue; Chris@1: Chris@1: if(v->assigned[j]==0){ Chris@1: unused++; Chris@1: continue; Chris@1: } Chris@1: Chris@1: localmin=v->max[j]+localmin/2; /* this gives us rough diameter */ Chris@1: if(min==-1 || localminmax)max=localmin; Chris@1: mean+=localmin; Chris@1: acc++; Chris@1: #ifdef NOISY Chris@1: spacings[count++]=localmin; Chris@1: #endif Chris@1: } Chris@1: Chris@1: fprintf(stderr,"cell diameter: %.03g::%.03g::%.03g (%ld unused/%ld dup)\n", Chris@1: min,mean/acc,max,unused,dup); Chris@1: Chris@1: #ifdef NOISY Chris@1: qsort(spacings,count,sizeof(float),directdsort); Chris@1: for(i=0;iquant)-1); Chris@1: Chris@1: int j,k; Chris@1: Chris@1: mindel=maxdel=_now(v,0)[0]; Chris@1: Chris@1: for(j=0;jentries;j++){ Chris@1: float last=0.f; Chris@1: for(k=0;kelements;k++){ Chris@1: if(mindel>_now(v,j)[k]-last)mindel=_now(v,j)[k]-last; Chris@1: if(maxdel<_now(v,j)[k]-last)maxdel=_now(v,j)[k]-last; Chris@1: if(q->sequencep)last=_now(v,j)[k]; Chris@1: } Chris@1: } Chris@1: Chris@1: Chris@1: /* first find the basic delta amount from the maximum span to be Chris@1: encoded. Loosen the delta slightly to allow for additional error Chris@1: during sequence quantization */ Chris@1: Chris@1: delta=(maxdel-mindel)/((1<quant)-1.5f); Chris@1: Chris@1: q->min=_float32_pack(mindel); Chris@1: q->delta=_float32_pack(delta); Chris@1: Chris@1: mindel=_float32_unpack(q->min); Chris@1: delta=_float32_unpack(q->delta); Chris@1: Chris@1: for(j=0;jentries;j++){ Chris@1: float last=0; Chris@1: for(k=0;kelements;k++){ Chris@1: float val=_now(v,j)[k]; Chris@1: float now=rint((val-last-mindel)/delta); Chris@1: Chris@1: _now(v,j)[k]=now; Chris@1: if(now<0){ Chris@1: /* be paranoid; this should be impossible */ Chris@1: fprintf(stderr,"fault; quantized value<0\n"); Chris@1: exit(1); Chris@1: } Chris@1: Chris@1: if(now>maxquant){ Chris@1: /* be paranoid; this should be impossible */ Chris@1: fprintf(stderr,"fault; quantized value>max\n"); Chris@1: exit(1); Chris@1: } Chris@1: if(q->sequencep)last=(now*delta)+mindel+last; Chris@1: } Chris@1: } Chris@1: } Chris@1: Chris@1: /* much easier :-). Unlike in the codebook, we don't un-log log Chris@1: scales; we just make sure they're properly offset. */ Chris@1: void vqgen_unquantize(vqgen *v,quant_meta *q){ Chris@1: long j,k; Chris@1: float mindel=_float32_unpack(q->min); Chris@1: float delta=_float32_unpack(q->delta); Chris@1: Chris@1: for(j=0;jentries;j++){ Chris@1: float last=0.f; Chris@1: for(k=0;kelements;k++){ Chris@1: float now=_now(v,j)[k]; Chris@1: now=fabs(now)*delta+last+mindel; Chris@1: if(q->sequencep)last=now; Chris@1: _now(v,j)[k]=now; Chris@1: } Chris@1: } Chris@1: } Chris@1: Chris@1: void vqgen_init(vqgen *v,int elements,int aux,int entries,float mindist, Chris@1: float (*metric)(vqgen *,float *, float *), Chris@1: float *(*weight)(vqgen *,float *),int centroid){ Chris@1: memset(v,0,sizeof(vqgen)); Chris@1: Chris@1: v->centroid=centroid; Chris@1: v->elements=elements; Chris@1: v->aux=aux; Chris@1: v->mindist=mindist; Chris@1: v->allocated=32768; Chris@1: v->pointlist=_ogg_malloc(v->allocated*(v->elements+v->aux)*sizeof(float)); Chris@1: Chris@1: v->entries=entries; Chris@1: v->entrylist=_ogg_malloc(v->entries*v->elements*sizeof(float)); Chris@1: v->assigned=_ogg_malloc(v->entries*sizeof(long)); Chris@1: v->bias=_ogg_calloc(v->entries,sizeof(float)); Chris@1: v->max=_ogg_calloc(v->entries,sizeof(float)); Chris@1: if(metric) Chris@1: v->metric_func=metric; Chris@1: else Chris@1: v->metric_func=_dist; Chris@1: if(weight) Chris@1: v->weight_func=weight; Chris@1: else Chris@1: v->weight_func=_weight_null; Chris@1: Chris@1: v->asciipoints=tmpfile(); Chris@1: Chris@1: } Chris@1: Chris@1: void vqgen_addpoint(vqgen *v, float *p,float *a){ Chris@1: int k; Chris@1: for(k=0;kelements;k++) Chris@1: fprintf(v->asciipoints,"%.12g\n",p[k]); Chris@1: for(k=0;kaux;k++) Chris@1: fprintf(v->asciipoints,"%.12g\n",a[k]); Chris@1: Chris@1: if(v->points>=v->allocated){ Chris@1: v->allocated*=2; Chris@1: v->pointlist=_ogg_realloc(v->pointlist,v->allocated*(v->elements+v->aux)* Chris@1: sizeof(float)); Chris@1: } Chris@1: Chris@1: memcpy(_point(v,v->points),p,sizeof(float)*v->elements); Chris@1: if(v->aux)memcpy(_point(v,v->points)+v->elements,a,sizeof(float)*v->aux); Chris@1: Chris@1: /* quantize to the density mesh if it's selected */ Chris@1: if(v->mindist>0.f){ Chris@1: /* quantize to the mesh */ Chris@1: for(k=0;kelements+v->aux;k++) Chris@1: _point(v,v->points)[k]= Chris@1: rint(_point(v,v->points)[k]/v->mindist)*v->mindist; Chris@1: } Chris@1: v->points++; Chris@1: if(!(v->points&0xff))spinnit("loading... ",v->points); Chris@1: } Chris@1: Chris@1: /* yes, not threadsafe. These utils aren't */ Chris@1: static int sortit=0; Chris@1: static int sortsize=0; Chris@1: static int meshcomp(const void *a,const void *b){ Chris@1: if(((sortit++)&0xfff)==0)spinnit("sorting mesh...",sortit); Chris@1: return(memcmp(a,b,sortsize)); Chris@1: } Chris@1: Chris@1: void vqgen_sortmesh(vqgen *v){ Chris@1: sortit=0; Chris@1: if(v->mindist>0.f){ Chris@1: long i,march=1; Chris@1: Chris@1: /* sort to make uniqueness detection trivial */ Chris@1: sortsize=(v->elements+v->aux)*sizeof(float); Chris@1: qsort(v->pointlist,v->points,sortsize,meshcomp); Chris@1: Chris@1: /* now march through and eliminate dupes */ Chris@1: for(i=1;ipoints;i++){ Chris@1: if(memcmp(_point(v,i),_point(v,i-1),sortsize)){ Chris@1: /* a new, unique entry. march it down */ Chris@1: if(i>march)memcpy(_point(v,march),_point(v,i),sortsize); Chris@1: march++; Chris@1: } Chris@1: spinnit("eliminating density... ",v->points-i); Chris@1: } Chris@1: Chris@1: /* we're done */ Chris@1: fprintf(stderr,"\r%ld training points remining out of %ld" Chris@1: " after density mesh (%ld%%)\n",march,v->points,march*100/v->points); Chris@1: v->points=march; Chris@1: Chris@1: } Chris@1: v->sorted=1; Chris@1: } Chris@1: Chris@1: float vqgen_iterate(vqgen *v,int biasp){ Chris@1: long i,j,k; Chris@1: Chris@1: float fdesired; Chris@1: long desired; Chris@1: long desired2; Chris@1: Chris@1: float asserror=0.f; Chris@1: float meterror=0.f; Chris@1: float *new; Chris@1: float *new2; Chris@1: long *nearcount; Chris@1: float *nearbias; Chris@1: #ifdef NOISY Chris@1: char buff[80]; Chris@1: FILE *assig; Chris@1: FILE *bias; Chris@1: FILE *cells; Chris@1: sprintf(buff,"cells%d.m",v->it); Chris@1: cells=fopen(buff,"w"); Chris@1: sprintf(buff,"assig%d.m",v->it); Chris@1: assig=fopen(buff,"w"); Chris@1: sprintf(buff,"bias%d.m",v->it); Chris@1: bias=fopen(buff,"w"); Chris@1: #endif Chris@1: Chris@1: Chris@1: if(v->entries<2){ Chris@1: fprintf(stderr,"generation requires at least two entries\n"); Chris@1: exit(1); Chris@1: } Chris@1: Chris@1: if(!v->sorted)vqgen_sortmesh(v); Chris@1: if(!v->seeded)_vqgen_seed(v); Chris@1: Chris@1: fdesired=(float)v->points/v->entries; Chris@1: desired=fdesired; Chris@1: desired2=desired*2; Chris@1: new=_ogg_malloc(sizeof(float)*v->entries*v->elements); Chris@1: new2=_ogg_malloc(sizeof(float)*v->entries*v->elements); Chris@1: nearcount=_ogg_malloc(v->entries*sizeof(long)); Chris@1: nearbias=_ogg_malloc(v->entries*desired2*sizeof(float)); Chris@1: Chris@1: /* fill in nearest points for entry biasing */ Chris@1: /*memset(v->bias,0,sizeof(float)*v->entries);*/ Chris@1: memset(nearcount,0,sizeof(long)*v->entries); Chris@1: memset(v->assigned,0,sizeof(long)*v->entries); Chris@1: if(biasp){ Chris@1: for(i=0;ipoints;i++){ Chris@1: float *ppt=v->weight_func(v,_point(v,i)); Chris@1: float firstmetric=v->metric_func(v,_now(v,0),ppt)+v->bias[0]; Chris@1: float secondmetric=v->metric_func(v,_now(v,1),ppt)+v->bias[1]; Chris@1: long firstentry=0; Chris@1: long secondentry=1; Chris@1: Chris@1: if(!(i&0xff))spinnit("biasing... ",v->points+v->points+v->entries-i); Chris@1: Chris@1: if(firstmetric>secondmetric){ Chris@1: float temp=firstmetric; Chris@1: firstmetric=secondmetric; Chris@1: secondmetric=temp; Chris@1: firstentry=1; Chris@1: secondentry=0; Chris@1: } Chris@1: Chris@1: for(j=2;jentries;j++){ Chris@1: float thismetric=v->metric_func(v,_now(v,j),ppt)+v->bias[j]; Chris@1: if(thismetricentries;j++){ Chris@1: Chris@1: float thismetric,localmetric; Chris@1: float *nearbiasptr=nearbias+desired2*j; Chris@1: long k=nearcount[j]; Chris@1: Chris@1: localmetric=v->metric_func(v,_now(v,j),ppt); Chris@1: /* 'thismetric' is to be the bias value necessary in the current Chris@1: arrangement for entry j to capture point i */ Chris@1: if(firstentry==j){ Chris@1: /* use the secondary entry as the threshhold */ Chris@1: thismetric=secondmetric-localmetric; Chris@1: }else{ Chris@1: /* use the primary entry as the threshhold */ Chris@1: thismetric=firstmetric-localmetric; Chris@1: } Chris@1: Chris@1: /* support the idea of 'minimum distance'... if we want the Chris@1: cells in a codebook to be roughly some minimum size (as with Chris@1: the low resolution residue books) */ Chris@1: Chris@1: /* a cute two-stage delayed sorting hack */ Chris@1: if(kpoints+v->points+v->entries-i); Chris@1: qsort(nearbiasptr,desired,sizeof(float),directdsort); Chris@1: } Chris@1: Chris@1: }else if(thismetric>nearbiasptr[desired-1]){ Chris@1: nearbiasptr[k]=thismetric; Chris@1: k++; Chris@1: if(k==desired2){ Chris@1: spinnit("biasing... ",v->points+v->points+v->entries-i); Chris@1: qsort(nearbiasptr,desired2,sizeof(float),directdsort); Chris@1: k=desired; Chris@1: } Chris@1: } Chris@1: nearcount[j]=k; Chris@1: } Chris@1: } Chris@1: Chris@1: /* inflate/deflate */ Chris@1: Chris@1: for(i=0;ientries;i++){ Chris@1: float *nearbiasptr=nearbias+desired2*i; Chris@1: Chris@1: spinnit("biasing... ",v->points+v->entries-i); Chris@1: Chris@1: /* due to the delayed sorting, we likely need to finish it off....*/ Chris@1: if(nearcount[i]>desired) Chris@1: qsort(nearbiasptr,nearcount[i],sizeof(float),directdsort); Chris@1: Chris@1: v->bias[i]=nearbiasptr[desired-1]; Chris@1: Chris@1: } Chris@1: }else{ Chris@1: memset(v->bias,0,v->entries*sizeof(float)); Chris@1: } Chris@1: Chris@1: /* Now assign with new bias and find new midpoints */ Chris@1: for(i=0;ipoints;i++){ Chris@1: float *ppt=v->weight_func(v,_point(v,i)); Chris@1: float firstmetric=v->metric_func(v,_now(v,0),ppt)+v->bias[0]; Chris@1: long firstentry=0; Chris@1: Chris@1: if(!(i&0xff))spinnit("centering... ",v->points-i); Chris@1: Chris@1: for(j=0;jentries;j++){ Chris@1: float thismetric=v->metric_func(v,_now(v,j),ppt)+v->bias[j]; Chris@1: if(thismetricbias[j]; Chris@1: meterror+=firstmetric; Chris@1: Chris@1: if(v->centroid==0){ Chris@1: /* set up midpoints for next iter */ Chris@1: if(v->assigned[j]++){ Chris@1: for(k=0;kelements;k++) Chris@1: vN(new,j)[k]+=ppt[k]; Chris@1: if(firstmetric>v->max[j])v->max[j]=firstmetric; Chris@1: }else{ Chris@1: for(k=0;kelements;k++) Chris@1: vN(new,j)[k]=ppt[k]; Chris@1: v->max[j]=firstmetric; Chris@1: } Chris@1: }else{ Chris@1: /* centroid */ Chris@1: if(v->assigned[j]++){ Chris@1: for(k=0;kelements;k++){ Chris@1: if(vN(new,j)[k]>ppt[k])vN(new,j)[k]=ppt[k]; Chris@1: if(vN(new2,j)[k]v->max[firstentry])v->max[j]=firstmetric; Chris@1: }else{ Chris@1: for(k=0;kelements;k++){ Chris@1: vN(new,j)[k]=ppt[k]; Chris@1: vN(new2,j)[k]=ppt[k]; Chris@1: } Chris@1: v->max[firstentry]=firstmetric; Chris@1: } Chris@1: } Chris@1: } Chris@1: Chris@1: /* assign midpoints */ Chris@1: Chris@1: for(j=0;jentries;j++){ Chris@1: #ifdef NOISY Chris@1: fprintf(assig,"%ld\n",v->assigned[j]); Chris@1: fprintf(bias,"%g\n",v->bias[j]); Chris@1: #endif Chris@1: asserror+=fabs(v->assigned[j]-fdesired); Chris@1: if(v->assigned[j]){ Chris@1: if(v->centroid==0){ Chris@1: for(k=0;kelements;k++) Chris@1: _now(v,j)[k]=vN(new,j)[k]/v->assigned[j]; Chris@1: }else{ Chris@1: for(k=0;kelements;k++) Chris@1: _now(v,j)[k]=(vN(new,j)[k]+vN(new2,j)[k])/2.f; Chris@1: } Chris@1: } Chris@1: } Chris@1: Chris@1: asserror/=(v->entries*fdesired); Chris@1: Chris@1: fprintf(stderr,"Pass #%d... ",v->it); Chris@1: fprintf(stderr,": dist %g(%g) metric error=%g \n", Chris@1: asserror,fdesired,meterror/v->points); Chris@1: v->it++; Chris@1: Chris@1: free(new); Chris@1: free(nearcount); Chris@1: free(nearbias); Chris@1: #ifdef NOISY Chris@1: fclose(assig); Chris@1: fclose(bias); Chris@1: fclose(cells); Chris@1: #endif Chris@1: return(asserror); Chris@1: } Chris@1: