cannam@86: /******************************************************************** cannam@86: * * cannam@86: * THIS FILE IS PART OF THE OggVorbis SOFTWARE CODEC SOURCE CODE. * cannam@86: * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS * cannam@86: * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE * cannam@86: * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING. * cannam@86: * * cannam@86: * THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2009 * cannam@86: * by the Xiph.Org Foundation http://www.xiph.org/ * cannam@86: * * cannam@86: ******************************************************************** cannam@86: cannam@86: function: basic shared codebook operations cannam@86: last mod: $Id: sharedbook.c 17030 2010-03-25 06:52:55Z xiphmont $ cannam@86: cannam@86: ********************************************************************/ cannam@86: cannam@86: #include cannam@86: #include cannam@86: #include cannam@86: #include cannam@86: #include "os.h" cannam@86: #include "misc.h" cannam@86: #include "vorbis/codec.h" cannam@86: #include "codebook.h" cannam@86: #include "scales.h" cannam@86: cannam@86: /**** pack/unpack helpers ******************************************/ cannam@86: int _ilog(unsigned int v){ cannam@86: int ret=0; cannam@86: while(v){ cannam@86: ret++; cannam@86: v>>=1; cannam@86: } cannam@86: return(ret); cannam@86: } cannam@86: cannam@86: /* 32 bit float (not IEEE; nonnormalized mantissa + cannam@86: biased exponent) : neeeeeee eeemmmmm mmmmmmmm mmmmmmmm cannam@86: Why not IEEE? It's just not that important here. */ cannam@86: cannam@86: #define VQ_FEXP 10 cannam@86: #define VQ_FMAN 21 cannam@86: #define VQ_FEXP_BIAS 768 /* bias toward values smaller than 1. */ cannam@86: cannam@86: /* doesn't currently guard under/overflow */ cannam@86: long _float32_pack(float val){ cannam@86: int sign=0; cannam@86: long exp; cannam@86: long mant; cannam@86: if(val<0){ cannam@86: sign=0x80000000; cannam@86: val= -val; cannam@86: } cannam@86: exp= floor(log(val)/log(2.f)+.001); //+epsilon cannam@86: mant=rint(ldexp(val,(VQ_FMAN-1)-exp)); cannam@86: exp=(exp+VQ_FEXP_BIAS)<>VQ_FMAN; cannam@86: if(sign)mant= -mant; cannam@86: return(ldexp(mant,exp-(VQ_FMAN-1)-VQ_FEXP_BIAS)); cannam@86: } cannam@86: cannam@86: /* given a list of word lengths, generate a list of codewords. Works cannam@86: for length ordered or unordered, always assigns the lowest valued cannam@86: codewords first. Extended to handle unused entries (length 0) */ cannam@86: ogg_uint32_t *_make_words(long *l,long n,long sparsecount){ cannam@86: long i,j,count=0; cannam@86: ogg_uint32_t marker[33]; cannam@86: ogg_uint32_t *r=_ogg_malloc((sparsecount?sparsecount:n)*sizeof(*r)); cannam@86: memset(marker,0,sizeof(marker)); cannam@86: cannam@86: for(i=0;i0){ cannam@86: ogg_uint32_t entry=marker[length]; cannam@86: cannam@86: /* when we claim a node for an entry, we also claim the nodes cannam@86: below it (pruning off the imagined tree that may have dangled cannam@86: from it) as well as blocking the use of any nodes directly cannam@86: above for leaves */ cannam@86: cannam@86: /* update ourself */ cannam@86: if(length<32 && (entry>>length)){ cannam@86: /* error condition; the lengths must specify an overpopulated tree */ cannam@86: _ogg_free(r); cannam@86: return(NULL); cannam@86: } cannam@86: r[count++]=entry; cannam@86: cannam@86: /* Look to see if the next shorter marker points to the node cannam@86: above. if so, update it and repeat. */ cannam@86: { cannam@86: for(j=length;j>0;j--){ cannam@86: cannam@86: if(marker[j]&1){ cannam@86: /* have to jump branches */ cannam@86: if(j==1) cannam@86: marker[1]++; cannam@86: else cannam@86: marker[j]=marker[j-1]<<1; cannam@86: break; /* invariant says next upper marker would already cannam@86: have been moved if it was on the same path */ cannam@86: } cannam@86: marker[j]++; cannam@86: } cannam@86: } cannam@86: cannam@86: /* prune the tree; the implicit invariant says all the longer cannam@86: markers were dangling from our just-taken node. Dangle them cannam@86: from our *new* node. */ cannam@86: for(j=length+1;j<33;j++) cannam@86: if((marker[j]>>1) == entry){ cannam@86: entry=marker[j]; cannam@86: marker[j]=marker[j-1]<<1; cannam@86: }else cannam@86: break; cannam@86: }else cannam@86: if(sparsecount==0)count++; cannam@86: } cannam@86: cannam@86: /* sanity check the huffman tree; an underpopulated tree must be cannam@86: rejected. The only exception is the one-node pseudo-nil tree, cannam@86: which appears to be underpopulated because the tree doesn't cannam@86: really exist; there's only one possible 'codeword' or zero bits, cannam@86: but the above tree-gen code doesn't mark that. */ cannam@86: if(sparsecount != 1){ cannam@86: for(i=1;i<33;i++) cannam@86: if(marker[i] & (0xffffffffUL>>(32-i))){ cannam@86: _ogg_free(r); cannam@86: return(NULL); cannam@86: } cannam@86: } cannam@86: cannam@86: /* bitreverse the words because our bitwise packer/unpacker is LSb cannam@86: endian */ cannam@86: for(i=0,count=0;i>j)&1; cannam@86: } cannam@86: cannam@86: if(sparsecount){ cannam@86: if(l[i]) cannam@86: r[count++]=temp; cannam@86: }else cannam@86: r[count++]=temp; cannam@86: } cannam@86: cannam@86: return(r); cannam@86: } cannam@86: cannam@86: /* there might be a straightforward one-line way to do the below cannam@86: that's portable and totally safe against roundoff, but I haven't cannam@86: thought of it. Therefore, we opt on the side of caution */ cannam@86: long _book_maptype1_quantvals(const static_codebook *b){ cannam@86: long vals=floor(pow((float)b->entries,1.f/b->dim)); cannam@86: cannam@86: /* the above *should* be reliable, but we'll not assume that FP is cannam@86: ever reliable when bitstream sync is at stake; verify via integer cannam@86: means that vals really is the greatest value of dim for which cannam@86: vals^b->bim <= b->entries */ cannam@86: /* treat the above as an initial guess */ cannam@86: while(1){ cannam@86: long acc=1; cannam@86: long acc1=1; cannam@86: int i; cannam@86: for(i=0;idim;i++){ cannam@86: acc*=vals; cannam@86: acc1*=vals+1; cannam@86: } cannam@86: if(acc<=b->entries && acc1>b->entries){ cannam@86: return(vals); cannam@86: }else{ cannam@86: if(acc>b->entries){ cannam@86: vals--; cannam@86: }else{ cannam@86: vals++; cannam@86: } cannam@86: } cannam@86: } cannam@86: } cannam@86: cannam@86: /* unpack the quantized list of values for encode/decode ***********/ cannam@86: /* we need to deal with two map types: in map type 1, the values are cannam@86: generated algorithmically (each column of the vector counts through cannam@86: the values in the quant vector). in map type 2, all the values came cannam@86: in in an explicit list. Both value lists must be unpacked */ cannam@86: float *_book_unquantize(const static_codebook *b,int n,int *sparsemap){ cannam@86: long j,k,count=0; cannam@86: if(b->maptype==1 || b->maptype==2){ cannam@86: int quantvals; cannam@86: float mindel=_float32_unpack(b->q_min); cannam@86: float delta=_float32_unpack(b->q_delta); cannam@86: float *r=_ogg_calloc(n*b->dim,sizeof(*r)); cannam@86: cannam@86: /* maptype 1 and 2 both use a quantized value vector, but cannam@86: different sizes */ cannam@86: switch(b->maptype){ cannam@86: case 1: cannam@86: /* most of the time, entries%dimensions == 0, but we need to be cannam@86: well defined. We define that the possible vales at each cannam@86: scalar is values == entries/dim. If entries%dim != 0, we'll cannam@86: have 'too few' values (values*dimentries;j++){ cannam@86: if((sparsemap && b->lengthlist[j]) || !sparsemap){ cannam@86: float last=0.f; cannam@86: int indexdiv=1; cannam@86: for(k=0;kdim;k++){ cannam@86: int index= (j/indexdiv)%quantvals; cannam@86: float val=b->quantlist[index]; cannam@86: val=fabs(val)*delta+mindel+last; cannam@86: if(b->q_sequencep)last=val; cannam@86: if(sparsemap) cannam@86: r[sparsemap[count]*b->dim+k]=val; cannam@86: else cannam@86: r[count*b->dim+k]=val; cannam@86: indexdiv*=quantvals; cannam@86: } cannam@86: count++; cannam@86: } cannam@86: cannam@86: } cannam@86: break; cannam@86: case 2: cannam@86: for(j=0;jentries;j++){ cannam@86: if((sparsemap && b->lengthlist[j]) || !sparsemap){ cannam@86: float last=0.f; cannam@86: cannam@86: for(k=0;kdim;k++){ cannam@86: float val=b->quantlist[j*b->dim+k]; cannam@86: val=fabs(val)*delta+mindel+last; cannam@86: if(b->q_sequencep)last=val; cannam@86: if(sparsemap) cannam@86: r[sparsemap[count]*b->dim+k]=val; cannam@86: else cannam@86: r[count*b->dim+k]=val; cannam@86: } cannam@86: count++; cannam@86: } cannam@86: } cannam@86: break; cannam@86: } cannam@86: cannam@86: return(r); cannam@86: } cannam@86: return(NULL); cannam@86: } cannam@86: cannam@86: void vorbis_staticbook_destroy(static_codebook *b){ cannam@86: if(b->allocedp){ cannam@86: if(b->quantlist)_ogg_free(b->quantlist); cannam@86: if(b->lengthlist)_ogg_free(b->lengthlist); cannam@86: memset(b,0,sizeof(*b)); cannam@86: _ogg_free(b); cannam@86: } /* otherwise, it is in static memory */ cannam@86: } cannam@86: cannam@86: void vorbis_book_clear(codebook *b){ cannam@86: /* static book is not cleared; we're likely called on the lookup and cannam@86: the static codebook belongs to the info struct */ cannam@86: if(b->valuelist)_ogg_free(b->valuelist); cannam@86: if(b->codelist)_ogg_free(b->codelist); cannam@86: cannam@86: if(b->dec_index)_ogg_free(b->dec_index); cannam@86: if(b->dec_codelengths)_ogg_free(b->dec_codelengths); cannam@86: if(b->dec_firsttable)_ogg_free(b->dec_firsttable); cannam@86: cannam@86: memset(b,0,sizeof(*b)); cannam@86: } cannam@86: cannam@86: int vorbis_book_init_encode(codebook *c,const static_codebook *s){ cannam@86: cannam@86: memset(c,0,sizeof(*c)); cannam@86: c->c=s; cannam@86: c->entries=s->entries; cannam@86: c->used_entries=s->entries; cannam@86: c->dim=s->dim; cannam@86: c->codelist=_make_words(s->lengthlist,s->entries,0); cannam@86: //c->valuelist=_book_unquantize(s,s->entries,NULL); cannam@86: c->quantvals=_book_maptype1_quantvals(s); cannam@86: c->minval=(int)rint(_float32_unpack(s->q_min)); cannam@86: c->delta=(int)rint(_float32_unpack(s->q_delta)); cannam@86: cannam@86: return(0); cannam@86: } cannam@86: cannam@86: static ogg_uint32_t bitreverse(ogg_uint32_t x){ cannam@86: x= ((x>>16)&0x0000ffffUL) | ((x<<16)&0xffff0000UL); cannam@86: x= ((x>> 8)&0x00ff00ffUL) | ((x<< 8)&0xff00ff00UL); cannam@86: x= ((x>> 4)&0x0f0f0f0fUL) | ((x<< 4)&0xf0f0f0f0UL); cannam@86: x= ((x>> 2)&0x33333333UL) | ((x<< 2)&0xccccccccUL); cannam@86: return((x>> 1)&0x55555555UL) | ((x<< 1)&0xaaaaaaaaUL); cannam@86: } cannam@86: cannam@86: static int sort32a(const void *a,const void *b){ cannam@86: return ( **(ogg_uint32_t **)a>**(ogg_uint32_t **)b)- cannam@86: ( **(ogg_uint32_t **)a<**(ogg_uint32_t **)b); cannam@86: } cannam@86: cannam@86: /* decode codebook arrangement is more heavily optimized than encode */ cannam@86: int vorbis_book_init_decode(codebook *c,const static_codebook *s){ cannam@86: int i,j,n=0,tabn; cannam@86: int *sortindex; cannam@86: memset(c,0,sizeof(*c)); cannam@86: cannam@86: /* count actually used entries */ cannam@86: for(i=0;ientries;i++) cannam@86: if(s->lengthlist[i]>0) cannam@86: n++; cannam@86: cannam@86: c->entries=s->entries; cannam@86: c->used_entries=n; cannam@86: c->dim=s->dim; cannam@86: cannam@86: if(n>0){ cannam@86: cannam@86: /* two different remappings go on here. cannam@86: cannam@86: First, we collapse the likely sparse codebook down only to cannam@86: actually represented values/words. This collapsing needs to be cannam@86: indexed as map-valueless books are used to encode original entry cannam@86: positions as integers. cannam@86: cannam@86: Second, we reorder all vectors, including the entry index above, cannam@86: by sorted bitreversed codeword to allow treeless decode. */ cannam@86: cannam@86: /* perform sort */ cannam@86: ogg_uint32_t *codes=_make_words(s->lengthlist,s->entries,c->used_entries); cannam@86: ogg_uint32_t **codep=alloca(sizeof(*codep)*n); cannam@86: cannam@86: if(codes==NULL)goto err_out; cannam@86: cannam@86: for(i=0;icodelist=_ogg_malloc(n*sizeof(*c->codelist)); cannam@86: /* the index is a reverse index */ cannam@86: for(i=0;icodelist[sortindex[i]]=codes[i]; cannam@86: _ogg_free(codes); cannam@86: cannam@86: cannam@86: c->valuelist=_book_unquantize(s,n,sortindex); cannam@86: c->dec_index=_ogg_malloc(n*sizeof(*c->dec_index)); cannam@86: cannam@86: for(n=0,i=0;ientries;i++) cannam@86: if(s->lengthlist[i]>0) cannam@86: c->dec_index[sortindex[n++]]=i; cannam@86: cannam@86: c->dec_codelengths=_ogg_malloc(n*sizeof(*c->dec_codelengths)); cannam@86: for(n=0,i=0;ientries;i++) cannam@86: if(s->lengthlist[i]>0) cannam@86: c->dec_codelengths[sortindex[n++]]=s->lengthlist[i]; cannam@86: cannam@86: c->dec_firsttablen=_ilog(c->used_entries)-4; /* this is magic */ cannam@86: if(c->dec_firsttablen<5)c->dec_firsttablen=5; cannam@86: if(c->dec_firsttablen>8)c->dec_firsttablen=8; cannam@86: cannam@86: tabn=1<dec_firsttablen; cannam@86: c->dec_firsttable=_ogg_calloc(tabn,sizeof(*c->dec_firsttable)); cannam@86: c->dec_maxlength=0; cannam@86: cannam@86: for(i=0;idec_maxlengthdec_codelengths[i]) cannam@86: c->dec_maxlength=c->dec_codelengths[i]; cannam@86: if(c->dec_codelengths[i]<=c->dec_firsttablen){ cannam@86: ogg_uint32_t orig=bitreverse(c->codelist[i]); cannam@86: for(j=0;j<(1<<(c->dec_firsttablen-c->dec_codelengths[i]));j++) cannam@86: c->dec_firsttable[orig|(j<dec_codelengths[i])]=i+1; cannam@86: } cannam@86: } cannam@86: cannam@86: /* now fill in 'unused' entries in the firsttable with hi/lo search cannam@86: hints for the non-direct-hits */ cannam@86: { cannam@86: ogg_uint32_t mask=0xfffffffeUL<<(31-c->dec_firsttablen); cannam@86: long lo=0,hi=0; cannam@86: cannam@86: for(i=0;idec_firsttablen); cannam@86: if(c->dec_firsttable[bitreverse(word)]==0){ cannam@86: while((lo+1)codelist[lo+1]<=word)lo++; cannam@86: while( hi=(c->codelist[hi]&mask))hi++; cannam@86: cannam@86: /* we only actually have 15 bits per hint to play with here. cannam@86: In order to overflow gracefully (nothing breaks, efficiency cannam@86: just drops), encode as the difference from the extremes. */ cannam@86: { cannam@86: unsigned long loval=lo; cannam@86: unsigned long hival=n-hi; cannam@86: cannam@86: if(loval>0x7fff)loval=0x7fff; cannam@86: if(hival>0x7fff)hival=0x7fff; cannam@86: c->dec_firsttable[bitreverse(word)]= cannam@86: 0x80000000UL | (loval<<15) | hival; cannam@86: } cannam@86: } cannam@86: } cannam@86: } cannam@86: } cannam@86: cannam@86: return(0); cannam@86: err_out: cannam@86: vorbis_book_clear(c); cannam@86: return(-1); cannam@86: } cannam@86: cannam@86: long vorbis_book_codeword(codebook *book,int entry){ cannam@86: if(book->c) /* only use with encode; decode optimizations are cannam@86: allowed to break this */ cannam@86: return book->codelist[entry]; cannam@86: return -1; cannam@86: } cannam@86: cannam@86: long vorbis_book_codelen(codebook *book,int entry){ cannam@86: if(book->c) /* only use with encode; decode optimizations are cannam@86: allowed to break this */ cannam@86: return book->c->lengthlist[entry]; cannam@86: return -1; cannam@86: } cannam@86: cannam@86: #ifdef _V_SELFTEST cannam@86: cannam@86: /* Unit tests of the dequantizer; this stuff will be OK cannam@86: cross-platform, I simply want to be sure that special mapping cases cannam@86: actually work properly; a bug could go unnoticed for a while */ cannam@86: cannam@86: #include cannam@86: cannam@86: /* cases: cannam@86: cannam@86: no mapping cannam@86: full, explicit mapping cannam@86: algorithmic mapping cannam@86: cannam@86: nonsequential cannam@86: sequential cannam@86: */ cannam@86: cannam@86: static long full_quantlist1[]={0,1,2,3, 4,5,6,7, 8,3,6,1}; cannam@86: static long partial_quantlist1[]={0,7,2}; cannam@86: cannam@86: /* no mapping */ cannam@86: static_codebook test1={ cannam@86: 4,16, cannam@86: NULL, cannam@86: 0, cannam@86: 0,0,0,0, cannam@86: NULL, cannam@86: 0 cannam@86: }; cannam@86: static float *test1_result=NULL; cannam@86: cannam@86: /* linear, full mapping, nonsequential */ cannam@86: static_codebook test2={ cannam@86: 4,3, cannam@86: NULL, cannam@86: 2, cannam@86: -533200896,1611661312,4,0, cannam@86: full_quantlist1, cannam@86: 0 cannam@86: }; cannam@86: static float test2_result[]={-3,-2,-1,0, 1,2,3,4, 5,0,3,-2}; cannam@86: cannam@86: /* linear, full mapping, sequential */ cannam@86: static_codebook test3={ cannam@86: 4,3, cannam@86: NULL, cannam@86: 2, cannam@86: -533200896,1611661312,4,1, cannam@86: full_quantlist1, cannam@86: 0 cannam@86: }; cannam@86: static float test3_result[]={-3,-5,-6,-6, 1,3,6,10, 5,5,8,6}; cannam@86: cannam@86: /* linear, algorithmic mapping, nonsequential */ cannam@86: static_codebook test4={ cannam@86: 3,27, cannam@86: NULL, cannam@86: 1, cannam@86: -533200896,1611661312,4,0, cannam@86: partial_quantlist1, cannam@86: 0 cannam@86: }; cannam@86: static float test4_result[]={-3,-3,-3, 4,-3,-3, -1,-3,-3, cannam@86: -3, 4,-3, 4, 4,-3, -1, 4,-3, cannam@86: -3,-1,-3, 4,-1,-3, -1,-1,-3, cannam@86: -3,-3, 4, 4,-3, 4, -1,-3, 4, cannam@86: -3, 4, 4, 4, 4, 4, -1, 4, 4, cannam@86: -3,-1, 4, 4,-1, 4, -1,-1, 4, cannam@86: -3,-3,-1, 4,-3,-1, -1,-3,-1, cannam@86: -3, 4,-1, 4, 4,-1, -1, 4,-1, cannam@86: -3,-1,-1, 4,-1,-1, -1,-1,-1}; cannam@86: cannam@86: /* linear, algorithmic mapping, sequential */ cannam@86: static_codebook test5={ cannam@86: 3,27, cannam@86: NULL, cannam@86: 1, cannam@86: -533200896,1611661312,4,1, cannam@86: partial_quantlist1, cannam@86: 0 cannam@86: }; cannam@86: static float test5_result[]={-3,-6,-9, 4, 1,-2, -1,-4,-7, cannam@86: -3, 1,-2, 4, 8, 5, -1, 3, 0, cannam@86: -3,-4,-7, 4, 3, 0, -1,-2,-5, cannam@86: -3,-6,-2, 4, 1, 5, -1,-4, 0, cannam@86: -3, 1, 5, 4, 8,12, -1, 3, 7, cannam@86: -3,-4, 0, 4, 3, 7, -1,-2, 2, cannam@86: -3,-6,-7, 4, 1, 0, -1,-4,-5, cannam@86: -3, 1, 0, 4, 8, 7, -1, 3, 2, cannam@86: -3,-4,-5, 4, 3, 2, -1,-2,-3}; cannam@86: cannam@86: void run_test(static_codebook *b,float *comp){ cannam@86: float *out=_book_unquantize(b,b->entries,NULL); cannam@86: int i; cannam@86: cannam@86: if(comp){ cannam@86: if(!out){ cannam@86: fprintf(stderr,"_book_unquantize incorrectly returned NULL\n"); cannam@86: exit(1); cannam@86: } cannam@86: cannam@86: for(i=0;ientries*b->dim;i++) cannam@86: if(fabs(out[i]-comp[i])>.0001){ cannam@86: fprintf(stderr,"disagreement in unquantized and reference data:\n" cannam@86: "position %d, %g != %g\n",i,out[i],comp[i]); cannam@86: exit(1); cannam@86: } cannam@86: cannam@86: }else{ cannam@86: if(out){ cannam@86: fprintf(stderr,"_book_unquantize returned a value array: \n" cannam@86: " correct result should have been NULL\n"); cannam@86: exit(1); cannam@86: } cannam@86: } cannam@86: } cannam@86: cannam@86: int main(){ cannam@86: /* run the nine dequant tests, and compare to the hand-rolled results */ cannam@86: fprintf(stderr,"Dequant test 1... "); cannam@86: run_test(&test1,test1_result); cannam@86: fprintf(stderr,"OK\nDequant test 2... "); cannam@86: run_test(&test2,test2_result); cannam@86: fprintf(stderr,"OK\nDequant test 3... "); cannam@86: run_test(&test3,test3_result); cannam@86: fprintf(stderr,"OK\nDequant test 4... "); cannam@86: run_test(&test4,test4_result); cannam@86: fprintf(stderr,"OK\nDequant test 5... "); cannam@86: run_test(&test5,test5_result); cannam@86: fprintf(stderr,"OK\n\n"); cannam@86: cannam@86: return(0); cannam@86: } cannam@86: cannam@86: #endif