1 /********************************************************************
3 * THIS FILE IS PART OF THE OggVorbis SOFTWARE CODEC SOURCE CODE. *
4 * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS *
5 * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE *
6 * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING. *
8 * THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2002 *
9 * by the XIPHOPHORUS Company http://www.xiph.org/ *
11 ********************************************************************
13 function: basic shared codebook operations
14 last mod: $Id: sharedbook.c,v 1.29 2002/10/11 07:44:28 xiphmont Exp $
16 ********************************************************************/
24 #include "vorbis/codec.h"
28 /**** pack/unpack helpers ******************************************/
29 int _ilog(unsigned int v){
38 /* 32 bit float (not IEEE; nonnormalized mantissa +
39 biased exponent) : neeeeeee eeemmmmm mmmmmmmm mmmmmmmm
40 Why not IEEE? It's just not that important here. */
44 #define VQ_FEXP_BIAS 768 /* bias toward values smaller than 1. */
46 /* doesn't currently guard under/overflow */
47 long _float32_pack(float val){
55 exp= floor(log(val)/log(2.f));
56 mant=rint(ldexp(val,(VQ_FMAN-1)-exp));
57 exp=(exp+VQ_FEXP_BIAS)<<VQ_FMAN;
59 return(sign|exp|mant);
62 float _float32_unpack(long val){
63 double mant=val&0x1fffff;
64 int sign=val&0x80000000;
65 long exp =(val&0x7fe00000L)>>VQ_FMAN;
67 return(ldexp(mant,exp-(VQ_FMAN-1)-VQ_FEXP_BIAS));
70 /* given a list of word lengths, generate a list of codewords. Works
71 for length ordered or unordered, always assigns the lowest valued
72 codewords first. Extended to handle unused entries (length 0) */
73 ogg_uint32_t *_make_words(long *l,long n,long sparsecount){
75 ogg_uint32_t marker[33];
76 ogg_uint32_t *r=_ogg_malloc((sparsecount?sparsecount:n)*sizeof(*r));
77 memset(marker,0,sizeof(marker));
82 ogg_uint32_t entry=marker[length];
84 /* when we claim a node for an entry, we also claim the nodes
85 below it (pruning off the imagined tree that may have dangled
86 from it) as well as blocking the use of any nodes directly
90 if(length<32 && (entry>>length)){
91 /* error condition; the lengths must specify an overpopulated tree */
97 /* Look to see if the next shorter marker points to the node
98 above. if so, update it and repeat. */
100 for(j=length;j>0;j--){
103 /* have to jump branches */
107 marker[j]=marker[j-1]<<1;
108 break; /* invariant says next upper marker would already
109 have been moved if it was on the same path */
115 /* prune the tree; the implicit invariant says all the longer
116 markers were dangling from our just-taken node. Dangle them
117 from our *new* node. */
118 for(j=length+1;j<33;j++)
119 if((marker[j]>>1) == entry){
121 marker[j]=marker[j-1]<<1;
125 if(sparsecount==0)count++;
128 /* bitreverse the words because our bitwise packer/unpacker is LSb
130 for(i=0,count=0;i<n;i++){
134 temp|=(r[count]>>j)&1;
147 /* there might be a straightforward one-line way to do the below
148 that's portable and totally safe against roundoff, but I haven't
149 thought of it. Therefore, we opt on the side of caution */
150 long _book_maptype1_quantvals(const static_codebook *b){
151 long vals=floor(pow((float)b->entries,1.f/b->dim));
153 /* the above *should* be reliable, but we'll not assume that FP is
154 ever reliable when bitstream sync is at stake; verify via integer
155 means that vals really is the greatest value of dim for which
156 vals^b->bim <= b->entries */
157 /* treat the above as an initial guess */
162 for(i=0;i<b->dim;i++){
166 if(acc<=b->entries && acc1>b->entries){
178 /* unpack the quantized list of values for encode/decode ***********/
179 /* we need to deal with two map types: in map type 1, the values are
180 generated algorithmically (each column of the vector counts through
181 the values in the quant vector). in map type 2, all the values came
182 in in an explicit list. Both value lists must be unpacked */
183 float *_book_unquantize(const static_codebook *b,int n,int *sparsemap){
185 if(b->maptype==1 || b->maptype==2){
187 float mindel=_float32_unpack(b->q_min);
188 float delta=_float32_unpack(b->q_delta);
189 float *r=_ogg_calloc(n*b->dim,sizeof(*r));
191 /* maptype 1 and 2 both use a quantized value vector, but
195 /* most of the time, entries%dimensions == 0, but we need to be
196 well defined. We define that the possible vales at each
197 scalar is values == entries/dim. If entries%dim != 0, we'll
198 have 'too few' values (values*dim<entries), which means that
199 we'll have 'left over' entries; left over entries use zeroed
200 values (and are wasted). So don't generate codebooks like
202 quantvals=_book_maptype1_quantvals(b);
203 for(j=0;j<b->entries;j++){
204 if((sparsemap && b->lengthlist[j]) || !sparsemap){
207 for(k=0;k<b->dim;k++){
208 int index= (j/indexdiv)%quantvals;
209 float val=b->quantlist[index];
210 val=fabs(val)*delta+mindel+last;
211 if(b->q_sequencep)last=val;
213 r[sparsemap[count]*b->dim+k]=val;
215 r[count*b->dim+k]=val;
224 for(j=0;j<b->entries;j++){
225 if((sparsemap && b->lengthlist[j]) || !sparsemap){
228 for(k=0;k<b->dim;k++){
229 float val=b->quantlist[j*b->dim+k];
230 val=fabs(val)*delta+mindel+last;
231 if(b->q_sequencep)last=val;
233 r[sparsemap[count]*b->dim+k]=val;
235 r[count*b->dim+k]=val;
248 void vorbis_staticbook_clear(static_codebook *b){
250 if(b->quantlist)_ogg_free(b->quantlist);
251 if(b->lengthlist)_ogg_free(b->lengthlist);
253 _ogg_free(b->nearest_tree->ptr0);
254 _ogg_free(b->nearest_tree->ptr1);
255 _ogg_free(b->nearest_tree->p);
256 _ogg_free(b->nearest_tree->q);
257 memset(b->nearest_tree,0,sizeof(*b->nearest_tree));
258 _ogg_free(b->nearest_tree);
261 _ogg_free(b->thresh_tree->quantthresh);
262 _ogg_free(b->thresh_tree->quantmap);
263 memset(b->thresh_tree,0,sizeof(*b->thresh_tree));
264 _ogg_free(b->thresh_tree);
267 memset(b,0,sizeof(*b));
271 void vorbis_staticbook_destroy(static_codebook *b){
273 vorbis_staticbook_clear(b);
278 void vorbis_book_clear(codebook *b){
279 /* static book is not cleared; we're likely called on the lookup and
280 the static codebook belongs to the info struct */
281 if(b->valuelist)_ogg_free(b->valuelist);
282 if(b->codelist)_ogg_free(b->codelist);
284 if(b->dec_index)_ogg_free(b->dec_index);
285 if(b->dec_codelengths)_ogg_free(b->dec_codelengths);
286 if(b->dec_firsttable)_ogg_free(b->dec_firsttable);
288 memset(b,0,sizeof(*b));
291 int vorbis_book_init_encode(codebook *c,const static_codebook *s){
293 memset(c,0,sizeof(*c));
295 c->entries=s->entries;
296 c->used_entries=s->entries;
298 c->codelist=_make_words(s->lengthlist,s->entries,0);
299 c->valuelist=_book_unquantize(s,s->entries,NULL);
304 static ogg_uint32_t bitreverse(ogg_uint32_t x){
305 x= ((x>>16)&0x0000ffffUL) | ((x<<16)&0xffff0000UL);
306 x= ((x>> 8)&0x00ff00ffUL) | ((x<< 8)&0xff00ff00UL);
307 x= ((x>> 4)&0x0f0f0f0fUL) | ((x<< 4)&0xf0f0f0f0UL);
308 x= ((x>> 2)&0x33333333UL) | ((x<< 2)&0xccccccccUL);
309 return((x>> 1)&0x55555555UL) | ((x<< 1)&0xaaaaaaaaUL);
312 static int sort32a(const void *a,const void *b){
313 return ( **(ogg_uint32_t **)a>**(ogg_uint32_t **)b)-
314 ( **(ogg_uint32_t **)a<**(ogg_uint32_t **)b);
317 /* decode codebook arrangement is more heavily optimized than encode */
318 int vorbis_book_init_decode(codebook *c,const static_codebook *s){
321 memset(c,0,sizeof(*c));
323 /* count actually used entries */
324 for(i=0;i<s->entries;i++)
325 if(s->lengthlist[i]>0)
328 c->entries=s->entries;
332 /* two different remappings go on here.
334 First, we collapse the likely sparse codebook down only to
335 actually represented values/words. This collapsing needs to be
336 indexed as map-valueless books are used to encode original entry
337 positions as integers.
339 Second, we reorder all vectors, including the entry index above,
340 by sorted bitreversed codeword to allow treeless decode. */
344 ogg_uint32_t *codes=_make_words(s->lengthlist,s->entries,c->used_entries);
345 ogg_uint32_t **codep=alloca(sizeof(*codep)*n);
347 if(codes==NULL)goto err_out;
350 codes[i]=bitreverse(codes[i]);
354 qsort(codep,n,sizeof(*codep),sort32a);
356 sortindex=alloca(n*sizeof(*sortindex));
357 c->codelist=_ogg_malloc(n*sizeof(*c->codelist));
358 /* the index is a reverse index */
360 int position=codep[i]-codes;
361 sortindex[position]=i;
365 c->codelist[sortindex[i]]=codes[i];
369 c->valuelist=_book_unquantize(s,n,sortindex);
370 c->dec_index=_ogg_malloc(n*sizeof(*c->dec_index));
372 for(n=0,i=0;i<s->entries;i++)
373 if(s->lengthlist[i]>0)
374 c->dec_index[sortindex[n++]]=i;
376 c->dec_codelengths=_ogg_malloc(n*sizeof(*c->dec_codelengths));
377 for(n=0,i=0;i<s->entries;i++)
378 if(s->lengthlist[i]>0)
379 c->dec_codelengths[sortindex[n++]]=s->lengthlist[i];
381 c->dec_firsttablen=_ilog(c->used_entries)-4; /* this is magic */
382 if(c->dec_firsttablen<5)c->dec_firsttablen=5;
383 if(c->dec_firsttablen>8)c->dec_firsttablen=8;
385 tabn=1<<c->dec_firsttablen;
386 c->dec_firsttable=_ogg_calloc(tabn,sizeof(*c->dec_firsttable));
390 if(c->dec_maxlength<c->dec_codelengths[i])
391 c->dec_maxlength=c->dec_codelengths[i];
392 if(c->dec_codelengths[i]<=c->dec_firsttablen){
393 ogg_uint32_t orig=bitreverse(c->codelist[i]);
394 for(j=0;j<(1<<(c->dec_firsttablen-c->dec_codelengths[i]));j++)
395 c->dec_firsttable[orig|(j<<c->dec_codelengths[i])]=i+1;
399 /* now fill in 'unused' entries in the firsttable with hi/lo search
400 hints for the non-direct-hits */
402 ogg_uint32_t mask=0xfffffffeUL<<(31-c->dec_firsttablen);
406 ogg_uint32_t word=i<<(32-c->dec_firsttablen);
407 if(c->dec_firsttable[bitreverse(word)]==0){
408 while((lo+1)<n && c->codelist[lo+1]<=word)lo++;
409 while( hi<n && word>=(c->codelist[hi]&mask))hi++;
411 /* we only actually have 15 bits per hint to play with here.
412 In order to overflow gracefully (nothing breaks, efficiency
413 just drops), encode as the difference from the extremes. */
415 unsigned long loval=lo;
416 unsigned long hival=n-hi;
418 if(loval>0x7fff)loval=0x7fff;
419 if(hival>0x7fff)hival=0x7fff;
420 c->dec_firsttable[bitreverse(word)]=
421 0x80000000UL | (loval<<15) | hival;
430 vorbis_book_clear(c);
434 static float _dist(int el,float *ref, float *b,int step){
438 float val=(ref[i]-b[i*step]);
444 int _best(codebook *book, float *a, int step){
445 encode_aux_threshmatch *tt=book->c->thresh_tree;
448 encode_aux_nearestmatch *nt=book->c->nearest_tree;
449 encode_aux_pigeonhole *pt=book->c->pigeon_tree;
456 /* do we have a threshhold encode hint? */
459 /* find the quant val of each scalar */
460 for(k=0,o=step*(dim-1);k<dim;k++,o-=step){
463 if(a[o]<tt->quantthresh[i]){
466 if(a[o]>=tt->quantthresh[i-1])
471 for(i++;i<tt->threshvals-1;i++)
472 if(a[o]<tt->quantthresh[i])break;
476 index=(index*tt->quantvals)+tt->quantmap[i];
478 /* regular lattices are easy :-) */
479 if(book->c->lengthlist[index]>0) /* is this unused? If so, we'll
480 use a decision tree after all
486 /* do we have a pigeonhole encode hint? */
488 const static_codebook *c=book->c;
493 /* dealing with sequentialness is a pain in the ass */
498 for(k=0,o=0;k<dim;k++,o+=step){
499 pv=(int)((a[o]-qlast-pt->min)/pt->del);
500 if(pv<0 || pv>=pt->mapentries)break;
501 entry+=pt->pigeonmap[pv]*mul;
503 qlast+=pv*pt->del+pt->min;
506 for(k=0,o=step*(dim-1);k<dim;k++,o-=step){
507 int pv=(int)((a[o]-pt->min)/pt->del);
508 if(pv<0 || pv>=pt->mapentries)break;
509 entry=entry*pt->quantvals+pt->pigeonmap[pv];
513 /* must be within the pigeonholable range; if we quant outside (or
514 in an entry that we define no list for), brute force it */
515 if(k==dim && pt->fitlength[entry]){
516 /* search the abbreviated list */
517 long *list=pt->fitlist+pt->fitmap[entry];
518 for(i=0;i<pt->fitlength[entry];i++){
519 float this=_dist(dim,book->valuelist+list[i]*dim,a,step);
520 if(besti==-1 || this<best){
531 /* optimized using the decision tree */
534 float *p=book->valuelist+nt->p[ptr];
535 float *q=book->valuelist+nt->q[ptr];
537 for(k=0,o=0;k<dim;k++,o+=step)
538 c+=(p[k]-q[k])*(a[o]-(p[k]+q[k])*.5);
550 /* brute force it! */
552 const static_codebook *c=book->c;
555 float *e=book->valuelist;
556 for(i=0;i<book->entries;i++){
557 if(c->lengthlist[i]>0){
558 float this=_dist(dim,e,a,step);
559 if(besti==-1 || this<best){
567 /*if(savebest!=-1 && savebest!=besti){
568 fprintf(stderr,"brute force/pigeonhole disagreement:\n"
570 for(i=0;i<dim*step;i+=step)fprintf(stderr,"%g,",a[i]);
572 "pigeonhole (entry %d, err %g):",savebest,saverr);
573 for(i=0;i<dim;i++)fprintf(stderr,"%g,",
574 (book->valuelist+savebest*dim)[i]);
576 "bruteforce (entry %d, err %g):",besti,best);
577 for(i=0;i<dim;i++)fprintf(stderr,"%g,",
578 (book->valuelist+besti*dim)[i]);
579 fprintf(stderr,"\n");
585 long vorbis_book_codeword(codebook *book,int entry){
586 if(book->c) /* only use with encode; decode optimizations are
587 allowed to break this */
588 return book->codelist[entry];
592 long vorbis_book_codelen(codebook *book,int entry){
593 if(book->c) /* only use with encode; decode optimizations are
594 allowed to break this */
595 return book->c->lengthlist[entry];
601 /* Unit tests of the dequantizer; this stuff will be OK
602 cross-platform, I simply want to be sure that special mapping cases
603 actually work properly; a bug could go unnoticed for a while */
610 full, explicit mapping
617 static long full_quantlist1[]={0,1,2,3, 4,5,6,7, 8,3,6,1};
618 static long partial_quantlist1[]={0,7,2};
621 static_codebook test1={
629 static float *test1_result=NULL;
631 /* linear, full mapping, nonsequential */
632 static_codebook test2={
636 -533200896,1611661312,4,0,
640 static float test2_result[]={-3,-2,-1,0, 1,2,3,4, 5,0,3,-2};
642 /* linear, full mapping, sequential */
643 static_codebook test3={
647 -533200896,1611661312,4,1,
651 static float test3_result[]={-3,-5,-6,-6, 1,3,6,10, 5,5,8,6};
653 /* linear, algorithmic mapping, nonsequential */
654 static_codebook test4={
658 -533200896,1611661312,4,0,
662 static float test4_result[]={-3,-3,-3, 4,-3,-3, -1,-3,-3,
663 -3, 4,-3, 4, 4,-3, -1, 4,-3,
664 -3,-1,-3, 4,-1,-3, -1,-1,-3,
665 -3,-3, 4, 4,-3, 4, -1,-3, 4,
666 -3, 4, 4, 4, 4, 4, -1, 4, 4,
667 -3,-1, 4, 4,-1, 4, -1,-1, 4,
668 -3,-3,-1, 4,-3,-1, -1,-3,-1,
669 -3, 4,-1, 4, 4,-1, -1, 4,-1,
670 -3,-1,-1, 4,-1,-1, -1,-1,-1};
672 /* linear, algorithmic mapping, sequential */
673 static_codebook test5={
677 -533200896,1611661312,4,1,
681 static float test5_result[]={-3,-6,-9, 4, 1,-2, -1,-4,-7,
682 -3, 1,-2, 4, 8, 5, -1, 3, 0,
683 -3,-4,-7, 4, 3, 0, -1,-2,-5,
684 -3,-6,-2, 4, 1, 5, -1,-4, 0,
685 -3, 1, 5, 4, 8,12, -1, 3, 7,
686 -3,-4, 0, 4, 3, 7, -1,-2, 2,
687 -3,-6,-7, 4, 1, 0, -1,-4,-5,
688 -3, 1, 0, 4, 8, 7, -1, 3, 2,
689 -3,-4,-5, 4, 3, 2, -1,-2,-3};
691 void run_test(static_codebook *b,float *comp){
692 float *out=_book_unquantize(b,b->entries,NULL);
697 fprintf(stderr,"_book_unquantize incorrectly returned NULL\n");
701 for(i=0;i<b->entries*b->dim;i++)
702 if(fabs(out[i]-comp[i])>.0001){
703 fprintf(stderr,"disagreement in unquantized and reference data:\n"
704 "position %d, %g != %g\n",i,out[i],comp[i]);
710 fprintf(stderr,"_book_unquantize returned a value array: \n"
711 " correct result should have been NULL\n");
718 /* run the nine dequant tests, and compare to the hand-rolled results */
719 fprintf(stderr,"Dequant test 1... ");
720 run_test(&test1,test1_result);
721 fprintf(stderr,"OK\nDequant test 2... ");
722 run_test(&test2,test2_result);
723 fprintf(stderr,"OK\nDequant test 3... ");
724 run_test(&test3,test3_result);
725 fprintf(stderr,"OK\nDequant test 4... ");
726 run_test(&test4,test4_result);
727 fprintf(stderr,"OK\nDequant test 5... ");
728 run_test(&test5,test5_result);
729 fprintf(stderr,"OK\n\n");