1 /********************************************************************
3 * THIS FILE IS PART OF THE OggVorbis SOFTWARE CODEC SOURCE CODE. *
4 * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS *
5 * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE *
6 * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING. *
8 * THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2001 *
9 * by the XIPHOPHORUS Company http://www.xiph.org/ *
11 ********************************************************************
13 function: utility functions for loading .vqh and .vqd files
14 last mod: $Id: bookutil.c,v 1.27 2002/06/28 22:19:56 xiphmont Exp $
16 ********************************************************************/
25 /* A few little utils for reading files */
26 /* read a line. Use global, persistent buffering */
27 static char *linebuffer=NULL;
28 static int lbufsize=0;
29 char *get_line(FILE *in){
31 if(feof(in))return NULL;
37 if(sofar+1>=lbufsize){
40 linebuffer=_ogg_malloc(lbufsize);
43 linebuffer=_ogg_realloc(linebuffer,lbufsize);
50 if(sofar==0)return(NULL);
51 /* fallthrough correct */
53 linebuffer[sofar]='\0';
57 linebuffer[sofar++]=c;
58 linebuffer[sofar]='\0';
64 if(linebuffer[0]=='#'){
72 /* read the next numerical value from the given file */
73 static char *value_line_buff=NULL;
75 int get_line_value(FILE *in,float *value){
78 if(!value_line_buff)return(-1);
80 *value=strtod(value_line_buff, &next);
81 if(next==value_line_buff){
86 while(*value_line_buff>44)value_line_buff++;
87 if(*value_line_buff==44)value_line_buff++;
92 int get_next_value(FILE *in,float *value){
94 if(get_line_value(in,value)){
95 value_line_buff=get_line(in);
96 if(!value_line_buff)return(-1);
103 int get_next_ivalue(FILE *in,long *ivalue){
105 int ret=get_next_value(in,&value);
110 static float sequence_base=0.f;
111 static int v_sofar=0;
112 void reset_next_value(void){
113 value_line_buff=NULL;
118 char *setup_line(FILE *in){
120 value_line_buff=get_line(in);
121 return(value_line_buff);
125 int get_vector(codebook *b,FILE *in,int start, int n,float *a){
127 const static_codebook *c=b->c;
131 if(v_sofar==n || get_line_value(in,a)){
133 if(get_next_value(in,a))
135 for(i=0;i<start;i++){
137 get_line_value(in,a);
141 for(i=1;i<c->dim;i++)
142 if(get_line_value(in,a+i))
146 float temp=a[c->dim-1];
147 for(i=0;i<c->dim;i++)a[i]-=sequence_base;
148 if(c->q_sequencep)sequence_base=temp;
158 /* read lines fromt he beginning until we find one containing the
160 char *find_seek_to(FILE *in,char *s){
163 char *line=get_line(in);
173 /* this reads the format as written by vqbuild/latticebuild; innocent
174 (legal) tweaking of the file that would not affect its valid
175 header-ness will break this routine */
177 codebook *codebook_load(char *filename){
178 codebook *b=_ogg_calloc(1,sizeof(codebook));
179 static_codebook *c=(static_codebook *)(b->c=_ogg_calloc(1,sizeof(static_codebook)));
180 encode_aux_nearestmatch *a=NULL;
181 encode_aux_threshmatch *t=NULL;
182 encode_aux_pigeonhole *p=NULL;
184 FILE *in=fopen(filename,"r");
189 fprintf(stderr,"Couldn't open codebook %s\n",filename);
193 /* find the codebook struct */
194 find_seek_to(in,"static static_codebook ");
196 /* get the major important values */
198 if(sscanf(line,"%ld, %ld,",
199 &(c->dim),&(c->entries))!=2){
200 fprintf(stderr,"1: syntax in %s in line:\t %s",filename,line);
205 if(sscanf(line,"%d, %ld, %ld, %d, %d,",
206 &(c->maptype),&(c->q_min),&(c->q_delta),&(c->q_quant),
207 &(c->q_sequencep))!=5){
208 fprintf(stderr,"1: syntax in %s in line:\t %s",filename,line);
212 /* find the auxiliary encode struct[s] (if any) */
213 if(find_seek_to(in,"static encode_aux_nearestmatch _vq_aux")){
215 c->nearest_tree=a=_ogg_calloc(1,sizeof(encode_aux_nearestmatch));
221 if(sscanf(line,"%ld, %ld",&(a->aux),&(a->alloc))!=2){
222 fprintf(stderr,"2: syntax in %s in line:\t %s",filename,line);
227 find_seek_to(in,"static long _vq_ptr0");
229 a->ptr0=_ogg_malloc(sizeof(long)*a->aux);
230 for(i=0;i<a->aux;i++)
231 if(get_next_ivalue(in,a->ptr0+i)){
232 fprintf(stderr,"out of data while reading codebook %s\n",filename);
237 find_seek_to(in,"static long _vq_ptr1");
239 a->ptr1=_ogg_malloc(sizeof(long)*a->aux);
240 for(i=0;i<a->aux;i++)
241 if(get_next_ivalue(in,a->ptr1+i)){
242 fprintf(stderr,"out of data while reading codebook %s\n",filename);
248 find_seek_to(in,"static long _vq_p_");
250 a->p=_ogg_malloc(sizeof(long)*a->aux);
251 for(i=0;i<a->aux;i++)
252 if(get_next_ivalue(in,a->p+i)){
253 fprintf(stderr,"out of data while reading codebook %s\n",filename);
258 find_seek_to(in,"static long _vq_q_");
260 a->q=_ogg_malloc(sizeof(long)*a->aux);
261 for(i=0;i<a->aux;i++)
262 if(get_next_ivalue(in,a->q+i)){
263 fprintf(stderr,"out of data while reading codebook %s\n",filename);
268 if(find_seek_to(in,"static encode_aux_threshmatch _vq_aux")){
270 c->thresh_tree=t=_ogg_calloc(1,sizeof(encode_aux_threshmatch));
274 if(sscanf(line,"%d",&(t->quantvals))!=1){
275 fprintf(stderr,"3: syntax in %s in line:\t %s",filename,line);
279 if(sscanf(line,"%d",&(t->threshvals))!=1){
280 fprintf(stderr,"4: syntax in %s in line:\t %s",filename,line);
283 /* load quantthresh */
284 find_seek_to(in,"static float _vq_quantthresh_");
286 t->quantthresh=_ogg_malloc(sizeof(float)*t->threshvals);
287 for(i=0;i<t->threshvals-1;i++)
288 if(get_next_value(in,t->quantthresh+i)){
289 fprintf(stderr,"out of data 1 while reading codebook %s\n",filename);
293 find_seek_to(in,"static long _vq_quantmap_");
295 t->quantmap=_ogg_malloc(sizeof(long)*t->threshvals);
296 for(i=0;i<t->threshvals;i++)
297 if(get_next_ivalue(in,t->quantmap+i)){
298 fprintf(stderr,"out of data 2 while reading codebook %s\n",filename);
303 if(find_seek_to(in,"static encode_aux_pigeonhole _vq_aux")){
306 c->pigeon_tree=p=_ogg_calloc(1,sizeof(encode_aux_pigeonhole));
308 if(sscanf(line,"%f, %f, %d, %d",&(p->min),&(p->del),
309 &(p->mapentries),&(p->quantvals))!=4){
310 fprintf(stderr,"5: syntax in %s in line:\t %s",filename,line);
315 if(sscanf(line,"%ld",&(p->fittotal))!=1){
316 fprintf(stderr,"6: syntax in %s in line:\t %s",filename,line);
320 find_seek_to(in,"static long _vq_pigeonmap_");
322 p->pigeonmap=_ogg_malloc(sizeof(long)*p->mapentries);
323 for(i=0;i<p->mapentries;i++)
324 if(get_next_ivalue(in,p->pigeonmap+i)){
325 fprintf(stderr,"out of data (pigeonmap) while reading codebook %s\n",filename);
329 find_seek_to(in,"static long _vq_fitlist_");
331 p->fitlist=_ogg_malloc(sizeof(long)*p->fittotal);
332 for(i=0;i<p->fittotal;i++)
333 if(get_next_ivalue(in,p->fitlist+i)){
334 fprintf(stderr,"out of data (fitlist) while reading codebook %s\n",filename);
338 find_seek_to(in,"static long _vq_fitmap_");
340 for(i=0;i<c->dim;i++)pigeons*=p->quantvals;
341 p->fitmap=_ogg_malloc(sizeof(long)*pigeons);
342 for(i=0;i<pigeons;i++)
343 if(get_next_ivalue(in,p->fitmap+i)){
344 fprintf(stderr,"out of data (fitmap) while reading codebook %s\n",filename);
349 find_seek_to(in,"static long _vq_fitlength_");
351 p->fitlength=_ogg_malloc(sizeof(long)*pigeons);
352 for(i=0;i<pigeons;i++)
353 if(get_next_ivalue(in,p->fitlength+i)){
354 fprintf(stderr,"out of data (fitlength) while reading codebook %s\n",filename);
364 quant_to_read=_book_maptype1_quantvals(c);
367 quant_to_read=c->entries*c->dim;
371 /* load the quantized entries */
372 find_seek_to(in,"static long _vq_quantlist_");
374 c->quantlist=_ogg_malloc(sizeof(long)*quant_to_read);
375 for(i=0;i<quant_to_read;i++)
376 if(get_next_ivalue(in,c->quantlist+i)){
377 fprintf(stderr,"out of data while reading codebook %s\n",filename);
381 /* load the lengthlist */
382 find_seek_to(in,"_lengthlist");
384 c->lengthlist=_ogg_malloc(sizeof(long)*c->entries);
385 for(i=0;i<c->entries;i++)
386 if(get_next_ivalue(in,c->lengthlist+i)){
387 fprintf(stderr,"out of data while reading codebook %s\n",filename);
394 vorbis_book_init_encode(b,c);
399 void spinnit(char *s,int n){
401 static long lasttime=0;
403 struct timeval thistime;
405 gettimeofday(&thistime,NULL);
406 test=thistime.tv_sec*10+thistime.tv_usec/100000;
410 fprintf(stderr,"%s%d ",s,n);
415 fprintf(stderr,"| \r");
418 fprintf(stderr,"/ \r");
421 fprintf(stderr,"- \r");
424 fprintf(stderr,"\\ \r");
431 void build_tree_from_lengths(int vals, long *hist, long *lengths){
433 long *membership=_ogg_malloc(vals*sizeof(long));
434 long *histsave=alloca(vals*sizeof(long));
435 memcpy(histsave,hist,vals*sizeof(long));
437 for(i=0;i<vals;i++)membership[i]=i;
439 /* find codeword lengths */
440 /* much more elegant means exist. Brute force n^2, minimum thought */
442 int first=-1,second=-1;
445 spinnit("building... ",i);
447 /* find the two nodes to join */
449 if(least==-1 || hist[j]<least){
455 if((least==-1 || hist[j]<least) && membership[j]!=first){
457 second=membership[j];
459 if(first==-1 || second==-1){
460 fprintf(stderr,"huffman fault; no free branch\n");
465 least=hist[first]+hist[second];
467 if(membership[j]==first || membership[j]==second){
473 for(i=0;i<vals-1;i++)
474 if(membership[i]!=membership[i+1]){
475 fprintf(stderr,"huffman fault; failed to build single tree\n");
479 /* for sanity check purposes: how many bits would it have taken to
480 encode the training set? */
485 bitsum+=(histsave[i]-1)*lengths[i];
486 samples+=histsave[i]-1;
490 fprintf(stderr,"\rTotal samples in training set: %ld \n",samples);
491 fprintf(stderr,"\rTotal bits used to represent training set: %ld\n",
499 /* wrap build_tree_from_lengths to allow zero entries in the histogram */
500 void build_tree_from_lengths0(int vals, long *hist, long *lengths){
502 /* pack the 'sparse' hit list into a dense list, then unpack
503 the lengths after the build */
506 long *lengthlist=_ogg_calloc(vals,sizeof(long));
507 long *newhist=alloca(vals*sizeof(long));
511 newhist[upper++]=hist[i];
514 fprintf(stderr,"\rEliminating %d unused entries; %d entries remain\n",
518 build_tree_from_lengths(upper,newhist,lengthlist);
523 lengths[i]=lengthlist[upper++];
530 void write_codebook(FILE *out,char *name,const static_codebook *c){
531 encode_aux_pigeonhole *p=c->pigeon_tree;
532 encode_aux_threshmatch *t=c->thresh_tree;
533 encode_aux_nearestmatch *n=c->nearest_tree;
536 /* save the book in C header form */
538 /* first, the static vectors, then the book structure to tie it together. */
541 long vals=(c->maptype==1?_book_maptype1_quantvals(c):c->entries*c->dim);
542 fprintf(out,"static long _vq_quantlist_%s[] = {\n",name);
544 fprintf(out,"\t%ld,\n",c->quantlist[j]);
546 fprintf(out,"};\n\n");
550 fprintf(out,"static long _vq_lengthlist_%s[] = {\n",name);
551 for(j=0;j<c->entries;){
553 for(k=0;k<16 && j<c->entries;k++,j++)
554 fprintf(out,"%2ld,",c->lengthlist[j]);
557 fprintf(out,"};\n\n");
561 fprintf(out,"static float _vq_quantthresh_%s[] = {\n",name);
562 for(j=0;j<t->threshvals-1;){
564 for(k=0;k<8 && j<t->threshvals-1;k++,j++)
565 fprintf(out,"%.5g, ",t->quantthresh[j]);
568 fprintf(out,"};\n\n");
571 fprintf(out,"static long _vq_quantmap_%s[] = {\n",name);
572 for(j=0;j<t->threshvals;){
574 for(k=0;k<8 && j<t->threshvals;k++,j++)
575 fprintf(out,"%5ld,",t->quantmap[j]);
578 fprintf(out,"};\n\n");
580 fprintf(out,"static encode_aux_threshmatch _vq_auxt_%s = {\n",name);
581 fprintf(out,"\t_vq_quantthresh_%s,\n",name);
582 fprintf(out,"\t_vq_quantmap_%s,\n",name);
583 fprintf(out,"\t%d,\n",t->quantvals);
584 fprintf(out,"\t%d\n};\n\n",t->threshvals);
589 for(i=0;i<c->dim;i++)pigeons*=p->quantvals;
592 fprintf(out,"static long _vq_pigeonmap_%s[] = {\n",name);
593 for(j=0;j<p->mapentries;){
595 for(k=0;k<8 && j<p->mapentries;k++,j++)
596 fprintf(out,"%5ld, ",p->pigeonmap[j]);
599 fprintf(out,"};\n\n");
601 fprintf(out,"static long _vq_fitlist_%s[] = {\n",name);
602 for(j=0;j<p->fittotal;){
604 for(k=0;k<8 && j<p->fittotal;k++,j++)
605 fprintf(out,"%5ld, ",p->fitlist[j]);
608 fprintf(out,"};\n\n");
610 fprintf(out,"static long _vq_fitmap_%s[] = {\n",name);
613 for(k=0;k<8 && j<pigeons;k++,j++)
614 fprintf(out,"%5ld, ",p->fitmap[j]);
617 fprintf(out,"};\n\n");
619 fprintf(out,"static long _vq_fitlength_%s[] = {\n",name);
622 for(k=0;k<8 && j<pigeons;k++,j++)
623 fprintf(out,"%5ld, ",p->fitlength[j]);
626 fprintf(out,"};\n\n");
628 fprintf(out,"static encode_aux_pigeonhole _vq_auxp_%s = {\n",name);
629 fprintf(out,"\t%g, %g, %d, %d,\n",
630 p->min,p->del,p->mapentries,p->quantvals);
632 fprintf(out,"\t_vq_pigeonmap_%s,\n",name);
634 fprintf(out,"\t%ld,\n",p->fittotal);
635 fprintf(out,"\t_vq_fitlist_%s,\n",name);
636 fprintf(out,"\t_vq_fitmap_%s,\n",name);
637 fprintf(out,"\t_vq_fitlength_%s\n};\n\n",name);
643 fprintf(out,"static long _vq_ptr0_%s[] = {\n",name);
646 for(k=0;k<8 && j<n->aux;k++,j++)
647 fprintf(out,"%6ld,",n->ptr0[j]);
650 fprintf(out,"};\n\n");
653 fprintf(out,"static long _vq_ptr1_%s[] = {\n",name);
656 for(k=0;k<8 && j<n->aux;k++,j++)
657 fprintf(out,"%6ld,",n->ptr1[j]);
660 fprintf(out,"};\n\n");
663 fprintf(out,"static long _vq_p_%s[] = {\n",name);
666 for(k=0;k<8 && j<n->aux;k++,j++)
667 fprintf(out,"%6ld,",n->p[j]*c->dim);
670 fprintf(out,"};\n\n");
673 fprintf(out,"static long _vq_q_%s[] = {\n",name);
676 for(k=0;k<8 && j<n->aux;k++,j++)
677 fprintf(out,"%6ld,",n->q[j]*c->dim);
680 fprintf(out,"};\n\n");
682 fprintf(out,"static encode_aux_nearestmatch _vq_auxn_%s = {\n",name);
683 fprintf(out,"\t_vq_ptr0_%s,\n",name);
684 fprintf(out,"\t_vq_ptr1_%s,\n",name);
685 fprintf(out,"\t_vq_p_%s,\n",name);
686 fprintf(out,"\t_vq_q_%s,\n",name);
687 fprintf(out,"\t%ld, %ld\n};\n\n",n->aux,n->aux);
690 /* tie it all together */
692 fprintf(out,"static static_codebook %s = {\n",name);
694 fprintf(out,"\t%ld, %ld,\n",c->dim,c->entries);
695 fprintf(out,"\t_vq_lengthlist_%s,\n",name);
696 fprintf(out,"\t%d, %ld, %ld, %d, %d,\n",
697 c->maptype,c->q_min,c->q_delta,c->q_quant,c->q_sequencep);
699 fprintf(out,"\t_vq_quantlist_%s,\n",name);
701 fprintf(out,"\tNULL,\n");
704 fprintf(out,"\t&_vq_auxn_%s,\n",name);
706 fprintf(out,"\tNULL,\n");
708 fprintf(out,"\t&_vq_auxt_%s,\n",name);
710 fprintf(out,"\tNULL,\n");
712 fprintf(out,"\t&_vq_auxp_%s,\n",name);
714 fprintf(out,"\tNULL,\n");
716 fprintf(out,"\t0\n};\n\n");