void vqext_quantize(vqgen *v,quant_meta *q){ int j,k; long dim=v->elements; long n=v->entries; float max=-1; float *test=alloca(sizeof(float)*dim); int moved=0; /* allow movement only to unoccupied coordinates on the coarse grid */ for(j=0;j<n;j++){ for(k=0;k<dim;k++){ float val=_now(v,j)[k]; float norm=rint(fabs(val)/scalequant); if(norm>max)max=norm; test[k]=norm; } /* allow move only if unoccupied */ if(quant_save){ for(k=0;k<n;k++) if(j!=k && memcmp(test,quant_save+dim*k,dim*sizeof(float))==0) break; if(k==n){ if(memcmp(test,quant_save+dim*j,dim*sizeof(float))) moved++; memcpy(quant_save+dim*j,test,sizeof(float)*dim); } }else{ memcpy(_now(v,j),test,sizeof(float)*dim); } } /* unlike the other trainers, we fill in our quantization information (as we know granularity beforehand and don't need to maximize it) */ q->min=_float32_pack(0.f); q->delta=_float32_pack(scalequant); q->quant=_ilog(max); if(quant_save){ memcpy(_now(v,0),quant_save,sizeof(float)*dim*n); fprintf(stderr,"cells shifted this iteration: %d\n",moved); } }
void vqext_quantize(vqgen *v,quant_meta *q){ float delta,mindel; float maxquant=((1<<q->quant)-1); int j,k; /* first find the basic delta amount from the maximum span to be encoded. Loosen the delta slightly to allow for additional error during sequence quantization */ delta=(global_maxdel-global_mindel)/((1<<q->quant)-1.5f); q->min=_float32_pack(global_mindel); q->delta=_float32_pack(delta); mindel=_float32_unpack(q->min); delta=_float32_unpack(q->delta); for(j=0;j<v->entries;j++){ float last=0; for(k=0;k<v->elements;k++){ float val=_now(v,j)[k]; float now=rint((val-last-mindel)/delta); _now(v,j)[k]=now; if(now<0){ /* be paranoid; this should be impossible */ fprintf(stderr,"fault; quantized value<0\n"); exit(1); } if(now>maxquant){ /* be paranoid; this should be impossible */ fprintf(stderr,"fault; quantized value>max\n"); exit(1); } last=(now*delta)+mindel+last; } } }
int main(int argc,char *argv[]){ codebook b; static_codebook c; double *quantlist; long *hits; int entries=-1,dim=-1,quantvals=-1,addmul=-1,sequencep=0; FILE *in=NULL; char *line,*name; long i,j; memset(&b,0,sizeof(b)); memset(&c,0,sizeof(c)); if(argv[1]==NULL){ fprintf(stderr,"Need a lattice description file on the command line.\n"); exit(1); } { char *ptr; char *filename=_ogg_calloc(strlen(argv[1])+4,1); strcpy(filename,argv[1]); in=fopen(filename,"r"); if(!in){ fprintf(stderr,"Could not open input file %s\n",filename); exit(1); } ptr=strrchr(filename,'.'); if(ptr){ *ptr='\0'; name=strdup(filename); }else{ name=strdup(filename); } } /* read the description */ line=get_line(in); if(sscanf(line,"%d %d %d %d",&quantvals,&dim,&addmul,&sequencep)!=4){ if(sscanf(line,"%d %d %d",&quantvals,&dim,&addmul)!=3){ fprintf(stderr,"Syntax error reading description file (line 1)\n"); exit(1); } } entries=pow(quantvals,dim); c.dim=dim; c.entries=entries; c.lengthlist=_ogg_malloc(entries*sizeof(long)); c.maptype=1; c.q_sequencep=sequencep; c.quantlist=_ogg_calloc(quantvals,sizeof(long)); quantlist=_ogg_malloc(sizeof(double)*c.dim*c.entries); hits=_ogg_malloc(c.entries*sizeof(long)); for(j=0;j<entries;j++)hits[j]=1; for(j=0;j<entries;j++)c.lengthlist[j]=1; reset_next_value(); line=setup_line(in); for(j=0;j<quantvals;j++){ char *temp; if(!line || sscanf(line,"%lf",quantlist+j)!=1){ fprintf(stderr,"Ran out of data on line 2 of description file\n"); exit(1); } temp=strchr(line,','); if(!temp)temp=strchr(line,' '); if(temp)temp++; line=temp; } /* gen a real quant list from the more easily human-grokked input */ { double min=quantlist[0]; double mindel=-1; int fac=1; for(j=1;j<quantvals;j++)if(quantlist[j]<min)min=quantlist[j]; for(j=0;j<quantvals;j++) for(i=j+1;i<quantvals;i++) if(mindel==-1 || fabs(quantlist[j]-quantlist[i])<mindel) mindel=fabs(quantlist[j]-quantlist[i]); j=0; while(j<quantvals){ for(j=0;j<quantvals;j++){ double test=fac*(quantlist[j]-min)/mindel; if( fabs(rint(test)-test)>.00001f) break; } if(fac>100)break; if(j<quantvals)fac++; } mindel/=fac; fprintf(stderr,"min=%g mindel=%g\n",min,mindel); c.q_min=_float32_pack(min); c.q_delta=_float32_pack(mindel); c.q_quant=0; min=_float32_unpack(c.q_min); mindel=_float32_unpack(c.q_delta); for(j=0;j<quantvals;j++){ c.quantlist[j]=rint((quantlist[j]-min)/mindel); if(ilog(c.quantlist[j])>c.q_quant)c.q_quant=ilog(c.quantlist[j]); } } /* build the [default] codeword lengths */ memset(c.lengthlist,0,sizeof(long)*entries); for(i=0;i<entries;i++)hits[i]=1; build_tree_from_lengths(entries,hits,c.lengthlist); /* save the book in C header form */ write_codebook(stdout,name,&c); fprintf(stderr,"\r " "\nDone.\n"); exit(0); }