void vqext_quantize(vqgen *v,quant_meta *q){
float delta,mindel;
float maxquant=((1<<q->quant)-1);
int j,k;
/* first find the basic delta amount from the maximum span to be
encoded. Loosen the delta slightly to allow for additional error
during sequence quantization */
delta=(global_maxdel-global_mindel)/((1<<q->quant)-1.5f);
q->min=_float32_pack(global_mindel);
q->delta=_float32_pack(delta);
mindel=_float32_unpack(q->min);
delta=_float32_unpack(q->delta);
for(j=0;j<v->entries;j++){
float last=0;
for(k=0;k<v->elements;k++){
float val=_now(v,j)[k];
float now=rint((val-last-mindel)/delta);
_now(v,j)[k]=now;
if(now<0){
/* be paranoid; this should be impossible */
fprintf(stderr,"fault; quantized value<0\n");
exit(1);
}
if(now>maxquant){
/* be paranoid; this should be impossible */
fprintf(stderr,"fault; quantized value>max\n");
exit(1);
}
last=(now*delta)+mindel+last;
}
}
}
int main(int argc,char *argv[]){
FILE *in;
long lines=0;
float min;
float max;
long bins=-1;
int flag=0;
long *countarray;
long total=0;
char *line;
if(argv[1]==NULL){
fprintf(stderr,"Usage: distribution {data.vqd [bins]| book.vqh} \n\n");
exit(1);
}
if(argv[2]!=NULL)
bins=atoi(argv[2])-1;
in=fopen(argv[1],"r");
if(!in){
fprintf(stderr,"Could not open input file %s\n",argv[1]);
exit(1);
}
if(strrchr(argv[1],'.') && strcmp(strrchr(argv[1],'.'),".vqh")==0){
/* load/decode a book */
codebook *b=codebook_load(argv[1]);
static_codebook *c=(static_codebook *)(b->c);
float delta;
int i;
fclose(in);
switch(c->maptype){
case 0:
printf("entropy codebook only; no mappings\n");
exit(0);
break;
case 1:
bins=_book_maptype1_quantvals(c);
break;
case 2:
bins=c->entries*c->dim;
break;
}
max=min=_float32_unpack(c->q_min);
delta=_float32_unpack(c->q_delta);
for(i=0;i<bins;i++){
float val=c->quantlist[i]*delta+min;
if(val>max)max=val;
}
printf("Minimum scalar value: %f\n",min);
printf("Maximum scalar value: %f\n",max);
switch(c->maptype){
case 1:
{
/* lattice codebook. dump it. */
int j,k;
long maxcount=0;
long **sort=calloc(bins,sizeof(long *));
long base=c->lengthlist[0];
countarray=calloc(bins,sizeof(long));
for(i=0;i<bins;i++)sort[i]=c->quantlist+i;
qsort(sort,bins,sizeof(long *),ascend);
for(i=0;i<b->entries;i++)
if(c->lengthlist[i]>base)base=c->lengthlist[i];
/* dump a full, correlated count */
for(j=0;j<b->entries;j++){
if(c->lengthlist[j]){
int indexdiv=1;
printf("%4d: ",j);
for(k=0;k<b->dim;k++){
int index= (j/indexdiv)%bins;
printf("%+3.1f,", c->quantlist[index]*_float32_unpack(c->q_delta)+
_float32_unpack(c->q_min));
indexdiv*=bins;
}
printf("\t|");
for(k=0;k<base-c->lengthlist[j];k++)printf("*");
printf("\n");
}
}
/* do a rough count */
for(j=0;j<b->entries;j++){
int indexdiv=1;
for(k=0;k<b->dim;k++){
if(c->lengthlist[j]){
int index= (j/indexdiv)%bins;
countarray[index]+=(1<<(base-c->lengthlist[j]));
indexdiv*=bins;
}
}
}
/* dump the count */
{
long maxcount=0,i,j;
for(i=0;i<bins;i++)
if(countarray[i]>maxcount)maxcount=countarray[i];
for(i=0;i<bins;i++){
int ptr=sort[i]-c->quantlist;
int stars=rint(50./maxcount*countarray[ptr]);
printf("%+08f (%8ld) |",c->quantlist[ptr]*delta+min,countarray[ptr]);
for(j=0;j<stars;j++)printf("*");
printf("\n");
}
}
}
break;
case 2:
{
/* trained, full mapping codebook. */
printf("Can't do probability dump of a trained [type 2] codebook (yet)\n");
}
break;
}
}else{
/* load/count a data file */
/* do it the simple way; two pass. */
line=setup_line(in);
while(line){
float code;
char buf[80];
lines++;
sprintf(buf,"getting min/max (%.2f::%.2f). lines...",min,max);
if(!(lines&0xff))spinnit(buf,lines);
while(!flag && sscanf(line,"%f",&code)==1){
line=strchr(line,',');
min=max=code;
flag=1;
}
while(line && sscanf(line,"%f",&code)==1){
line=strchr(line,',');
if(line)line++;
if(code<min)min=code;
if(code>max)max=code;
}
line=setup_line(in);
}
if(bins<1){
if((int)(max-min)==min-max){
bins=max-min;
}else{
bins=25;
}
}
printf("\r \r");
printf("Minimum scalar value: %f\n",min);
printf("Maximum scalar value: %f\n",max);
if(argv[2]){
printf("\n counting hits into %ld bins...\n",bins+1);
countarray=calloc(bins+1,sizeof(long));
rewind(in);
line=setup_line(in);
while(line){
float code;
lines--;
if(!(lines&0xff))spinnit("counting distribution. lines so far...",lines);
while(line && sscanf(line,"%f",&code)==1){
line=strchr(line,',');
if(line)line++;
code-=min;
code/=(max-min);
code*=bins;
countarray[(int)rint(code)]++;
total++;
}
line=setup_line(in);
}
/* make a pretty graph */
{
long maxcount=0,i,j;
for(i=0;i<bins+1;i++)
if(countarray[i]>maxcount)maxcount=countarray[i];
printf("\r \r");
printf("Total scalars: %ld\n",total);
for(i=0;i<bins+1;i++){
int stars=rint(50./maxcount*countarray[i]);
printf("%08f (%8ld) |",(max-min)/bins*i+min,countarray[i]);
for(j=0;j<stars;j++)printf("*");
printf("\n");
}
}
}
fclose(in);
}
printf("\nDone.\n");
exit(0);
}
int main(int argc,char *argv[]){
codebook *b;
static_codebook *c;
long *lengths;
long *hits;
int entries=-1,dim=-1,guard=1;
FILE *in=NULL;
char *line,*name;
long j;
if(argv[1]==NULL){
fprintf(stderr,"Need a lattice codebook on the command line.\n");
exit(1);
}
if(argv[2]==NULL){
fprintf(stderr,"Need a codeword data file on the command line.\n");
exit(1);
}
if(argv[3]!=NULL)guard=0;
{
char *ptr;
char *filename=strdup(argv[1]);
b=codebook_load(filename);
c=(static_codebook *)(b->c);
ptr=strrchr(filename,'.');
if(ptr){
*ptr='\0';
name=strdup(filename);
}else{
name=strdup(filename);
}
}
if(c->maptype!=1){
fprintf(stderr,"Provided book is not a latticebook.\n");
exit(1);
}
entries=b->entries;
dim=b->dim;
hits=_ogg_malloc(entries*sizeof(long));
lengths=_ogg_calloc(entries,sizeof(long));
for(j=0;j<entries;j++)hits[j]=guard;
in=fopen(argv[2],"r");
if(!in){
fprintf(stderr,"Could not open input file %s\n",argv[2]);
exit(1);
}
if(!strrcmp_i(argv[0],"latticetune")){
long lines=0;
line=setup_line(in);
while(line){
long code;
lines++;
if(!(lines&0xfff))spinnit("codewords so far...",lines);
if(sscanf(line,"%ld",&code)==1)
hits[code]++;
line=setup_line(in);
}
}
/* now we simply count already collated by-entry data */
if(!strrcmp_i(argv[0],"res0tune") || !strrcmp_i(argv[0],"res1tune")){
line=setup_line(in);
while(line){
/* code:hits\n */
/* likely to have multiple listing for each code entry; must
accumulate */
char *pos=strchr(line,':');
if(pos){
long code=atol(line);
long val=atol(pos+1);
hits[code]+=val;
}
line=setup_line(in);
}
}
fclose(in);
/* build the codeword lengths */
build_tree_from_lengths0(entries,hits,lengths);
c->lengthlist=lengths;
write_codebook(stdout,name,c);
{
long bins=_book_maptype1_quantvals(c);
long i,k,base=c->lengthlist[0];
for(i=0;i<entries;i++)
if(c->lengthlist[i]>base)base=c->lengthlist[i];
for(j=0;j<entries;j++){
if(c->lengthlist[j]){
int indexdiv=1;
fprintf(stderr,"%4ld: ",j);
for(k=0;k<c->dim;k++){
int index= (j/indexdiv)%bins;
fprintf(stderr,"%+3.1f,", c->quantlist[index]*_float32_unpack(c->q_delta)+
_float32_unpack(c->q_min));
indexdiv*=bins;
}
fprintf(stderr,"\t|");
for(k=0;k<base-c->lengthlist[j];k++)fprintf(stderr,"*");
fprintf(stderr,"\n");
}
}
}
fprintf(stderr,"\r "
"\nDone.\n");
exit(0);
}
int main(int argc,char *argv[]) {
codebook *b;
static_codebook *c;
int entries=-1,dim=-1;
float min,del;
char *name;
long i,j;
float *suggestions;
int suggcount=0;
if(argv[1]==NULL) {
fprintf(stderr,"Need a lattice book on the command line.\n");
exit(1);
}
{
char *ptr;
char *filename=strdup(argv[1]);
b=codebook_load(filename);
c=(static_codebook *)(b->c);
ptr=strrchr(filename,'.');
if(ptr) {
*ptr='\0';
name=strdup(filename);
} else {
name=strdup(filename);
}
}
if(c->maptype!=1) {
fprintf(stderr,"Provided book is not a latticebook.\n");
exit(1);
}
entries=b->entries;
dim=b->dim;
min=_float32_unpack(c->q_min);
del=_float32_unpack(c->q_delta);
/* Do we want to gen a threshold hint? */
if(c->q_sequencep==0) {
/* yes. Discard any preexisting threshhold hint */
long quantvals=_book_maptype1_quantvals(c);
long **quantsort=alloca(quantvals*sizeof(long *));
encode_aux_threshmatch *t=_ogg_calloc(1,sizeof(encode_aux_threshmatch));
c->thresh_tree=t;
fprintf(stderr,"Adding threshold hint to %s...\n",name);
/* partial/complete suggestions */
if(argv[2]) {
char *ptr=strdup(argv[2]);
suggestions=alloca(sizeof(float)*quantvals);
for(suggcount=0; ptr && suggcount<quantvals; suggcount++) {
char *ptr2=strchr(ptr,',');
if(ptr2)*ptr2++='\0';
suggestions[suggcount]=atof(ptr);
ptr=ptr2;
}
}
/* simplest possible threshold hint only */
t->quantthresh=_ogg_calloc(quantvals-1,sizeof(float));
t->quantmap=_ogg_calloc(quantvals,sizeof(int));
t->threshvals=quantvals;
t->quantvals=quantvals;
/* the quantvals may not be in order; sort em first */
for(i=0; i<quantvals; i++)quantsort[i]=c->quantlist+i;
qsort(quantsort,quantvals,sizeof(long *),longsort);
/* ok, gen the map and thresholds */
for(i=0; i<quantvals; i++)t->quantmap[i]=quantsort[i]-c->quantlist;
for(i=0; i<quantvals-1; i++) {
float v1=*(quantsort[i])*del+min;
float v2=*(quantsort[i+1])*del+min;
for(j=0; j<suggcount; j++)
if(v1<suggestions[j] && suggestions[j]<v2) {
t->quantthresh[i]=suggestions[j];
break;
}
if(j==suggcount) {
t->quantthresh[i]=(v1+v2)*.5;
}
}
}
/* Do we want to gen a pigeonhole hint? */
#if 0
for(i=0; i<entries; i++)if(c->lengthlist[i]==0)break;
if(c->q_sequencep || i<entries) {
long **tempstack;
long *tempcount;
long *temptrack;
float *tempmin;
float *tempmax;
long totalstack=0;
long pigeons;
long subpigeons;
long quantvals=_book_maptype1_quantvals(c);
int changep=1,factor;
encode_aux_pigeonhole *p=_ogg_calloc(1,sizeof(encode_aux_pigeonhole));
c->pigeon_tree=p;
fprintf(stderr,"Adding pigeonhole hint to %s...\n",name);
/* the idea is that we quantize uniformly, even in a nonuniform
lattice, so that quantization of one scalar has a predictable
result on the next sequential scalar in a greedy matching
algorithm. We generate a lookup based on the quantization of
the vector (pigeonmap groups quantized entries together) and
list the entries that could possible be the best fit for any
given member of that pigeonhole. The encode process then has a
much smaller list to brute force */
/* find our pigeonhole-specific quantization values, fill in the
quant value->pigeonhole map */
factor=3;
p->del=del;
p->min=min;
p->quantvals=quantvals;
{
int max=0;
for(i=0; i<quantvals; i++)if(max<c->quantlist[i])max=c->quantlist[i];
p->mapentries=max;
}
p->pigeonmap=_ogg_malloc(p->mapentries*sizeof(long));
p->quantvals=(quantvals+factor-1)/factor;
/* pigeonhole roughly on the boundaries of the quantvals; the
exact pigeonhole grouping is an optimization issue, not a
correctness issue */
for(i=0; i<p->mapentries; i++) {
float thisval=del*i+min; /* middle of the quant zone */
int quant=0;
float err=fabs(c->quantlist[0]*del+min-thisval);
for(j=1; j<quantvals; j++) {
float thiserr=fabs(c->quantlist[j]*del+min-thisval);
if(thiserr<err) {
quant=j/factor;
err=thiserr;
}
}
p->pigeonmap[i]=quant;
}
/* pigeonmap complete. Now do the grungy business of finding the
entries that could possibly be the best fit for a value appearing
in the pigeonhole. The trick that allows the below to work is the
uniform quantization; even though the scalars may be 'sequential'
(each a delta from the last), the uniform quantization means that
the error variance is *not* dependant. Given a pigeonhole and an
entry, we can find the minimum and maximum possible errors
(relative to the entry) for any point that could appear in the
pigeonhole */
/* must iterate over both pigeonholes and entries */
/* temporarily (in order to avoid thinking hard), we grow each
pigeonhole separately, the build a stack of 'em later */
pigeons=1;
subpigeons=1;
for(i=0; i<dim; i++)subpigeons*=p->mapentries;
for(i=0; i<dim; i++)pigeons*=p->quantvals;
temptrack=_ogg_calloc(dim,sizeof(long));
tempmin=_ogg_calloc(dim,sizeof(float));
tempmax=_ogg_calloc(dim,sizeof(float));
tempstack=_ogg_calloc(pigeons,sizeof(long *));
tempcount=_ogg_calloc(pigeons,sizeof(long));
while(1) {
float errorpost=-1;
char buffer[80];
/* map our current pigeonhole to a 'big pigeonhole' so we know
what list we're after */
int entry=0;
for(i=dim-1; i>=0; i--)entry=entry*p->quantvals+p->pigeonmap[temptrack[i]];
setvals(dim,p,temptrack,tempmin,tempmax,c->q_sequencep);
sprintf(buffer,"Building pigeonhole search list [%ld]...",totalstack);
/* Search all entries to find the one with the minimum possible
maximum error. Record that error */
for(i=0; i<entries; i++) {
if(c->lengthlist[i]>0) {
float this=maxerror(dim,b->valuelist+i*dim,p,
temptrack,tempmin,tempmax);
if(errorpost==-1 || this<errorpost)errorpost=this;
spinnit(buffer,subpigeons);
}
}
/* Our search list will contain all entries with a minimum
possible error <= our errorpost */
for(i=0; i<entries; i++)
if(c->lengthlist[i]>0) {
spinnit(buffer,subpigeons);
if(minerror(dim,b->valuelist+i*dim,p,
temptrack,tempmin,tempmax)<errorpost)
totalstack+=addtosearch(entry,tempstack,tempcount,i);
}
for(i=0; i<dim; i++) {
temptrack[i]++;
if(temptrack[i]<p->mapentries)break;
temptrack[i]=0;
}
if(i==dim)break;
subpigeons--;
}
int main(int argc,char *argv[]){
codebook b;
static_codebook c;
double *quantlist;
long *hits;
int entries=-1,dim=-1,quantvals=-1,addmul=-1,sequencep=0;
FILE *in=NULL;
char *line,*name;
long i,j;
memset(&b,0,sizeof(b));
memset(&c,0,sizeof(c));
if(argv[1]==NULL){
fprintf(stderr,"Need a lattice description file on the command line.\n");
exit(1);
}
{
char *ptr;
char *filename=_ogg_calloc(strlen(argv[1])+4,1);
strcpy(filename,argv[1]);
in=fopen(filename,"r");
if(!in){
fprintf(stderr,"Could not open input file %s\n",filename);
exit(1);
}
ptr=strrchr(filename,'.');
if(ptr){
*ptr='\0';
name=strdup(filename);
}else{
name=strdup(filename);
}
}
/* read the description */
line=get_line(in);
if(sscanf(line,"%d %d %d %d",&quantvals,&dim,&addmul,&sequencep)!=4){
if(sscanf(line,"%d %d %d",&quantvals,&dim,&addmul)!=3){
fprintf(stderr,"Syntax error reading description file (line 1)\n");
exit(1);
}
}
entries=pow(quantvals,dim);
c.dim=dim;
c.entries=entries;
c.lengthlist=_ogg_malloc(entries*sizeof(long));
c.maptype=1;
c.q_sequencep=sequencep;
c.quantlist=_ogg_calloc(quantvals,sizeof(long));
quantlist=_ogg_malloc(sizeof(double)*c.dim*c.entries);
hits=_ogg_malloc(c.entries*sizeof(long));
for(j=0;j<entries;j++)hits[j]=1;
for(j=0;j<entries;j++)c.lengthlist[j]=1;
reset_next_value();
line=setup_line(in);
for(j=0;j<quantvals;j++){
char *temp;
if(!line || sscanf(line,"%lf",quantlist+j)!=1){
fprintf(stderr,"Ran out of data on line 2 of description file\n");
exit(1);
}
temp=strchr(line,',');
if(!temp)temp=strchr(line,' ');
if(temp)temp++;
line=temp;
}
/* gen a real quant list from the more easily human-grokked input */
{
double min=quantlist[0];
double mindel=-1;
int fac=1;
for(j=1;j<quantvals;j++)if(quantlist[j]<min)min=quantlist[j];
for(j=0;j<quantvals;j++)
for(i=j+1;i<quantvals;i++)
if(mindel==-1 || fabs(quantlist[j]-quantlist[i])<mindel)
mindel=fabs(quantlist[j]-quantlist[i]);
j=0;
while(j<quantvals){
for(j=0;j<quantvals;j++){
double test=fac*(quantlist[j]-min)/mindel;
if( fabs(rint(test)-test)>.00001f) break;
}
if(fac>100)break;
if(j<quantvals)fac++;
}
mindel/=fac;
fprintf(stderr,"min=%g mindel=%g\n",min,mindel);
c.q_min=_float32_pack(min);
c.q_delta=_float32_pack(mindel);
c.q_quant=0;
min=_float32_unpack(c.q_min);
mindel=_float32_unpack(c.q_delta);
for(j=0;j<quantvals;j++){
c.quantlist[j]=rint((quantlist[j]-min)/mindel);
if(ilog(c.quantlist[j])>c.q_quant)c.q_quant=ilog(c.quantlist[j]);
}
}
/* build the [default] codeword lengths */
memset(c.lengthlist,0,sizeof(long)*entries);
for(i=0;i<entries;i++)hits[i]=1;
build_tree_from_lengths(entries,hits,c.lengthlist);
/* save the book in C header form */
write_codebook(stdout,name,&c);
fprintf(stderr,"\r "
"\nDone.\n");
exit(0);
}
