SVECTOR* add_list_sort_ss(SVECTOR *a)
/* computes the linear combination of the SVECTOR list weighted
by the factor of each SVECTOR. This should be a lot faster
than add_list_ss for long lists. */
{
return(add_list_sort_ss_r(a,0));
}
SVECTOR* add_dual_list_sort_ss_r(SVECTOR *a, SVECTOR *b, double min_non_zero)
/* computes the linear combination of the two SVECTOR lists weighted
by the factor of each SVECTOR */
{
SVECTOR *f,*sum;
for(f=a;f->next;f=f->next); /* find end of first vector list */
f->next=b; /* temporarily append the two vector lists */
sum=add_list_sort_ss_r(a,min_non_zero);
f->next=NULL; /* undo append */
return(sum);
}
void svm_learn_struct_joint(SAMPLE sample, STRUCT_LEARN_PARM *sparm,
LEARN_PARM *lparm, KERNEL_PARM *kparm,
STRUCTMODEL *sm, int alg_type)
{
int i,j;
int numIt=0;
long argmax_count=0;
long totconstraints=0;
long kernel_type_org;
double epsilon,epsilon_cached;
double lhsXw,rhs_i;
double rhs=0;
double slack,ceps;
double dualitygap,modellength,alphasum;
long sizePsi;
double *alpha=NULL;
long *alphahist=NULL,optcount=0;
CONSTSET cset;
SVECTOR *diff=NULL;
double *lhs_n=NULL;
SVECTOR *fy, *fydelta, **fycache, *lhs;
MODEL *svmModel=NULL;
DOC *doc;
long n=sample.n;
EXAMPLE *ex=sample.examples;
double rt_total=0,rt_opt=0,rt_init=0,rt_psi=0,rt_viol=0,rt_kernel=0;
double rt_cacheupdate=0,rt_cacheconst=0,rt_cacheadd=0,rt_cachesum=0;
double rt1=0,rt2=0;
long progress;
/*
SVECTOR ***fydelta_cache=NULL;
double **loss_cache=NULL;
int cache_size=0;
*/
CCACHE *ccache=NULL;
int cached_constraint;
double viol,viol_est,epsilon_est=0;
long uptr=0;
long *randmapping=NULL;
long batch_size=n;
rt1=get_runtime();
if(sparm->batch_size<100)
batch_size=sparm->batch_size*n/100.0;
init_struct_model(sample,sm,sparm,lparm,kparm);
sizePsi=sm->sizePsi+1; /* sm must contain size of psi on return */
if(sparm->slack_norm == 1) {
lparm->svm_c=sparm->C; /* set upper bound C */
lparm->sharedslack=1;
}
else if(sparm->slack_norm == 2) {
printf("ERROR: The joint algorithm does not apply to L2 slack norm!");
fflush(stdout);
exit(0);
}
else {
printf("ERROR: Slack norm must be L1 or L2!"); fflush(stdout);
exit(0);
}
lparm->biased_hyperplane=0; /* set threshold to zero */
epsilon=100.0; /* start with low precision and
increase later */
epsilon_cached=epsilon; /* epsilon to use for iterations
using constraints constructed
from the constraint cache */
cset=init_struct_constraints(sample, sm, sparm);
if(cset.m > 0) {
alpha=(double *)realloc(alpha,sizeof(double)*cset.m);
alphahist=(long *)realloc(alphahist,sizeof(long)*cset.m);
for(i=0; i<cset.m; i++) {
alpha[i]=0;
alphahist[i]=-1; /* -1 makes sure these constraints are never removed */
}
}
kparm->gram_matrix=NULL;
if((alg_type == ONESLACK_DUAL_ALG) || (alg_type == ONESLACK_DUAL_CACHE_ALG))
kparm->gram_matrix=init_kernel_matrix(&cset,kparm);
/* set initial model and slack variables */
svmModel=(MODEL *)my_malloc(sizeof(MODEL));
lparm->epsilon_crit=epsilon;
svm_learn_optimization(cset.lhs,cset.rhs,cset.m,sizePsi,
lparm,kparm,NULL,svmModel,alpha);
add_weight_vector_to_linear_model(svmModel);
sm->svm_model=svmModel;
sm->w=svmModel->lin_weights; /* short cut to weight vector */
/* create a cache of the feature vectors for the correct labels */
fycache=(SVECTOR **)my_malloc(n*sizeof(SVECTOR *));
for(i=0;i<n;i++) {
if(USE_FYCACHE) {
fy=psi(ex[i].x,ex[i].y,sm,sparm);
if(kparm->kernel_type == LINEAR_KERNEL) { /* store difference vector directly */
diff=add_list_sort_ss_r(fy,COMPACT_ROUNDING_THRESH);
free_svector(fy);
fy=diff;
}
}
else
fy=NULL;
fycache[i]=fy;
}
/* initialize the constraint cache */
if(alg_type == ONESLACK_DUAL_CACHE_ALG) {
ccache=create_constraint_cache(sample,sparm,sm);
/* NOTE: */
for(i=0;i<n;i++)
if(loss(ex[i].y,ex[i].y,sparm) != 0) {
printf("ERROR: Loss function returns non-zero value loss(y_%d,y_%d)\n",i,i);
printf(" W4 algorithm assumes that loss(y_i,y_i)=0 for all i.\n");
exit(1);
}
}
if(kparm->kernel_type == LINEAR_KERNEL)
lhs_n=create_nvector(sm->sizePsi);
/* randomize order or training examples */
if(batch_size<n)
randmapping=random_order(n);
rt_init+=MAX(get_runtime()-rt1,0);
rt_total+=rt_init;
/*****************/
/*** main loop ***/
/*****************/
do { /* iteratively find and add constraints to working set */
if(struct_verbosity>=1) {
printf("Iter %i: ",++numIt);
fflush(stdout);
}
rt1=get_runtime();
/**** compute current slack ****/
alphasum=0;
for(j=0;(j<cset.m);j++)
alphasum+=alpha[j];
for(j=0,slack=-1;(j<cset.m) && (slack==-1);j++)
if(alpha[j] > alphasum/cset.m)
slack=MAX(0,cset.rhs[j]-classify_example(svmModel,cset.lhs[j]));
slack=MAX(0,slack);
rt_total+=MAX(get_runtime()-rt1,0);
/**** find a violated joint constraint ****/
lhs=NULL;
rhs=0;
if(alg_type == ONESLACK_DUAL_CACHE_ALG) {
rt1=get_runtime();
/* Compute violation of constraints in cache for current w */
if(struct_verbosity>=2) rt2=get_runtime();
update_constraint_cache_for_model(ccache, svmModel);
if(struct_verbosity>=2) rt_cacheupdate+=MAX(get_runtime()-rt2,0);
/* Is there is a sufficiently violated constraint in cache? */
viol=compute_violation_of_constraint_in_cache(ccache,epsilon_est/2);
if(viol-slack > MAX(epsilon_est/10,sparm->epsilon)) {
/* There is a sufficiently violated constraint in cache, so
use this constraint in this iteration. */
if(struct_verbosity>=2) rt2=get_runtime();
viol=find_most_violated_joint_constraint_in_cache(ccache,
epsilon_est/2,lhs_n,&lhs,&rhs);
if(struct_verbosity>=2) rt_cacheconst+=MAX(get_runtime()-rt2,0);
cached_constraint=1;
}
else {
/* There is no sufficiently violated constraint in cache, so
update cache by computing most violated constraint
explicitly for batch_size examples. */
viol_est=0;
progress=0;
viol=compute_violation_of_constraint_in_cache(ccache,0);
for(j=0;(j<batch_size) || ((j<n)&&(viol-slack<sparm->epsilon));j++) {
if(struct_verbosity>=1)
print_percent_progress(&progress,n,10,".");
uptr=uptr % n;
if(randmapping)
i=randmapping[uptr];
else
i=uptr;
/* find most violating fydelta=fy-fybar and rhs for example i */
find_most_violated_constraint(&fydelta,&rhs_i,&ex[i],
fycache[i],n,sm,sparm,
&rt_viol,&rt_psi,&argmax_count);
/* add current fy-fybar and loss to cache */
if(struct_verbosity>=2) rt2=get_runtime();
viol+=add_constraint_to_constraint_cache(ccache,sm->svm_model,
i,fydelta,rhs_i,0.0001*sparm->epsilon/n,
sparm->ccache_size,&rt_cachesum);
if(struct_verbosity>=2) rt_cacheadd+=MAX(get_runtime()-rt2,0);
viol_est+=ccache->constlist[i]->viol;
uptr++;
}
cached_constraint=(j<n);
if(struct_verbosity>=2) rt2=get_runtime();
if(cached_constraint)
viol=find_most_violated_joint_constraint_in_cache(ccache,
epsilon_est/2,lhs_n,&lhs,&rhs);
else
viol=find_most_violated_joint_constraint_in_cache(ccache,0,lhs_n,
&lhs,&rhs);
if(struct_verbosity>=2) rt_cacheconst+=MAX(get_runtime()-rt2,0);
viol_est*=((double)n/j);
epsilon_est=(1-(double)j/n)*epsilon_est+(double)j/n*(viol_est-slack);
if((struct_verbosity >= 1) && (j!=n))
printf("(upd=%5.1f%%,eps^=%.4f,eps*=%.4f)",
100.0*j/n,viol_est-slack,epsilon_est);
}
lhsXw=rhs-viol;
rt_total+=MAX(get_runtime()-rt1,0);
}
else {
/* do not use constraint from cache */
rt1=get_runtime();
cached_constraint=0;
if(kparm->kernel_type == LINEAR_KERNEL)
clear_nvector(lhs_n,sm->sizePsi);
progress=0;
rt_total+=MAX(get_runtime()-rt1,0);
for(i=0; i<n; i++) {
rt1=get_runtime();
if(struct_verbosity>=1)
print_percent_progress(&progress,n,10,".");
/* compute most violating fydelta=fy-fybar and rhs for example i */
find_most_violated_constraint(&fydelta,&rhs_i,&ex[i],fycache[i],n,
sm,sparm,&rt_viol,&rt_psi,&argmax_count);
/* add current fy-fybar to lhs of constraint */
if(kparm->kernel_type == LINEAR_KERNEL) {
add_list_n_ns(lhs_n,fydelta,1.0); /* add fy-fybar to sum */
free_svector(fydelta);
}
else {
append_svector_list(fydelta,lhs); /* add fy-fybar to vector list */
lhs=fydelta;
}
rhs+=rhs_i; /* add loss to rhs */
rt_total+=MAX(get_runtime()-rt1,0);
} /* end of example loop */
rt1=get_runtime();
/* create sparse vector from dense sum */
if(kparm->kernel_type == LINEAR_KERNEL)
lhs=create_svector_n_r(lhs_n,sm->sizePsi,NULL,1.0,
COMPACT_ROUNDING_THRESH);
doc=create_example(cset.m,0,1,1,lhs);
lhsXw=classify_example(svmModel,doc);
free_example(doc,0);
viol=rhs-lhsXw;
rt_total+=MAX(get_runtime()-rt1,0);
} /* end of finding most violated joint constraint */
rt1=get_runtime();
/**** if `error', then add constraint and recompute QP ****/
if(slack > (rhs-lhsXw+0.000001)) {
printf("\nWARNING: Slack of most violated constraint is smaller than slack of working\n");
printf(" set! There is probably a bug in 'find_most_violated_constraint_*'.\n");
printf("slack=%f, newslack=%f\n",slack,rhs-lhsXw);
/* exit(1); */
}
ceps=MAX(0,rhs-lhsXw-slack);
if((ceps > sparm->epsilon) || cached_constraint) {
/**** resize constraint matrix and add new constraint ****/
cset.lhs=(DOC **)realloc(cset.lhs,sizeof(DOC *)*(cset.m+1));
cset.lhs[cset.m]=create_example(cset.m,0,1,1,lhs);
cset.rhs=(double *)realloc(cset.rhs,sizeof(double)*(cset.m+1));
cset.rhs[cset.m]=rhs;
alpha=(double *)realloc(alpha,sizeof(double)*(cset.m+1));
alpha[cset.m]=0;
alphahist=(long *)realloc(alphahist,sizeof(long)*(cset.m+1));
alphahist[cset.m]=optcount;
cset.m++;
totconstraints++;
if((alg_type == ONESLACK_DUAL_ALG)
|| (alg_type == ONESLACK_DUAL_CACHE_ALG)) {
if(struct_verbosity>=2) rt2=get_runtime();
kparm->gram_matrix=update_kernel_matrix(kparm->gram_matrix,cset.m-1,
&cset,kparm);
if(struct_verbosity>=2) rt_kernel+=MAX(get_runtime()-rt2,0);
}
/**** get new QP solution ****/
if(struct_verbosity>=1) {
printf("*");fflush(stdout);
}
if(struct_verbosity>=2) rt2=get_runtime();
/* set svm precision so that higher than eps of most violated constr */
if(cached_constraint) {
epsilon_cached=MIN(epsilon_cached,ceps);
lparm->epsilon_crit=epsilon_cached/2;
}
else {
epsilon=MIN(epsilon,ceps); /* best eps so far */
lparm->epsilon_crit=epsilon/2;
epsilon_cached=epsilon;
}
free_model(svmModel,0);
svmModel=(MODEL *)my_malloc(sizeof(MODEL));
/* Run the QP solver on cset. */
kernel_type_org=kparm->kernel_type;
if((alg_type == ONESLACK_DUAL_ALG)
|| (alg_type == ONESLACK_DUAL_CACHE_ALG))
kparm->kernel_type=GRAM; /* use kernel stored in kparm */
svm_learn_optimization(cset.lhs,cset.rhs,cset.m,sizePsi,
lparm,kparm,NULL,svmModel,alpha);
kparm->kernel_type=kernel_type_org;
svmModel->kernel_parm.kernel_type=kernel_type_org;
/* Always add weight vector, in case part of the kernel is
linear. If not, ignore the weight vector since its
content is bogus. */
add_weight_vector_to_linear_model(svmModel);
sm->svm_model=svmModel;
sm->w=svmModel->lin_weights; /* short cut to weight vector */
optcount++;
/* keep track of when each constraint was last
active. constraints marked with -1 are not updated */
for(j=0;j<cset.m;j++)
if((alphahist[j]>-1) && (alpha[j] != 0))
alphahist[j]=optcount;
if(struct_verbosity>=2) rt_opt+=MAX(get_runtime()-rt2,0);
/* Check if some of the linear constraints have not been
active in a while. Those constraints are then removed to
avoid bloating the working set beyond necessity. */
if(struct_verbosity>=3)
printf("Reducing working set...");fflush(stdout);
remove_inactive_constraints(&cset,alpha,optcount,alphahist,50);
if(struct_verbosity>=3)
printf("done. ");
}
else {
free_svector(lhs);
}
if(struct_verbosity>=1)
printf("(NumConst=%d, SV=%ld, CEps=%.4f, QPEps=%.4f)\n",cset.m,
svmModel->sv_num-1,ceps,svmModel->maxdiff);
rt_total+=MAX(get_runtime()-rt1,0);
} while(finalize_iteration(ceps,cached_constraint,sample,sm,cset,alpha,sparm)|| cached_constraint || (ceps > sparm->epsilon) );
// originally like below ... finalize_iteration was not called because of short-circuit evaluation
// } while(cached_constraint || (ceps > sparm->epsilon) ||
// finalize_iteration(ceps,cached_constraint,sample,sm,cset,alpha,sparm)
// );
if(struct_verbosity>=1) {
printf("Final epsilon on KKT-Conditions: %.5f\n",
MAX(svmModel->maxdiff,ceps));
slack=0;
for(j=0;j<cset.m;j++)
slack=MAX(slack,
cset.rhs[j]-classify_example(svmModel,cset.lhs[j]));
alphasum=0;
for(i=0; i<cset.m; i++)
alphasum+=alpha[i]*cset.rhs[i];
if(kparm->kernel_type == LINEAR_KERNEL)
modellength=model_length_n(svmModel);
else
modellength=model_length_s(svmModel);
dualitygap=(0.5*modellength*modellength+sparm->C*viol)
-(alphasum-0.5*modellength*modellength);
printf("Upper bound on duality gap: %.5f\n", dualitygap);
printf("Dual objective value: dval=%.5f\n",
alphasum-0.5*modellength*modellength);
printf("Primal objective value: pval=%.5f\n",
0.5*modellength*modellength+sparm->C*viol);
printf("Total number of constraints in final working set: %i (of %i)\n",(int)cset.m,(int)totconstraints);
printf("Number of iterations: %d\n",numIt);
printf("Number of calls to 'find_most_violated_constraint': %ld\n",argmax_count);
printf("Number of SV: %ld \n",svmModel->sv_num-1);
printf("Norm of weight vector: |w|=%.5f\n",modellength);
printf("Value of slack variable (on working set): xi=%.5f\n",slack);
printf("Value of slack variable (global): xi=%.5f\n",viol);
printf("Norm of longest difference vector: ||Psi(x,y)-Psi(x,ybar)||=%.5f\n",
length_of_longest_document_vector(cset.lhs,cset.m,kparm));
if(struct_verbosity>=2)
printf("Runtime in cpu-seconds: %.2f (%.2f%% for QP, %.2f%% for kernel, %.2f%% for Argmax, %.2f%% for Psi, %.2f%% for init, %.2f%% for cache update, %.2f%% for cache const, %.2f%% for cache add (incl. %.2f%% for sum))\n",
rt_total/100.0, (100.0*rt_opt)/rt_total, (100.0*rt_kernel)/rt_total,
(100.0*rt_viol)/rt_total, (100.0*rt_psi)/rt_total,
(100.0*rt_init)/rt_total,(100.0*rt_cacheupdate)/rt_total,
(100.0*rt_cacheconst)/rt_total,(100.0*rt_cacheadd)/rt_total,
(100.0*rt_cachesum)/rt_total);
else if(struct_verbosity==1)
printf("Runtime in cpu-seconds: %.2f\n",rt_total/100.0);
}
if(ccache) {
long cnum=0;
CCACHEELEM *celem;
for(i=0;i<n;i++)
for(celem=ccache->constlist[i];celem;celem=celem->next)
cnum++;
printf("Final number of constraints in cache: %ld\n",cnum);
}
if(struct_verbosity>=4)
printW(sm->w,sizePsi,n,lparm->svm_c);
if(svmModel) {
sm->svm_model=copy_model(svmModel);
sm->w=sm->svm_model->lin_weights; /* short cut to weight vector */
free_model(svmModel,0);
}
print_struct_learning_stats(sample,sm,cset,alpha,sparm);
if(lhs_n)
free_nvector(lhs_n);
if(ccache)
free_constraint_cache(ccache);
for(i=0;i<n;i++)
if(fycache[i])
free_svector(fycache[i]);
free(fycache);
free(alpha);
free(alphahist);
free(cset.rhs);
for(i=0;i<cset.m;i++)
free_example(cset.lhs[i],1);
free(cset.lhs);
if(kparm->gram_matrix)
free_matrix(kparm->gram_matrix);
}
double add_constraint_to_constraint_cache(CCACHE *ccache, MODEL *svmModel, int exnum, SVECTOR *fydelta, double rhs, double gainthresh, int maxconst, double *rt_cachesum)
/* add new constraint fydelta*w>rhs for example exnum to cache,
if it is more violated (by gainthresh) than the currently most
violated constraint in cache. if this grows the number of
cached constraints for this example beyond maxconst, then the
least recently used constraint is deleted. the function
assumes that update_constraint_cache_for_model has been
run. */
{
double viol,viol_gain,viol_gain_trunc;
double dist_ydelta;
DOC *doc_fydelta;
SVECTOR *fydelta_new;
CCACHEELEM *celem;
int cnum;
double rt2=0;
/* compute violation of new constraint */
doc_fydelta=create_example(1,0,1,1,fydelta);
dist_ydelta=classify_example(svmModel,doc_fydelta);
free_example(doc_fydelta,0);
viol=rhs-dist_ydelta;
viol_gain=viol-ccache->constlist[exnum]->viol;
viol_gain_trunc=viol-MAX(ccache->constlist[exnum]->viol,0);
ccache->avg_viol_gain[exnum]=viol_gain;
/* check if violation of new constraint is larger than that of the
best cache element */
if(viol_gain > gainthresh) {
fydelta_new=fydelta;
if(struct_verbosity>=2) rt2=get_runtime();
if(svmModel->kernel_parm.kernel_type == LINEAR_KERNEL) {
if(COMPACT_CACHED_VECTORS == 1) { /* eval sum for linear */
fydelta_new=add_list_sort_ss_r(fydelta,COMPACT_ROUNDING_THRESH);
free_svector(fydelta);
}
else if(COMPACT_CACHED_VECTORS == 2) {
fydelta_new=add_list_ss_r(fydelta,COMPACT_ROUNDING_THRESH);
free_svector(fydelta);
}
else if(COMPACT_CACHED_VECTORS == 3) {
fydelta_new=add_list_ns_r(fydelta,COMPACT_ROUNDING_THRESH);
free_svector(fydelta);
}
}
if(struct_verbosity>=2) (*rt_cachesum)+=MAX(get_runtime()-rt2,0);
celem=ccache->constlist[exnum];
ccache->constlist[exnum]=(CCACHEELEM *)my_malloc(sizeof(CCACHEELEM));
ccache->constlist[exnum]->next=celem;
ccache->constlist[exnum]->fydelta=fydelta_new;
ccache->constlist[exnum]->rhs=rhs;
ccache->constlist[exnum]->viol=viol;
ccache->changed[exnum]+=2;
/* remove last constraint in list, if list is longer than maxconst */
cnum=2;
for(celem=ccache->constlist[exnum];celem && celem->next && celem->next->next;celem=celem->next)
cnum++;
if(cnum>maxconst) {
free_svector(celem->next->fydelta);
free(celem->next);
celem->next=NULL;
}
}
else {
free_svector(fydelta);
}
return(viol_gain_trunc);
}
