MODEL *compact_linear_model(MODEL *model)
/* Makes a copy of model where the support vectors are replaced
with a single linear weight vector. */
/* NOTE: It adds the linear weight vector also to newmodel->lin_weights */
/* WARNING: This is correct only for linear models! */
{
MODEL *newmodel;
newmodel=(MODEL *)my_malloc(sizeof(MODEL));
(*newmodel)=(*model);
add_weight_vector_to_linear_model(newmodel);
newmodel->supvec = (DOC **)my_malloc(sizeof(DOC *)*2);
newmodel->alpha = (double *)my_malloc(sizeof(double)*2);
newmodel->index = NULL; /* index is not copied */
newmodel->supvec[0] = NULL;
newmodel->alpha[0] = 0.0;
newmodel->supvec[1] = create_example(-1,0,0,0,
create_svector_n(newmodel->lin_weights,
newmodel->totwords,
NULL,1.0));
newmodel->alpha[1] = 1.0;
newmodel->sv_num=2;
return(newmodel);
}
int main (int argc, char* argv[])
{
MODEL *model;
read_input_parameters(argc,argv,modelfile,outfile, &verbosity, &format);
if (format) {
model=read_binary_model(modelfile);
} else {
model=read_model(modelfile);
if(model->kernel_parm.kernel_type == 0) { /* linear kernel */
/* compute weight vector */
add_weight_vector_to_linear_model(model);
}
}
if(model->kernel_parm.kernel_type == 0) { /* linear kernel */
FILE* modelfl = fopen (outfile, "wb");
if (modelfl==NULL)
{ perror (modelfile); exit (1); }
if (verbosity > 1)
fprintf(modelfl,"B=%.32g\n",model->b);
long i=0;
for (i= 0; i< model->totwords; ++i)
fprintf(modelfl,"%.32g\n",model->lin_weights[i]);
} else {
fprintf(stderr,"No output besides linear models\n");
}
free_model(model,1);
return(0);
}
int SVMModule::LoadSVMModelFile(std::string modelFile)
{
MODEL* model = read_model((char*)modelFile.c_str());
if(model == NULL)
return -1;
add_weight_vector_to_linear_model(model);
m_pTrainModel = model;
return 0;
}
void svm_learn_struct(SAMPLE sample, STRUCT_LEARN_PARM *sparm,
LEARN_PARM *lparm, KERNEL_PARM *kparm,
STRUCTMODEL *sm)
{
int i,j;
int numIt=0;
long newconstraints=0, activenum=0;
int opti_round, *opti;
long old_numConst=0;
double epsilon;
long tolerance;
double lossval,factor;
double margin=0;
double slack, *slacks, slacksum;
long sizePsi;
double *alpha=NULL;
CONSTSET cset;
SVECTOR *diff=NULL;
SVECTOR *fy, *fybar, *f;
SVECTOR *slackvec;
WORD slackv[2];
MODEL *svmModel=NULL;
KERNEL_CACHE *kcache=NULL;
LABEL ybar;
DOC *doc;
long n=sample.n;
EXAMPLE *ex=sample.examples;
double rt_total=0.0, rt_opt=0.0;
long rt1,rt2;
init_struct_model(sample,sm,sparm);
sizePsi=sm->sizePsi+1; /* sm must contain size of psi on return */
/* initialize example selection heuristic */
opti=(int*)my_malloc(n*sizeof(int));
for(i=0;i<n;i++) {
opti[i]=0;
}
opti_round=0;
if(sparm->slack_norm == 1) {
lparm->svm_c=sparm->C; /* set upper bound C */
lparm->sharedslack=1;
}
else if(sparm->slack_norm == 2) {
lparm->svm_c=999999999999999.0; /* upper bound C must never be reached */
lparm->sharedslack=0;
if(kparm->kernel_type != LINEAR) {
printf("ERROR: Kernels are not implemented for L2 slack norm!");
fflush(stdout);
exit(0);
}
}
else {
printf("ERROR: Slack norm must be L1 or L2!"); fflush(stdout);
exit(0);
}
epsilon=1.0; /* start with low precision and
increase later */
tolerance=n/100; /* increase precision, whenever less
than that number of constraints
is not fulfilled */
lparm->biased_hyperplane=0; /* set threshold to zero */
cset=init_struct_constraints(sample, sm, sparm);
if(cset.m > 0) {
alpha=realloc(alpha,sizeof(double)*cset.m);
for(i=0; i<cset.m; i++)
alpha[i]=0;
}
/* set initial model and slack variables*/
svmModel=(MODEL *)my_malloc(sizeof(MODEL));
svm_learn_optimization(cset.lhs,cset.rhs,cset.m,sizePsi+n,
lparm,kparm,NULL,svmModel,alpha);
add_weight_vector_to_linear_model(svmModel);
sm->svm_model=svmModel;
sm->w=svmModel->lin_weights; /* short cut to weight vector */
printf("Starting Iterations\n");
/*****************/
/*** main loop ***/
/*****************/
do { /* iteratively increase precision */
epsilon=MAX(epsilon*0.09999999999,sparm->epsilon);
if(epsilon == sparm->epsilon) /* for final precision, find all SV */
tolerance=0;
lparm->epsilon_crit=epsilon/2; /* svm precision must be higher than eps */
if(struct_verbosity>=1)
printf("Setting current working precision to %g.\n",epsilon);
do { /* iteration until (approx) all SV are found for current
precision and tolerance */
old_numConst=cset.m;
opti_round++;
activenum=n;
do { /* go through examples that keep producing new constraints */
if(struct_verbosity>=1) {
printf("--Iteration %i (%ld active): ",++numIt,activenum);
fflush(stdout);
}
for(i=0; i<n; i++) { /*** example loop ***/
rt1=get_runtime();
if(opti[i] != opti_round) {/* if the example is not shrunk
away, then see if it is necessary to
add a new constraint */
if(sparm->loss_type == SLACK_RESCALING)
ybar=find_most_violated_constraint_slackrescaling(ex[i].x,
ex[i].y,sm,
sparm);
else
ybar=find_most_violated_constraint_marginrescaling(ex[i].x,
ex[i].y,sm,
sparm);
if(empty_label(ybar)) {
if(opti[i] != opti_round) {
activenum--;
opti[i]=opti_round;
}
if(struct_verbosity>=2)
printf("no-incorrect-found(%i) ",i);
continue;
}
/**** get psi(y)-psi(ybar) ****/
fy=psi(ex[i].x,ex[i].y,sm,sparm);
fybar=psi(ex[i].x,ybar,sm,sparm);
/**** scale feature vector and margin by loss ****/
lossval=loss(ex[i].y,ybar,sparm);
if(sparm->slack_norm == 2)
lossval=sqrt(lossval);
if(sparm->loss_type == SLACK_RESCALING)
factor=lossval;
else /* do not rescale vector for */
factor=1.0; /* margin rescaling loss type */
for(f=fy;f;f=f->next)
f->factor*=factor;
for(f=fybar;f;f=f->next)
f->factor*=-factor;
margin=lossval;
/**** create constraint for current ybar ****/
append_svector_list(fy,fybar);/* append the two vector lists */
doc=create_example(cset.m,0,i+1,1,fy);
/**** compute slack for this example ****/
slack=0;
for(j=0;j<cset.m;j++)
if(cset.lhs[j]->slackid == i+1) {
if(sparm->slack_norm == 2) /* works only for linear kernel */
slack=MAX(slack,cset.rhs[j]
-(classify_example(svmModel,cset.lhs[j])
-sm->w[sizePsi+i]/(sqrt(2*sparm->C))));
else
slack=MAX(slack,
cset.rhs[j]-classify_example(svmModel,cset.lhs[j]));
}
/**** if `error' add constraint and recompute ****/
if((classify_example(svmModel,doc)+slack)<(margin-epsilon)) {
if(struct_verbosity>=2)
{printf("(%i) ",i); fflush(stdout);}
if(struct_verbosity==1)
{printf("."); fflush(stdout);}
/**** resize constraint matrix and add new constraint ****/
cset.m++;
cset.lhs=realloc(cset.lhs,sizeof(DOC *)*cset.m);
if(kparm->kernel_type == LINEAR) {
diff=add_list_ss(fy); /* store difference vector directly */
if(sparm->slack_norm == 1)
cset.lhs[cset.m-1]=create_example(cset.m-1,0,i+1,1,
copy_svector(diff));
else if(sparm->slack_norm == 2) {
/**** add squared slack variable to feature vector ****/
slackv[0].wnum=sizePsi+i;
slackv[0].weight=1/(sqrt(2*sparm->C));
slackv[1].wnum=0; /*terminator*/
slackvec=create_svector(slackv,"",1.0);
cset.lhs[cset.m-1]=create_example(cset.m-1,0,i+1,1,
add_ss(diff,slackvec));
free_svector(slackvec);
}
free_svector(diff);
}
else { /* kernel is used */
if(sparm->slack_norm == 1)
cset.lhs[cset.m-1]=create_example(cset.m-1,0,i+1,1,
copy_svector(fy));
else if(sparm->slack_norm == 2)
exit(1);
}
cset.rhs=realloc(cset.rhs,sizeof(double)*cset.m);
cset.rhs[cset.m-1]=margin;
alpha=realloc(alpha,sizeof(double)*cset.m);
alpha[cset.m-1]=0;
newconstraints++;
}
else {
printf("+"); fflush(stdout);
if(opti[i] != opti_round) {
activenum--;
opti[i]=opti_round;
}
}
free_example(doc,0);
free_svector(fy); /* this also free's fybar */
free_label(ybar);
}
/**** get new QP solution ****/
if((newconstraints >= sparm->newconstretrain)
|| ((newconstraints > 0) && (i == n-1))) {
if(struct_verbosity>=1) {
printf("*");fflush(stdout);
}
rt2=get_runtime();
free_model(svmModel,0);
svmModel=(MODEL *)my_malloc(sizeof(MODEL));
/* Always get a new kernel cache. It is not possible to use the
same cache for two different training runs */
if(kparm->kernel_type != LINEAR)
kcache=kernel_cache_init(cset.m,lparm->kernel_cache_size);
/* Run the QP solver on cset. */
svm_learn_optimization(cset.lhs,cset.rhs,cset.m,sizePsi+n,
lparm,kparm,kcache,svmModel,alpha);
if(kcache)
kernel_cache_cleanup(kcache);
/* Always add weight vector, in case part of the kernel is
linear. If not, ignore the weight vector since its
content is bogus. */
add_weight_vector_to_linear_model(svmModel);
sm->svm_model=svmModel;
sm->w=svmModel->lin_weights; /* short cut to weight vector */
rt_opt+=MAX(get_runtime()-rt2,0);
newconstraints=0;
}
rt_total+=MAX(get_runtime()-rt1,0);
} /* end of example loop */
if(struct_verbosity>=1)
printf("(NumConst=%d, SV=%ld, Eps=%.4f)\n",cset.m,svmModel->sv_num-1,
svmModel->maxdiff);
} while(activenum > 0); /* repeat until all examples produced no
constraint at least once */
} while((cset.m - old_numConst) > tolerance) ;
} while(epsilon > sparm->epsilon);
if(struct_verbosity>=1) {
/**** compute sum of slacks ****/
slacks=(double *)my_malloc(sizeof(double)*(n+1));
for(i=0; i<=n; i++) {
slacks[i]=0;
}
if(sparm->slack_norm == 1) {
for(j=0;j<cset.m;j++)
slacks[cset.lhs[j]->slackid]=MAX(slacks[cset.lhs[j]->slackid],
cset.rhs[j]-classify_example(svmModel,cset.lhs[j]));
}
else if(sparm->slack_norm == 2) {
for(j=0;j<cset.m;j++)
slacks[cset.lhs[j]->slackid]=MAX(slacks[cset.lhs[j]->slackid],
cset.rhs[j]
-(classify_example(svmModel,cset.lhs[j])
-sm->w[sizePsi+cset.lhs[j]->slackid-1]/(sqrt(2*sparm->C))));
}
slacksum=0;
for(i=0; i<=n; i++)
slacksum+=slacks[i];
free(slacks);
printf("Final epsilon on KKT-Conditions: %.5f\n",
MAX(svmModel->maxdiff,epsilon));
printf("Total number of constraints added: %i\n",(int)cset.m);
if(sparm->slack_norm == 1) {
printf("Number of SV: %ld \n",svmModel->sv_num-1);
printf("Number of non-zero slack variables: %ld (out of %ld)\n",
svmModel->at_upper_bound,n);
printf("Norm of weight vector: |w|=%.5f\n",
model_length_s(svmModel,kparm));
}
else if(sparm->slack_norm == 2){
printf("Number of SV: %ld (including %ld at upper bound)\n",
svmModel->sv_num-1,svmModel->at_upper_bound);
printf("Norm of weight vector (including L2-loss): |w|=%.5f\n",
model_length_s(svmModel,kparm));
}
printf("Sum of slack variables: sum(xi_i)=%.5f\n",slacksum);
printf("Norm of longest difference vector: ||Psi(x,y)-Psi(x,ybar)||=%.5f\n",
length_of_longest_document_vector(cset.lhs,cset.m,kparm));
printf("Runtime in cpu-seconds: %.2f (%.2f%% for SVM optimization)\n",
rt_total/100.0, 100.0*rt_opt/rt_total);
}
if(struct_verbosity>=4)
printW(sm->w,sizePsi,n,lparm->svm_c);
if(svmModel) {
sm->svm_model=copy_model(svmModel);
sm->w=sm->svm_model->lin_weights; /* short cut to weight vector */
}
print_struct_learning_stats(sample,sm,cset,alpha,sparm);
if(svmModel)
free_model(svmModel,0);
free(alpha);
free(opti);
free(cset.rhs);
for(i=0;i<cset.m;i++)
free_example(cset.lhs[i],1);
free(cset.lhs);
}
void svm_learn_struct_joint(SAMPLE sample, STRUCT_LEARN_PARM *sparm,
LEARN_PARM *lparm, KERNEL_PARM *kparm,
STRUCTMODEL *sm, int alg_type)
{
int i,j;
int numIt=0;
long argmax_count=0;
long totconstraints=0;
long kernel_type_org;
double epsilon,epsilon_cached;
double lhsXw,rhs_i;
double rhs=0;
double slack,ceps;
double dualitygap,modellength,alphasum;
long sizePsi;
double *alpha=NULL;
long *alphahist=NULL,optcount=0;
CONSTSET cset;
SVECTOR *diff=NULL;
double *lhs_n=NULL;
SVECTOR *fy, *fydelta, **fycache, *lhs;
MODEL *svmModel=NULL;
DOC *doc;
long n=sample.n;
EXAMPLE *ex=sample.examples;
double rt_total=0,rt_opt=0,rt_init=0,rt_psi=0,rt_viol=0,rt_kernel=0;
double rt_cacheupdate=0,rt_cacheconst=0,rt_cacheadd=0,rt_cachesum=0;
double rt1=0,rt2=0;
long progress;
/*
SVECTOR ***fydelta_cache=NULL;
double **loss_cache=NULL;
int cache_size=0;
*/
CCACHE *ccache=NULL;
int cached_constraint;
double viol,viol_est,epsilon_est=0;
long uptr=0;
long *randmapping=NULL;
long batch_size=n;
rt1=get_runtime();
if(sparm->batch_size<100)
batch_size=sparm->batch_size*n/100.0;
init_struct_model(sample,sm,sparm,lparm,kparm);
sizePsi=sm->sizePsi+1; /* sm must contain size of psi on return */
if(sparm->slack_norm == 1) {
lparm->svm_c=sparm->C; /* set upper bound C */
lparm->sharedslack=1;
}
else if(sparm->slack_norm == 2) {
printf("ERROR: The joint algorithm does not apply to L2 slack norm!");
fflush(stdout);
exit(0);
}
else {
printf("ERROR: Slack norm must be L1 or L2!"); fflush(stdout);
exit(0);
}
lparm->biased_hyperplane=0; /* set threshold to zero */
epsilon=100.0; /* start with low precision and
increase later */
epsilon_cached=epsilon; /* epsilon to use for iterations
using constraints constructed
from the constraint cache */
cset=init_struct_constraints(sample, sm, sparm);
if(cset.m > 0) {
alpha=(double *)realloc(alpha,sizeof(double)*cset.m);
alphahist=(long *)realloc(alphahist,sizeof(long)*cset.m);
for(i=0; i<cset.m; i++) {
alpha[i]=0;
alphahist[i]=-1; /* -1 makes sure these constraints are never removed */
}
}
kparm->gram_matrix=NULL;
if((alg_type == ONESLACK_DUAL_ALG) || (alg_type == ONESLACK_DUAL_CACHE_ALG))
kparm->gram_matrix=init_kernel_matrix(&cset,kparm);
/* set initial model and slack variables */
svmModel=(MODEL *)my_malloc(sizeof(MODEL));
lparm->epsilon_crit=epsilon;
svm_learn_optimization(cset.lhs,cset.rhs,cset.m,sizePsi,
lparm,kparm,NULL,svmModel,alpha);
add_weight_vector_to_linear_model(svmModel);
sm->svm_model=svmModel;
sm->w=svmModel->lin_weights; /* short cut to weight vector */
/* create a cache of the feature vectors for the correct labels */
fycache=(SVECTOR **)my_malloc(n*sizeof(SVECTOR *));
for(i=0;i<n;i++) {
if(USE_FYCACHE) {
fy=psi(ex[i].x,ex[i].y,sm,sparm);
if(kparm->kernel_type == LINEAR_KERNEL) { /* store difference vector directly */
diff=add_list_sort_ss_r(fy,COMPACT_ROUNDING_THRESH);
free_svector(fy);
fy=diff;
}
}
else
fy=NULL;
fycache[i]=fy;
}
/* initialize the constraint cache */
if(alg_type == ONESLACK_DUAL_CACHE_ALG) {
ccache=create_constraint_cache(sample,sparm,sm);
/* NOTE: */
for(i=0;i<n;i++)
if(loss(ex[i].y,ex[i].y,sparm) != 0) {
printf("ERROR: Loss function returns non-zero value loss(y_%d,y_%d)\n",i,i);
printf(" W4 algorithm assumes that loss(y_i,y_i)=0 for all i.\n");
exit(1);
}
}
if(kparm->kernel_type == LINEAR_KERNEL)
lhs_n=create_nvector(sm->sizePsi);
/* randomize order or training examples */
if(batch_size<n)
randmapping=random_order(n);
rt_init+=MAX(get_runtime()-rt1,0);
rt_total+=rt_init;
/*****************/
/*** main loop ***/
/*****************/
do { /* iteratively find and add constraints to working set */
if(struct_verbosity>=1) {
printf("Iter %i: ",++numIt);
fflush(stdout);
}
rt1=get_runtime();
/**** compute current slack ****/
alphasum=0;
for(j=0;(j<cset.m);j++)
alphasum+=alpha[j];
for(j=0,slack=-1;(j<cset.m) && (slack==-1);j++)
if(alpha[j] > alphasum/cset.m)
slack=MAX(0,cset.rhs[j]-classify_example(svmModel,cset.lhs[j]));
slack=MAX(0,slack);
rt_total+=MAX(get_runtime()-rt1,0);
/**** find a violated joint constraint ****/
lhs=NULL;
rhs=0;
if(alg_type == ONESLACK_DUAL_CACHE_ALG) {
rt1=get_runtime();
/* Compute violation of constraints in cache for current w */
if(struct_verbosity>=2) rt2=get_runtime();
update_constraint_cache_for_model(ccache, svmModel);
if(struct_verbosity>=2) rt_cacheupdate+=MAX(get_runtime()-rt2,0);
/* Is there is a sufficiently violated constraint in cache? */
viol=compute_violation_of_constraint_in_cache(ccache,epsilon_est/2);
if(viol-slack > MAX(epsilon_est/10,sparm->epsilon)) {
/* There is a sufficiently violated constraint in cache, so
use this constraint in this iteration. */
if(struct_verbosity>=2) rt2=get_runtime();
viol=find_most_violated_joint_constraint_in_cache(ccache,
epsilon_est/2,lhs_n,&lhs,&rhs);
if(struct_verbosity>=2) rt_cacheconst+=MAX(get_runtime()-rt2,0);
cached_constraint=1;
}
else {
/* There is no sufficiently violated constraint in cache, so
update cache by computing most violated constraint
explicitly for batch_size examples. */
viol_est=0;
progress=0;
viol=compute_violation_of_constraint_in_cache(ccache,0);
for(j=0;(j<batch_size) || ((j<n)&&(viol-slack<sparm->epsilon));j++) {
if(struct_verbosity>=1)
print_percent_progress(&progress,n,10,".");
uptr=uptr % n;
if(randmapping)
i=randmapping[uptr];
else
i=uptr;
/* find most violating fydelta=fy-fybar and rhs for example i */
find_most_violated_constraint(&fydelta,&rhs_i,&ex[i],
fycache[i],n,sm,sparm,
&rt_viol,&rt_psi,&argmax_count);
/* add current fy-fybar and loss to cache */
if(struct_verbosity>=2) rt2=get_runtime();
viol+=add_constraint_to_constraint_cache(ccache,sm->svm_model,
i,fydelta,rhs_i,0.0001*sparm->epsilon/n,
sparm->ccache_size,&rt_cachesum);
if(struct_verbosity>=2) rt_cacheadd+=MAX(get_runtime()-rt2,0);
viol_est+=ccache->constlist[i]->viol;
uptr++;
}
cached_constraint=(j<n);
if(struct_verbosity>=2) rt2=get_runtime();
if(cached_constraint)
viol=find_most_violated_joint_constraint_in_cache(ccache,
epsilon_est/2,lhs_n,&lhs,&rhs);
else
viol=find_most_violated_joint_constraint_in_cache(ccache,0,lhs_n,
&lhs,&rhs);
if(struct_verbosity>=2) rt_cacheconst+=MAX(get_runtime()-rt2,0);
viol_est*=((double)n/j);
epsilon_est=(1-(double)j/n)*epsilon_est+(double)j/n*(viol_est-slack);
if((struct_verbosity >= 1) && (j!=n))
printf("(upd=%5.1f%%,eps^=%.4f,eps*=%.4f)",
100.0*j/n,viol_est-slack,epsilon_est);
}
lhsXw=rhs-viol;
rt_total+=MAX(get_runtime()-rt1,0);
}
else {
/* do not use constraint from cache */
rt1=get_runtime();
cached_constraint=0;
if(kparm->kernel_type == LINEAR_KERNEL)
clear_nvector(lhs_n,sm->sizePsi);
progress=0;
rt_total+=MAX(get_runtime()-rt1,0);
for(i=0; i<n; i++) {
rt1=get_runtime();
if(struct_verbosity>=1)
print_percent_progress(&progress,n,10,".");
/* compute most violating fydelta=fy-fybar and rhs for example i */
find_most_violated_constraint(&fydelta,&rhs_i,&ex[i],fycache[i],n,
sm,sparm,&rt_viol,&rt_psi,&argmax_count);
/* add current fy-fybar to lhs of constraint */
if(kparm->kernel_type == LINEAR_KERNEL) {
add_list_n_ns(lhs_n,fydelta,1.0); /* add fy-fybar to sum */
free_svector(fydelta);
}
else {
append_svector_list(fydelta,lhs); /* add fy-fybar to vector list */
lhs=fydelta;
}
rhs+=rhs_i; /* add loss to rhs */
rt_total+=MAX(get_runtime()-rt1,0);
} /* end of example loop */
rt1=get_runtime();
/* create sparse vector from dense sum */
if(kparm->kernel_type == LINEAR_KERNEL)
lhs=create_svector_n_r(lhs_n,sm->sizePsi,NULL,1.0,
COMPACT_ROUNDING_THRESH);
doc=create_example(cset.m,0,1,1,lhs);
lhsXw=classify_example(svmModel,doc);
free_example(doc,0);
viol=rhs-lhsXw;
rt_total+=MAX(get_runtime()-rt1,0);
} /* end of finding most violated joint constraint */
rt1=get_runtime();
/**** if `error', then add constraint and recompute QP ****/
if(slack > (rhs-lhsXw+0.000001)) {
printf("\nWARNING: Slack of most violated constraint is smaller than slack of working\n");
printf(" set! There is probably a bug in 'find_most_violated_constraint_*'.\n");
printf("slack=%f, newslack=%f\n",slack,rhs-lhsXw);
/* exit(1); */
}
ceps=MAX(0,rhs-lhsXw-slack);
if((ceps > sparm->epsilon) || cached_constraint) {
/**** resize constraint matrix and add new constraint ****/
cset.lhs=(DOC **)realloc(cset.lhs,sizeof(DOC *)*(cset.m+1));
cset.lhs[cset.m]=create_example(cset.m,0,1,1,lhs);
cset.rhs=(double *)realloc(cset.rhs,sizeof(double)*(cset.m+1));
cset.rhs[cset.m]=rhs;
alpha=(double *)realloc(alpha,sizeof(double)*(cset.m+1));
alpha[cset.m]=0;
alphahist=(long *)realloc(alphahist,sizeof(long)*(cset.m+1));
alphahist[cset.m]=optcount;
cset.m++;
totconstraints++;
if((alg_type == ONESLACK_DUAL_ALG)
|| (alg_type == ONESLACK_DUAL_CACHE_ALG)) {
if(struct_verbosity>=2) rt2=get_runtime();
kparm->gram_matrix=update_kernel_matrix(kparm->gram_matrix,cset.m-1,
&cset,kparm);
if(struct_verbosity>=2) rt_kernel+=MAX(get_runtime()-rt2,0);
}
/**** get new QP solution ****/
if(struct_verbosity>=1) {
printf("*");fflush(stdout);
}
if(struct_verbosity>=2) rt2=get_runtime();
/* set svm precision so that higher than eps of most violated constr */
if(cached_constraint) {
epsilon_cached=MIN(epsilon_cached,ceps);
lparm->epsilon_crit=epsilon_cached/2;
}
else {
epsilon=MIN(epsilon,ceps); /* best eps so far */
lparm->epsilon_crit=epsilon/2;
epsilon_cached=epsilon;
}
free_model(svmModel,0);
svmModel=(MODEL *)my_malloc(sizeof(MODEL));
/* Run the QP solver on cset. */
kernel_type_org=kparm->kernel_type;
if((alg_type == ONESLACK_DUAL_ALG)
|| (alg_type == ONESLACK_DUAL_CACHE_ALG))
kparm->kernel_type=GRAM; /* use kernel stored in kparm */
svm_learn_optimization(cset.lhs,cset.rhs,cset.m,sizePsi,
lparm,kparm,NULL,svmModel,alpha);
kparm->kernel_type=kernel_type_org;
svmModel->kernel_parm.kernel_type=kernel_type_org;
/* Always add weight vector, in case part of the kernel is
linear. If not, ignore the weight vector since its
content is bogus. */
add_weight_vector_to_linear_model(svmModel);
sm->svm_model=svmModel;
sm->w=svmModel->lin_weights; /* short cut to weight vector */
optcount++;
/* keep track of when each constraint was last
active. constraints marked with -1 are not updated */
for(j=0;j<cset.m;j++)
if((alphahist[j]>-1) && (alpha[j] != 0))
alphahist[j]=optcount;
if(struct_verbosity>=2) rt_opt+=MAX(get_runtime()-rt2,0);
/* Check if some of the linear constraints have not been
active in a while. Those constraints are then removed to
avoid bloating the working set beyond necessity. */
if(struct_verbosity>=3)
printf("Reducing working set...");fflush(stdout);
remove_inactive_constraints(&cset,alpha,optcount,alphahist,50);
if(struct_verbosity>=3)
printf("done. ");
}
else {
free_svector(lhs);
}
if(struct_verbosity>=1)
printf("(NumConst=%d, SV=%ld, CEps=%.4f, QPEps=%.4f)\n",cset.m,
svmModel->sv_num-1,ceps,svmModel->maxdiff);
rt_total+=MAX(get_runtime()-rt1,0);
} while(finalize_iteration(ceps,cached_constraint,sample,sm,cset,alpha,sparm)|| cached_constraint || (ceps > sparm->epsilon) );
// originally like below ... finalize_iteration was not called because of short-circuit evaluation
// } while(cached_constraint || (ceps > sparm->epsilon) ||
// finalize_iteration(ceps,cached_constraint,sample,sm,cset,alpha,sparm)
// );
if(struct_verbosity>=1) {
printf("Final epsilon on KKT-Conditions: %.5f\n",
MAX(svmModel->maxdiff,ceps));
slack=0;
for(j=0;j<cset.m;j++)
slack=MAX(slack,
cset.rhs[j]-classify_example(svmModel,cset.lhs[j]));
alphasum=0;
for(i=0; i<cset.m; i++)
alphasum+=alpha[i]*cset.rhs[i];
if(kparm->kernel_type == LINEAR_KERNEL)
modellength=model_length_n(svmModel);
else
modellength=model_length_s(svmModel);
dualitygap=(0.5*modellength*modellength+sparm->C*viol)
-(alphasum-0.5*modellength*modellength);
printf("Upper bound on duality gap: %.5f\n", dualitygap);
printf("Dual objective value: dval=%.5f\n",
alphasum-0.5*modellength*modellength);
printf("Primal objective value: pval=%.5f\n",
0.5*modellength*modellength+sparm->C*viol);
printf("Total number of constraints in final working set: %i (of %i)\n",(int)cset.m,(int)totconstraints);
printf("Number of iterations: %d\n",numIt);
printf("Number of calls to 'find_most_violated_constraint': %ld\n",argmax_count);
printf("Number of SV: %ld \n",svmModel->sv_num-1);
printf("Norm of weight vector: |w|=%.5f\n",modellength);
printf("Value of slack variable (on working set): xi=%.5f\n",slack);
printf("Value of slack variable (global): xi=%.5f\n",viol);
printf("Norm of longest difference vector: ||Psi(x,y)-Psi(x,ybar)||=%.5f\n",
length_of_longest_document_vector(cset.lhs,cset.m,kparm));
if(struct_verbosity>=2)
printf("Runtime in cpu-seconds: %.2f (%.2f%% for QP, %.2f%% for kernel, %.2f%% for Argmax, %.2f%% for Psi, %.2f%% for init, %.2f%% for cache update, %.2f%% for cache const, %.2f%% for cache add (incl. %.2f%% for sum))\n",
rt_total/100.0, (100.0*rt_opt)/rt_total, (100.0*rt_kernel)/rt_total,
(100.0*rt_viol)/rt_total, (100.0*rt_psi)/rt_total,
(100.0*rt_init)/rt_total,(100.0*rt_cacheupdate)/rt_total,
(100.0*rt_cacheconst)/rt_total,(100.0*rt_cacheadd)/rt_total,
(100.0*rt_cachesum)/rt_total);
else if(struct_verbosity==1)
printf("Runtime in cpu-seconds: %.2f\n",rt_total/100.0);
}
if(ccache) {
long cnum=0;
CCACHEELEM *celem;
for(i=0;i<n;i++)
for(celem=ccache->constlist[i];celem;celem=celem->next)
cnum++;
printf("Final number of constraints in cache: %ld\n",cnum);
}
if(struct_verbosity>=4)
printW(sm->w,sizePsi,n,lparm->svm_c);
if(svmModel) {
sm->svm_model=copy_model(svmModel);
sm->w=sm->svm_model->lin_weights; /* short cut to weight vector */
free_model(svmModel,0);
}
print_struct_learning_stats(sample,sm,cset,alpha,sparm);
if(lhs_n)
free_nvector(lhs_n);
if(ccache)
free_constraint_cache(ccache);
for(i=0;i<n;i++)
if(fycache[i])
free_svector(fycache[i]);
free(fycache);
free(alpha);
free(alphahist);
free(cset.rhs);
for(i=0;i<cset.m;i++)
free_example(cset.lhs[i],1);
free(cset.lhs);
if(kparm->gram_matrix)
free_matrix(kparm->gram_matrix);
}
void svm_learn_struct(SAMPLE sample, STRUCT_LEARN_PARM *sparm,
LEARN_PARM *lparm, KERNEL_PARM *kparm,
STRUCTMODEL *sm, int alg_type)
{
int i,j;
int numIt=0;
long argmax_count=0;
long newconstraints=0, totconstraints=0, activenum=0;
int opti_round, *opti, fullround, use_shrinking;
long old_totconstraints=0;
double epsilon,svmCnorm;
long tolerance,new_precision=1,dont_stop=0;
double lossval,factor,dist;
double margin=0;
double slack, *slacks, slacksum, ceps;
double dualitygap,modellength,alphasum;
long sizePsi;
double *alpha=NULL;
long *alphahist=NULL,optcount=0,lastoptcount=0;
CONSTSET cset;
SVECTOR *diff=NULL;
SVECTOR *fy, *fybar, *f, **fycache=NULL;
SVECTOR *slackvec;
WORD slackv[2];
MODEL *svmModel=NULL;
KERNEL_CACHE *kcache=NULL;
LABEL ybar;
DOC *doc;
long n=sample.n;
EXAMPLE *ex=sample.examples;
double rt_total=0, rt_opt=0, rt_init=0, rt_psi=0, rt_viol=0;
double rt1,rt2;
rt1=get_runtime();
init_struct_model(sample,sm,sparm,lparm,kparm);
sizePsi=sm->sizePsi+1; /* sm must contain size of psi on return */
/* initialize shrinking-style example selection heuristic */
if(alg_type == NSLACK_SHRINK_ALG)
use_shrinking=1;
else
use_shrinking=0;
opti=(int*)my_malloc(n*sizeof(int));
for(i=0;i<n;i++) {
opti[i]=0;
}
opti_round=0;
/* normalize regularization parameter C by the number of training examples */
svmCnorm=sparm->C/n;
if(sparm->slack_norm == 1) {
lparm->svm_c=svmCnorm; /* set upper bound C */
lparm->sharedslack=1;
}
else if(sparm->slack_norm == 2) {
lparm->svm_c=999999999999999.0; /* upper bound C must never be reached */
lparm->sharedslack=0;
if(kparm->kernel_type != LINEAR_KERNEL) {
printf("ERROR: Kernels are not implemented for L2 slack norm!");
fflush(stdout);
exit(0);
}
}
else {
printf("ERROR: Slack norm must be L1 or L2!"); fflush(stdout);
exit(0);
}
epsilon=100.0; /* start with low precision and
increase later */
tolerance=MIN(n/3,MAX(n/100,5));/* increase precision, whenever less
than that number of constraints
is not fulfilled */
lparm->biased_hyperplane=0; /* set threshold to zero */
cset=init_struct_constraints(sample, sm, sparm);
if(cset.m > 0) {
alpha=(double *)realloc(alpha,sizeof(double)*cset.m);
alphahist=(long *)realloc(alphahist,sizeof(long)*cset.m);
for(i=0; i<cset.m; i++) {
alpha[i]=0;
alphahist[i]=-1; /* -1 makes sure these constraints are never removed */
}
}
/* set initial model and slack variables*/
svmModel=(MODEL *)my_malloc(sizeof(MODEL));
lparm->epsilon_crit=epsilon;
if(kparm->kernel_type != LINEAR_KERNEL)
kcache=kernel_cache_init(MAX(cset.m,1),lparm->kernel_cache_size);
svm_learn_optimization(cset.lhs,cset.rhs,cset.m,sizePsi+n,
lparm,kparm,kcache,svmModel,alpha);
if(kcache)
kernel_cache_cleanup(kcache);
add_weight_vector_to_linear_model(svmModel);
sm->svm_model=svmModel;
sm->w=svmModel->lin_weights; /* short cut to weight vector */
/* create a cache of the feature vectors for the correct labels */
if(USE_FYCACHE) {
fycache=(SVECTOR **)my_malloc(n*sizeof(SVECTOR *));
for(i=0;i<n;i++) {
fy=psi(ex[i].x,ex[i].y,sm,sparm);
if(kparm->kernel_type == LINEAR_KERNEL) {
diff=add_list_ss(fy); /* store difference vector directly */
free_svector(fy);
fy=diff;
}
fycache[i]=fy;
}
}
rt_init+=MAX(get_runtime()-rt1,0);
rt_total+=MAX(get_runtime()-rt1,0);
/*****************/
/*** main loop ***/
/*****************/
do { /* iteratively increase precision */
epsilon=MAX(epsilon*0.49999999999,sparm->epsilon);
new_precision=1;
if(epsilon == sparm->epsilon) /* for final precision, find all SV */
tolerance=0;
lparm->epsilon_crit=epsilon/2; /* svm precision must be higher than eps */
if(struct_verbosity>=1)
printf("Setting current working precision to %g.\n",epsilon);
do { /* iteration until (approx) all SV are found for current
precision and tolerance */
opti_round++;
activenum=n;
dont_stop=0;
old_totconstraints=totconstraints;
do { /* with shrinking turned on, go through examples that keep
producing new constraints */
if(struct_verbosity>=1) {
printf("Iter %i (%ld active): ",++numIt,activenum);
fflush(stdout);
}
ceps=0;
fullround=(activenum == n);
for(i=0; i<n; i++) { /*** example loop ***/
rt1=get_runtime();
if((!use_shrinking) || (opti[i] != opti_round)) {
/* if the example is not shrunk
away, then see if it is necessary to
add a new constraint */
rt2=get_runtime();
argmax_count++;
if(sparm->loss_type == SLACK_RESCALING)
ybar=find_most_violated_constraint_slackrescaling(ex[i].x,
ex[i].y,sm,
sparm);
else
ybar=find_most_violated_constraint_marginrescaling(ex[i].x,
ex[i].y,sm,
sparm);
rt_viol+=MAX(get_runtime()-rt2,0);
if(empty_label(ybar)) {
if(opti[i] != opti_round) {
activenum--;
opti[i]=opti_round;
}
if(struct_verbosity>=2)
printf("no-incorrect-found(%i) ",i);
continue;
}
/**** get psi(y)-psi(ybar) ****/
rt2=get_runtime();
if(fycache)
fy=copy_svector(fycache[i]);
else
fy=psi(ex[i].x,ex[i].y,sm,sparm);
fybar=psi(ex[i].x,ybar,sm,sparm);
rt_psi+=MAX(get_runtime()-rt2,0);
/**** scale feature vector and margin by loss ****/
lossval=loss(ex[i].y,ybar,sparm);
if(sparm->slack_norm == 2)
lossval=sqrt(lossval);
if(sparm->loss_type == SLACK_RESCALING)
factor=lossval;
else /* do not rescale vector for */
factor=1.0; /* margin rescaling loss type */
for(f=fy;f;f=f->next)
f->factor*=factor;
for(f=fybar;f;f=f->next)
f->factor*=-factor;
margin=lossval;
/**** create constraint for current ybar ****/
append_svector_list(fy,fybar);/* append the two vector lists */
doc=create_example(cset.m,0,i+1,1,fy);
/**** compute slack for this example ****/
slack=0;
for(j=0;j<cset.m;j++)
if(cset.lhs[j]->slackid == i+1) {
if(sparm->slack_norm == 2) /* works only for linear kernel */
slack=MAX(slack,cset.rhs[j]
-(classify_example(svmModel,cset.lhs[j])
-sm->w[sizePsi+i]/(sqrt(2*svmCnorm))));
else
slack=MAX(slack,
cset.rhs[j]-classify_example(svmModel,cset.lhs[j]));
}
/**** if `error' add constraint and recompute ****/
dist=classify_example(svmModel,doc);
ceps=MAX(ceps,margin-dist-slack);
if(slack > (margin-dist+0.0001)) {
printf("\nWARNING: Slack of most violated constraint is smaller than slack of working\n");
printf(" set! There is probably a bug in 'find_most_violated_constraint_*'.\n");
printf("Ex %d: slack=%f, newslack=%f\n",i,slack,margin-dist);
/* exit(1); */
}
if((dist+slack)<(margin-epsilon)) {
if(struct_verbosity>=2)
{printf("(%i,eps=%.2f) ",i,margin-dist-slack); fflush(stdout);}
if(struct_verbosity==1)
{printf("."); fflush(stdout);}
/**** resize constraint matrix and add new constraint ****/
cset.m++;
cset.lhs=(DOC **)realloc(cset.lhs,sizeof(DOC *)*cset.m);
if(kparm->kernel_type == LINEAR_KERNEL) {
diff=add_list_ss(fy); /* store difference vector directly */
if(sparm->slack_norm == 1)
cset.lhs[cset.m-1]=create_example(cset.m-1,0,i+1,1,
copy_svector(diff));
else if(sparm->slack_norm == 2) {
/**** add squared slack variable to feature vector ****/
slackv[0].wnum=sizePsi+i;
slackv[0].weight=1/(sqrt(2*svmCnorm));
slackv[1].wnum=0; /*terminator*/
slackvec=create_svector(slackv,NULL,1.0);
cset.lhs[cset.m-1]=create_example(cset.m-1,0,i+1,1,
add_ss(diff,slackvec));
free_svector(slackvec);
}
free_svector(diff);
}
else { /* kernel is used */
if(sparm->slack_norm == 1)
cset.lhs[cset.m-1]=create_example(cset.m-1,0,i+1,1,
copy_svector(fy));
else if(sparm->slack_norm == 2)
exit(1);
}
cset.rhs=(double *)realloc(cset.rhs,sizeof(double)*cset.m);
cset.rhs[cset.m-1]=margin;
alpha=(double *)realloc(alpha,sizeof(double)*cset.m);
alpha[cset.m-1]=0;
alphahist=(long *)realloc(alphahist,sizeof(long)*cset.m);
alphahist[cset.m-1]=optcount;
newconstraints++;
totconstraints++;
}
else {
printf("+"); fflush(stdout);
if(opti[i] != opti_round) {
activenum--;
opti[i]=opti_round;
}
}
free_example(doc,0);
free_svector(fy); /* this also free's fybar */
free_label(ybar);
}
/**** get new QP solution ****/
if((newconstraints >= sparm->newconstretrain)
|| ((newconstraints > 0) && (i == n-1))
|| (new_precision && (i == n-1))) {
if(struct_verbosity>=1) {
printf("*");fflush(stdout);
}
rt2=get_runtime();
free_model(svmModel,0);
svmModel=(MODEL *)my_malloc(sizeof(MODEL));
/* Always get a new kernel cache. It is not possible to use the
same cache for two different training runs */
if(kparm->kernel_type != LINEAR_KERNEL)
kcache=kernel_cache_init(MAX(cset.m,1),lparm->kernel_cache_size);
/* Run the QP solver on cset. */
svm_learn_optimization(cset.lhs,cset.rhs,cset.m,sizePsi+n,
lparm,kparm,kcache,svmModel,alpha);
if(kcache)
kernel_cache_cleanup(kcache);
/* Always add weight vector, in case part of the kernel is
linear. If not, ignore the weight vector since its
content is bogus. */
add_weight_vector_to_linear_model(svmModel);
sm->svm_model=svmModel;
sm->w=svmModel->lin_weights; /* short cut to weight vector */
optcount++;
/* keep track of when each constraint was last
active. constraints marked with -1 are not updated */
for(j=0;j<cset.m;j++)
if((alphahist[j]>-1) && (alpha[j] != 0))
alphahist[j]=optcount;
rt_opt+=MAX(get_runtime()-rt2,0);
if(new_precision && (epsilon <= sparm->epsilon))
dont_stop=1; /* make sure we take one final pass */
new_precision=0;
newconstraints=0;
}
rt_total+=MAX(get_runtime()-rt1,0);
} /* end of example loop */
rt1=get_runtime();
if(struct_verbosity>=1)
printf("(NumConst=%d, SV=%ld, CEps=%.4f, QPEps=%.4f)\n",cset.m,
svmModel->sv_num-1,ceps,svmModel->maxdiff);
/* Check if some of the linear constraints have not been
active in a while. Those constraints are then removed to
avoid bloating the working set beyond necessity. */
if(struct_verbosity>=2)
printf("Reducing working set...");fflush(stdout);
remove_inactive_constraints(&cset,alpha,optcount,alphahist,
MAX(50,optcount-lastoptcount));
lastoptcount=optcount;
if(struct_verbosity>=2)
printf("done. (NumConst=%d)\n",cset.m);
rt_total+=MAX(get_runtime()-rt1,0);
} while(use_shrinking && (activenum > 0)); /* when using shrinking,
repeat until all examples
produced no constraint at
least once */
} while(((totconstraints - old_totconstraints) > tolerance) || dont_stop);
} while((epsilon > sparm->epsilon)
|| finalize_iteration(ceps,0,sample,sm,cset,alpha,sparm));
if(struct_verbosity>=1) {
/**** compute sum of slacks ****/
/**** WARNING: If positivity constraints are used, then the
maximum slack id is larger than what is allocated
below ****/
slacks=(double *)my_malloc(sizeof(double)*(n+1));
for(i=0; i<=n; i++) {
slacks[i]=0;
}
if(sparm->slack_norm == 1) {
for(j=0;j<cset.m;j++)
slacks[cset.lhs[j]->slackid]=MAX(slacks[cset.lhs[j]->slackid],
cset.rhs[j]-classify_example(svmModel,cset.lhs[j]));
}
else if(sparm->slack_norm == 2) {
for(j=0;j<cset.m;j++)
slacks[cset.lhs[j]->slackid]=MAX(slacks[cset.lhs[j]->slackid],
cset.rhs[j]
-(classify_example(svmModel,cset.lhs[j])
-sm->w[sizePsi+cset.lhs[j]->slackid-1]/(sqrt(2*svmCnorm))));
}
slacksum=0;
for(i=1; i<=n; i++)
slacksum+=slacks[i];
free(slacks);
alphasum=0;
for(i=0; i<cset.m; i++)
alphasum+=alpha[i]*cset.rhs[i];
modellength=model_length_s(svmModel);
dualitygap=(0.5*modellength*modellength+svmCnorm*(slacksum+n*ceps))
-(alphasum-0.5*modellength*modellength);
printf("Final epsilon on KKT-Conditions: %.5f\n",
MAX(svmModel->maxdiff,epsilon));
printf("Upper bound on duality gap: %.5f\n", dualitygap);
printf("Dual objective value: dval=%.5f\n",
alphasum-0.5*modellength*modellength);
printf("Total number of constraints in final working set: %i (of %i)\n",(int)cset.m,(int)totconstraints);
printf("Number of iterations: %d\n",numIt);
printf("Number of calls to 'find_most_violated_constraint': %ld\n",argmax_count);
if(sparm->slack_norm == 1) {
printf("Number of SV: %ld \n",svmModel->sv_num-1);
printf("Number of non-zero slack variables: %ld (out of %ld)\n",
svmModel->at_upper_bound,n);
printf("Norm of weight vector: |w|=%.5f\n",modellength);
}
else if(sparm->slack_norm == 2){
printf("Number of SV: %ld (including %ld at upper bound)\n",
svmModel->sv_num-1,svmModel->at_upper_bound);
printf("Norm of weight vector (including L2-loss): |w|=%.5f\n",
modellength);
}
printf("Norm. sum of slack variables (on working set): sum(xi_i)/n=%.5f\n",slacksum/n);
printf("Norm of longest difference vector: ||Psi(x,y)-Psi(x,ybar)||=%.5f\n",
length_of_longest_document_vector(cset.lhs,cset.m,kparm));
printf("Runtime in cpu-seconds: %.2f (%.2f%% for QP, %.2f%% for Argmax, %.2f%% for Psi, %.2f%% for init)\n",
rt_total/100.0, (100.0*rt_opt)/rt_total, (100.0*rt_viol)/rt_total,
(100.0*rt_psi)/rt_total, (100.0*rt_init)/rt_total);
}
if(struct_verbosity>=4)
printW(sm->w,sizePsi,n,lparm->svm_c);
if(svmModel) {
sm->svm_model=copy_model(svmModel);
sm->w=sm->svm_model->lin_weights; /* short cut to weight vector */
}
print_struct_learning_stats(sample,sm,cset,alpha,sparm);
if(fycache) {
for(i=0;i<n;i++)
free_svector(fycache[i]);
free(fycache);
}
if(svmModel)
free_model(svmModel,0);
free(alpha);
free(alphahist);
free(opti);
free(cset.rhs);
for(i=0;i<cset.m;i++)
free_example(cset.lhs[i],1);
free(cset.lhs);
}
int main_classify (int argc, char* argv[])
{
DOC *doc; /* test example */
WORDSVM *words;
long max_docs,max_words_doc,lld;
long totdoc=0,queryid,slackid;
long correct=0,incorrect=0,no_accuracy=0;
long res_a=0,res_b=0,res_c=0,res_d=0,wnum,pred_format;
long j;
double t1,runtime=0;
double dist,doc_label,costfactor;
char *line,*comment;
FILE *predfl,*docfl;
MODEL *model;
read_input_parameters(argc,argv,docfile,modelfile,predictionsfile,
&verbosity,&pred_format);
nol_ll(docfile,&max_docs,&max_words_doc,&lld); /* scan size of input file */
max_words_doc+=2;
lld+=2;
line = (char *)my_malloc(sizeof(char)*lld);
words = (WORDSVM *)my_malloc(sizeof(WORDSVM)*(max_words_doc+10));
model=read_model(modelfile);
if(model->kernel_parm.kernel_type == 0) { /* linear kernel */
/* compute weight vector */
add_weight_vector_to_linear_model(model);
}
if(verbosity>=2) {
printf("Classifying test examples.."); fflush(stdout);
}
if ((docfl = fopen (docfile, "r")) == NULL)
{ perror (docfile); exit (1); }
if ((predfl = fopen (predictionsfile, "w")) == NULL)
{ perror (predictionsfile); exit (1); }
while((!feof(docfl)) && fgets(line,(int)lld,docfl)) {
if(line[0] == '#') continue; /* line contains comments */
parse_document(line,words,&doc_label,&queryid,&slackid,&costfactor,&wnum,
max_words_doc,&comment);
totdoc++;
if(model->kernel_parm.kernel_type == 0) { /* linear kernel */
for(j=0;(words[j]).wnum != 0;j++) { /* Check if feature numbers */
if((words[j]).wnum>model->totwords) /* are not larger than in */
(words[j]).wnum=0; /* model. Remove feature if */
} /* necessary. */
doc = create_example(-1,0,0,0.0,create_svector(words,comment,1.0));
t1=get_runtime();
dist=classify_example_linear(model,doc);
runtime+=(get_runtime()-t1);
free_example(doc,1);
}
else { /* non-linear kernel */
doc = create_example(-1,0,0,0.0,create_svector(words,comment,1.0));
t1=get_runtime();
dist=classify_example(model,doc);
runtime+=(get_runtime()-t1);
free_example(doc,1);
}
if(dist>0) {
if(pred_format==0) { /* old weired output format */
fprintf(predfl,"%.8g:+1 %.8g:-1\n",dist,-dist);
}
if(doc_label>0) correct++; else incorrect++;
if(doc_label>0) res_a++; else res_b++;
}
else {
if(pred_format==0) { /* old weired output format */
fprintf(predfl,"%.8g:-1 %.8g:+1\n",-dist,dist);
}
if(doc_label<0) correct++; else incorrect++;
if(doc_label>0) res_c++; else res_d++;
}
if(pred_format==1) { /* output the value of decision function */
fprintf(predfl,"%.8g\n",dist);
}
if((int)(0.01+(doc_label*doc_label)) != 1)
{ no_accuracy=1; } /* test data is not binary labeled */
if(verbosity>=2) {
if(totdoc % 100 == 0) {
printf("%ld..",totdoc); fflush(stdout);
}
}
}
free(line);
free(words);
free_model(model,1);
if(verbosity>=2) {
printf("done\n");
/* Note by Gary Boone Date: 29 April 2000 */
/* o Timing is inaccurate. The timer has 0.01 second resolution. */
/* Because classification of a single vector takes less than */
/* 0.01 secs, the timer was underflowing. */
printf("Runtime (without IO) in cpu-seconds: %.2f\n",
(float)(runtime/100.0));
}
if((!no_accuracy) && (verbosity>=1)) {
printf("Accuracy on test set: %.2f%% (%ld correct, %ld incorrect, %ld total)\n",(float)(correct)*100.0/totdoc,correct,incorrect,totdoc);
printf("Precision/recall on test set: %.2f%%/%.2f%%\n",(float)(res_a)*100.0/(res_a+res_b),(float)(res_a)*100.0/(res_a+res_c));
}
return(0);
}
int main (int argc, char* argv[])
{
long correct=0,incorrect=0,no_accuracy=0;
long i;
double t1,runtime=0;
double avgloss=0,l;
FILE *predfl;
STRUCTMODEL model;
STRUCT_LEARN_PARM sparm;
STRUCT_TEST_STATS teststats;
SAMPLE testsample;
LABEL y;
svm_struct_classify_api_init(argc,argv);
read_input_parameters(argc,argv,testfile,modelfile,predictionsfile,&sparm,
&verbosity,&struct_verbosity);
if(struct_verbosity>=1) {
printf("Reading model..."); fflush(stdout);
}
model=read_struct_model(modelfile,&sparm);
if(struct_verbosity>=1) {
fprintf(stdout, "done.\n");
}
if(model.svm_model->kernel_parm.kernel_type == LINEAR) { /* linear kernel */
/* compute weight vector */
add_weight_vector_to_linear_model(model.svm_model);
model.w=model.svm_model->lin_weights;
}
if(struct_verbosity>=1) {
printf("Reading test examples..."); fflush(stdout);
}
testsample=read_struct_examples(testfile,&sparm);
if(struct_verbosity>=1) {
printf("done.\n"); fflush(stdout);
}
if(struct_verbosity>=1) {
printf("Classifying test examples..."); fflush(stdout);
}
if ((predfl = fopen (predictionsfile, "w")) == NULL)
{ perror (predictionsfile); exit (1); }
for(i=0;i<testsample.n;i++) {
t1=get_runtime();
y=classify_struct_example(testsample.examples[i].x,&model,&sparm);
runtime+=(get_runtime()-t1);
write_label(predfl,y);
l=loss(testsample.examples[i].y,y,&sparm);
avgloss+=l;
if(l == 0)
correct++;
else
incorrect++;
eval_prediction(i,testsample.examples[i],y,&model,&sparm,&teststats);
if(empty_label(testsample.examples[i].y))
{ no_accuracy=1; } /* test data is not labeled */
if(struct_verbosity>=2) {
if((i+1) % 100 == 0) {
printf("%ld..",i+1); fflush(stdout);
}
}
free_label(y);
}
avgloss/=testsample.n;
fclose(predfl);
if(struct_verbosity>=1) {
printf("done\n");
printf("Runtime (without IO) in cpu-seconds: %.2f\n",
(float)(runtime/100.0));
}
if((!no_accuracy) && (struct_verbosity>=1)) {
printf("Average loss on test set: %.4f\n",(float)avgloss);
printf("Zero/one-error on test set: %.2f%% (%ld correct, %ld incorrect, %d total)\n",(float)100.0*incorrect/testsample.n,correct,incorrect,testsample.n);
}
print_struct_testing_stats(testsample,&model,&sparm,&teststats);
free_struct_sample(testsample);
free_struct_model(model);
svm_struct_classify_api_exit();
return(0);
}
void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[])
#endif
{
long correct=0,incorrect=0,no_accuracy=0;
long i;
double t1,runtime=0;
double avgloss=0,l;
#ifndef COMPILE_MEX_INTERFACE
FILE *predfl;
#endif
STRUCTMODEL model;
STRUCT_LEARN_PARM sparm;
STRUCT_TEST_STATS teststats;
SAMPLE testsample;
LABEL y;
#ifdef COMPILE_MEX_INTERFACE
int argc;
char **argv;
if (nrhs < 3) {
print_help();
return;
}
else if (nrhs==3) {
argc=1;
argv=(char **)my_malloc(MAX_ARGVS*sizeof(char *));
argv[0]="OLR";
}
else
create_argc_argv(prhs[3],&argc,&argv);
#endif
svm_struct_classify_api_init(argc,argv);
#ifndef COMPILE_MEX_INTERFACE
read_input_parameters(argc,argv,testfile,modelfile,predictionsfile,&sparm,
&verbosity,&struct_verbosity);
#else
read_input_parameters(argc,argv,&sparm,&verbosity,&struct_verbosity);
#endif
if(struct_verbosity>=1) {
printf("Reading model..."); fflush(stdout);
}
#ifndef COMPILE_MEX_INTERFACE
model=read_struct_model(modelfile,&sparm);
#else
model=read_struct_model(prhs[2],&sparm);
#endif
if(struct_verbosity>=1) {
fprintf(stdout, "done.\n");
}
if(model.svm_model->kernel_parm.kernel_type == LINEAR) { /* linear kernel */
/* compute weight vector */
add_weight_vector_to_linear_model(model.svm_model);
model.w=model.svm_model->lin_weights;
}
if(struct_verbosity>=1) {
printf("Reading test examples..."); fflush(stdout);
}
#ifndef COMPILE_MEX_INTERFACE
testsample=read_struct_examples(testfile,&sparm);
#else
testsample=read_struct_examples(prhs,&sparm);
#endif
if(struct_verbosity>=1) {
printf("done.\n"); fflush(stdout);
}
if(struct_verbosity>=1) {
printf("Classifying test examples..."); fflush(stdout);
}
#ifndef COMPILE_MEX_INTERFACE
if ((predfl = fopen (predictionsfile, "w")) == NULL)
{ perror (predictionsfile); exit (1); }
#else
mwSize rows=mxGetM(prhs[0]);
mxArray *predictions=mxCreateDoubleMatrix(rows,1,mxREAL);
double *pred_ptr=mxGetPr(predictions);
#endif
for(i=0;i<testsample.n;i++) {
t1=get_runtime();
y=classify_struct_example(testsample.examples[i].x,&model,&sparm);
runtime+=(get_runtime()-t1);
#ifndef COMPILE_MEX_INTERFACE
write_label(predfl,y);
#else
write_label(&pred_ptr,y);
#endif
l=loss(testsample.examples[i].y,y,&sparm);
avgloss+=l;
if(l == 0)
correct++;
else
incorrect++;
eval_prediction(i,testsample.examples[i],y,&model,&sparm,&teststats);
if(empty_label(testsample.examples[i].y))
{ no_accuracy=1; } /* test data is not labeled */
if(struct_verbosity>=2) {
if((i+1) % 100 == 0) {
printf("%ld..",i+1); fflush(stdout);
}
}
free_label(y);
}
avgloss/=testsample.n;
#ifndef COMPILE_MEX_INTERFACE
fclose(predfl);
#endif
if(struct_verbosity>=1) {
printf("done\n");
printf("Runtime (without IO) in cpu-seconds: %.2f\n",
(float)(runtime/100.0));
}
if((!no_accuracy) && (struct_verbosity>=1)) {
printf("Average loss on test set: %.4f\n",(float)avgloss);
printf("Zero/one-error on test set: %.2f%% (%ld correct, %ld incorrect, %d total)\n",(float)100.0*incorrect/testsample.n,correct,incorrect,testsample.n);
}
print_struct_testing_stats(testsample,&model,&sparm,&teststats);
free_struct_sample(testsample);
free_struct_model(model);
svm_struct_classify_api_exit();
#ifndef COMPILE_MEX_INTERFACE
return(0);
#else
plhs[0]=predictions;
#endif
}
int SVMLightRunner::librarySVMClassifyMain(
int argc, char **argv, bool use_gmumr, SVMConfiguration &config
) {
LOG(
config.log,
LogLevel::DEBUG_LEVEL,
__debug_prefix__ + ".librarySVMClassifyMain() Started."
);
DOC *doc; /* test example */
WORD *words;
long max_docs,max_words_doc,lld;
long totdoc=0,queryid,slackid;
long correct=0,incorrect=0,no_accuracy=0;
long res_a=0,res_b=0,res_c=0,res_d=0,wnum,pred_format;
long j;
double t1,runtime=0;
double dist,doc_label,costfactor;
char *line,*comment;
FILE *predfl,*docfl;
MODEL *model;
// GMUM.R changes {
librarySVMClassifyReadInputParameters(
argc, argv, docfile, modelfile, predictionsfile, &verbosity,
&pred_format, use_gmumr, config);
if (!use_gmumr) {
nol_ll(docfile,&max_docs,&max_words_doc,&lld); /* scan size of input file */
lld+=2;
line = (char *)my_malloc(sizeof(char)*lld);
} else {
max_docs = config.target.n_rows;
max_words_doc = config.getDataDim();
config.result = arma::zeros<arma::vec>(max_docs);
// Prevent writing to the file
pred_format = -1;
// lld used only for file reading
}
max_words_doc+=2;
words = (WORD *)my_malloc(sizeof(WORD)*(max_words_doc+10));
// GMUM.R changes }
model=libraryReadModel(modelfile, use_gmumr, config);
// GMUM.R changes }
if(model->kernel_parm.kernel_type == 0) { /* linear kernel */
/* compute weight vector */
add_weight_vector_to_linear_model(model);
}
if(verbosity>=2) {
C_PRINTF("Classifying test examples.."); C_FFLUSH(stdout);
}
// GMUM.R changes {
bool newline;
if (!use_gmumr) {
if ((predfl = fopen (predictionsfile, "w")) == NULL)
{ perror (predictionsfile); EXIT (1); }
if ((docfl = fopen (docfile, "r")) == NULL)
{ perror (docfile); EXIT (1); }
newline = (!feof(docfl)) && fgets(line,(int)lld,docfl);
} else {
newline = false;
if (totdoc < config.getDataExamplesNumber()) {
newline = true;
std::string str = SVMConfigurationToSVMLightLearnInputLine(config, totdoc);
line = new char[str.size() + 1];
std::copy(str.begin(), str.end(), line);
line[str.size()] = '\0';
}
}
while(newline) {
if (use_gmumr) {
std::string stringline = "";
}
// GMUM.R changes }
if(line[0] == '#') continue; /* line contains comments */
parse_document(line,words,&doc_label,&queryid,&slackid,&costfactor,&wnum,
max_words_doc,&comment);
totdoc++;
if(model->kernel_parm.kernel_type == 0) { /* linear kernel */
for(j=0;(words[j]).wnum != 0;j++) { /* Check if feature numbers */
if((words[j]).wnum>model->totwords) /* are not larger than in */
(words[j]).wnum=0; /* model. Remove feature if */
} /* necessary. */
doc = create_example(-1,0,0,0.0,create_svector(words,comment,1.0));
t1=get_runtime();
dist=classify_example_linear(model,doc);
runtime+=(get_runtime()-t1);
free_example(doc,1);
}
else { /* non-linear kernel */
doc = create_example(-1,0,0,0.0,create_svector(words,comment,1.0));
t1=get_runtime();
dist=classify_example(model,doc);
runtime+=(get_runtime()-t1);
free_example(doc,1);
}
if(dist>0) {
if(pred_format==0) { /* old weired output format */
C_FPRINTF(predfl,"%.8g:+1 %.8g:-1\n",dist,-dist);
}
if(doc_label>0) correct++; else incorrect++;
if(doc_label>0) res_a++; else res_b++;
}
else {
if(pred_format==0) { /* old weired output format */
C_FPRINTF(predfl,"%.8g:-1 %.8g:+1\n",-dist,dist);
}
if(doc_label<0) correct++; else incorrect++;
if(doc_label>0) res_c++; else res_d++;
}
if(pred_format==1) { /* output the value of decision function */
C_FPRINTF(predfl,"%.8g\n",dist);
}
if((int)(0.01+(doc_label*doc_label)) != 1)
{ no_accuracy=1; } /* test data is not binary labeled */
if(verbosity>=2) {
if(totdoc % 100 == 0) {
C_PRINTF("%ld..",totdoc); C_FFLUSH(stdout);
}
}
// GMUM.R changes {
if (!use_gmumr) {
newline = (!feof(docfl)) && fgets(line,(int)lld,docfl);
} else {
newline = false;
// Store prediction result in config
config.result[totdoc-1] = dist;
// Read next line
if (totdoc < config.getDataExamplesNumber()) {
newline = true;
std::string str = SVMConfigurationToSVMLightLearnInputLine(config, totdoc);
line = new char[str.size() + 1];
std::copy(str.begin(), str.end(), line);
line[str.size()] = '\0';
}
}
}
if (!use_gmumr) {
fclose(predfl);
fclose(docfl);
free(line);
}
// GMUM.R changes }
free(words);
free_model(model,1);
if(verbosity>=2) {
C_PRINTF("done\n");
/* Note by Gary Boone Date: 29 April 2000 */
/* o Timing is inaccurate. The timer has 0.01 second resolution. */
/* Because classification of a single vector takes less than */
/* 0.01 secs, the timer was underflowing. */
C_PRINTF("Runtime (without IO) in cpu-seconds: %.2f\n",
(float)(runtime/100.0));
}
if((!no_accuracy) && (verbosity>=1)) {
C_PRINTF("Accuracy on test set: %.2f%% (%ld correct, %ld incorrect, %ld total)\n",(float)(correct)*100.0/totdoc,correct,incorrect,totdoc);
C_PRINTF("Precision/recall on test set: %.2f%%/%.2f%%\n",(float)(res_a)*100.0/(res_a+res_b),(float)(res_a)*100.0/(res_a+res_c));
}
return(0);
}
Classifier::Classifier(string output, Trainer::KernelType kernel,
Location* le, Location* re, Location* n,
roiFnT roiFn) {
// check if the output directory exists, or else bail
DIR* dir;
dir = opendir(output.c_str());
if (dir == NULL) {
string err = "Trainer::Trainer. The directory " + output + " does not exist. Bailing out.";
throw (err);
}
closedir(dir);
outputDirectory = output;
roiFunction = roiFn;
kernelType = kernel;
// Build a set of feature extraction objects and stuff them into the
// extractor vector
Feature* f = (Feature*) new FeatureLX();
featureExtractors.push_back(f);
f = (Feature*) new FeatureRX();
featureExtractors.push_back(f);
f = (Feature*) new FeatureNX();
featureExtractors.push_back(f);
f = (Feature*) new FeatureLRDist();
featureExtractors.push_back(f);
f = (Feature*) new FeatureLNDist();
featureExtractors.push_back(f);
f = (Feature*) new FeatureRNDist();
featureExtractors.push_back(f);
f = (Feature*) new FeatureLNAngle();
featureExtractors.push_back(f);
f = (Feature*) new FeatureRNAngle();
featureExtractors.push_back(f);
f = (Feature*) new FeatureLRNArea();
featureExtractors.push_back(f);
nFeatures = (int)Feature::End - 1;
// read extremal feature values and other data generated during
// training that we need for classification
readParameters();
// create location extractors
leftEye = le;
rightEye = re;
nose = n;
// load models
for (unsigned int i = 0; i < Globals::numZones; i++) {
char buffer[Globals::smallBufferSize];
sprintf(buffer, "%d.model", i + 1);
string str = buffer;
string modelFileName = outputDirectory + "/" + Globals::modelNamePrefix + str;
MODEL* model = read_model((char*)modelFileName.c_str());
if (!model) {
string err = "Classifier::Classifier. read_model returned NULL.";
throw (err);
}
if (model->kernel_parm.kernel_type == 0) { /* linear kernel */
/* compute weight vector */
add_weight_vector_to_linear_model(model);
}
models.push_back(model);
}
}
void svm_learn_struct_joint(SAMPLE sample, STRUCT_LEARN_PARM *sparm,
LEARN_PARM *lparm, KERNEL_PARM *kparm,
STRUCTMODEL *sm, int alg_type)
{
int i,j;
int numIt=0;
long argmax_count=0;
long totconstraints=0;
long kernel_type_org;
double epsilon,epsilon_cached;
double lossval,factor,dist;
double margin=0;
double slack, slacksum, ceps;
double dualitygap,modellength,alphasum;
long sizePsi;
double *alpha=NULL;
long *alphahist=NULL,optcount=0;
CONSTSET cset;
SVECTOR *diff=NULL;
double *diff_n=NULL;
SVECTOR *fy, *fybar, *f, **fycache, *lhs;
MODEL *svmModel=NULL;
LABEL ybar;
DOC *doc;
long n=sample.n;
EXAMPLE *ex=sample.examples;
double rt_total=0,rt_opt=0,rt_init=0,rt_psi=0,rt_viol=0,rt_kernel=0;
double rt1,rt2;
double progress,progress_old;
/*
SVECTOR ***fydelta_cache=NULL;
double **loss_cache=NULL;
int cache_size=0;
*/
CCACHE *ccache=NULL;
int cached_constraint;
rt1=get_runtime();
init_struct_model(sample,sm,sparm,lparm,kparm);
sizePsi=sm->sizePsi+1; /* sm must contain size of psi on return */
if(sparm->slack_norm == 1) {
lparm->svm_c=sparm->C; /* set upper bound C */
lparm->sharedslack=1;
}
else if(sparm->slack_norm == 2) {
printf("ERROR: The joint algorithm does not apply to L2 slack norm!");
fflush(stdout);
exit(0);
}
else {
printf("ERROR: Slack norm must be L1 or L2!"); fflush(stdout);
exit(0);
}
lparm->biased_hyperplane=0; /* set threshold to zero */
epsilon=100.0; /* start with low precision and
increase later */
epsilon_cached=epsilon; /* epsilon to use for iterations
using constraints constructed
from the constraint cache */
cset=init_struct_constraints(sample, sm, sparm);
if(cset.m > 0) {
alpha=(double *)realloc(alpha,sizeof(double)*cset.m);
alphahist=(long *)realloc(alphahist,sizeof(long)*cset.m);
for(i=0; i<cset.m; i++) {
alpha[i]=0;
alphahist[i]=-1; /* -1 makes sure these constraints are never removed */
}
}
kparm->gram_matrix=NULL;
if((alg_type == DUAL_ALG) || (alg_type == DUAL_CACHE_ALG))
kparm->gram_matrix=init_kernel_matrix(&cset,kparm);
/* set initial model and slack variables */
svmModel=(MODEL *)my_malloc(sizeof(MODEL));
lparm->epsilon_crit=epsilon;
svm_learn_optimization(cset.lhs,cset.rhs,cset.m,sizePsi+n,
lparm,kparm,NULL,svmModel,alpha);
add_weight_vector_to_linear_model(svmModel);
sm->svm_model=svmModel;
sm->w=svmModel->lin_weights; /* short cut to weight vector */
/* create a cache of the feature vectors for the correct labels */
fycache=(SVECTOR **)malloc(n*sizeof(SVECTOR *));
for(i=0;i<n;i++) {
fy=psi(ex[i].x,ex[i].y,sm,sparm);
if(kparm->kernel_type == LINEAR) {
diff=add_list_ss(fy); /* store difference vector directly */
free_svector(fy);
fy=diff;
}
fycache[i]=fy;
}
/* initialize the constraint cache */
if(alg_type == DUAL_CACHE_ALG) {
ccache=create_constraint_cache(sample,sparm);
}
rt_init+=MAX(get_runtime()-rt1,0);
rt_total+=MAX(get_runtime()-rt1,0);
/*****************/
/*** main loop ***/
/*****************/
do { /* iteratively find and add constraints to working set */
if(struct_verbosity>=1) {
printf("Iter %i: ",++numIt);
fflush(stdout);
}
rt1=get_runtime();
/**** compute current slack ****/
slack=0;
for(j=0;j<cset.m;j++)
slack=MAX(slack,cset.rhs[j]-classify_example(svmModel,cset.lhs[j]));
/**** find a violated joint constraint ****/
lhs=NULL;
dist=0;
if(alg_type == DUAL_CACHE_ALG) {
/* see if it is possible to construct violated constraint from cache */
update_constraint_cache_for_model(ccache, svmModel);
dist=find_most_violated_joint_constraint_in_cache(ccache,&lhs,&margin);
}
rt_total+=MAX(get_runtime()-rt1,0);
/* Is there a sufficiently violated constraint in cache? */
if(dist-slack > MAX(epsilon/10,sparm->epsilon)) {
/* use constraint from cache */
rt1=get_runtime();
cached_constraint=1;
if(kparm->kernel_type == LINEAR) {
diff=add_list_ns(lhs); /* Linear case: compute weighted sum */
free_svector_shallow(lhs);
}
else { /* Non-linear case: make sure we have deep copy for cset */
diff=copy_svector(lhs);
free_svector_shallow(lhs);
}
rt_total+=MAX(get_runtime()-rt1,0);
}
else {
/* do not use constraint from cache */
rt1=get_runtime();
cached_constraint=0;
if(lhs)
free_svector_shallow(lhs);
lhs=NULL;
if(kparm->kernel_type == LINEAR) {
diff_n=create_nvector(sm->sizePsi);
clear_nvector(diff_n,sm->sizePsi);
}
margin=0;
progress=0;
progress_old=progress;
rt_total+=MAX(get_runtime()-rt1,0);
/**** find most violated joint constraint ***/
for(i=0; i<n; i++) {
rt1=get_runtime();
progress+=10.0/n;
if((struct_verbosity==1) && (((int)progress_old) != ((int)progress)))
{printf(".");fflush(stdout); progress_old=progress;}
if(struct_verbosity>=2)
{printf("."); fflush(stdout);}
rt2=get_runtime();
argmax_count++;
if(sparm->loss_type == SLACK_RESCALING)
ybar=find_most_violated_constraint_slackrescaling(ex[i].x,
ex[i].y,sm,
sparm);
else
ybar=find_most_violated_constraint_marginrescaling(ex[i].x,
ex[i].y,sm,
sparm);
rt_viol+=MAX(get_runtime()-rt2,0);
if(empty_label(ybar)) {
printf("ERROR: empty label was returned for example (%i)\n",i);
/* exit(1); */
continue;
}
/**** get psi(x,y) and psi(x,ybar) ****/
rt2=get_runtime();
fy=copy_svector(fycache[i]); /*<= fy=psi(ex[i].x,ex[i].y,sm,sparm);*/
fybar=psi(ex[i].x,ybar,sm,sparm);
rt_psi+=MAX(get_runtime()-rt2,0);
lossval=loss(ex[i].y,ybar,sparm);
free_label(ybar);
/**** scale feature vector and margin by loss ****/
if(sparm->loss_type == SLACK_RESCALING)
factor=lossval/n;
else /* do not rescale vector for */
factor=1.0/n; /* margin rescaling loss type */
for(f=fy;f;f=f->next)
f->factor*=factor;
for(f=fybar;f;f=f->next)
f->factor*=-factor;
append_svector_list(fybar,fy); /* compute fy-fybar */
/**** add current fy-fybar and loss to cache ****/
if(alg_type == DUAL_CACHE_ALG) {
if(kparm->kernel_type == LINEAR)
add_constraint_to_constraint_cache(ccache,svmModel,i,
add_list_ss(fybar),
lossval/n,sparm->ccache_size);
else
add_constraint_to_constraint_cache(ccache,svmModel,i,
copy_svector(fybar),
lossval/n,sparm->ccache_size);
}
/**** add current fy-fybar to constraint and margin ****/
if(kparm->kernel_type == LINEAR) {
add_list_n_ns(diff_n,fybar,1.0); /* add fy-fybar to sum */
free_svector(fybar);
}
else {
append_svector_list(fybar,lhs); /* add fy-fybar to vector list */
lhs=fybar;
}
margin+=lossval/n; /* add loss to rhs */
rt_total+=MAX(get_runtime()-rt1,0);
} /* end of example loop */
rt1=get_runtime();
/* create sparse vector from dense sum */
if(kparm->kernel_type == LINEAR) {
diff=create_svector_n(diff_n,sm->sizePsi,"",1.0);
free_nvector(diff_n);
}
else {
diff=lhs;
}
rt_total+=MAX(get_runtime()-rt1,0);
} /* end of finding most violated joint constraint */
rt1=get_runtime();
/**** if `error', then add constraint and recompute QP ****/
doc=create_example(cset.m,0,1,1,diff);
dist=classify_example(svmModel,doc);
ceps=MAX(0,margin-dist-slack);
if(slack > (margin-dist+0.000001)) {
printf("\nWARNING: Slack of most violated constraint is smaller than slack of working\n");
printf(" set! There is probably a bug in 'find_most_violated_constraint_*'.\n");
printf("slack=%f, newslack=%f\n",slack,margin-dist);
/* exit(1); */
}
if(ceps > sparm->epsilon) {
/**** resize constraint matrix and add new constraint ****/
cset.lhs=(DOC **)realloc(cset.lhs,sizeof(DOC *)*(cset.m+1));
if(sparm->slack_norm == 1)
cset.lhs[cset.m]=create_example(cset.m,0,1,1,diff);
else if(sparm->slack_norm == 2)
exit(1);
cset.rhs=(double *)realloc(cset.rhs,sizeof(double)*(cset.m+1));
cset.rhs[cset.m]=margin;
alpha=(double *)realloc(alpha,sizeof(double)*(cset.m+1));
alpha[cset.m]=0;
alphahist=(long *)realloc(alphahist,sizeof(long)*(cset.m+1));
alphahist[cset.m]=optcount;
cset.m++;
totconstraints++;
if((alg_type == DUAL_ALG) || (alg_type == DUAL_CACHE_ALG)) {
if(struct_verbosity>=1) {
printf(":");fflush(stdout);
}
rt2=get_runtime();
kparm->gram_matrix=update_kernel_matrix(kparm->gram_matrix,cset.m-1,
&cset,kparm);
rt_kernel+=MAX(get_runtime()-rt2,0);
}
/**** get new QP solution ****/
if(struct_verbosity>=1) {
printf("*");fflush(stdout);
}
rt2=get_runtime();
/* set svm precision so that higher than eps of most violated constr */
if(cached_constraint) {
epsilon_cached=MIN(epsilon_cached,MAX(ceps,sparm->epsilon));
lparm->epsilon_crit=epsilon_cached/2;
}
else {
epsilon=MIN(epsilon,MAX(ceps,sparm->epsilon)); /* best eps so far */
lparm->epsilon_crit=epsilon/2;
epsilon_cached=epsilon;
}
free_model(svmModel,0);
svmModel=(MODEL *)my_malloc(sizeof(MODEL));
/* Run the QP solver on cset. */
kernel_type_org=kparm->kernel_type;
if((alg_type == DUAL_ALG) || (alg_type == DUAL_CACHE_ALG))
kparm->kernel_type=GRAM; /* use kernel stored in kparm */
svm_learn_optimization(cset.lhs,cset.rhs,cset.m,sizePsi+n,
lparm,kparm,NULL,svmModel,alpha);
kparm->kernel_type=kernel_type_org;
svmModel->kernel_parm.kernel_type=kernel_type_org;
/* Always add weight vector, in case part of the kernel is
linear. If not, ignore the weight vector since its
content is bogus. */
add_weight_vector_to_linear_model(svmModel);
sm->svm_model=svmModel;
sm->w=svmModel->lin_weights; /* short cut to weight vector */
optcount++;
/* keep track of when each constraint was last
active. constraints marked with -1 are not updated */
for(j=0;j<cset.m;j++)
if((alphahist[j]>-1) && (alpha[j] != 0))
alphahist[j]=optcount;
rt_opt+=MAX(get_runtime()-rt2,0);
/* Check if some of the linear constraints have not been
active in a while. Those constraints are then removed to
avoid bloating the working set beyond necessity. */
if(struct_verbosity>=2)
printf("Reducing working set...");fflush(stdout);
remove_inactive_constraints(&cset,alpha,optcount,alphahist,50);
if(struct_verbosity>=2)
printf("done. (NumConst=%d) ",cset.m);
}
else {
free_svector(diff);
}
if(struct_verbosity>=1)
printf("(NumConst=%d, SV=%ld, CEps=%.4f, QPEps=%.4f)\n",cset.m,
svmModel->sv_num-1,ceps,svmModel->maxdiff);
free_example(doc,0);
rt_total+=MAX(get_runtime()-rt1,0);
} while((ceps > sparm->epsilon) ||
finalize_iteration(ceps,cached_constraint,sample,sm,cset,alpha,sparm)
);
if(struct_verbosity>=1) {
/**** compute sum of slacks ****/
/**** WARNING: If positivity constraints are used, then the
maximum slack id is larger than what is allocated
below ****/
slacksum=0;
if(sparm->slack_norm == 1) {
for(j=0;j<cset.m;j++)
slacksum=MAX(slacksum,
cset.rhs[j]-classify_example(svmModel,cset.lhs[j]));
}
else if(sparm->slack_norm == 2) {
exit(1);
}
alphasum=0;
for(i=0; i<cset.m; i++)
alphasum+=alpha[i]*cset.rhs[i];
modellength=model_length_s(svmModel,kparm);
dualitygap=(0.5*modellength*modellength+sparm->C*(slacksum+ceps))
-(alphasum-0.5*modellength*modellength);
printf("Final epsilon on KKT-Conditions: %.5f\n",
MAX(svmModel->maxdiff,ceps));
printf("Upper bound on duality gap: %.5f\n", dualitygap);
printf("Dual objective value: dval=%.5f\n",
alphasum-0.5*modellength*modellength);
printf("Total number of constraints in final working set: %i (of %i)\n",(int)cset.m,(int)totconstraints);
printf("Number of iterations: %d\n",numIt);
printf("Number of calls to 'find_most_violated_constraint': %ld\n",argmax_count);
if(sparm->slack_norm == 1) {
printf("Number of SV: %ld \n",svmModel->sv_num-1);
printf("Norm of weight vector: |w|=%.5f\n",
model_length_s(svmModel,kparm));
}
else if(sparm->slack_norm == 2){
printf("Number of SV: %ld (including %ld at upper bound)\n",
svmModel->sv_num-1,svmModel->at_upper_bound);
printf("Norm of weight vector (including L2-loss): |w|=%.5f\n",
model_length_s(svmModel,kparm));
}
printf("Value of slack variable (on working set): xi=%.5f\n",slacksum);
printf("Norm of longest difference vector: ||Psi(x,y)-Psi(x,ybar)||=%.5f\n",
length_of_longest_document_vector(cset.lhs,cset.m,kparm));
printf("Runtime in cpu-seconds: %.2f (%.2f%% for QP, %.2f%% for kernel, %.2f%% for Argmax, %.2f%% for Psi, %.2f%% for init)\n",
rt_total/100.0, (100.0*rt_opt)/rt_total, (100.0*rt_kernel)/rt_total,
(100.0*rt_viol)/rt_total, (100.0*rt_psi)/rt_total,
(100.0*rt_init)/rt_total);
}
if(ccache) {
long cnum=0;
CCACHEELEM *celem;
for(i=0;i<n;i++)
for(celem=ccache->constlist[i];celem;celem=celem->next)
cnum++;
printf("Final number of constraints in cache: %ld\n",cnum);
}
if(struct_verbosity>=4)
printW(sm->w,sizePsi,n,lparm->svm_c);
if(svmModel) {
sm->svm_model=copy_model(svmModel);
sm->w=sm->svm_model->lin_weights; /* short cut to weight vector */
}
print_struct_learning_stats(sample,sm,cset,alpha,sparm);
if(ccache)
free_constraint_cache(ccache);
for(i=0;i<n;i++)
free_svector(fycache[i]);
free(fycache);
if(svmModel)
free_model(svmModel,0);
free(alpha);
free(alphahist);
free(cset.rhs);
for(i=0;i<cset.m;i++)
free_example(cset.lhs[i],1);
free(cset.lhs);
if(kparm->gram_matrix)
free_matrix(kparm->gram_matrix);
}
