void write_struct_model(char *file, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm) { /* Writes structural model sm to file file. */ PyObject *pFunc, *pValue; // Reduce the support vectors if appropriate. if (sm->w && sm->lin_reduce) { MODEL *m = sm->svm_model; // Get rid of the old support vectors. int i; for (i=1; i<m->sv_num; ++i) { free_example(m->supvec[i], 1); } free(m->supvec); free(m->alpha); // Create the new model sv structurs. m->supvec = (DOC**)my_malloc(sizeof(DOC*)*2); m->supvec[0] = NULL; m->alpha = (double*)my_malloc(sizeof(double)*2); m->alpha[0]=0.0; m->alpha[1]=1.0; m->sv_num = 2; // Create the new support vector. SVECTOR *sv = create_svector_n(sm->w, sm->sizePsi,"",1.0); m->supvec[1] = create_example(0,0,1,1.0,sv); } // Call the relevant Python function. pFunc = getFunction(PYTHON_WRITE_MODEL); pValue = PyObject_CallFunction (pFunc,"sNN",file,StructModel_FromStructModel(sm),Sparm_FromSparm(sparm)); PY_RUNCHECK; Py_DECREF(pValue); }
void remove_inactive_constraints(CONSTSET *cset, double *alpha, long currentiter, long *alphahist, long mininactive) /* removes the constraints from cset (and alpha) for which alphahist indicates that they have not been active for at least mininactive iterations */ { long i,m; m=0; for(i=0;i<cset->m;i++) { if((alphahist[i]<0) || ((currentiter-alphahist[i]) < mininactive)) { /* keep constraints that are marked as -1 or which have recently been active */ cset->lhs[m]=cset->lhs[i]; cset->lhs[m]->docnum=m; cset->rhs[m]=cset->rhs[i]; alpha[m]=alpha[i]; alphahist[m]=alphahist[i]; m++; } else { free_example(cset->lhs[i],1); } } if(cset->m != m) { cset->m=m; cset->lhs=(DOC **)realloc(cset->lhs,sizeof(DOC *)*cset->m); cset->rhs=(double *)realloc(cset->rhs,sizeof(double)*cset->m); /* alpha=realloc(alpha,sizeof(double)*cset->m); */ /* alphahist=realloc(alphahist,sizeof(long)*cset->m); */ } }
/* save constraints in structmodel for training the full model in the future */ void save_constraints(STRUCT_LEARN_PARM *sparm, STRUCTMODEL *sm) { int i, count; CONSTSET c; /* read the constraints */ c = read_constraints(sparm->confile, sm); count = 0; for(i = 0; i < c.m; i++) { sparm->cset.lhs = (DOC**)realloc(sparm->cset.lhs, sizeof(DOC*)*(sparm->cset.m+1)); sparm->cset.lhs[sparm->cset.m] = create_example(sparm->cset.m, 0, 1000000+sparm->cset.m, 1, create_svector(c.lhs[i]->fvec->words, "", 1.0)); sparm->cset.rhs = (double*)realloc(sparm->cset.rhs, sizeof(double)*(sparm->cset.m+1)); sparm->cset.rhs[sparm->cset.m] = c.rhs[i]; sparm->cset.m++; count++; } /* clean up */ free(c.rhs); for(i = 0; i < c.m; i++) free_example(c.lhs[i], 1); free(c.lhs); printf("%d constraints added, totally %d constraints\n", count, sparm->cset.m); }
double SVMModule::classify(std::vector<float>& instance) { if(m_pTrainModel == NULL) return -1; WORD *words; int vecLen = instance.size(); words = (WORD*)my_malloc(sizeof(WORD)*(vecLen + 1)); for(int i = 0; i < vecLen; i++) { words[i].wnum = (i+1); words[i].weight = instance[i]; } words[vecLen].wnum = 0; // make svector SVECTOR* svec = create_svector(words, "", 1.0); DOC* doc = create_example(-1, 0, 0, 0.0, svec); double prob = classify_example(m_pTrainModel, doc); free_example(doc, 1); return prob; }
void freeModelMem(MODEL *model) { long i; if(model->supvec) { for(i=1;i<model->sv_num;i++) { if (model->supvec[i]) free_example(model->supvec[i],1); } } }
double find_most_violated_joint_constraint_in_cache_old(int n, int cache_size, SVECTOR ***fydelta_cache, double **loss_cache, MODEL *svmModel, SVECTOR **lhs, double *margin) { int i,j; double progress,progress_old; double maxviol=0,sumviol,viol,lossval; double dist_ydelta; SVECTOR *fydelta; DOC *doc_fydelta; (*lhs)=NULL; (*margin)=0; sumviol=0; progress=0; progress_old=progress; for(i=0; i<n; i++) { /*** example loop ***/ progress+=10.0/n; if((struct_verbosity==1) && (((int)progress_old) != ((int)progress))) {printf("+");fflush(stdout); progress_old=progress;} if(struct_verbosity>=2) {printf("+"); fflush(stdout);} fydelta=NULL; lossval=0; for(j=0;j<cache_size;j++) { doc_fydelta=create_example(1,0,1,1,fydelta_cache[j][i]); dist_ydelta=classify_example(svmModel,doc_fydelta); free_example(doc_fydelta,0); viol=loss_cache[j][i]-dist_ydelta; if((viol > maxviol) || (!fydelta)) { fydelta=fydelta_cache[j][i]; lossval=loss_cache[j][i]; maxviol=viol; } } /**** add current fydelta to joint constraint ****/ fydelta=copy_svector(fydelta); append_svector_list(fydelta,(*lhs)); /* add fydelta to lhs */ (*lhs)=fydelta; (*margin)+=lossval; /* add loss to rhs */ sumviol+=maxviol; } return(sumviol); }
void free_model(MODEL *model, int deep) { long i; if(model->supvec) { if(deep) { for(i=1;i<model->sv_num;i++) { free_example(model->supvec[i],1); } } my_free(model->supvec); } if(model->alpha) my_free(model->alpha); if(model->index) my_free(model->index); if(model->lin_weights) my_free(model->lin_weights); my_free(model); }
void update_constraint_cache_for_model(CCACHE *ccache, MODEL *svmModel) /* update the violation scores according to svmModel and find the most violated constraints for each example */ { int i; double progress=0,progress_old=0; double maxviol=0; double dist_ydelta; DOC *doc_fydelta; CCACHEELEM *celem,*prev,*maxviol_celem,*maxviol_prev; for(i=0; i<ccache->n; i++) { /*** example loop ***/ progress+=10.0/ccache->n; if((struct_verbosity==1) && (((int)progress_old) != ((int)progress))) {printf("+");fflush(stdout); progress_old=progress;} if(struct_verbosity>=2) {printf("+"); fflush(stdout);} maxviol=0; prev=NULL; maxviol_celem=NULL; maxviol_prev=NULL; for(celem=ccache->constlist[i];celem;celem=celem->next) { doc_fydelta=create_example(1,0,1,1,celem->fydelta); dist_ydelta=classify_example(svmModel,doc_fydelta); free_example(doc_fydelta,0); celem->viol=celem->rhs-dist_ydelta; if((celem->viol > maxviol) || (!maxviol_celem)) { maxviol=celem->viol; maxviol_celem=celem; maxviol_prev=prev; } prev=celem; } if(maxviol_prev) { /* move max violated constraint to the top of list */ maxviol_prev->next=maxviol_celem->next; maxviol_celem->next=ccache->constlist[i]; ccache->constlist[i]=maxviol_celem; } } }
void update_constraint_cache_for_model(CCACHE *ccache, MODEL *svmModel) /* update the violation scores according to svmModel and find the most violated constraints for each example */ { int i; long progress=0; double maxviol=0; double dist_ydelta; DOC *doc_fydelta; CCACHEELEM *celem,*prev,*maxviol_celem,*maxviol_prev; doc_fydelta=create_example(1,0,1,1,NULL); for(i=0; i<ccache->n; i++) { /*** example loop ***/ if(struct_verbosity>=3) print_percent_progress(&progress,ccache->n,10,"+"); maxviol=0; prev=NULL; maxviol_celem=NULL; maxviol_prev=NULL; for(celem=ccache->constlist[i];celem;celem=celem->next) { doc_fydelta->fvec=celem->fydelta; dist_ydelta=classify_example(svmModel,doc_fydelta); celem->viol=celem->rhs-dist_ydelta; if((celem->viol > maxviol) || (!maxviol_celem)) { maxviol=celem->viol; maxviol_celem=celem; maxviol_prev=prev; } prev=celem; } ccache->changed[i]=0; if(maxviol_prev) { /* move max violated constraint to the top of list */ maxviol_prev->next=maxviol_celem->next; maxviol_celem->next=ccache->constlist[i]; ccache->constlist[i]=maxviol_celem; ccache->changed[i]=1; } } free_example(doc_fydelta,0); }
void add_constraint_to_constraint_cache(CCACHE *ccache, MODEL *svmModel, int exnum, SVECTOR *fydelta, double rhs, int maxconst) /* add new constraint fydelta*w>rhs for example exnum to cache, if it is more violated than the currently most violated constraint in cache. if this grows the number of constraint for this example beyond maxconst, then the most unused constraint is deleted. the funciton assumes that update_constraint_cache_for_model has been run. */ { double viol; double dist_ydelta; DOC *doc_fydelta; CCACHEELEM *celem; int cnum; doc_fydelta=create_example(1,0,1,1,fydelta); dist_ydelta=classify_example(svmModel,doc_fydelta); free_example(doc_fydelta,0); viol=rhs-dist_ydelta; if((viol-0.000000000001) > ccache->constlist[exnum]->viol) { celem=ccache->constlist[exnum]; ccache->constlist[exnum]=(CCACHEELEM *)malloc(sizeof(CCACHEELEM)); ccache->constlist[exnum]->next=celem; ccache->constlist[exnum]->fydelta=fydelta; ccache->constlist[exnum]->rhs=rhs; ccache->constlist[exnum]->viol=viol; /* remove last constraint in list, if list is longer than maxconst */ cnum=2; for(celem=ccache->constlist[exnum];celem && celem->next && celem->next->next;celem=celem->next) cnum++; if(cnum>maxconst) { free_svector(celem->next->fydelta); free(celem->next); celem->next=NULL; } } else { free_svector(fydelta); } }
IMP_S32 ipProcessTargetClassifierInternal( IpTargetClassifier *pstClassifier ) { IpTrackedTarget *pstTarget; IpTargetPosition *pstPos0; IMP_U8 *value1,*value2; IMP_S32 i,j,m,n,cnt,c,targetType; IpClassFeature *astClassFeature; WORDSVM words[15]; svm_node x[15]; DOC *doc; IMP_RECT_S *rc; IMP_RECT_S pstRectImg; IMP_S32 j13;//=blobObjCur.pt.y+blobObjCur.blobHeight/2-blobObjCur.blobHeight/3; IMP_S32 j23;//=blobObjCur.pt.y+blobObjCur.blobHeight/2-2*blobObjCur.blobHeight/3; IMP_S32 is_possible_human,is_possible_vehicle; IMP_FLOAT type; IMP_S32 m13,m23,mBG,mFG,mPerimeter; PEA_RESULT_S *pstResult = pstClassifier->pstResult; PEA_DETECTED_REGIONSET_S *pstRegions = &pstResult->stDRegionSet; GRAY_IMAGE_S *pstImgFgOrg = pstRegions->pstImgFgOrg; GRAY_IMAGE_S *pstImgMediate = pstRegions->pstImgMediate; IMP_S32 s32Width = pstImgFgOrg->s32W; IMP_S32 s32Height = pstImgFgOrg->s32H; IMP_S32 blobWidth,blobHeight; IpClassifierPara *pstParams=&pstClassifier->stPara; IMP_S32 s32UseBorderConstrain=pstParams->s32UseBorderConstrain; IMP_S32 s32SceneMode = 0; pstTarget = pstResult->stTrackedTargetSet.astTargets; cnt = pstResult->stTrackedTargetSet.s32UsedTotal; astClassFeature=pstResult->stTrackedTargetSet.astClassFeature; if (pstParams->pstRule->stEvrmentTgtPara.stScenePara.s32SceneLmt) { s32SceneMode = pstParams->pstRule->stEvrmentTgtPara.stScenePara.s32SceneMode; } if (pstParams->s8SvmFuncType==DISABLE) { return -1; } IMP_GrayImageClone(pstImgFgOrg,pstImgMediate); for( i=0, j=0; i<IMP_MAX_TGT_CNT; i++ ) { if (ipTrackedTargetIsActive(pstTarget)) { #if 0 {//added by mzhang, just for debug. pstTarget->stTargetInfo.s32HumanLikehood = 100; pstTarget->stTargetInfo.s32VehicleLikehood = -1; pstTarget->stTargetInfo.u32Type=IMP_TGT_TYPE_HUMAN; continue; } #endif { targetType = 0; if(!targetType) { pstPos0 = ipTargetTrajectoryGetPosition( &pstTarget->stTrajectory, 0 ); rc=&pstPos0->stRg; if( s32UseBorderConstrain ) { pstRectImg.s16X1 = 0; pstRectImg.s16Y1 = 0; pstRectImg.s16X2 = s32Width - 1; pstRectImg.s16Y2 = s32Height - 1; } if (!s32UseBorderConstrain || s32UseBorderConstrain && !ipBoundaryConditionsJudgment( rc, &pstRectImg, s32UseBorderConstrain )//加入了限定要求并且满足边界条件 ) { m13=0; m23=0; mBG=0; mFG=0; mPerimeter=0; j13=rc->s16Y2-(rc->s16Y2-rc->s16Y1)/3; j23=rc->s16Y2-2*(rc->s16Y2-rc->s16Y1)/3; for (m=rc->s16X1;m<rc->s16X2;m++) { value1=pstImgFgOrg->pu8Data+ s32Width*j13 + m; value2=pstImgFgOrg->pu8Data+ s32Width*j23 + m; if(*value1>0) { m13++; } if(*value2>0) { m23++; } } for (m=rc->s16Y1;m<rc->s16Y2;m++) { for (n=rc->s16X1;n<rc->s16X2;n++) { value1=pstImgFgOrg->pu8Data+ s32Width*m + n; if (*value1==0) { mBG++; } } } for (m=rc->s16Y1;m<rc->s16Y2;m++) { for (n=rc->s16X1;n<rc->s16X2;n++) { value1=pstImgMediate->pu8Data+ s32Width*m + n; if (*value1>0) { mPerimeter++; *value1=100; break; } } for (n=rc->s16X2;n<rc->s16X1;n--) { value1=pstImgMediate->pu8Data+ s32Width*m + n; if (*value1==100) { break; } else if (*value1>0) { mPerimeter++; *value1=100; break; } } } for (n=rc->s16X1;n<rc->s16X2;n++) { for (m=rc->s16Y1;m<rc->s16Y2;m++) { value1=pstImgMediate->pu8Data+ s32Width*m + n; if (*value1==100) break; if (*value1>0) { mPerimeter++; *value1=100; break; } } for (m=rc->s16Y2;m<rc->s16Y1;m--) { value1=pstImgMediate->pu8Data+ s32Width*m + n; if (*value1==100) break; if (*value1>0) { mPerimeter++; *value1=100; break; } } } blobWidth=rc->s16X2-rc->s16X1+1; blobHeight=rc->s16Y2-rc->s16Y1+1; mFG=blobWidth*blobHeight-mBG; astClassFeature->_P=(IMP_DOUBLE)(blobWidth)/blobHeight; astClassFeature->_P13=(IMP_DOUBLE)(m13)/blobHeight; astClassFeature->_P23=(IMP_DOUBLE)(m23)/blobHeight; astClassFeature->_I=(IMP_DOUBLE)(mBG)/(blobWidth*blobHeight); astClassFeature->_D=(IMP_DOUBLE)mPerimeter/mFG; ipComputeHuMomentInvariants(pstImgFgOrg,rc,s32Width,s32Height,astClassFeature->_Hu,&astClassFeature->_Axis); astClassFeature->_Delta=0; astClassFeature->label=pstParams->s32ClassType; if (pstParams->s8SvmFuncType==CLASSIFY) { for (i=0;i<=13;i++) { x[i].index = i+1; } x[14].index = -1; c=-1; x[0].value = astClassFeature->_Axis; x[1].value = astClassFeature->_D; x[2].value = astClassFeature->_Delta; x[3].value = astClassFeature->_Hu[0]; x[4].value = astClassFeature->_Hu[1]; x[5].value = astClassFeature->_Hu[2]; x[6].value = astClassFeature->_Hu[3]; x[7].value = astClassFeature->_Hu[4]; x[8].value = astClassFeature->_Hu[5]; x[9].value = astClassFeature->_Hu[6]; x[10].value = astClassFeature->_I; x[11].value = astClassFeature->_P; x[12].value = astClassFeature->_P13; x[13].value = astClassFeature->_P23; // (*label)[i]=classFeatureTemp.label; // (queryid)=0; // (slackid)=0; // (costfactor)=1; words[0].wnum = 1; words[0].weight = (float)astClassFeature->_Axis; words[1].wnum = 2; words[1].weight = (float)astClassFeature->_D; words[2].wnum = 3; words[2].weight = (float)astClassFeature->_Delta; words[3].wnum = 4; words[3].weight = (float)astClassFeature->_Hu[0]; words[4].wnum = 5; words[4].weight = (float)astClassFeature->_Hu[1]; words[5].wnum = 6; words[5].weight = (float)astClassFeature->_Hu[2]; words[6].wnum = 7; words[6].weight = (float)astClassFeature->_Hu[3]; words[7].wnum = 8; words[7].weight = (float)astClassFeature->_Hu[4]; words[8].wnum = 9; words[8].weight = (float)astClassFeature->_Hu[5]; words[9].wnum = 10; words[9].weight = (float)astClassFeature->_Hu[6]; words[10].wnum = 11; words[10].weight = (float)astClassFeature->_I; words[11].wnum = 12; words[11].weight = (float)astClassFeature->_P; words[12].wnum = 13; words[12].weight = (float)astClassFeature->_P13; words[13].wnum = 14; words[13].weight = (float)astClassFeature->_P23; words[14].wnum=0; //c= (IMP_S32)svm_predict(pstParams->m_model, x); doc = create_example(-1,0,0,0.0,create_svector(words,"",1.0)); type=(float)classify_example(pstParams->pstModel,doc); if (type>0 && type <2) { c=IMP_TGT_TYPE_HUMAN; pstTarget->stTargetInfo.s32HumanLikehood++; pstTarget->stTargetInfo.s32VehicleLikehood--; } else if (type>2 && type <4) { c=IMP_TGT_TYPE_VEHICLE; pstTarget->stTargetInfo.s32VehicleLikehood++; pstTarget->stTargetInfo.s32HumanLikehood--; } free_example(doc,1); } } } } if (pstTarget->stTargetInfo.s32VehicleLikehood>100) { pstTarget->stTargetInfo.s32VehicleLikehood=100; } else if(pstTarget->stTargetInfo.s32VehicleLikehood<-1) { pstTarget->stTargetInfo.s32VehicleLikehood=-1; } if (pstTarget->stTargetInfo.s32VehicleLikehood>100) { pstTarget->stTargetInfo.s32VehicleLikehood=100; } else if(pstTarget->stTargetInfo.s32VehicleLikehood<-1) { pstTarget->stTargetInfo.s32VehicleLikehood=-1; } if (pstTarget->stTargetInfo.s32HumanLikehood && pstTarget->stTargetInfo.s32HumanLikehood >= pstTarget->stTargetInfo.s32VehicleLikehood ) { pstTarget->stTargetInfo.u32Type=IMP_TGT_TYPE_HUMAN; //printf("target Type is HUMAN\n"); } else if (pstTarget->stTargetInfo.s32VehicleLikehood && pstTarget->stTargetInfo.s32VehicleLikehood >= pstTarget->stTargetInfo.s32HumanLikehood ) { pstTarget->stTargetInfo.u32Type=IMP_TGT_TYPE_VEHICLE; //printf("target Type is VEHICLE\n"); } else { pstTarget->stTargetInfo.u32Type=IMP_TGT_TYPE_UNKNOWN; //printf("target Type is UNKNOWN\n"); } } j += pstTarget->s32Used ? 1 : 0; if( j>=cnt ) break; astClassFeature++; pstTarget++; } return 1; }
void svm_learn_struct_joint(SAMPLE sample, STRUCT_LEARN_PARM *sparm, LEARN_PARM *lparm, KERNEL_PARM *kparm, STRUCTMODEL *sm, int alg_type) { int i,j; int numIt=0; long argmax_count=0; long totconstraints=0; long kernel_type_org; double epsilon,epsilon_cached; double lhsXw,rhs_i; double rhs=0; double slack,ceps; double dualitygap,modellength,alphasum; long sizePsi; double *alpha=NULL; long *alphahist=NULL,optcount=0; CONSTSET cset; SVECTOR *diff=NULL; double *lhs_n=NULL; SVECTOR *fy, *fydelta, **fycache, *lhs; MODEL *svmModel=NULL; DOC *doc; long n=sample.n; EXAMPLE *ex=sample.examples; double rt_total=0,rt_opt=0,rt_init=0,rt_psi=0,rt_viol=0,rt_kernel=0; double rt_cacheupdate=0,rt_cacheconst=0,rt_cacheadd=0,rt_cachesum=0; double rt1=0,rt2=0; long progress; /* SVECTOR ***fydelta_cache=NULL; double **loss_cache=NULL; int cache_size=0; */ CCACHE *ccache=NULL; int cached_constraint; double viol,viol_est,epsilon_est=0; long uptr=0; long *randmapping=NULL; long batch_size=n; rt1=get_runtime(); if(sparm->batch_size<100) batch_size=sparm->batch_size*n/100.0; init_struct_model(sample,sm,sparm,lparm,kparm); sizePsi=sm->sizePsi+1; /* sm must contain size of psi on return */ if(sparm->slack_norm == 1) { lparm->svm_c=sparm->C; /* set upper bound C */ lparm->sharedslack=1; } else if(sparm->slack_norm == 2) { printf("ERROR: The joint algorithm does not apply to L2 slack norm!"); fflush(stdout); exit(0); } else { printf("ERROR: Slack norm must be L1 or L2!"); fflush(stdout); exit(0); } lparm->biased_hyperplane=0; /* set threshold to zero */ epsilon=100.0; /* start with low precision and increase later */ epsilon_cached=epsilon; /* epsilon to use for iterations using constraints constructed from the constraint cache */ cset=init_struct_constraints(sample, sm, sparm); if(cset.m > 0) { alpha=(double *)realloc(alpha,sizeof(double)*cset.m); alphahist=(long *)realloc(alphahist,sizeof(long)*cset.m); for(i=0; i<cset.m; i++) { alpha[i]=0; alphahist[i]=-1; /* -1 makes sure these constraints are never removed */ } } kparm->gram_matrix=NULL; if((alg_type == ONESLACK_DUAL_ALG) || (alg_type == ONESLACK_DUAL_CACHE_ALG)) kparm->gram_matrix=init_kernel_matrix(&cset,kparm); /* set initial model and slack variables */ svmModel=(MODEL *)my_malloc(sizeof(MODEL)); lparm->epsilon_crit=epsilon; svm_learn_optimization(cset.lhs,cset.rhs,cset.m,sizePsi, lparm,kparm,NULL,svmModel,alpha); add_weight_vector_to_linear_model(svmModel); sm->svm_model=svmModel; sm->w=svmModel->lin_weights; /* short cut to weight vector */ /* create a cache of the feature vectors for the correct labels */ fycache=(SVECTOR **)my_malloc(n*sizeof(SVECTOR *)); for(i=0;i<n;i++) { if(USE_FYCACHE) { fy=psi(ex[i].x,ex[i].y,sm,sparm); if(kparm->kernel_type == LINEAR_KERNEL) { /* store difference vector directly */ diff=add_list_sort_ss_r(fy,COMPACT_ROUNDING_THRESH); free_svector(fy); fy=diff; } } else fy=NULL; fycache[i]=fy; } /* initialize the constraint cache */ if(alg_type == ONESLACK_DUAL_CACHE_ALG) { ccache=create_constraint_cache(sample,sparm,sm); /* NOTE: */ for(i=0;i<n;i++) if(loss(ex[i].y,ex[i].y,sparm) != 0) { printf("ERROR: Loss function returns non-zero value loss(y_%d,y_%d)\n",i,i); printf(" W4 algorithm assumes that loss(y_i,y_i)=0 for all i.\n"); exit(1); } } if(kparm->kernel_type == LINEAR_KERNEL) lhs_n=create_nvector(sm->sizePsi); /* randomize order or training examples */ if(batch_size<n) randmapping=random_order(n); rt_init+=MAX(get_runtime()-rt1,0); rt_total+=rt_init; /*****************/ /*** main loop ***/ /*****************/ do { /* iteratively find and add constraints to working set */ if(struct_verbosity>=1) { printf("Iter %i: ",++numIt); fflush(stdout); } rt1=get_runtime(); /**** compute current slack ****/ alphasum=0; for(j=0;(j<cset.m);j++) alphasum+=alpha[j]; for(j=0,slack=-1;(j<cset.m) && (slack==-1);j++) if(alpha[j] > alphasum/cset.m) slack=MAX(0,cset.rhs[j]-classify_example(svmModel,cset.lhs[j])); slack=MAX(0,slack); rt_total+=MAX(get_runtime()-rt1,0); /**** find a violated joint constraint ****/ lhs=NULL; rhs=0; if(alg_type == ONESLACK_DUAL_CACHE_ALG) { rt1=get_runtime(); /* Compute violation of constraints in cache for current w */ if(struct_verbosity>=2) rt2=get_runtime(); update_constraint_cache_for_model(ccache, svmModel); if(struct_verbosity>=2) rt_cacheupdate+=MAX(get_runtime()-rt2,0); /* Is there is a sufficiently violated constraint in cache? */ viol=compute_violation_of_constraint_in_cache(ccache,epsilon_est/2); if(viol-slack > MAX(epsilon_est/10,sparm->epsilon)) { /* There is a sufficiently violated constraint in cache, so use this constraint in this iteration. */ if(struct_verbosity>=2) rt2=get_runtime(); viol=find_most_violated_joint_constraint_in_cache(ccache, epsilon_est/2,lhs_n,&lhs,&rhs); if(struct_verbosity>=2) rt_cacheconst+=MAX(get_runtime()-rt2,0); cached_constraint=1; } else { /* There is no sufficiently violated constraint in cache, so update cache by computing most violated constraint explicitly for batch_size examples. */ viol_est=0; progress=0; viol=compute_violation_of_constraint_in_cache(ccache,0); for(j=0;(j<batch_size) || ((j<n)&&(viol-slack<sparm->epsilon));j++) { if(struct_verbosity>=1) print_percent_progress(&progress,n,10,"."); uptr=uptr % n; if(randmapping) i=randmapping[uptr]; else i=uptr; /* find most violating fydelta=fy-fybar and rhs for example i */ find_most_violated_constraint(&fydelta,&rhs_i,&ex[i], fycache[i],n,sm,sparm, &rt_viol,&rt_psi,&argmax_count); /* add current fy-fybar and loss to cache */ if(struct_verbosity>=2) rt2=get_runtime(); viol+=add_constraint_to_constraint_cache(ccache,sm->svm_model, i,fydelta,rhs_i,0.0001*sparm->epsilon/n, sparm->ccache_size,&rt_cachesum); if(struct_verbosity>=2) rt_cacheadd+=MAX(get_runtime()-rt2,0); viol_est+=ccache->constlist[i]->viol; uptr++; } cached_constraint=(j<n); if(struct_verbosity>=2) rt2=get_runtime(); if(cached_constraint) viol=find_most_violated_joint_constraint_in_cache(ccache, epsilon_est/2,lhs_n,&lhs,&rhs); else viol=find_most_violated_joint_constraint_in_cache(ccache,0,lhs_n, &lhs,&rhs); if(struct_verbosity>=2) rt_cacheconst+=MAX(get_runtime()-rt2,0); viol_est*=((double)n/j); epsilon_est=(1-(double)j/n)*epsilon_est+(double)j/n*(viol_est-slack); if((struct_verbosity >= 1) && (j!=n)) printf("(upd=%5.1f%%,eps^=%.4f,eps*=%.4f)", 100.0*j/n,viol_est-slack,epsilon_est); } lhsXw=rhs-viol; rt_total+=MAX(get_runtime()-rt1,0); } else { /* do not use constraint from cache */ rt1=get_runtime(); cached_constraint=0; if(kparm->kernel_type == LINEAR_KERNEL) clear_nvector(lhs_n,sm->sizePsi); progress=0; rt_total+=MAX(get_runtime()-rt1,0); for(i=0; i<n; i++) { rt1=get_runtime(); if(struct_verbosity>=1) print_percent_progress(&progress,n,10,"."); /* compute most violating fydelta=fy-fybar and rhs for example i */ find_most_violated_constraint(&fydelta,&rhs_i,&ex[i],fycache[i],n, sm,sparm,&rt_viol,&rt_psi,&argmax_count); /* add current fy-fybar to lhs of constraint */ if(kparm->kernel_type == LINEAR_KERNEL) { add_list_n_ns(lhs_n,fydelta,1.0); /* add fy-fybar to sum */ free_svector(fydelta); } else { append_svector_list(fydelta,lhs); /* add fy-fybar to vector list */ lhs=fydelta; } rhs+=rhs_i; /* add loss to rhs */ rt_total+=MAX(get_runtime()-rt1,0); } /* end of example loop */ rt1=get_runtime(); /* create sparse vector from dense sum */ if(kparm->kernel_type == LINEAR_KERNEL) lhs=create_svector_n_r(lhs_n,sm->sizePsi,NULL,1.0, COMPACT_ROUNDING_THRESH); doc=create_example(cset.m,0,1,1,lhs); lhsXw=classify_example(svmModel,doc); free_example(doc,0); viol=rhs-lhsXw; rt_total+=MAX(get_runtime()-rt1,0); } /* end of finding most violated joint constraint */ rt1=get_runtime(); /**** if `error', then add constraint and recompute QP ****/ if(slack > (rhs-lhsXw+0.000001)) { printf("\nWARNING: Slack of most violated constraint is smaller than slack of working\n"); printf(" set! There is probably a bug in 'find_most_violated_constraint_*'.\n"); printf("slack=%f, newslack=%f\n",slack,rhs-lhsXw); /* exit(1); */ } ceps=MAX(0,rhs-lhsXw-slack); if((ceps > sparm->epsilon) || cached_constraint) { /**** resize constraint matrix and add new constraint ****/ cset.lhs=(DOC **)realloc(cset.lhs,sizeof(DOC *)*(cset.m+1)); cset.lhs[cset.m]=create_example(cset.m,0,1,1,lhs); cset.rhs=(double *)realloc(cset.rhs,sizeof(double)*(cset.m+1)); cset.rhs[cset.m]=rhs; alpha=(double *)realloc(alpha,sizeof(double)*(cset.m+1)); alpha[cset.m]=0; alphahist=(long *)realloc(alphahist,sizeof(long)*(cset.m+1)); alphahist[cset.m]=optcount; cset.m++; totconstraints++; if((alg_type == ONESLACK_DUAL_ALG) || (alg_type == ONESLACK_DUAL_CACHE_ALG)) { if(struct_verbosity>=2) rt2=get_runtime(); kparm->gram_matrix=update_kernel_matrix(kparm->gram_matrix,cset.m-1, &cset,kparm); if(struct_verbosity>=2) rt_kernel+=MAX(get_runtime()-rt2,0); } /**** get new QP solution ****/ if(struct_verbosity>=1) { printf("*");fflush(stdout); } if(struct_verbosity>=2) rt2=get_runtime(); /* set svm precision so that higher than eps of most violated constr */ if(cached_constraint) { epsilon_cached=MIN(epsilon_cached,ceps); lparm->epsilon_crit=epsilon_cached/2; } else { epsilon=MIN(epsilon,ceps); /* best eps so far */ lparm->epsilon_crit=epsilon/2; epsilon_cached=epsilon; } free_model(svmModel,0); svmModel=(MODEL *)my_malloc(sizeof(MODEL)); /* Run the QP solver on cset. */ kernel_type_org=kparm->kernel_type; if((alg_type == ONESLACK_DUAL_ALG) || (alg_type == ONESLACK_DUAL_CACHE_ALG)) kparm->kernel_type=GRAM; /* use kernel stored in kparm */ svm_learn_optimization(cset.lhs,cset.rhs,cset.m,sizePsi, lparm,kparm,NULL,svmModel,alpha); kparm->kernel_type=kernel_type_org; svmModel->kernel_parm.kernel_type=kernel_type_org; /* Always add weight vector, in case part of the kernel is linear. If not, ignore the weight vector since its content is bogus. */ add_weight_vector_to_linear_model(svmModel); sm->svm_model=svmModel; sm->w=svmModel->lin_weights; /* short cut to weight vector */ optcount++; /* keep track of when each constraint was last active. constraints marked with -1 are not updated */ for(j=0;j<cset.m;j++) if((alphahist[j]>-1) && (alpha[j] != 0)) alphahist[j]=optcount; if(struct_verbosity>=2) rt_opt+=MAX(get_runtime()-rt2,0); /* Check if some of the linear constraints have not been active in a while. Those constraints are then removed to avoid bloating the working set beyond necessity. */ if(struct_verbosity>=3) printf("Reducing working set...");fflush(stdout); remove_inactive_constraints(&cset,alpha,optcount,alphahist,50); if(struct_verbosity>=3) printf("done. "); } else { free_svector(lhs); } if(struct_verbosity>=1) printf("(NumConst=%d, SV=%ld, CEps=%.4f, QPEps=%.4f)\n",cset.m, svmModel->sv_num-1,ceps,svmModel->maxdiff); rt_total+=MAX(get_runtime()-rt1,0); } while(finalize_iteration(ceps,cached_constraint,sample,sm,cset,alpha,sparm)|| cached_constraint || (ceps > sparm->epsilon) ); // originally like below ... finalize_iteration was not called because of short-circuit evaluation // } while(cached_constraint || (ceps > sparm->epsilon) || // finalize_iteration(ceps,cached_constraint,sample,sm,cset,alpha,sparm) // ); if(struct_verbosity>=1) { printf("Final epsilon on KKT-Conditions: %.5f\n", MAX(svmModel->maxdiff,ceps)); slack=0; for(j=0;j<cset.m;j++) slack=MAX(slack, cset.rhs[j]-classify_example(svmModel,cset.lhs[j])); alphasum=0; for(i=0; i<cset.m; i++) alphasum+=alpha[i]*cset.rhs[i]; if(kparm->kernel_type == LINEAR_KERNEL) modellength=model_length_n(svmModel); else modellength=model_length_s(svmModel); dualitygap=(0.5*modellength*modellength+sparm->C*viol) -(alphasum-0.5*modellength*modellength); printf("Upper bound on duality gap: %.5f\n", dualitygap); printf("Dual objective value: dval=%.5f\n", alphasum-0.5*modellength*modellength); printf("Primal objective value: pval=%.5f\n", 0.5*modellength*modellength+sparm->C*viol); printf("Total number of constraints in final working set: %i (of %i)\n",(int)cset.m,(int)totconstraints); printf("Number of iterations: %d\n",numIt); printf("Number of calls to 'find_most_violated_constraint': %ld\n",argmax_count); printf("Number of SV: %ld \n",svmModel->sv_num-1); printf("Norm of weight vector: |w|=%.5f\n",modellength); printf("Value of slack variable (on working set): xi=%.5f\n",slack); printf("Value of slack variable (global): xi=%.5f\n",viol); printf("Norm of longest difference vector: ||Psi(x,y)-Psi(x,ybar)||=%.5f\n", length_of_longest_document_vector(cset.lhs,cset.m,kparm)); if(struct_verbosity>=2) printf("Runtime in cpu-seconds: %.2f (%.2f%% for QP, %.2f%% for kernel, %.2f%% for Argmax, %.2f%% for Psi, %.2f%% for init, %.2f%% for cache update, %.2f%% for cache const, %.2f%% for cache add (incl. %.2f%% for sum))\n", rt_total/100.0, (100.0*rt_opt)/rt_total, (100.0*rt_kernel)/rt_total, (100.0*rt_viol)/rt_total, (100.0*rt_psi)/rt_total, (100.0*rt_init)/rt_total,(100.0*rt_cacheupdate)/rt_total, (100.0*rt_cacheconst)/rt_total,(100.0*rt_cacheadd)/rt_total, (100.0*rt_cachesum)/rt_total); else if(struct_verbosity==1) printf("Runtime in cpu-seconds: %.2f\n",rt_total/100.0); } if(ccache) { long cnum=0; CCACHEELEM *celem; for(i=0;i<n;i++) for(celem=ccache->constlist[i];celem;celem=celem->next) cnum++; printf("Final number of constraints in cache: %ld\n",cnum); } if(struct_verbosity>=4) printW(sm->w,sizePsi,n,lparm->svm_c); if(svmModel) { sm->svm_model=copy_model(svmModel); sm->w=sm->svm_model->lin_weights; /* short cut to weight vector */ free_model(svmModel,0); } print_struct_learning_stats(sample,sm,cset,alpha,sparm); if(lhs_n) free_nvector(lhs_n); if(ccache) free_constraint_cache(ccache); for(i=0;i<n;i++) if(fycache[i]) free_svector(fycache[i]); free(fycache); free(alpha); free(alphahist); free(cset.rhs); for(i=0;i<cset.m;i++) free_example(cset.lhs[i],1); free(cset.lhs); if(kparm->gram_matrix) free_matrix(kparm->gram_matrix); }
int resize_cleanup(int size_active, int **ptr_idle, double **ptr_alpha, double **ptr_delta, double **ptr_gammaG0, double **ptr_proximal_rhs, double ***ptr_G, DOC ***ptr_dXc, double **ptr_cut_error) { int i, j, new_size_active; long k; int *idle = *ptr_idle; double *alpha = *ptr_alpha; double *delta = *ptr_delta; double *gammaG0 = *ptr_gammaG0; double *proximal_rhs = *ptr_proximal_rhs; double **G = *ptr_G; DOC **dXc = *ptr_dXc; double *cut_error = *ptr_cut_error; i = 0; while ((i < size_active) && (idle[i] < IDLE_ITER)) i++; j = i; while ((j < size_active) && (idle[j] >= IDLE_ITER)) j++; while (j < size_active) { /* copying */ alpha[i] = alpha[j]; delta[i] = delta[j]; gammaG0[i] = gammaG0[j]; cut_error[i] = cut_error[j]; free(G[i]); G[i] = G[j]; G[j] = NULL; free_example(dXc[i], 0); dXc[i] = dXc[j]; dXc[j] = NULL; i++; j++; while ((j < size_active) && (idle[j] >= IDLE_ITER)) j++; } for (k = i; k < size_active; k++) { if (G[k] != NULL) free(G[k]); if (dXc[k] != NULL) free_example(dXc[k], 0); } new_size_active = i; alpha = (double *) realloc(alpha, sizeof(double) * new_size_active); delta = (double *) realloc(delta, sizeof(double) * new_size_active); gammaG0 = (double *) realloc(gammaG0, sizeof(double) * new_size_active); proximal_rhs = (double *) realloc(proximal_rhs, sizeof(double) * new_size_active); G = (double **) realloc(G, sizeof(double *) * new_size_active); dXc = (DOC **) realloc(dXc, sizeof(DOC *) * new_size_active); // DEBUG //printf("new_size_active: %d\n", new_size_active); assert(dXc != NULL); // END DEBUG cut_error = (double *) realloc(cut_error, sizeof(double) * new_size_active); /* resize G and idle */ i = 0; while ((i < size_active) && (idle[i] < IDLE_ITER)) i++; j = i; while ((j < size_active) && (idle[j] >= IDLE_ITER)) j++; while (j < size_active) { idle[i] = idle[j]; for (k = 0; k < new_size_active; k++) { G[k][i] = G[k][j]; } i++; j++; while ((j < size_active) && (idle[j] >= IDLE_ITER)) j++; } idle = (int *) realloc(idle, sizeof(int) * new_size_active); for (k = 0; k < new_size_active; k++) { G[k] = (double *) realloc(G[k], sizeof(double) * new_size_active); } *ptr_idle = idle; *ptr_alpha = alpha; *ptr_delta = delta; *ptr_gammaG0 = gammaG0; *ptr_proximal_rhs = proximal_rhs; *ptr_G = G; *ptr_dXc = dXc; *ptr_cut_error = cut_error; return (new_size_active); }
void free_pattern(PATTERNX x) { /* Frees the memory of x. */ free_example(x.doc,1); }
void svm_learn_struct(SAMPLE sample, STRUCT_LEARN_PARM *sparm, LEARN_PARM *lparm, KERNEL_PARM *kparm, STRUCTMODEL *sm, int alg_type) { int i,j; int numIt=0; long argmax_count=0; long newconstraints=0, totconstraints=0, activenum=0; int opti_round, *opti, fullround, use_shrinking; long old_totconstraints=0; double epsilon,svmCnorm; long tolerance,new_precision=1,dont_stop=0; double lossval,factor,dist; double margin=0; double slack, *slacks, slacksum, ceps; double dualitygap,modellength,alphasum; long sizePsi; double *alpha=NULL; long *alphahist=NULL,optcount=0,lastoptcount=0; CONSTSET cset; SVECTOR *diff=NULL; SVECTOR *fy, *fybar, *f, **fycache=NULL; SVECTOR *slackvec; WORD slackv[2]; MODEL *svmModel=NULL; KERNEL_CACHE *kcache=NULL; LABEL ybar; DOC *doc; long n=sample.n; EXAMPLE *ex=sample.examples; double rt_total=0, rt_opt=0, rt_init=0, rt_psi=0, rt_viol=0; double rt1,rt2; rt1=get_runtime(); init_struct_model(sample,sm,sparm,lparm,kparm); sizePsi=sm->sizePsi+1; /* sm must contain size of psi on return */ /* initialize shrinking-style example selection heuristic */ if(alg_type == NSLACK_SHRINK_ALG) use_shrinking=1; else use_shrinking=0; opti=(int*)my_malloc(n*sizeof(int)); for(i=0;i<n;i++) { opti[i]=0; } opti_round=0; /* normalize regularization parameter C by the number of training examples */ svmCnorm=sparm->C/n; if(sparm->slack_norm == 1) { lparm->svm_c=svmCnorm; /* set upper bound C */ lparm->sharedslack=1; } else if(sparm->slack_norm == 2) { lparm->svm_c=999999999999999.0; /* upper bound C must never be reached */ lparm->sharedslack=0; if(kparm->kernel_type != LINEAR_KERNEL) { printf("ERROR: Kernels are not implemented for L2 slack norm!"); fflush(stdout); exit(0); } } else { printf("ERROR: Slack norm must be L1 or L2!"); fflush(stdout); exit(0); } epsilon=100.0; /* start with low precision and increase later */ tolerance=MIN(n/3,MAX(n/100,5));/* increase precision, whenever less than that number of constraints is not fulfilled */ lparm->biased_hyperplane=0; /* set threshold to zero */ cset=init_struct_constraints(sample, sm, sparm); if(cset.m > 0) { alpha=(double *)realloc(alpha,sizeof(double)*cset.m); alphahist=(long *)realloc(alphahist,sizeof(long)*cset.m); for(i=0; i<cset.m; i++) { alpha[i]=0; alphahist[i]=-1; /* -1 makes sure these constraints are never removed */ } } /* set initial model and slack variables*/ svmModel=(MODEL *)my_malloc(sizeof(MODEL)); lparm->epsilon_crit=epsilon; if(kparm->kernel_type != LINEAR_KERNEL) kcache=kernel_cache_init(MAX(cset.m,1),lparm->kernel_cache_size); svm_learn_optimization(cset.lhs,cset.rhs,cset.m,sizePsi+n, lparm,kparm,kcache,svmModel,alpha); if(kcache) kernel_cache_cleanup(kcache); add_weight_vector_to_linear_model(svmModel); sm->svm_model=svmModel; sm->w=svmModel->lin_weights; /* short cut to weight vector */ /* create a cache of the feature vectors for the correct labels */ if(USE_FYCACHE) { fycache=(SVECTOR **)my_malloc(n*sizeof(SVECTOR *)); for(i=0;i<n;i++) { fy=psi(ex[i].x,ex[i].y,sm,sparm); if(kparm->kernel_type == LINEAR_KERNEL) { diff=add_list_ss(fy); /* store difference vector directly */ free_svector(fy); fy=diff; } fycache[i]=fy; } } rt_init+=MAX(get_runtime()-rt1,0); rt_total+=MAX(get_runtime()-rt1,0); /*****************/ /*** main loop ***/ /*****************/ do { /* iteratively increase precision */ epsilon=MAX(epsilon*0.49999999999,sparm->epsilon); new_precision=1; if(epsilon == sparm->epsilon) /* for final precision, find all SV */ tolerance=0; lparm->epsilon_crit=epsilon/2; /* svm precision must be higher than eps */ if(struct_verbosity>=1) printf("Setting current working precision to %g.\n",epsilon); do { /* iteration until (approx) all SV are found for current precision and tolerance */ opti_round++; activenum=n; dont_stop=0; old_totconstraints=totconstraints; do { /* with shrinking turned on, go through examples that keep producing new constraints */ if(struct_verbosity>=1) { printf("Iter %i (%ld active): ",++numIt,activenum); fflush(stdout); } ceps=0; fullround=(activenum == n); for(i=0; i<n; i++) { /*** example loop ***/ rt1=get_runtime(); if((!use_shrinking) || (opti[i] != opti_round)) { /* if the example is not shrunk away, then see if it is necessary to add a new constraint */ rt2=get_runtime(); argmax_count++; if(sparm->loss_type == SLACK_RESCALING) ybar=find_most_violated_constraint_slackrescaling(ex[i].x, ex[i].y,sm, sparm); else ybar=find_most_violated_constraint_marginrescaling(ex[i].x, ex[i].y,sm, sparm); rt_viol+=MAX(get_runtime()-rt2,0); if(empty_label(ybar)) { if(opti[i] != opti_round) { activenum--; opti[i]=opti_round; } if(struct_verbosity>=2) printf("no-incorrect-found(%i) ",i); continue; } /**** get psi(y)-psi(ybar) ****/ rt2=get_runtime(); if(fycache) fy=copy_svector(fycache[i]); else fy=psi(ex[i].x,ex[i].y,sm,sparm); fybar=psi(ex[i].x,ybar,sm,sparm); rt_psi+=MAX(get_runtime()-rt2,0); /**** scale feature vector and margin by loss ****/ lossval=loss(ex[i].y,ybar,sparm); if(sparm->slack_norm == 2) lossval=sqrt(lossval); if(sparm->loss_type == SLACK_RESCALING) factor=lossval; else /* do not rescale vector for */ factor=1.0; /* margin rescaling loss type */ for(f=fy;f;f=f->next) f->factor*=factor; for(f=fybar;f;f=f->next) f->factor*=-factor; margin=lossval; /**** create constraint for current ybar ****/ append_svector_list(fy,fybar);/* append the two vector lists */ doc=create_example(cset.m,0,i+1,1,fy); /**** compute slack for this example ****/ slack=0; for(j=0;j<cset.m;j++) if(cset.lhs[j]->slackid == i+1) { if(sparm->slack_norm == 2) /* works only for linear kernel */ slack=MAX(slack,cset.rhs[j] -(classify_example(svmModel,cset.lhs[j]) -sm->w[sizePsi+i]/(sqrt(2*svmCnorm)))); else slack=MAX(slack, cset.rhs[j]-classify_example(svmModel,cset.lhs[j])); } /**** if `error' add constraint and recompute ****/ dist=classify_example(svmModel,doc); ceps=MAX(ceps,margin-dist-slack); if(slack > (margin-dist+0.0001)) { printf("\nWARNING: Slack of most violated constraint is smaller than slack of working\n"); printf(" set! There is probably a bug in 'find_most_violated_constraint_*'.\n"); printf("Ex %d: slack=%f, newslack=%f\n",i,slack,margin-dist); /* exit(1); */ } if((dist+slack)<(margin-epsilon)) { if(struct_verbosity>=2) {printf("(%i,eps=%.2f) ",i,margin-dist-slack); fflush(stdout);} if(struct_verbosity==1) {printf("."); fflush(stdout);} /**** resize constraint matrix and add new constraint ****/ cset.m++; cset.lhs=(DOC **)realloc(cset.lhs,sizeof(DOC *)*cset.m); if(kparm->kernel_type == LINEAR_KERNEL) { diff=add_list_ss(fy); /* store difference vector directly */ if(sparm->slack_norm == 1) cset.lhs[cset.m-1]=create_example(cset.m-1,0,i+1,1, copy_svector(diff)); else if(sparm->slack_norm == 2) { /**** add squared slack variable to feature vector ****/ slackv[0].wnum=sizePsi+i; slackv[0].weight=1/(sqrt(2*svmCnorm)); slackv[1].wnum=0; /*terminator*/ slackvec=create_svector(slackv,NULL,1.0); cset.lhs[cset.m-1]=create_example(cset.m-1,0,i+1,1, add_ss(diff,slackvec)); free_svector(slackvec); } free_svector(diff); } else { /* kernel is used */ if(sparm->slack_norm == 1) cset.lhs[cset.m-1]=create_example(cset.m-1,0,i+1,1, copy_svector(fy)); else if(sparm->slack_norm == 2) exit(1); } cset.rhs=(double *)realloc(cset.rhs,sizeof(double)*cset.m); cset.rhs[cset.m-1]=margin; alpha=(double *)realloc(alpha,sizeof(double)*cset.m); alpha[cset.m-1]=0; alphahist=(long *)realloc(alphahist,sizeof(long)*cset.m); alphahist[cset.m-1]=optcount; newconstraints++; totconstraints++; } else { printf("+"); fflush(stdout); if(opti[i] != opti_round) { activenum--; opti[i]=opti_round; } } free_example(doc,0); free_svector(fy); /* this also free's fybar */ free_label(ybar); } /**** get new QP solution ****/ if((newconstraints >= sparm->newconstretrain) || ((newconstraints > 0) && (i == n-1)) || (new_precision && (i == n-1))) { if(struct_verbosity>=1) { printf("*");fflush(stdout); } rt2=get_runtime(); free_model(svmModel,0); svmModel=(MODEL *)my_malloc(sizeof(MODEL)); /* Always get a new kernel cache. It is not possible to use the same cache for two different training runs */ if(kparm->kernel_type != LINEAR_KERNEL) kcache=kernel_cache_init(MAX(cset.m,1),lparm->kernel_cache_size); /* Run the QP solver on cset. */ svm_learn_optimization(cset.lhs,cset.rhs,cset.m,sizePsi+n, lparm,kparm,kcache,svmModel,alpha); if(kcache) kernel_cache_cleanup(kcache); /* Always add weight vector, in case part of the kernel is linear. If not, ignore the weight vector since its content is bogus. */ add_weight_vector_to_linear_model(svmModel); sm->svm_model=svmModel; sm->w=svmModel->lin_weights; /* short cut to weight vector */ optcount++; /* keep track of when each constraint was last active. constraints marked with -1 are not updated */ for(j=0;j<cset.m;j++) if((alphahist[j]>-1) && (alpha[j] != 0)) alphahist[j]=optcount; rt_opt+=MAX(get_runtime()-rt2,0); if(new_precision && (epsilon <= sparm->epsilon)) dont_stop=1; /* make sure we take one final pass */ new_precision=0; newconstraints=0; } rt_total+=MAX(get_runtime()-rt1,0); } /* end of example loop */ rt1=get_runtime(); if(struct_verbosity>=1) printf("(NumConst=%d, SV=%ld, CEps=%.4f, QPEps=%.4f)\n",cset.m, svmModel->sv_num-1,ceps,svmModel->maxdiff); /* Check if some of the linear constraints have not been active in a while. Those constraints are then removed to avoid bloating the working set beyond necessity. */ if(struct_verbosity>=2) printf("Reducing working set...");fflush(stdout); remove_inactive_constraints(&cset,alpha,optcount,alphahist, MAX(50,optcount-lastoptcount)); lastoptcount=optcount; if(struct_verbosity>=2) printf("done. (NumConst=%d)\n",cset.m); rt_total+=MAX(get_runtime()-rt1,0); } while(use_shrinking && (activenum > 0)); /* when using shrinking, repeat until all examples produced no constraint at least once */ } while(((totconstraints - old_totconstraints) > tolerance) || dont_stop); } while((epsilon > sparm->epsilon) || finalize_iteration(ceps,0,sample,sm,cset,alpha,sparm)); if(struct_verbosity>=1) { /**** compute sum of slacks ****/ /**** WARNING: If positivity constraints are used, then the maximum slack id is larger than what is allocated below ****/ slacks=(double *)my_malloc(sizeof(double)*(n+1)); for(i=0; i<=n; i++) { slacks[i]=0; } if(sparm->slack_norm == 1) { for(j=0;j<cset.m;j++) slacks[cset.lhs[j]->slackid]=MAX(slacks[cset.lhs[j]->slackid], cset.rhs[j]-classify_example(svmModel,cset.lhs[j])); } else if(sparm->slack_norm == 2) { for(j=0;j<cset.m;j++) slacks[cset.lhs[j]->slackid]=MAX(slacks[cset.lhs[j]->slackid], cset.rhs[j] -(classify_example(svmModel,cset.lhs[j]) -sm->w[sizePsi+cset.lhs[j]->slackid-1]/(sqrt(2*svmCnorm)))); } slacksum=0; for(i=1; i<=n; i++) slacksum+=slacks[i]; free(slacks); alphasum=0; for(i=0; i<cset.m; i++) alphasum+=alpha[i]*cset.rhs[i]; modellength=model_length_s(svmModel); dualitygap=(0.5*modellength*modellength+svmCnorm*(slacksum+n*ceps)) -(alphasum-0.5*modellength*modellength); printf("Final epsilon on KKT-Conditions: %.5f\n", MAX(svmModel->maxdiff,epsilon)); printf("Upper bound on duality gap: %.5f\n", dualitygap); printf("Dual objective value: dval=%.5f\n", alphasum-0.5*modellength*modellength); printf("Total number of constraints in final working set: %i (of %i)\n",(int)cset.m,(int)totconstraints); printf("Number of iterations: %d\n",numIt); printf("Number of calls to 'find_most_violated_constraint': %ld\n",argmax_count); if(sparm->slack_norm == 1) { printf("Number of SV: %ld \n",svmModel->sv_num-1); printf("Number of non-zero slack variables: %ld (out of %ld)\n", svmModel->at_upper_bound,n); printf("Norm of weight vector: |w|=%.5f\n",modellength); } else if(sparm->slack_norm == 2){ printf("Number of SV: %ld (including %ld at upper bound)\n", svmModel->sv_num-1,svmModel->at_upper_bound); printf("Norm of weight vector (including L2-loss): |w|=%.5f\n", modellength); } printf("Norm. sum of slack variables (on working set): sum(xi_i)/n=%.5f\n",slacksum/n); printf("Norm of longest difference vector: ||Psi(x,y)-Psi(x,ybar)||=%.5f\n", length_of_longest_document_vector(cset.lhs,cset.m,kparm)); printf("Runtime in cpu-seconds: %.2f (%.2f%% for QP, %.2f%% for Argmax, %.2f%% for Psi, %.2f%% for init)\n", rt_total/100.0, (100.0*rt_opt)/rt_total, (100.0*rt_viol)/rt_total, (100.0*rt_psi)/rt_total, (100.0*rt_init)/rt_total); } if(struct_verbosity>=4) printW(sm->w,sizePsi,n,lparm->svm_c); if(svmModel) { sm->svm_model=copy_model(svmModel); sm->w=sm->svm_model->lin_weights; /* short cut to weight vector */ } print_struct_learning_stats(sample,sm,cset,alpha,sparm); if(fycache) { for(i=0;i<n;i++) free_svector(fycache[i]); free(fycache); } if(svmModel) free_model(svmModel,0); free(alpha); free(alphahist); free(opti); free(cset.rhs); for(i=0;i<cset.m;i++) free_example(cset.lhs[i],1); free(cset.lhs); }
int main (int argc, char* argv[]) { int o, i, j, cad_num; char filename[256]; CAD **cads, *cad; SAMPLE sample, sample_part; /* training sample */ LEARN_PARM learn_parm; KERNEL_PARM kernel_parm; STRUCT_LEARN_PARM struct_parm; STRUCTMODEL structmodel; int alg_type; int rank; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); /* select GPU */ select_gpu(rank); svm_struct_learn_api_init(argc,argv); read_input_parameters(argc, argv, trainfile, cadfile, testfile, &verbosity, &struct_verbosity, &struct_parm, &learn_parm, &kernel_parm, &alg_type); /* read cad models */ cads = read_cad_model(cadfile, &cad_num, 0, &struct_parm); /* if cad_num == 1, train the final model */ if(cad_num == 1) { printf("Train the hierarchical model\n"); /* set the cad model for structmodel */ structmodel.cad_num = 1; structmodel.cads = cads; printf("Read training samples\n"); sample = read_struct_examples(trainfile, &struct_parm, &structmodel); printf("Read training samples done\n"); if(struct_parm.is_root == 1 || struct_parm.is_aspectlet == 1) { /* first train weights for root parts */ printf("Train root models\n"); cad = cads[0]; struct_parm.cad_index = 0; /* for each part */ for(i = 0; i < cad->part_num; i++) { /* choose what parts to train */ if((cad->roots[i] == -1 && struct_parm.is_root == 1) || (cad->roots[i] == 1 && struct_parm.is_aspectlet == 1)) { printf("Train part %d\n", i); /* training iteration index */ struct_parm.iter = 0; struct_parm.part_index = i; /* select training samples for part */ if(rank == 0) select_examples_part(trainfile, sample, cad, 0, i); MPI_Barrier(MPI_COMM_WORLD); sample_part = read_struct_examples("temp_part.dat", &struct_parm, &structmodel); /* train the root template */ struct_parm.deep = 0; /* Do the learning and return structmodel. */ if(alg_type == 1) svm_learn_struct(sample_part, &struct_parm, &learn_parm, &kernel_parm, &structmodel); else if(alg_type == 2) svm_learn_struct_joint(sample_part, &struct_parm, &learn_parm, &kernel_parm, &structmodel, PRIMAL_ALG); else if(alg_type == 3) svm_learn_struct_joint(sample_part, &struct_parm, &learn_parm, &kernel_parm, &structmodel, DUAL_ALG); else if(alg_type == 4) svm_learn_struct_joint(sample_part, &struct_parm, &learn_parm, &kernel_parm, &structmodel, DUAL_CACHE_ALG); else exit(1); /* if aspectlet, train the subtree */ if(cad->roots[i] == 1 && struct_parm.is_aspectlet == 1) { printf("Train subtree for part %d\n", i); struct_parm.deep = 1; struct_parm.iter++; /* select training samples for part */ if(rank == 0) select_examples_part(trainfile, sample, cad, 0, i); MPI_Barrier(MPI_COMM_WORLD); free_struct_sample(sample_part); sample_part = read_struct_examples("temp_part.dat", &struct_parm, &structmodel); free_struct_model(structmodel); /* set the cad model for structmodel */ structmodel.cad_num = 1; structmodel.cads = cads; if(alg_type == 1) svm_learn_struct(sample_part, &struct_parm, &learn_parm, &kernel_parm, &structmodel); else if(alg_type == 2) svm_learn_struct_joint(sample_part, &struct_parm, &learn_parm, &kernel_parm, &structmodel, PRIMAL_ALG); else if(alg_type == 3) svm_learn_struct_joint(sample_part, &struct_parm, &learn_parm, &kernel_parm, &structmodel, DUAL_ALG); else if(alg_type == 4) svm_learn_struct_joint(sample_part, &struct_parm, &learn_parm, &kernel_parm, &structmodel, DUAL_CACHE_ALG); else exit(1); } /* data mining hard examples */ for(j = 0; j < struct_parm.hard_negative; j++) { /* increase iteration number */ struct_parm.iter++; data_mining_hard_examples("temp_part.dat", testfile, &struct_parm, &structmodel); /* read the training examples */ if(struct_verbosity>=1) { printf("Reading training examples..."); fflush(stdout); } free_struct_sample(sample_part); sample_part = read_struct_examples("temp.dat", &struct_parm, &structmodel); if(struct_verbosity>=1) { printf("done\n"); fflush(stdout); } /* Do the learning and return structmodel. */ free_struct_model(structmodel); /* set the cad model for structmodel */ structmodel.cad_num = 1; structmodel.cads = cads; if(alg_type == 1) svm_learn_struct(sample_part, &struct_parm, &learn_parm, &kernel_parm, &structmodel); else if(alg_type == 2) svm_learn_struct_joint(sample_part, &struct_parm, &learn_parm, &kernel_parm, &structmodel, PRIMAL_ALG); else if(alg_type == 3) svm_learn_struct_joint(sample_part, &struct_parm, &learn_parm, &kernel_parm, &structmodel, DUAL_ALG); else if(alg_type == 4) svm_learn_struct_joint(sample_part, &struct_parm, &learn_parm, &kernel_parm, &structmodel, DUAL_CACHE_ALG); else exit(1); } /* save constraints for training the full model */ if(rank == 0) save_constraints(&struct_parm, &structmodel); MPI_Barrier(MPI_COMM_WORLD); free_struct_sample(sample_part); free_struct_model(structmodel); /* set the cad model for structmodel */ structmodel.cad_num = 1; structmodel.cads = cads; } } /* end for each part */ if(rank == 0) { write_constraints(struct_parm.cset, &struct_parm); free(struct_parm.cset.rhs); for(i = 0; i < struct_parm.cset.m; i++) free_example(struct_parm.cset.lhs[i], 1); free(struct_parm.cset.lhs); } MPI_Barrier(MPI_COMM_WORLD); } /* end if is_root == 1 */ /* train the full model */ printf("Train the full model\n"); struct_parm.iter = 0; struct_parm.cad_index = -1; struct_parm.part_index = -1; struct_parm.deep = 1; /* Do the learning and return structmodel. */ if(alg_type == 1) svm_learn_struct(sample, &struct_parm, &learn_parm, &kernel_parm, &structmodel); else if(alg_type == 2) svm_learn_struct_joint(sample, &struct_parm, &learn_parm, &kernel_parm, &structmodel, PRIMAL_ALG); else if(alg_type == 3) svm_learn_struct_joint(sample, &struct_parm, &learn_parm, &kernel_parm, &structmodel, DUAL_ALG); else if(alg_type == 4) svm_learn_struct_joint(sample, &struct_parm, &learn_parm, &kernel_parm, &structmodel, DUAL_CACHE_ALG); else exit(1); /* data mining hard examples */ while(struct_parm.hard_negative > 0) { /* increase iteration number */ struct_parm.iter++; data_mining_hard_examples(trainfile, testfile, &struct_parm, &structmodel); /* read the training examples */ if(struct_verbosity>=1) { printf("Reading training examples..."); fflush(stdout); } free_struct_sample(sample); sample = read_struct_examples("temp.dat", &struct_parm, &structmodel); if(struct_verbosity>=1) { printf("done\n"); fflush(stdout); } /* Do the learning and return structmodel. */ free_struct_model(structmodel); /* set the cad model for structmodel */ structmodel.cad_num = 1; structmodel.cads = cads; if(alg_type == 1) svm_learn_struct(sample, &struct_parm, &learn_parm, &kernel_parm, &structmodel); else if(alg_type == 2) svm_learn_struct_joint(sample, &struct_parm, &learn_parm, &kernel_parm, &structmodel, PRIMAL_ALG); else if(alg_type == 3) svm_learn_struct_joint(sample, &struct_parm, &learn_parm, &kernel_parm, &structmodel, DUAL_ALG); else if(alg_type == 4) svm_learn_struct_joint(sample, &struct_parm, &learn_parm, &kernel_parm, &structmodel, DUAL_CACHE_ALG); else exit(1); struct_parm.hard_negative--; } if(rank == 0) { if(struct_verbosity >= 1) { printf("Writing learned model..."); fflush(stdout); } sprintf(modelfile, "%s.mod", struct_parm.cls); write_struct_model(modelfile, &structmodel, &struct_parm); if(struct_verbosity>=1) { printf("done\n"); fflush(stdout); } } MPI_Barrier(MPI_COMM_WORLD); free_struct_sample(sample); free_struct_model(structmodel); } /* train weights for aspectlets */ /* start with the second aspectlet, the first one is the whole object */ for(o = 1; o < cad_num; o++) { printf("Train aspectlet %d\n", o); cad = cads[o]; /* set the cad model for structmodel */ structmodel.cad_num = 1; structmodel.cads = &cad; /* training iteration index */ struct_parm.iter = 0; struct_parm.cad_index = -1; struct_parm.part_index = -1; struct_parm.deep = 1; /* select training samples for the aspectlet */ if(rank == 0) select_examples_aspectlet(trainfile, cads, o); MPI_Barrier(MPI_COMM_WORLD); printf("Read training samples\n"); sample_part = read_struct_examples("temp_part.dat", &struct_parm, &structmodel); printf("Read training samples done\n"); /* Do the learning and return structmodel. */ if(alg_type == 1) svm_learn_struct(sample_part, &struct_parm, &learn_parm, &kernel_parm, &structmodel); else if(alg_type == 2) svm_learn_struct_joint(sample_part, &struct_parm, &learn_parm, &kernel_parm, &structmodel, PRIMAL_ALG); else if(alg_type == 3) svm_learn_struct_joint(sample_part, &struct_parm, &learn_parm, &kernel_parm, &structmodel, DUAL_ALG); else if(alg_type == 4) svm_learn_struct_joint(sample_part, &struct_parm, &learn_parm, &kernel_parm, &structmodel, DUAL_CACHE_ALG); else exit(1); /* data mining hard examples */ for(i = 0; i < struct_parm.hard_negative; i++) { /* increase iteration number */ struct_parm.iter++; data_mining_hard_examples("temp_part.dat", testfile, &struct_parm, &structmodel); /* read the training examples */ if(struct_verbosity>=1) { printf("Reading training examples..."); fflush(stdout); } free_struct_sample(sample_part); sample_part = read_struct_examples("temp.dat", &struct_parm, &structmodel); if(struct_verbosity>=1) { printf("done\n"); fflush(stdout); } /* Do the learning and return structmodel. */ free_struct_model(structmodel); /* set the cad model for structmodel */ structmodel.cad_num = 1; structmodel.cads = &cad; if(alg_type == 1) svm_learn_struct(sample_part, &struct_parm, &learn_parm, &kernel_parm, &structmodel); else if(alg_type == 2) svm_learn_struct_joint(sample_part, &struct_parm, &learn_parm, &kernel_parm, &structmodel, PRIMAL_ALG); else if(alg_type == 3) svm_learn_struct_joint(sample_part, &struct_parm, &learn_parm, &kernel_parm, &structmodel, DUAL_ALG); else if(alg_type == 4) svm_learn_struct_joint(sample_part, &struct_parm, &learn_parm, &kernel_parm, &structmodel, DUAL_CACHE_ALG); else exit(1); } if(rank == 0) { if(struct_verbosity>=1) { printf("Writing learned model..."); fflush(stdout); } sprintf(modelfile, "%s_cad%03d.mod", struct_parm.cls, o); write_struct_model(modelfile, &structmodel, &struct_parm); if(struct_verbosity>=1) { printf("done\n"); fflush(stdout); } } MPI_Barrier(MPI_COMM_WORLD); free_struct_sample(sample_part); free_struct_model(structmodel); } /* end for each cad model */ for(i = 0; i < cad_num; i++) destroy_cad(cads[i]); free(cads); svm_struct_learn_api_exit(); MPI_Finalize(); return 0; }
double add_constraint_to_constraint_cache(CCACHE *ccache, MODEL *svmModel, int exnum, SVECTOR *fydelta, double rhs, double gainthresh, int maxconst, double *rt_cachesum) /* add new constraint fydelta*w>rhs for example exnum to cache, if it is more violated (by gainthresh) than the currently most violated constraint in cache. if this grows the number of cached constraints for this example beyond maxconst, then the least recently used constraint is deleted. the function assumes that update_constraint_cache_for_model has been run. */ { double viol,viol_gain,viol_gain_trunc; double dist_ydelta; DOC *doc_fydelta; SVECTOR *fydelta_new; CCACHEELEM *celem; int cnum; double rt2=0; /* compute violation of new constraint */ doc_fydelta=create_example(1,0,1,1,fydelta); dist_ydelta=classify_example(svmModel,doc_fydelta); free_example(doc_fydelta,0); viol=rhs-dist_ydelta; viol_gain=viol-ccache->constlist[exnum]->viol; viol_gain_trunc=viol-MAX(ccache->constlist[exnum]->viol,0); ccache->avg_viol_gain[exnum]=viol_gain; /* check if violation of new constraint is larger than that of the best cache element */ if(viol_gain > gainthresh) { fydelta_new=fydelta; if(struct_verbosity>=2) rt2=get_runtime(); if(svmModel->kernel_parm.kernel_type == LINEAR_KERNEL) { if(COMPACT_CACHED_VECTORS == 1) { /* eval sum for linear */ fydelta_new=add_list_sort_ss_r(fydelta,COMPACT_ROUNDING_THRESH); free_svector(fydelta); } else if(COMPACT_CACHED_VECTORS == 2) { fydelta_new=add_list_ss_r(fydelta,COMPACT_ROUNDING_THRESH); free_svector(fydelta); } else if(COMPACT_CACHED_VECTORS == 3) { fydelta_new=add_list_ns_r(fydelta,COMPACT_ROUNDING_THRESH); free_svector(fydelta); } } if(struct_verbosity>=2) (*rt_cachesum)+=MAX(get_runtime()-rt2,0); celem=ccache->constlist[exnum]; ccache->constlist[exnum]=(CCACHEELEM *)my_malloc(sizeof(CCACHEELEM)); ccache->constlist[exnum]->next=celem; ccache->constlist[exnum]->fydelta=fydelta_new; ccache->constlist[exnum]->rhs=rhs; ccache->constlist[exnum]->viol=viol; ccache->changed[exnum]+=2; /* remove last constraint in list, if list is longer than maxconst */ cnum=2; for(celem=ccache->constlist[exnum];celem && celem->next && celem->next->next;celem=celem->next) cnum++; if(cnum>maxconst) { free_svector(celem->next->fydelta); free(celem->next); celem->next=NULL; } } else { free_svector(fydelta); } return(viol_gain_trunc); }
double cutting_plane_algorithm(double *w, long m, int MAX_ITER, double C, double epsilon, SVECTOR **fycache, EXAMPLE *ex, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm, int *valid_examples) { long i,j,t; double *alpha; DOC **dXc; /* constraint matrix */ double *delta; /* rhs of constraints */ SVECTOR *new_constraint; int iter, size_active; double value; double threshold = 0.0; double margin; double primal_obj, cur_obj; double *cur_slack = NULL; int mv_iter; int *idle = NULL; double **psiDiffs = NULL; SVECTOR *f; int r; long fnum, last_wnum; /* set parameters for hideo solver */ LEARN_PARM lparm; KERNEL_PARM kparm; MODEL *svm_model=NULL; lparm.biased_hyperplane = 0; lparm.epsilon_crit = MIN(epsilon,0.001); lparm.svm_c = C; lparm.sharedslack = 1; kparm.kernel_type = LINEAR; lparm.remove_inconsistent=0; lparm.skip_final_opt_check=0; lparm.svm_maxqpsize=10; lparm.svm_newvarsinqp=0; lparm.svm_iter_to_shrink=-9999; lparm.maxiter=100000; lparm.kernel_cache_size=40; lparm.eps = epsilon; lparm.transduction_posratio=-1.0; lparm.svm_costratio=1.0; lparm.svm_costratio_unlab=1.0; lparm.svm_unlabbound=1E-5; lparm.epsilon_a=1E-10; /* changed from 1e-15 */ lparm.compute_loo=0; lparm.rho=1.0; lparm.xa_depth=0; strcpy(lparm.alphafile,""); kparm.poly_degree=3; kparm.rbf_gamma=1.0; kparm.coef_lin=1; kparm.coef_const=1; strcpy(kparm.custom,"empty"); iter = 0; size_active = 0; alpha = NULL; dXc = NULL; delta = NULL; printf("Running structural SVM solver: "); fflush(stdout); new_constraint = find_cutting_plane(ex, fycache, &margin, m, sm, sparm, valid_examples); value = margin - sprod_ns(w, new_constraint); while((value>threshold+epsilon)&&(iter<MAX_ITER)) { iter+=1; size_active+=1; printf("."); fflush(stdout); /* add constraint */ dXc = (DOC**)realloc(dXc, sizeof(DOC*)*size_active); assert(dXc!=NULL); dXc[size_active-1] = (DOC*)malloc(sizeof(DOC)); dXc[size_active-1]->fvec = new_constraint; dXc[size_active-1]->slackid = 1; // only one common slackid (one-slack) dXc[size_active-1]->costfactor = 1.0; delta = (double*)realloc(delta, sizeof(double)*size_active); assert(delta!=NULL); delta[size_active-1] = margin; /*alpha = (double*)realloc(alpha, sizeof(double)*size_active); assert(alpha!=NULL); alpha[size_active-1] = 0.0;*/ /*idle = (int *) realloc(idle, sizeof(int)*size_active); assert(idle!=NULL); idle[size_active-1] = 0;*/ /* update Gram matrix */ psiDiffs = (double **) realloc(psiDiffs, sizeof(double *)*size_active); assert(psiDiffs!=NULL); psiDiffs[size_active-1] = NULL; psiDiffs[size_active-1] = (double *) realloc(psiDiffs[size_active-1], sizeof(double)*((sparm->phi1_size+sparm->phi2_size)*3)); assert(psiDiffs[size_active-1]!=NULL); fnum = 0; last_wnum = 0; while(dXc[size_active-1]->fvec->words[fnum].wnum) { for (t = last_wnum+1; t < dXc[size_active-1]->fvec->words[fnum].wnum; t++) { psiDiffs[size_active-1][t-1] = 0; } psiDiffs[size_active-1][dXc[size_active-1]->fvec->words[fnum].wnum-1] = dXc[size_active-1]->fvec->words[fnum].weight; /*if((psiDiffs[size_active-1][dXc[size_active-1]->fvec->words[fnum].wnum-1]<EQUALITY_EPSILON) && (psiDiffs[size_active-1][dXc[size_active-1]->fvec->words[fnum].wnum-1]>(-1*EQUALITY_EPSILON))){ psiDiffs[size_active-1][dXc[size_active-1]->fvec->words[fnum].wnum-1] = 0; }*/ last_wnum = dXc[size_active-1]->fvec->words[fnum].wnum; fnum++; } for (t = (last_wnum+1); t <= (sparm->phi1_size+sparm->phi2_size)*3; t++) { psiDiffs[size_active-1][t-1] = 0; } /* solve QP to update w */ clear_nvector(w,sm->sizePsi); //cur_slack = (double *) realloc(cur_slack,sizeof(double)*size_active); cur_slack = (double *) realloc(cur_slack,sizeof(double)); r = mosek_qp_optimize(psiDiffs, delta, w, cur_slack, (long) size_active, C, &cur_obj, (sparm->phi1_size+sparm->phi2_size)*3, (sparm->phi1_size+sparm->phi2_size)*2); if(r >= 1293 && r <= 1296) { printf("r:%d. G might not be psd due to numerical errors.\n",r); exit(1); } else if(r) { printf("Error %d in mosek_qp_optimize: Check ${MOSEKHOME}/${VERSION}/tools/platform/${PLATFORM}/h/mosek.h\n",r); exit(1); } for(j = 1; j <= (sparm->phi1_size+sparm->phi2_size)*3; j++) { if((w[j]<EQUALITY_EPSILON) && (w[j]>(-1*EQUALITY_EPSILON))){ w[j] = 0; } } /*for (j=0;j<size_active;j++) { if (cur_slack[j]>ALPHA_THRESHOLD) { idle[j] = 0; } else idle[j]++; }*/ /*mv_iter = 0; if(size_active > 1) { for(j = 0; j < size_active; j++) { if(cur_slack[j] >= cur_slack[mv_iter]) mv_iter = j; } }*/ if(size_active > 1) //threshold = cur_slack[mv_iter]; threshold = cur_slack[0]; else threshold = 0.0; new_constraint = find_cutting_plane(ex, fycache, &margin, m, sm, sparm, valid_examples); value = margin - sprod_ns(w, new_constraint); /*if((iter % CLEANUP_CHECK) == 0) { printf("+"); fflush(stdout); size_active = resize_cleanup(size_active, &idle, &cur_slack, &delta, &dXc, &psiDiffs, &mv_iter); }*/ } // end cutting plane while loop primal_obj = current_obj_val(ex, fycache, m, sm, sparm, C, valid_examples); printf(" Inner loop optimization finished.\n"); fflush(stdout); /* free memory */ for (j=0;j<size_active;j++) { free(psiDiffs[j]); free_example(dXc[j],1); } free(psiDiffs); free(dXc); //free(alpha); free(delta); free_svector(new_constraint); free(cur_slack); //free(idle); if (svm_model!=NULL) free_model(svm_model,0); return(primal_obj); }
void cutting_plane_algorithm_dual(double *w, long m, int MAX_ITER, double C, double epsilon, SVECTOR **fycache, EXAMPLE *ex, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm, int *valid_examples) { long i,j; double *alpha; DOC **dXc; // constraint matrix double *delta; // rhs of constraints SVECTOR *new_constraint; int iter, size_active; double value; double threshold = 0.0; double margin; double primal_obj, cur_obj; double *cur_slack = NULL; int mv_iter; int *idle = NULL; double **G = NULL; double **G2 = NULL; double **qmatrix = NULL; SVECTOR *f; int r; // set parameters for hideo solver LEARN_PARM lparm; KERNEL_PARM kparm; MODEL *svm_model=NULL; lparm.biased_hyperplane = 0; lparm.epsilon_crit = MIN(epsilon,0.001); lparm.svm_c = C; lparm.sharedslack = 1; kparm.kernel_type = LINEAR; lparm.remove_inconsistent=0; lparm.skip_final_opt_check=0; lparm.svm_maxqpsize=10; lparm.svm_newvarsinqp=0; lparm.svm_iter_to_shrink=-9999; lparm.maxiter=100000; lparm.kernel_cache_size=40; lparm.eps = epsilon; lparm.transduction_posratio=-1.0; lparm.svm_costratio=1.0; lparm.svm_costratio_unlab=1.0; lparm.svm_unlabbound=1E-5; lparm.epsilon_a=1E-10; // changed from 1e-15 lparm.compute_loo=0; lparm.rho=1.0; lparm.xa_depth=0; strcpy(lparm.alphafile,""); kparm.poly_degree=3; kparm.rbf_gamma=1.0; kparm.coef_lin=1; kparm.coef_const=1; strcpy(kparm.custom,"empty"); iter = 0; size_active = 0; alpha = NULL; dXc = NULL; delta = NULL; //qmatrix = (double **) malloc(sizeof(double *)*10); //assert(qmatrix!=NULL); printf("Running structural SVM solver: "); fflush(stdout); new_constraint = find_cutting_plane(ex, fycache, &margin, m, sm, sparm, valid_examples); value = margin - sprod_ns(w, new_constraint); while((value>threshold+epsilon)&&(iter<MAX_ITER)) { iter+=1; size_active+=1; printf("."); fflush(stdout); // add constraint dXc = (DOC**)realloc(dXc, sizeof(DOC*)*size_active); assert(dXc!=NULL); dXc[size_active-1] = (DOC*)malloc(sizeof(DOC)); dXc[size_active-1]->fvec = new_constraint; dXc[size_active-1]->slackid = 1; // only one common slackid (one-slack) dXc[size_active-1]->costfactor = 1.0; delta = (double*)realloc(delta, sizeof(double)*size_active); assert(delta!=NULL); delta[size_active-1] = margin; //alpha = (double*)malloc(sizeof(double)*(size_active+(sparm->phi1_size+sparm->phi2_size))); //assert(alpha!=NULL); //for(j=0; j<(sparm->phi1_size+sparm->phi2_size)+size_active; j++){ // alpha[j] = 0.0; //} alpha = (double*)realloc(alpha, sizeof(double)*(size_active+(sparm->phi1_size+sparm->phi2_size))); assert(alpha!=NULL); alpha[size_active-1] = 0.0; idle = (int *) realloc(idle, sizeof(int)*size_active); assert(idle!=NULL); idle[size_active-1] = 0; qmatrix = (double **) realloc(qmatrix, sizeof(double *)*size_active); assert(qmatrix!=NULL); qmatrix[size_active-1] = malloc(sizeof(double)*(sparm->phi1_size+sparm->phi2_size)); for(j = 0; j < (sparm->phi1_size+sparm->phi2_size); j++){ qmatrix[size_active-1][j] = (-1)*returnWeightAtIndex(dXc[size_active-1]->fvec->words, ((sparm->phi1_size+sparm->phi2_size)*2+j+1)); } // update Gram matrix G = (double **) realloc(G, sizeof(double *)*size_active); assert(G!=NULL); G[size_active-1] = NULL; for(j = 0; j < size_active; j++) { G[j] = (double *) realloc(G[j], sizeof(double)*size_active); assert(G[j]!=NULL); } for(j = 0; j < size_active-1; j++) { G[size_active-1][j] = sprod_ss(dXc[size_active-1]->fvec, dXc[j]->fvec); G[size_active-1][j] = G[size_active-1][j]/2; G[j][size_active-1] = G[size_active-1][j]; } G[size_active-1][size_active-1] = sprod_ss(dXc[size_active-1]->fvec,dXc[size_active-1]->fvec); // hack: add a constant to the diagonal to make sure G is PSD G[size_active-1][size_active-1] += 1e-6; // solve QP to update alpha //r = mosek_qp_optimize(G, delta, alpha, (long) size_active, C, &cur_obj, dXc, (sparm->phi1_size+sparm->phi2_size)*2, (sparm->phi1_size+sparm->phi2_size)); r = mosek_qp_optimize_dual(G, qmatrix, delta, alpha, (long) size_active, (long) (sparm->phi1_size+sparm->phi2_size), C, &cur_obj, 0, 0); if(r >= 1293 && r <= 1296) { printf("r:%d. G might not be psd due to numerical errors.\n",r); fflush(stdout); //exit(1); while(r==1295) { printf("r:%d. G might not be psd due to numerical errors. Gram Reg=%0.7f\n",r, sparm->gram_regularization); fflush(stdout); for(i=0;i<size_active;i++) { G[i][i] += 10*sparm->gram_regularization-sparm->gram_regularization; } sparm->gram_regularization *= 10; r = mosek_qp_optimize_dual(G, qmatrix, delta, alpha, (long) size_active, (long) (sparm->phi1_size+sparm->phi2_size), C, &cur_obj, sparm->gram_regularization, sparm->gram_regularization*0.1); } } else if(r) { printf("Error %d in mosek_qp_optimize: Check ${MOSEKHOME}/${VERSION}/tools/platform/${PLATFORM}/h/mosek.h\n",r); exit(1); } clear_nvector(w,sm->sizePsi); for (j=0;j<size_active;j++) { if (alpha[j]>C*ALPHA_THRESHOLD) { add_vector_ns(w,dXc[j]->fvec,alpha[j]); idle[j] = 0; } else idle[j]++; } for(j=0; j<(sparm->phi1_size+sparm->phi2_size);j++){ if (alpha[size_active+j] > EQUALITY_EPSILON){ w[j+1+(sparm->phi1_size+sparm->phi2_size)*2] = w[j+1+(sparm->phi1_size+sparm->phi2_size)*2] - alpha[size_active+j]; } } for(j=1; j<=(sparm->phi1_size+sparm->phi2_size)*3;j++){ if((w[j]<EQUALITY_EPSILON) && (w[j]>(-1*EQUALITY_EPSILON))){ w[j] = 0; } } for(j=(sparm->phi1_size+sparm->phi2_size)*2+1; j<=(sparm->phi1_size+sparm->phi2_size)*3;j++){ //assert(w[j] <= 0); if(w[j]>0){ printf("j = %ld, w[j] = %0.6f\n", j, w[j]); fflush(stdout); } } cur_slack = (double *) realloc(cur_slack,sizeof(double)*size_active); for(i = 0; i < size_active; i++) { cur_slack[i] = 0.0; for(f = dXc[i]->fvec; f; f = f->next) { j = 0; while(f->words[j].wnum) { cur_slack[i] += w[f->words[j].wnum]*f->words[j].weight; j++; } } if(cur_slack[i] >= delta[i]) cur_slack[i] = 0.0; else cur_slack[i] = delta[i]-cur_slack[i]; } mv_iter = 0; if(size_active > 1) { for(j = 0; j < size_active; j++) { if(cur_slack[j] >= cur_slack[mv_iter]) mv_iter = j; } } if(size_active > 1) threshold = cur_slack[mv_iter]; else threshold = 0.0; new_constraint = find_cutting_plane(ex, fycache, &margin, m, sm, sparm, valid_examples); value = margin - sprod_ns(w, new_constraint); if((iter % CLEANUP_CHECK) == 0) { printf("+"); fflush(stdout); size_active = resize_cleanup(size_active, &idle, &alpha, &delta, &dXc, &G, &mv_iter); } free(alpha); alpha=NULL; } // end cutting plane while loop //primal_obj = current_obj_val(ex, fycache, m, sm, sparm, C, valid_examples); printf(" Inner loop optimization finished.\n"); fflush(stdout); // free memory for (j=0;j<size_active;j++) { free(G[j]); free_example(dXc[j],1); } free(G); free(dXc); free(alpha); free(delta); free_svector(new_constraint); free(cur_slack); free(idle); if (svm_model!=NULL) free_model(svm_model,0); //return(primal_obj); return; }
CONSTSET init_struct_constraints(SAMPLE sample, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm) { /* Initializes the optimization problem. Typically, you do not need to change this function, since you want to start with an empty set of constraints. However, if for example you have constraints that certain weights need to be positive, you might put that in here. The constraints are represented as lhs[i]*w >= rhs[i]. lhs is an array of feature vectors, rhs is an array of doubles. m is the number of constraints. The function returns the initial set of constraints. */ CONSTSET c; PyObject *pFunc,*pValue,*pSeq; int i; // Set defaults of no constraints. c.m=0; c.lhs=NULL; c.rhs=NULL; // Call the relevant Python function. pFunc = getFunction(PYTHON_INIT_CONSTRAINTS); pValue = PyObject_CallFunction(pFunc, "NNN", Sample_FromSample(sample), StructModel_FromStructModel(sm), Sparm_FromSparm(sparm)); //Py_DECREF(pArgs); PY_RUNCHECK; // Check for None possibility. if (pValue == Py_None) { // No constraints if no return value. Py_DECREF(pValue); return c; } // Convert this into a sequence. pSeq = PySequence_Fast(pValue, "constraints not a sequence"); Py_DECREF(pValue); if (pSeq == NULL) { PyErr_Print(); Py_Exit(1); } // Read the examples from the sequence. c.m = PySequence_Size(pSeq); if (c.m==0) { // Empty constraint set. Py_DECREF(pSeq); return c; } // Initialize the constraint data structures. c.lhs = (DOC **)my_malloc(sizeof(DOC*)*c.m); c.rhs = (double*)my_malloc(sizeof(double)*c.m); bzero(c.lhs, sizeof(DOC*)*c.m); bzero(c.rhs, sizeof(double)*c.m); // Try to iteratively extract the constraints. for (i=0; i<c.m; ++i) { PyObject *pConst = PySequence_Fast_GET_ITEM(pSeq, i); PyObject *pRHS, *pLHS; if (!pConst || !PySequence_Check(pConst) || PySequence_Size(pConst)<2){ fprintf(stderr, "%s's item %d is not a sequence element of " "at least two items!\n", PYTHON_INIT_CONSTRAINTS, i); goto error; } // Extract the right hand side. pRHS = PySequence_GetItem(pConst, 1); c.rhs[i] = PyFloat_AsDouble(pRHS); Py_XDECREF(pRHS); if (PyErr_Occurred()) { Py_DECREF(pConst); goto error; } // Extract the left hand side. pLHS = PySequence_GetItem(pConst, 0); c.lhs[i] = Document_AsDocument(pLHS); c.lhs[i]->docnum = i; Py_XDECREF(pLHS); //Py_DECREF(pConst); if (PyErr_Occurred()) { goto error; } } Py_DECREF(pSeq); return(c); error: // Free what document structures we have copied. Py_DECREF(pSeq); for (i=0; i<c.m; ++i) { if (c.lhs[i]) free_example(c.lhs[i],1); } free(c.lhs); free(c.rhs); PY_RUNCHECK; fprintf(stderr, PYTHON_INIT_CONSTRAINTS": should not be here\n"); Py_Exit(2); return c; }
int resize_cleanup(int size_active, int **ptr_idle, double **ptr_cur_slack, double **ptr_delta, DOC ***ptr_dXc, double ***ptr_psiDiffs, int *mv_iter) { int i,j, new_size_active; long k; int *idle=*ptr_idle; double *cur_slack=*ptr_cur_slack; double *delta=*ptr_delta; DOC **dXc = *ptr_dXc; double **psiDiffs = *ptr_psiDiffs; int new_mv_iter; i=0; while ((i<size_active)&&(idle[i]<IDLE_ITER)) i++; j=i; while((j<size_active)&&(idle[j]>=IDLE_ITER)) j++; while (j<size_active) { // copying cur_slack[i] = cur_slack[j]; delta[i] = delta[j]; free(psiDiffs[i]); psiDiffs[i] = psiDiffs[j]; psiDiffs[j] = NULL; free_example(dXc[i],1); dXc[i] = dXc[j]; dXc[j] = NULL; if(j == *mv_iter) new_mv_iter = i; i++; j++; while((j<size_active)&&(idle[j]>=IDLE_ITER)) j++; } for (k=i;k<size_active;k++) { if (psiDiffs[k]!=NULL) free(psiDiffs[k]); if (dXc[k]!=NULL) free_example(dXc[k],1); } *mv_iter = new_mv_iter; new_size_active = i; cur_slack = (double*)realloc(cur_slack, sizeof(double)*new_size_active); delta = (double*)realloc(delta, sizeof(double)*new_size_active); psiDiffs = (double **) realloc(psiDiffs, sizeof(double *)*new_size_active); dXc = (DOC**)realloc(dXc, sizeof(DOC*)*new_size_active); assert(dXc!=NULL); // resize idle i=0; while ((i<size_active)&&(idle[i]<IDLE_ITER)) i++; j=i; while((j<size_active)&&(idle[j]>=IDLE_ITER)) j++; while (j<size_active) { idle[i] = idle[j]; i++; j++; while((j<size_active)&&(idle[j]>=IDLE_ITER)) j++; } idle = (int*)realloc(idle, sizeof(int)*new_size_active); *ptr_idle = idle; *ptr_cur_slack = cur_slack; *ptr_delta = delta; *ptr_psiDiffs = psiDiffs; *ptr_dXc = dXc; return(new_size_active); }
int Classifier::getZone(IplImage* frame, double& confidence, FrameAnnotation& fa) { if (!leftEye || !rightEye || !nose) { string err = "Classifier::getZone. Location extractors malformed."; throw (err); } // the roi offset CvPoint offset; // LOIs CvPoint leftEyeLocation; CvPoint rightEyeLocation; CvPoint noseLocation; // computing the confidence of the location identification double leftPSR; double rightPSR; double nosePSR; CvPoint center = fa.getLOI(Annotations::Face); if (!center.x || !center.y) { center.x = Globals::imgWidth / 2; center.y = Globals::imgHeight / 2; fa.setFace(center); } offset.x = offset.y = 0; IplImage* roi = (roiFunction)? roiFunction(frame, fa, offset, Annotations::Face) : 0; // all location extractors do identical preprocessing. Therefore, preprocess // once using say the left eye extractor and re-use it for all three extractors fftw_complex* preprocessedImage = leftEye->getPreprocessedImage((roi)? roi : frame); #pragma omp parallel sections num_threads(2) { #pragma omp section { leftEye->setImage(preprocessedImage); leftEye->apply(); leftEye->getMaxLocation(leftEyeLocation, leftPSR); leftEyeLocation.x += offset.x; leftEyeLocation.y += offset.y; } #pragma omp section { // get the location of the right eye rightEye->setImage(preprocessedImage); rightEye->apply(); rightEye->getMaxLocation(rightEyeLocation, rightPSR); rightEyeLocation.x += offset.x; rightEyeLocation.y += offset.y; } } if (roi) cvReleaseImage(&roi); center.x = (leftEyeLocation.x + rightEyeLocation.x) / 2; center.y = leftEyeLocation.y + Globals::noseDrop; fa.setNose(center); offset.x = offset.y = 0; roi = (roiFunction)? roiFunction(frame, fa, offset, Annotations::Nose) : 0; // free the preprocessed image fftw_free(preprocessedImage); // all location extractors do identical preprocessing. Therefore, preprocess // once using say the left eye extractor and re-use it for all three extractors preprocessedImage = nose->getPreprocessedImage((roi)? roi : frame); // get the location of the nose nose->setImage(preprocessedImage); nose->apply(); nose->getMaxLocation(noseLocation, nosePSR); noseLocation.x += offset.x; noseLocation.y += offset.y; // free the preprocessed image fftw_free(preprocessedImage); fa.setLeftIris(leftEyeLocation); fa.setRightIris(rightEyeLocation); fa.setNose(noseLocation); // we are done with the images at this point. Free roi if not zero if (roi) cvReleaseImage(&roi); // cout << "Confidence (L, R, N) = (" << leftPSR << ", " << // rightPSR << ")" << endl; // extract features vector vector<double> data; for (int i = 0; i < nFeatures; i++) { double value = featureExtractors[i]->extract(&fa); data.push_back(value); } // normalize normalize(data); // create SVM Light objects to classify DOC* doc; WORD* words = (WORD*)malloc(sizeof(WORD) * (nFeatures + 1)); for (int i = 0; i < nFeatures; i++) { words[i].wnum = featureExtractors[i]->getId(); words[i].weight = data[i]; } // SVM Light expects that the features vector has a zero element // to indicate termination and hence words[nFeatures].wnum = 0; words[nFeatures].weight = 0.0; // create doc string comment = "Gaze SVM"; doc = create_example(-1, 0, 0, 0.0, create_svector(words, (char*)comment.c_str(), 1.0)); int maxIndex = 0; confidence = -FLT_MAX; double dists[Globals::numZones]; // classify using each zone model #pragma omp parallel for num_threads(Globals::numZones) for (unsigned int i = 0; i < Globals::numZones; i++) { if (kernelType == Trainer::Linear) dists[i] = classify_example_linear(models[i], doc); else dists[i] = classify_example(models[i], doc); } for (unsigned int i = 0; i < Globals::numZones; i++) { if (confidence < dists[i]) { confidence = dists[i]; maxIndex = i + 1; } } free_example(doc, 1); free(words); return maxIndex; }
int SVMLightRunner::librarySVMClassifyMain( int argc, char **argv, bool use_gmumr, SVMConfiguration &config ) { LOG( config.log, LogLevel::DEBUG_LEVEL, __debug_prefix__ + ".librarySVMClassifyMain() Started." ); DOC *doc; /* test example */ WORD *words; long max_docs,max_words_doc,lld; long totdoc=0,queryid,slackid; long correct=0,incorrect=0,no_accuracy=0; long res_a=0,res_b=0,res_c=0,res_d=0,wnum,pred_format; long j; double t1,runtime=0; double dist,doc_label,costfactor; char *line,*comment; FILE *predfl,*docfl; MODEL *model; // GMUM.R changes { librarySVMClassifyReadInputParameters( argc, argv, docfile, modelfile, predictionsfile, &verbosity, &pred_format, use_gmumr, config); if (!use_gmumr) { nol_ll(docfile,&max_docs,&max_words_doc,&lld); /* scan size of input file */ lld+=2; line = (char *)my_malloc(sizeof(char)*lld); } else { max_docs = config.target.n_rows; max_words_doc = config.getDataDim(); config.result = arma::zeros<arma::vec>(max_docs); // Prevent writing to the file pred_format = -1; // lld used only for file reading } max_words_doc+=2; words = (WORD *)my_malloc(sizeof(WORD)*(max_words_doc+10)); // GMUM.R changes } model=libraryReadModel(modelfile, use_gmumr, config); // GMUM.R changes } if(model->kernel_parm.kernel_type == 0) { /* linear kernel */ /* compute weight vector */ add_weight_vector_to_linear_model(model); } if(verbosity>=2) { C_PRINTF("Classifying test examples.."); C_FFLUSH(stdout); } // GMUM.R changes { bool newline; if (!use_gmumr) { if ((predfl = fopen (predictionsfile, "w")) == NULL) { perror (predictionsfile); EXIT (1); } if ((docfl = fopen (docfile, "r")) == NULL) { perror (docfile); EXIT (1); } newline = (!feof(docfl)) && fgets(line,(int)lld,docfl); } else { newline = false; if (totdoc < config.getDataExamplesNumber()) { newline = true; std::string str = SVMConfigurationToSVMLightLearnInputLine(config, totdoc); line = new char[str.size() + 1]; std::copy(str.begin(), str.end(), line); line[str.size()] = '\0'; } } while(newline) { if (use_gmumr) { std::string stringline = ""; } // GMUM.R changes } if(line[0] == '#') continue; /* line contains comments */ parse_document(line,words,&doc_label,&queryid,&slackid,&costfactor,&wnum, max_words_doc,&comment); totdoc++; if(model->kernel_parm.kernel_type == 0) { /* linear kernel */ for(j=0;(words[j]).wnum != 0;j++) { /* Check if feature numbers */ if((words[j]).wnum>model->totwords) /* are not larger than in */ (words[j]).wnum=0; /* model. Remove feature if */ } /* necessary. */ doc = create_example(-1,0,0,0.0,create_svector(words,comment,1.0)); t1=get_runtime(); dist=classify_example_linear(model,doc); runtime+=(get_runtime()-t1); free_example(doc,1); } else { /* non-linear kernel */ doc = create_example(-1,0,0,0.0,create_svector(words,comment,1.0)); t1=get_runtime(); dist=classify_example(model,doc); runtime+=(get_runtime()-t1); free_example(doc,1); } if(dist>0) { if(pred_format==0) { /* old weired output format */ C_FPRINTF(predfl,"%.8g:+1 %.8g:-1\n",dist,-dist); } if(doc_label>0) correct++; else incorrect++; if(doc_label>0) res_a++; else res_b++; } else { if(pred_format==0) { /* old weired output format */ C_FPRINTF(predfl,"%.8g:-1 %.8g:+1\n",-dist,dist); } if(doc_label<0) correct++; else incorrect++; if(doc_label>0) res_c++; else res_d++; } if(pred_format==1) { /* output the value of decision function */ C_FPRINTF(predfl,"%.8g\n",dist); } if((int)(0.01+(doc_label*doc_label)) != 1) { no_accuracy=1; } /* test data is not binary labeled */ if(verbosity>=2) { if(totdoc % 100 == 0) { C_PRINTF("%ld..",totdoc); C_FFLUSH(stdout); } } // GMUM.R changes { if (!use_gmumr) { newline = (!feof(docfl)) && fgets(line,(int)lld,docfl); } else { newline = false; // Store prediction result in config config.result[totdoc-1] = dist; // Read next line if (totdoc < config.getDataExamplesNumber()) { newline = true; std::string str = SVMConfigurationToSVMLightLearnInputLine(config, totdoc); line = new char[str.size() + 1]; std::copy(str.begin(), str.end(), line); line[str.size()] = '\0'; } } } if (!use_gmumr) { fclose(predfl); fclose(docfl); free(line); } // GMUM.R changes } free(words); free_model(model,1); if(verbosity>=2) { C_PRINTF("done\n"); /* Note by Gary Boone Date: 29 April 2000 */ /* o Timing is inaccurate. The timer has 0.01 second resolution. */ /* Because classification of a single vector takes less than */ /* 0.01 secs, the timer was underflowing. */ C_PRINTF("Runtime (without IO) in cpu-seconds: %.2f\n", (float)(runtime/100.0)); } if((!no_accuracy) && (verbosity>=1)) { C_PRINTF("Accuracy on test set: %.2f%% (%ld correct, %ld incorrect, %ld total)\n",(float)(correct)*100.0/totdoc,correct,incorrect,totdoc); C_PRINTF("Precision/recall on test set: %.2f%%/%.2f%%\n",(float)(res_a)*100.0/(res_a+res_b),(float)(res_a)*100.0/(res_a+res_c)); } return(0); }
int SVMLightRunner::librarySVMLearnMain( int argc, char **argv, bool use_gmumr, SVMConfiguration &config ) { LOG( config.log, LogLevel::DEBUG_LEVEL, __debug_prefix__ + ".librarySVMLearnMain() Started." ); DOC **docs; /* training examples */ long totwords,totdoc,i; double *target; double *alpha_in=NULL; KERNEL_CACHE *kernel_cache; LEARN_PARM learn_parm; KERNEL_PARM kernel_parm; MODEL *model=(MODEL *)my_malloc(sizeof(MODEL)); // GMUM.R changes { librarySVMLearnReadInputParameters( argc, argv, docfile, modelfile, restartfile, &verbosity, &learn_parm, &kernel_parm, use_gmumr, config ); kernel_parm.kernel_type = static_cast<long int>(config.kernel_type); libraryReadDocuments( docfile, &docs, &target, &totwords, &totdoc, use_gmumr, config ); // GMUM.R changes } if(restartfile[0]) alpha_in=read_alphas(restartfile,totdoc); if(kernel_parm.kernel_type == LINEAR) { /* don't need the cache */ kernel_cache=NULL; } else { /* Always get a new kernel cache. It is not possible to use the * same cache for two different training runs */ kernel_cache=kernel_cache_init(totdoc,learn_parm.kernel_cache_size); } //gmum.r init_global_params_QP(); if(learn_parm.type == CLASSIFICATION) { svm_learn_classification(docs,target,totdoc,totwords,&learn_parm, &kernel_parm,kernel_cache,model,alpha_in); } else if(learn_parm.type == REGRESSION) { svm_learn_regression(docs,target,totdoc,totwords,&learn_parm, &kernel_parm,&kernel_cache,model); } else if(learn_parm.type == RANKING) { svm_learn_ranking(docs,target,totdoc,totwords,&learn_parm, &kernel_parm,&kernel_cache,model); } else if(learn_parm.type == OPTIMIZATION) { svm_learn_optimization(docs,target,totdoc,totwords,&learn_parm, &kernel_parm,kernel_cache,model,alpha_in); } //gmum.r config.iter = learn_parm.iterations; if(kernel_cache) { /* Free the memory used for the cache. */ kernel_cache_cleanup(kernel_cache); } /* Warning: The model contains references to the original data 'docs'. If you want to free the original data, and only keep the model, you have to make a deep copy of 'model'. */ /* deep_copy_of_model=copy_model(model); */ // GMUM.R changes { if (!use_gmumr) { write_model(modelfile,model); } else { SVMLightModelToSVMConfiguration(model, config); } // GMUM.R changes } free(alpha_in); free_model(model,0); for(i=0;i<totdoc;i++) free_example(docs[i],1); free(docs); free(target); LOG( config.log, LogLevel::DEBUG_LEVEL, __debug_prefix__ + ".librarySVMLearnMain() Done." ); return(0); }
int resize_cleanup(int size_active, int *idle, double *alpha, double *delta, double *gammaG0, double *proximal_rhs, double **G, DOC **dXc, double *cut_error) { int i,j, new_size_active; long k; i=0; while ((i<size_active)&&(idle[i]<IDLE_ITER)) i++; j=i; while((j<size_active)&&(idle[j]>=IDLE_ITER)) j++; while (j<size_active) { /* copying */ alpha[i] = alpha[j]; delta[i] = delta[j]; gammaG0[i] = gammaG0[j]; cut_error[i] = cut_error[j]; free(G[i]); G[i] = G[j]; G[j] = NULL; free_example(dXc[i],0); dXc[i] = dXc[j]; dXc[j] = NULL; i++; j++; while((j<size_active)&&(idle[j]>=IDLE_ITER)) j++; } for (k=i;k<size_active;k++) { if (G[k]!=NULL) free(G[k]); if (dXc[k]!=NULL) free_example(dXc[k],0); } new_size_active = i; alpha = (double*)realloc(alpha, sizeof(double)*new_size_active); delta = (double*)realloc(delta, sizeof(double)*new_size_active); gammaG0 = (double*)realloc(gammaG0, sizeof(double)*new_size_active); proximal_rhs = (double*)realloc(proximal_rhs, sizeof(double)*new_size_active); G = (double**)realloc(G, sizeof(double*)*new_size_active); dXc = (DOC**)realloc(dXc, sizeof(DOC*)*new_size_active); cut_error = (double*)realloc(cut_error, sizeof(double)*new_size_active); /* resize G and idle */ i=0; while ((i<size_active)&&(idle[i]<IDLE_ITER)) i++; j=i; while((j<size_active)&&(idle[j]>=IDLE_ITER)) j++; while (j<size_active) { idle[i] = idle[j]; for (k=0;k<new_size_active;k++) { G[k][i] = G[k][j]; } i++; j++; while((j<size_active)&&(idle[j]>=IDLE_ITER)) j++; } idle = (int*)realloc(idle, sizeof(int)*new_size_active); for (k=0;k<new_size_active;k++) { G[k] = (double*)realloc(G[k], sizeof(double)*new_size_active); } return(new_size_active); }
int _svm_learn (int argc, char* argv[]) { char docfile[200]; /* file with training examples */ char modelfile[200]; /* file for resulting classifier */ char restartfile[200]; /* file with initial alphas */ DOC **docs; /* training examples */ long totwords,totdoc,i; double *target; double *alpha_in=NULL; KERNEL_CACHE *kernel_cache; LEARN_PARM learn_parm; KERNEL_PARM kernel_parm; MODEL *model=(MODEL *)my_malloc(sizeof(MODEL)); HIDEO_ENV *hideo_env=create_env(); model->td_pred=NULL; model->n_td_pred=0; _read_input_parameters(argc,argv,docfile,modelfile,restartfile,&verbosity, &learn_parm,&kernel_parm); read_documents(docfile,&docs,&target,&totwords,&totdoc); if(restartfile[0]) alpha_in=read_alphas(restartfile,totdoc); if(kernel_parm.kernel_type == LINEAR) { /* don't need the cache */ kernel_cache=NULL; } else { /* Always get a new kernel cache. It is not possible to use the same cache for two different training runs */ kernel_cache=kernel_cache_init(totdoc,learn_parm.kernel_cache_size); } if(learn_parm.type == CLASSIFICATION) { svm_learn_classification(docs,target,totdoc,totwords,&learn_parm, &kernel_parm,kernel_cache,model,alpha_in,hideo_env); } else if(learn_parm.type == REGRESSION) { svm_learn_regression(docs,target,totdoc,totwords,&learn_parm, &kernel_parm,&kernel_cache,model,hideo_env); } else if(learn_parm.type == RANKING) { svm_learn_ranking(docs,target,totdoc,totwords,&learn_parm, &kernel_parm,&kernel_cache,model,hideo_env); } else if(learn_parm.type == OPTIMIZATION) { svm_learn_optimization(docs,target,totdoc,totwords,&learn_parm, &kernel_parm,kernel_cache,model,alpha_in,hideo_env); } if(kernel_cache) { /* Free the memory used for the cache. */ kernel_cache_cleanup(kernel_cache); } /* Warning: The model contains references to the original data 'docs'. If you want to free the original data, and only keep the model, you have to make a deep copy of 'model'. */ /* deep_copy_of_model=copy_model(model); */ write_model(modelfile,model); free(alpha_in); free_model(model,0); for(i=0;i<totdoc;i++) free_example(docs[i],1); free(docs); free(target); free_env(hideo_env); return(0); }
double cutting_plane_algorithm(double *w, long m, int MAX_ITER, double C, double epsilon, SVECTOR **fycache, EXAMPLE *ex, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm, char *tmpdir, char * trainfile, double frac_sim, double Fweight, char *dataset_stats_file, double rho_admm, long isExhaustive, long isLPrelaxation, double Cdash, int datasetStartIdx, int chunkSz, int eid, int chunkid, double *w_prev, int numChunks) { // printf("Addr. of w (inside cp_algo) %x\t%x\n",w,sm->w); long i,j; double xi; double *alpha; double **G; /* Gram matrix */ DOC **dXc; /* constraint matrix */ double *delta; /* rhs of constraints */ SVECTOR *new_constraint; double dual_obj, alphasum; int iter, size_active; double value; int r; int *idle; /* for cleaning up */ double margin; double primal_obj; double *proximal_rhs; double *gammaG0=NULL; double min_rho = 0.001; double max_rho; double serious_counter=0; double rho = 1.0; /* temporarily set it to 1 first */ double expected_descent, primal_obj_b=-1, reg_master_obj; int null_step=1; double *w_b; double kappa=0.1; double temp_var; double proximal_term, primal_lower_bound; double v_k; double obj_difference; double *cut_error; // cut_error[i] = alpha_{k,i} at current center x_k double sigma_k; double m2 = 0.2; double m3 = 0.9; double gTd; double last_sigma_k=0; double initial_primal_obj; int suff_decrease_cond=0; double decrease_proportion = 0.2; // start from 0.2 first double z_k_norm; double last_z_k_norm=0; w_b = create_nvector(sm->sizePsi); clear_nvector(w_b,sm->sizePsi); /* warm start */ for (i=1;i<sm->sizePsi+1;i++) { w_b[i] = w[i]; } iter = 0; size_active = 0; xi = 0.0; alpha = NULL; G = NULL; dXc = NULL; delta = NULL; idle = NULL; proximal_rhs = NULL; cut_error = NULL; printf("ITER 0 \n(before cutting plane) \n"); double margin2; new_constraint = find_cutting_plane (ex, fycache, &margin, m, sm, sparm, tmpdir, trainfile, frac_sim, Fweight, dataset_stats_file, rho_admm, isExhaustive, isLPrelaxation, &margin2, datasetStartIdx, chunkSz, eid, chunkid); value = margin2 - sprod_ns(w, new_constraint); margin -= sprod_ns(w_prev, new_constraint); //(Ajay: ONLINE LEARNING) IMPT NOTE --> constant addition to the loss .. // model score using w_prev values ('-' is used because the terms are reversed in the code) primal_obj_b = 0.5*sprod_nn(w_b,w_b,sm->sizePsi)+C*value + Cdash*margin/numChunks; // Ajay: Change in obj involing both hamming and F1 loss primal_obj = 0.5*sprod_nn(w,w,sm->sizePsi)+C*value + Cdash*margin/numChunks; // Ajay: Change in obj involing both hamming and F1 loss; primal_lower_bound = 0; expected_descent = -primal_obj_b; initial_primal_obj = primal_obj_b; max_rho = C; printf("Running CCCP inner loop solver: \n"); fflush(stdout); time_t iter_start, iter_end; while ((!suff_decrease_cond)&&(expected_descent<-epsilon)&&(iter<MAX_ITER)) { iter+=1; size_active+=1; time(&iter_start); #if (DEBUG_LEVEL>0) printf("ITER %d\n", iter); #endif printf("."); fflush(stdout); /* add constraint */ dXc = (DOC**)realloc(dXc, sizeof(DOC*)*size_active); assert(dXc!=NULL); dXc[size_active-1] = (DOC*)malloc(sizeof(DOC)); dXc[size_active-1]->fvec = new_constraint; dXc[size_active-1]->slackid = 1; // only one common slackid (one-slack) dXc[size_active-1]->costfactor = 1.0; delta = (double*)realloc(delta, sizeof(double)*size_active); assert(delta!=NULL); delta[size_active-1] = margin2; // Ajay: changing for the formulation combining hamming and F1loss alpha = (double*)realloc(alpha, sizeof(double)*size_active); assert(alpha!=NULL); alpha[size_active-1] = 0.0; idle = (int*)realloc(idle, sizeof(int)*size_active); assert(idle!=NULL); idle[size_active-1] = 0; /* proximal point */ proximal_rhs = (double*)realloc(proximal_rhs, sizeof(double)*size_active); assert(proximal_rhs!=NULL); cut_error = (double*)realloc(cut_error, sizeof(double)*size_active); assert(cut_error!=NULL); // note g_i = - new_constraint cut_error[size_active-1] = C*(sprod_ns(w_b, new_constraint) - sprod_ns(w, new_constraint)); cut_error[size_active-1] += (primal_obj_b - 0.5*sprod_nn(w_b,w_b,sm->sizePsi)); cut_error[size_active-1] -= (primal_obj - 0.5*sprod_nn(w,w,sm->sizePsi)); gammaG0 = (double*)realloc(gammaG0, sizeof(double)*size_active); assert(gammaG0!=NULL); /* update Gram matrix */ G = (double**)realloc(G, sizeof(double*)*size_active); assert(G!=NULL); G[size_active-1] = NULL; for (j=0;j<size_active;j++) { G[j] = (double*)realloc(G[j], sizeof(double)*size_active); assert(G[j]!=NULL); } for (j=0;j<size_active-1;j++) { G[size_active-1][j] = sprod_ss(dXc[size_active-1]->fvec, dXc[j]->fvec); G[j][size_active-1] = G[size_active-1][j]; } G[size_active-1][size_active-1] = sprod_ss(dXc[size_active-1]->fvec,dXc[size_active-1]->fvec); /* update gammaG0 */ if (null_step==1) { gammaG0[size_active-1] = sprod_ns(w_b, dXc[size_active-1]->fvec); } else { for (i=0;i<size_active;i++) { gammaG0[i] = sprod_ns(w_b, dXc[i]->fvec); } } /* update proximal_rhs */ for (i=0;i<size_active;i++) { proximal_rhs[i] = delta[i] - rho/(1+rho)*gammaG0[i]; } /* solve QP to update alpha */ dual_obj = 0; time_t mosek_start, mosek_end; time(&mosek_start); r = mosek_qp_optimize(G, proximal_rhs, alpha, (long) size_active, C, &dual_obj,rho); time(&mosek_end); #if(DEBUG_LEVEL == 1) print_time(mosek_start, mosek_end, "Mosek solver"); #endif /* DEBUG */ //printf("r: %d\n", r); fflush(stdout); /* END DEBUG */ clear_nvector(w,sm->sizePsi); for (j=0;j<size_active;j++) { if (alpha[j]>C*ALPHA_THRESHOLD) { add_vector_ns(w,dXc[j]->fvec,alpha[j]/(1+rho)); } } z_k_norm = sqrt(sprod_nn(w,w,sm->sizePsi)); add_vector_nn(w, w_b, sm->sizePsi, rho/(1+rho)); /* detect if step size too small */ sigma_k = 0; alphasum = 0; for (j=0;j<size_active;j++) { sigma_k += alpha[j]*cut_error[j]; alphasum+=alpha[j]; } sigma_k/=C; gTd = -C*(sprod_ns(w,new_constraint) - sprod_ns(w_b,new_constraint)); #if (DEBUG_LEVEL>0) for (j=0;j<size_active;j++) { printf("alpha[%d]: %.8g, cut_error[%d]: %.8g\n", j, alpha[j], j, cut_error[j]); } printf("sigma_k: %.8g\n", sigma_k); printf("alphasum: %.8g\n", alphasum); printf("g^T d: %.8g\n", gTd); fflush(stdout); #endif /* update cleanup information */ for (j=0;j<size_active;j++) { if (alpha[j]<ALPHA_THRESHOLD*C) { idle[j]++; } else { idle[j]=0; } } new_constraint = find_cutting_plane(ex, fycache, &margin, m, sm, sparm, tmpdir, trainfile, frac_sim, Fweight, dataset_stats_file, rho_admm, isExhaustive, isLPrelaxation, &margin2, datasetStartIdx, chunkSz, eid, chunkid); // new_constraint = find_cutting_plane(ex, fycache, &margin, m, sm, sparm, tmpdir, trainfile, frac_sim, Fweight, dataset_stats_file, rho); value = margin2 - sprod_ns(w, new_constraint); margin -= sprod_ns(w_prev, new_constraint); //(Ajay: ONLINE LEARNING) IMPT NOTE --> constant addition to the loss .. // model score using w_prev values ('-' is used because the terms are reversed in the code) /* print primal objective */ primal_obj = 0.5*sprod_nn(w,w,sm->sizePsi)+C*value + Cdash*margin/numChunks; // Ajay: Change in obj involing both hamming and F1 loss; #if (DEBUG_LEVEL>0) printf("ITER PRIMAL_OBJ %.4f\n", primal_obj); fflush(stdout); #endif temp_var = sprod_nn(w_b,w_b,sm->sizePsi); proximal_term = 0.0; for (i=1;i<sm->sizePsi+1;i++) { proximal_term += (w[i]-w_b[i])*(w[i]-w_b[i]); } reg_master_obj = -dual_obj+0.5*rho*temp_var/(1+rho); expected_descent = reg_master_obj - primal_obj_b; v_k = (reg_master_obj - proximal_term*rho/2) - primal_obj_b; primal_lower_bound = MAX(primal_lower_bound, reg_master_obj - 0.5*rho*(1+rho)*proximal_term); #if (DEBUG_LEVEL>0) printf("ITER REG_MASTER_OBJ: %.4f\n", reg_master_obj); printf("ITER EXPECTED_DESCENT: %.4f\n", expected_descent); printf("ITER PRIMLA_OBJ_B: %.4f\n", primal_obj_b); printf("ITER RHO: %.4f\n", rho); printf("ITER ||w-w_b||^2: %.4f\n", proximal_term); printf("ITER PRIMAL_LOWER_BOUND: %.4f\n", primal_lower_bound); printf("ITER V_K: %.4f\n", v_k); #endif obj_difference = primal_obj - primal_obj_b; if (primal_obj<primal_obj_b+kappa*expected_descent) { /* extra condition to be met */ if ((gTd>m2*v_k)||(rho<min_rho+1E-8)) { #if (DEBUG_LEVEL>0) printf("SERIOUS STEP\n"); #endif /* update cut_error */ for (i=0;i<size_active;i++) { cut_error[i] -= (primal_obj_b - 0.5*sprod_nn(w_b,w_b,sm->sizePsi)); cut_error[i] -= C*sprod_ns(w_b, dXc[i]->fvec); cut_error[i] += (primal_obj - 0.5*sprod_nn(w,w,sm->sizePsi)); cut_error[i] += C*sprod_ns(w, dXc[i]->fvec); } primal_obj_b = primal_obj; for (i=1;i<sm->sizePsi+1;i++) { w_b[i] = w[i]; } null_step = 0; serious_counter++; } else { /* increase step size */ #if (DEBUG_LEVEL>0) printf("NULL STEP: SS(ii) FAILS.\n"); #endif serious_counter--; rho = MAX(rho/10,min_rho); } } else { /* no sufficient decrease */ serious_counter--; if ((cut_error[size_active-1]>m3*last_sigma_k)&&(fabs(obj_difference)>last_z_k_norm+last_sigma_k)) { #if (DEBUG_LEVEL>0) printf("NULL STEP: NS(ii) FAILS.\n"); #endif rho = MIN(10*rho,max_rho); } #if (DEBUG_LEVEL>0) else printf("NULL STEP\n"); #endif } /* update last_sigma_k */ last_sigma_k = sigma_k; last_z_k_norm = z_k_norm; /* break away from while loop if more than certain proportioal decrease in primal objective */ if (primal_obj_b/initial_primal_obj<1-decrease_proportion) { suff_decrease_cond = 1; } /* clean up */ if (iter % CLEANUP_CHECK == 0) { size_active = resize_cleanup(size_active, idle, alpha, delta, gammaG0, proximal_rhs, G, dXc, cut_error); } time(&iter_end); #if (DEBUG_LEVEL==1) char msg[20]; sprintf(msg,"ITER %d",iter); print_time(iter_start, iter_end, msg); #endif } // end cutting plane while loop printf(" Inner loop optimization finished.\n"); fflush(stdout); /* free memory */ for (j=0;j<size_active;j++) { free(G[j]); free_example(dXc[j],0); } free(G); free(dXc); free(alpha); free(delta); free_svector(new_constraint); free(idle); free(gammaG0); free(proximal_rhs); free(cut_error); /* copy and free */ for (i=1;i<sm->sizePsi+1;i++) { w[i] = w_b[i]; } free(w_b); return(primal_obj_b); }
double cutting_plane_algorithm(double *w, long m, int MAX_ITER, double C, /*double epsilon,*/ SVECTOR **fycache, EXAMPLE *ex, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm) { long i,j; double *alpha; double **G; /* Gram matrix */ DOC **dXc; /* constraint matrix */ double *delta; /* rhs of constraints */ SVECTOR *new_constraint; double dual_obj/*, alphasum*/; int iter, size_active, no_violation_iter; double value; //int r; //int *idle; /* for cleaning up */ double margin; //double primal_obj; double lower_bound, approx_upper_bound; double *proximal_rhs; //double *gammaG0=NULL; //double min_rho = 0.001; //double max_rho; //double serious_counter=0; //double rho = 1.0; //double expected_descent, primal_obj_b=-1, reg_master_obj; //int null_step=1; //double *w_b; //double kappa=0.01; //double temp_var; //double proximal_term, primal_lower_bound; //double v_k; //double obj_difference; // double *cut_error; // cut_error[i] = alpha_{k,i} at current center x_k //double sigma_k; //double m2 = 0.2; //double m3 = 0.9; //double gTd; //double last_sigma_k=0; //double initial_primal_obj; //int suff_decrease_cond=0; //double decrease_proportion = 0.2; // start from 0.2 first //double z_k_norm; //double last_z_k_norm=0; /* w_b = create_nvector(sm->sizePsi); clear_nvector(w_b,sm->sizePsi); // warm start for (i=1;i<sm->sizePsi+1;i++) { w_b[i] = w[i]; }*/ iter = 0; no_violation_iter = 0; size_active = 0; alpha = NULL; G = NULL; dXc = NULL; delta = NULL; //idle = NULL; proximal_rhs = NULL; //cut_error = NULL; new_constraint = find_cutting_plane(ex, fycache, &margin, m, sm, sparm); value = margin - sprod_ns(w, new_constraint); //primal_obj_b = 0.5*sprod_nn(w_b,w_b,sm->sizePsi)+C*value; //primal_obj = 0.5*sprod_nn(w,w,sm->sizePsi)+C*value; //primal_lower_bound = 0; //expected_descent = -primal_obj_b; //initial_primal_obj = primal_obj_b; //max_rho = C; // Non negative weight constraints int nNonNeg = sm->sizePsi - sm->firstNonNegWeightIndex + 1; G = (double**)malloc(sizeof(double*)*nNonNeg); for (j=0; j<nNonNeg; j++) { G[j] = (double*)malloc(sizeof(double)*nNonNeg); for (int k=0; k<nNonNeg; k++) { G[j][k] = 0; } G[j][j] = 1.0; } double* alphabeta = NULL; while (/*(!suff_decrease_cond)&&(expected_descent<-epsilon)&&*/(iter<MAX_ITER)&&(no_violation_iter<MAX_INNER_ITER_NO_VIOLATION)) { LearningTracker::NextInnerIteration(); iter+=1; size_active+=1; #if (DEBUG_LEVEL>0) printf("INNER ITER %d\n", iter); #endif /* add constraint */ dXc = (DOC**)realloc(dXc, sizeof(DOC*)*size_active); assert(dXc!=NULL); dXc[size_active-1] = (DOC*)malloc(sizeof(DOC)); dXc[size_active-1]->fvec = new_constraint; dXc[size_active-1]->slackid = 1; // only one common slackid (one-slack) dXc[size_active-1]->costfactor = 1.0; delta = (double*)realloc(delta, sizeof(double)*size_active); assert(delta!=NULL); delta[size_active-1] = margin; alphabeta = (double*)realloc(alphabeta, sizeof(double)*(size_active+nNonNeg)); assert(alphabeta!=NULL); alphabeta[size_active+nNonNeg-1] = 0.0; /*idle = (int*)realloc(idle, sizeof(int)*size_active); assert(idle!=NULL); idle[size_active-1] = 0;*/ /* proximal point */ proximal_rhs = (double*)realloc(proximal_rhs, sizeof(double)*(size_active+nNonNeg)); assert(proximal_rhs!=NULL); /*cut_error = (double*)realloc(cut_error, sizeof(double)*size_active); assert(cut_error!=NULL); // note g_i = - new_constraint cut_error[size_active-1] = C*(sprod_ns(w_b, new_constraint) - sprod_ns(w, new_constraint)); cut_error[size_active-1] += (primal_obj_b - 0.5*sprod_nn(w_b,w_b,sm->sizePsi)); cut_error[size_active-1] -= (primal_obj - 0.5*sprod_nn(w,w,sm->sizePsi)); */ /*gammaG0 = (double*)realloc(gammaG0, sizeof(double)*size_active); assert(gammaG0!=NULL);*/ /* update Gram matrix */ G = (double**)realloc(G, sizeof(double*)*(size_active+nNonNeg)); assert(G!=NULL); G[size_active+nNonNeg-1] = NULL; for (j=0; j<size_active+nNonNeg; j++) { G[j] = (double*)realloc(G[j], sizeof(double)*(size_active+nNonNeg)); assert(G[j]!=NULL); } for (j=0; j<size_active-1; j++) { G[size_active+nNonNeg-1][j+nNonNeg] = sprod_ss(dXc[size_active-1]->fvec, dXc[j]->fvec); G[j+nNonNeg][size_active+nNonNeg-1] = G[size_active+nNonNeg-1][j+nNonNeg]; } G[size_active+nNonNeg-1][size_active+nNonNeg-1] = sprod_ss(dXc[size_active-1]->fvec,dXc[size_active-1]->fvec); for (j=0; j<nNonNeg; j++) { WORD indicator[2]; indicator[0].wnum = j + sm->firstNonNegWeightIndex; indicator[0].weight = 1.0; indicator[1].wnum = 0; indicator[1].weight = 0.0; SVECTOR* indicator_vec = create_svector(indicator, NULL, 1.0); G[size_active+nNonNeg-1][j] = sprod_ss(dXc[size_active-1]->fvec, indicator_vec); G[j][size_active+nNonNeg-1] = G[size_active+nNonNeg-1][j]; free_svector(indicator_vec); } /* update gammaG0 */ /*if (null_step==1) { gammaG0[size_active-1] = sprod_ns(w_b, dXc[size_active-1]->fvec); } else { for (i=0;i<size_active;i++) { gammaG0[i] = sprod_ns(w_b, dXc[i]->fvec); } }*/ /* update proximal_rhs */ for (i=0; i<size_active; i++) { proximal_rhs[i+nNonNeg] = -delta[i]; //(1+rho) * (rho * gammaG0[i] - (1 + rho) * delta[i]); } for (i=0; i<nNonNeg; i++) { proximal_rhs[i] = 0; //w_b[i + 1]*rho * (1+rho); } /* DEBUG */ /* for (i = 0; i < size_active + nNonNeg; ++i) { printf("G[%d]=", i); for (j = 0; j < size_active + nNonNeg; ++j) { printf("%.4f ", G[i][j]); } printf("\n"); } printf("\n"); for (i = 0; i < size_active + nNonNeg; ++i) printf("proximal_rhs[%d]=%.4f\n", i, proximal_rhs[i]); */ /* solve QP to update alpha */ dual_obj = 0; mosek_qp_optimize(G, proximal_rhs, alphabeta, (long) size_active+nNonNeg, C, &dual_obj, nNonNeg); printf("dual_obj=%.4lf\n", dual_obj); alpha = alphabeta + nNonNeg; clear_nvector(w,sm->sizePsi); for (i = 0; i < nNonNeg; i++) { w[sm->firstNonNegWeightIndex + i] = alphabeta[i];//alphabeta[i]/(1+rho); // add betas } for (j=0; j<size_active; j++) { if (alpha[j]>C*ALPHA_THRESHOLD) { //add_vector_ns(w,dXc[j]->fvec,alpha[j]/(1+rho)); add_vector_ns(w,dXc[j]->fvec,alpha[j]); } } //z_k_norm = sqrt(sprod_nn(w,w,sm->sizePsi)); //add_vector_nn(w, w_b, sm->sizePsi, rho/(1+rho)); LearningTracker::ReportWeights(w, sm->sizePsi); /* detect if step size too small */ /* sigma_k = 0; alphasum = 0; for (j=0;j<size_active;j++) { sigma_k += alpha[j]*cut_error[j]; alphasum+=alpha[j]; } sigma_k/=C; gTd = -C*(sprod_ns(w,new_constraint) - sprod_ns(w_b,new_constraint)); #if (DEBUG_LEVEL>0) for (j=0;j<size_active;j++) { printf("alpha[%d]: %.8g, cut_error[%d]: %.8g\n", j, alpha[j], j, cut_error[j]); } printf("sigma_k: %.8g\n", sigma_k); printf("alphasum: %.8g\n", alphasum); printf("g^T d: %.8g\n", gTd); fflush(stdout); #endif */ /* update cleanup information */ /* for (j=0;j<size_active;j++) { if (alpha[j]<ALPHA_THRESHOLD*C) { idle[j]++; } else { idle[j]=0; } } */ // update lower bound double xi = -1e+20; for (i = 0; i < size_active; ++i) { xi = MAX(xi, delta[i] - sprod_ns(w, dXc[i]->fvec)); } lower_bound = 0.5*sprod_nn(w,w,sm->sizePsi)+C*xi; printf("lower_bound=%.4lf\n", lower_bound); assert(fabs(lower_bound + dual_obj) < 1e-6); LearningTracker::ReportLowerBound(lower_bound); // find new constraint new_constraint = find_cutting_plane(ex, fycache, &margin, m, sm, sparm); value = margin - sprod_ns(w, new_constraint); double violation = value - xi; if (violation > CUTTING_PLANE_EPS) { printf("New constraint is violated by %.4lf\n", violation); no_violation_iter = 0; } else { ++no_violation_iter; printf("New constraint is underviolated by %.4lf\n", violation); printf("%d more such constraints to stop\n", MAX_INNER_ITER_NO_VIOLATION - no_violation_iter); } // update upper bound approx_upper_bound = 0.5*sprod_nn(w,w,sm->sizePsi)+C*value; printf("approx_upper_bound=%.4lf\n", approx_upper_bound); LearningTracker::ReportUpperBound(approx_upper_bound); /* temp_var = sprod_nn(w_b,w_b,sm->sizePsi); proximal_term = 0.0; for (i=1;i<sm->sizePsi+1;i++) { proximal_term += (w[i]-w_b[i])*(w[i]-w_b[i]); } reg_master_obj = -dual_obj+0.5*rho*temp_var/(1+rho); expected_descent = reg_master_obj - primal_obj_b; v_k = (reg_master_obj - proximal_term*rho/2) - primal_obj_b; primal_lower_bound = MAX(primal_lower_bound, reg_master_obj - 0.5*rho*(1+rho)*proximal_term); LearningTracker::ReportLowerBoundValue(reg_master_obj); #if (DEBUG_LEVEL>0) printf("ITER REG_MASTER_OBJ: %.4f\n", reg_master_obj); printf("ITER EXPECTED_DESCENT: %.4f\n", expected_descent); printf("ITER PRIMAL_OBJ_B: %.4f\n", primal_obj_b); printf("ITER RHO: %.4f\n", rho); printf("ITER ||w-w_b||^2: %.4f\n", proximal_term); printf("ITER PRIMAL_LOWER_BOUND: %.4f\n", primal_lower_bound); printf("ITER V_K: %.4f\n", v_k); #endif obj_difference = primal_obj - primal_obj_b; if (primal_obj<primal_obj_b+kappa*expected_descent) { // extra condition to be met if ((gTd>m2*v_k)||(rho<min_rho+1E-8)) { #if (DEBUG_LEVEL>0) printf("SERIOUS STEP\n"); #endif // update cut_error for (i=0;i<size_active;i++) { cut_error[i] -= (primal_obj_b - 0.5*sprod_nn(w_b,w_b,sm->sizePsi)); cut_error[i] -= C*sprod_ns(w_b, dXc[i]->fvec); cut_error[i] += (primal_obj - 0.5*sprod_nn(w,w,sm->sizePsi)); cut_error[i] += C*sprod_ns(w, dXc[i]->fvec); } primal_obj_b = primal_obj; for (i=1;i<sm->sizePsi+1;i++) { w_b[i] = w[i]; } null_step = 0; serious_counter++; } else { // increase step size #if (DEBUG_LEVEL>0) printf("NULL STEP: SS(ii) FAILS.\n"); #endif serious_counter--; rho = MAX(rho/10,min_rho); } } else { // no sufficient decrease serious_counter--; if ((cut_error[size_active-1]>m3*last_sigma_k)&&(fabs(obj_difference)>last_z_k_norm+last_sigma_k)) { #if (DEBUG_LEVEL>0) printf("NULL STEP: NS(ii) FAILS.\n"); #endif rho = MIN(10*rho,max_rho); } #if (DEBUG_LEVEL>0) else printf("NULL STEP\n"); #endif } // update last_sigma_k last_sigma_k = sigma_k; last_z_k_norm = z_k_norm; // break away from while loop if more than certain proportioal decrease in primal objective if (primal_obj_b/initial_primal_obj<1-decrease_proportion) { suff_decrease_cond = 1; } // clean up if (iter % CLEANUP_CHECK == 0) { size_active = resize_cleanup(size_active, idle, alpha, delta, gammaG0, proximal_rhs, G, dXc, cut_error); } */ } // end cutting plane while loop printf("Inner loop optimization finished.\n"); fflush(stdout); /* free memory */ for (j=0; j<size_active; j++) { free(G[j]); free_example(dXc[j],0); } free(G); free(dXc); free(alphabeta); free(delta); free_svector(new_constraint); //free(idle); //free(gammaG0); free(proximal_rhs); //free(cut_error); /* copy and free */ /*for (i=1;i<sm->sizePsi+1;i++) { w[i] = w_b[i]; } free(w_b);*/ //return(primal_obj_b); return lower_bound; }
void svm_learn_struct(SAMPLE sample, STRUCT_LEARN_PARM *sparm, LEARN_PARM *lparm, KERNEL_PARM *kparm, STRUCTMODEL *sm) { int i,j; int numIt=0; long newconstraints=0, activenum=0; int opti_round, *opti; long old_numConst=0; double epsilon; long tolerance; double lossval,factor; double margin=0; double slack, *slacks, slacksum; long sizePsi; double *alpha=NULL; CONSTSET cset; SVECTOR *diff=NULL; SVECTOR *fy, *fybar, *f; SVECTOR *slackvec; WORD slackv[2]; MODEL *svmModel=NULL; KERNEL_CACHE *kcache=NULL; LABEL ybar; DOC *doc; long n=sample.n; EXAMPLE *ex=sample.examples; double rt_total=0.0, rt_opt=0.0; long rt1,rt2; init_struct_model(sample,sm,sparm); sizePsi=sm->sizePsi+1; /* sm must contain size of psi on return */ /* initialize example selection heuristic */ opti=(int*)my_malloc(n*sizeof(int)); for(i=0;i<n;i++) { opti[i]=0; } opti_round=0; if(sparm->slack_norm == 1) { lparm->svm_c=sparm->C; /* set upper bound C */ lparm->sharedslack=1; } else if(sparm->slack_norm == 2) { lparm->svm_c=999999999999999.0; /* upper bound C must never be reached */ lparm->sharedslack=0; if(kparm->kernel_type != LINEAR) { printf("ERROR: Kernels are not implemented for L2 slack norm!"); fflush(stdout); exit(0); } } else { printf("ERROR: Slack norm must be L1 or L2!"); fflush(stdout); exit(0); } epsilon=1.0; /* start with low precision and increase later */ tolerance=n/100; /* increase precision, whenever less than that number of constraints is not fulfilled */ lparm->biased_hyperplane=0; /* set threshold to zero */ cset=init_struct_constraints(sample, sm, sparm); if(cset.m > 0) { alpha=realloc(alpha,sizeof(double)*cset.m); for(i=0; i<cset.m; i++) alpha[i]=0; } /* set initial model and slack variables*/ svmModel=(MODEL *)my_malloc(sizeof(MODEL)); svm_learn_optimization(cset.lhs,cset.rhs,cset.m,sizePsi+n, lparm,kparm,NULL,svmModel,alpha); add_weight_vector_to_linear_model(svmModel); sm->svm_model=svmModel; sm->w=svmModel->lin_weights; /* short cut to weight vector */ printf("Starting Iterations\n"); /*****************/ /*** main loop ***/ /*****************/ do { /* iteratively increase precision */ epsilon=MAX(epsilon*0.09999999999,sparm->epsilon); if(epsilon == sparm->epsilon) /* for final precision, find all SV */ tolerance=0; lparm->epsilon_crit=epsilon/2; /* svm precision must be higher than eps */ if(struct_verbosity>=1) printf("Setting current working precision to %g.\n",epsilon); do { /* iteration until (approx) all SV are found for current precision and tolerance */ old_numConst=cset.m; opti_round++; activenum=n; do { /* go through examples that keep producing new constraints */ if(struct_verbosity>=1) { printf("--Iteration %i (%ld active): ",++numIt,activenum); fflush(stdout); } for(i=0; i<n; i++) { /*** example loop ***/ rt1=get_runtime(); if(opti[i] != opti_round) {/* if the example is not shrunk away, then see if it is necessary to add a new constraint */ if(sparm->loss_type == SLACK_RESCALING) ybar=find_most_violated_constraint_slackrescaling(ex[i].x, ex[i].y,sm, sparm); else ybar=find_most_violated_constraint_marginrescaling(ex[i].x, ex[i].y,sm, sparm); if(empty_label(ybar)) { if(opti[i] != opti_round) { activenum--; opti[i]=opti_round; } if(struct_verbosity>=2) printf("no-incorrect-found(%i) ",i); continue; } /**** get psi(y)-psi(ybar) ****/ fy=psi(ex[i].x,ex[i].y,sm,sparm); fybar=psi(ex[i].x,ybar,sm,sparm); /**** scale feature vector and margin by loss ****/ lossval=loss(ex[i].y,ybar,sparm); if(sparm->slack_norm == 2) lossval=sqrt(lossval); if(sparm->loss_type == SLACK_RESCALING) factor=lossval; else /* do not rescale vector for */ factor=1.0; /* margin rescaling loss type */ for(f=fy;f;f=f->next) f->factor*=factor; for(f=fybar;f;f=f->next) f->factor*=-factor; margin=lossval; /**** create constraint for current ybar ****/ append_svector_list(fy,fybar);/* append the two vector lists */ doc=create_example(cset.m,0,i+1,1,fy); /**** compute slack for this example ****/ slack=0; for(j=0;j<cset.m;j++) if(cset.lhs[j]->slackid == i+1) { if(sparm->slack_norm == 2) /* works only for linear kernel */ slack=MAX(slack,cset.rhs[j] -(classify_example(svmModel,cset.lhs[j]) -sm->w[sizePsi+i]/(sqrt(2*sparm->C)))); else slack=MAX(slack, cset.rhs[j]-classify_example(svmModel,cset.lhs[j])); } /**** if `error' add constraint and recompute ****/ if((classify_example(svmModel,doc)+slack)<(margin-epsilon)) { if(struct_verbosity>=2) {printf("(%i) ",i); fflush(stdout);} if(struct_verbosity==1) {printf("."); fflush(stdout);} /**** resize constraint matrix and add new constraint ****/ cset.m++; cset.lhs=realloc(cset.lhs,sizeof(DOC *)*cset.m); if(kparm->kernel_type == LINEAR) { diff=add_list_ss(fy); /* store difference vector directly */ if(sparm->slack_norm == 1) cset.lhs[cset.m-1]=create_example(cset.m-1,0,i+1,1, copy_svector(diff)); else if(sparm->slack_norm == 2) { /**** add squared slack variable to feature vector ****/ slackv[0].wnum=sizePsi+i; slackv[0].weight=1/(sqrt(2*sparm->C)); slackv[1].wnum=0; /*terminator*/ slackvec=create_svector(slackv,"",1.0); cset.lhs[cset.m-1]=create_example(cset.m-1,0,i+1,1, add_ss(diff,slackvec)); free_svector(slackvec); } free_svector(diff); } else { /* kernel is used */ if(sparm->slack_norm == 1) cset.lhs[cset.m-1]=create_example(cset.m-1,0,i+1,1, copy_svector(fy)); else if(sparm->slack_norm == 2) exit(1); } cset.rhs=realloc(cset.rhs,sizeof(double)*cset.m); cset.rhs[cset.m-1]=margin; alpha=realloc(alpha,sizeof(double)*cset.m); alpha[cset.m-1]=0; newconstraints++; } else { printf("+"); fflush(stdout); if(opti[i] != opti_round) { activenum--; opti[i]=opti_round; } } free_example(doc,0); free_svector(fy); /* this also free's fybar */ free_label(ybar); } /**** get new QP solution ****/ if((newconstraints >= sparm->newconstretrain) || ((newconstraints > 0) && (i == n-1))) { if(struct_verbosity>=1) { printf("*");fflush(stdout); } rt2=get_runtime(); free_model(svmModel,0); svmModel=(MODEL *)my_malloc(sizeof(MODEL)); /* Always get a new kernel cache. It is not possible to use the same cache for two different training runs */ if(kparm->kernel_type != LINEAR) kcache=kernel_cache_init(cset.m,lparm->kernel_cache_size); /* Run the QP solver on cset. */ svm_learn_optimization(cset.lhs,cset.rhs,cset.m,sizePsi+n, lparm,kparm,kcache,svmModel,alpha); if(kcache) kernel_cache_cleanup(kcache); /* Always add weight vector, in case part of the kernel is linear. If not, ignore the weight vector since its content is bogus. */ add_weight_vector_to_linear_model(svmModel); sm->svm_model=svmModel; sm->w=svmModel->lin_weights; /* short cut to weight vector */ rt_opt+=MAX(get_runtime()-rt2,0); newconstraints=0; } rt_total+=MAX(get_runtime()-rt1,0); } /* end of example loop */ if(struct_verbosity>=1) printf("(NumConst=%d, SV=%ld, Eps=%.4f)\n",cset.m,svmModel->sv_num-1, svmModel->maxdiff); } while(activenum > 0); /* repeat until all examples produced no constraint at least once */ } while((cset.m - old_numConst) > tolerance) ; } while(epsilon > sparm->epsilon); if(struct_verbosity>=1) { /**** compute sum of slacks ****/ slacks=(double *)my_malloc(sizeof(double)*(n+1)); for(i=0; i<=n; i++) { slacks[i]=0; } if(sparm->slack_norm == 1) { for(j=0;j<cset.m;j++) slacks[cset.lhs[j]->slackid]=MAX(slacks[cset.lhs[j]->slackid], cset.rhs[j]-classify_example(svmModel,cset.lhs[j])); } else if(sparm->slack_norm == 2) { for(j=0;j<cset.m;j++) slacks[cset.lhs[j]->slackid]=MAX(slacks[cset.lhs[j]->slackid], cset.rhs[j] -(classify_example(svmModel,cset.lhs[j]) -sm->w[sizePsi+cset.lhs[j]->slackid-1]/(sqrt(2*sparm->C)))); } slacksum=0; for(i=0; i<=n; i++) slacksum+=slacks[i]; free(slacks); printf("Final epsilon on KKT-Conditions: %.5f\n", MAX(svmModel->maxdiff,epsilon)); printf("Total number of constraints added: %i\n",(int)cset.m); if(sparm->slack_norm == 1) { printf("Number of SV: %ld \n",svmModel->sv_num-1); printf("Number of non-zero slack variables: %ld (out of %ld)\n", svmModel->at_upper_bound,n); printf("Norm of weight vector: |w|=%.5f\n", model_length_s(svmModel,kparm)); } else if(sparm->slack_norm == 2){ printf("Number of SV: %ld (including %ld at upper bound)\n", svmModel->sv_num-1,svmModel->at_upper_bound); printf("Norm of weight vector (including L2-loss): |w|=%.5f\n", model_length_s(svmModel,kparm)); } printf("Sum of slack variables: sum(xi_i)=%.5f\n",slacksum); printf("Norm of longest difference vector: ||Psi(x,y)-Psi(x,ybar)||=%.5f\n", length_of_longest_document_vector(cset.lhs,cset.m,kparm)); printf("Runtime in cpu-seconds: %.2f (%.2f%% for SVM optimization)\n", rt_total/100.0, 100.0*rt_opt/rt_total); } if(struct_verbosity>=4) printW(sm->w,sizePsi,n,lparm->svm_c); if(svmModel) { sm->svm_model=copy_model(svmModel); sm->w=sm->svm_model->lin_weights; /* short cut to weight vector */ } print_struct_learning_stats(sample,sm,cset,alpha,sparm); if(svmModel) free_model(svmModel,0); free(alpha); free(opti); free(cset.rhs); for(i=0;i<cset.m;i++) free_example(cset.lhs[i],1); free(cset.lhs); }
int main_classify (int argc, char* argv[]) { DOC *doc; /* test example */ WORDSVM *words; long max_docs,max_words_doc,lld; long totdoc=0,queryid,slackid; long correct=0,incorrect=0,no_accuracy=0; long res_a=0,res_b=0,res_c=0,res_d=0,wnum,pred_format; long j; double t1,runtime=0; double dist,doc_label,costfactor; char *line,*comment; FILE *predfl,*docfl; MODEL *model; read_input_parameters(argc,argv,docfile,modelfile,predictionsfile, &verbosity,&pred_format); nol_ll(docfile,&max_docs,&max_words_doc,&lld); /* scan size of input file */ max_words_doc+=2; lld+=2; line = (char *)my_malloc(sizeof(char)*lld); words = (WORDSVM *)my_malloc(sizeof(WORDSVM)*(max_words_doc+10)); model=read_model(modelfile); if(model->kernel_parm.kernel_type == 0) { /* linear kernel */ /* compute weight vector */ add_weight_vector_to_linear_model(model); } if(verbosity>=2) { printf("Classifying test examples.."); fflush(stdout); } if ((docfl = fopen (docfile, "r")) == NULL) { perror (docfile); exit (1); } if ((predfl = fopen (predictionsfile, "w")) == NULL) { perror (predictionsfile); exit (1); } while((!feof(docfl)) && fgets(line,(int)lld,docfl)) { if(line[0] == '#') continue; /* line contains comments */ parse_document(line,words,&doc_label,&queryid,&slackid,&costfactor,&wnum, max_words_doc,&comment); totdoc++; if(model->kernel_parm.kernel_type == 0) { /* linear kernel */ for(j=0;(words[j]).wnum != 0;j++) { /* Check if feature numbers */ if((words[j]).wnum>model->totwords) /* are not larger than in */ (words[j]).wnum=0; /* model. Remove feature if */ } /* necessary. */ doc = create_example(-1,0,0,0.0,create_svector(words,comment,1.0)); t1=get_runtime(); dist=classify_example_linear(model,doc); runtime+=(get_runtime()-t1); free_example(doc,1); } else { /* non-linear kernel */ doc = create_example(-1,0,0,0.0,create_svector(words,comment,1.0)); t1=get_runtime(); dist=classify_example(model,doc); runtime+=(get_runtime()-t1); free_example(doc,1); } if(dist>0) { if(pred_format==0) { /* old weired output format */ fprintf(predfl,"%.8g:+1 %.8g:-1\n",dist,-dist); } if(doc_label>0) correct++; else incorrect++; if(doc_label>0) res_a++; else res_b++; } else { if(pred_format==0) { /* old weired output format */ fprintf(predfl,"%.8g:-1 %.8g:+1\n",-dist,dist); } if(doc_label<0) correct++; else incorrect++; if(doc_label>0) res_c++; else res_d++; } if(pred_format==1) { /* output the value of decision function */ fprintf(predfl,"%.8g\n",dist); } if((int)(0.01+(doc_label*doc_label)) != 1) { no_accuracy=1; } /* test data is not binary labeled */ if(verbosity>=2) { if(totdoc % 100 == 0) { printf("%ld..",totdoc); fflush(stdout); } } } free(line); free(words); free_model(model,1); if(verbosity>=2) { printf("done\n"); /* Note by Gary Boone Date: 29 April 2000 */ /* o Timing is inaccurate. The timer has 0.01 second resolution. */ /* Because classification of a single vector takes less than */ /* 0.01 secs, the timer was underflowing. */ printf("Runtime (without IO) in cpu-seconds: %.2f\n", (float)(runtime/100.0)); } if((!no_accuracy) && (verbosity>=1)) { printf("Accuracy on test set: %.2f%% (%ld correct, %ld incorrect, %ld total)\n",(float)(correct)*100.0/totdoc,correct,incorrect,totdoc); printf("Precision/recall on test set: %.2f%%/%.2f%%\n",(float)(res_a)*100.0/(res_a+res_b),(float)(res_a)*100.0/(res_a+res_c)); } return(0); }