CLatentLabels* CLatentSVM::apply() { if (!m_model) SG_ERROR("LatentModel is not set!\n"); if (!features) return NULL; index_t num_examples = m_model->get_num_vectors(); CLatentLabels* hs = new CLatentLabels(num_examples); CBinaryLabels* ys = new CBinaryLabels(num_examples); hs->set_labels(ys); m_model->set_labels(hs); for (index_t i = 0; i < num_examples; ++i) { /* find h for the example */ CData* h = m_model->infer_latent_variable(w, i); hs->add_latent_label(h); } /* compute the y labels */ CDotFeatures* x = m_model->get_psi_feature_vectors(); x->dense_dot_range(ys->get_labels().vector, 0, num_examples, NULL, w.vector, w.vlen, 0.0); return hs; }
void CExactInferenceMethod::update_all() { if (m_labels) m_label_vector = ((CRegressionLabels*) m_labels)->get_labels().clone(); if (m_features && m_features->has_property(FP_DOT) && m_features->get_num_vectors()) m_feature_matrix = ((CDotFeatures*)m_features)->get_computed_dot_feature_matrix(); else if (m_features && m_features->get_feature_class() == C_COMBINED) { CDotFeatures* feat = (CDotFeatures*)((CCombinedFeatures*)m_features)-> get_first_feature_obj(); if (feat->get_num_vectors()) m_feature_matrix = feat->get_computed_dot_feature_matrix(); SG_UNREF(feat); } update_data_means(); if (m_kernel) update_train_kernel(); if (m_ktrtr.num_cols*m_ktrtr.num_rows) { update_chol(); update_alpha(); } }
void CExactInferenceMethod::check_members() { if (!m_labels) SG_ERROR("No labels set\n") if (m_labels->get_label_type() != LT_REGRESSION) SG_ERROR("Expected RegressionLabels\n") if (!m_features) SG_ERROR("No features set!\n") if (m_labels->get_num_labels() != m_features->get_num_vectors()) SG_ERROR("Number of training vectors does not match number of labels\n") if(m_features->get_feature_class() == C_COMBINED) { CDotFeatures* feat = (CDotFeatures*)((CCombinedFeatures*)m_features)-> get_first_feature_obj(); if (!feat->has_property(FP_DOT)) SG_ERROR("Specified features are not of type CFeatures\n") if (feat->get_feature_class() != C_DENSE) SG_ERROR("Expected Simple Features\n") if (feat->get_feature_type() != F_DREAL) SG_ERROR("Expected Real Features\n") SG_UNREF(feat); }
void CInferenceMethod::set_latent_features(CFeatures* feat) { SG_REF(feat); SG_UNREF(m_latent_features); m_latent_features=feat; if (m_latent_features && m_latent_features->has_property(FP_DOT) && m_latent_features->get_num_vectors()) m_latent_matrix = ((CDotFeatures*)m_latent_features)->get_computed_dot_feature_matrix(); else if (m_latent_features && m_latent_features->get_feature_class() == C_COMBINED) { CDotFeatures* subfeat = (CDotFeatures*)((CCombinedFeatures*)m_latent_features)-> get_first_feature_obj(); if (m_latent_features->get_num_vectors()) m_latent_matrix = subfeat->get_computed_dot_feature_matrix(); SG_UNREF(subfeat); } update_data_means(); update_train_kernel(); update_chol(); update_alpha(); }
CRegressionLabels* CGaussianProcessRegression::apply_regression(CFeatures* data) { if (data) { if(data->get_feature_class() == C_COMBINED) { CDotFeatures* feat = (CDotFeatures*)((CCombinedFeatures*)data)-> get_first_feature_obj(); if (!feat->has_property(FP_DOT)) SG_ERROR("Specified features are not of type CFeatures\n") if (feat->get_feature_class() != C_DENSE) SG_ERROR("Expected Simple Features\n") if (feat->get_feature_type() != F_DREAL) SG_ERROR("Expected Real Features\n") SG_UNREF(feat); } else { if (!data->has_property(FP_DOT))
CResultSet* CMulticlassModel::argmax( SGVector< float64_t > w, int32_t feat_idx, bool const training) { CDotFeatures* df = (CDotFeatures*) m_features; int32_t feats_dim = df->get_dim_feature_space(); if ( training ) { CMulticlassSOLabels* ml = (CMulticlassSOLabels*) m_labels; m_num_classes = ml->get_num_classes(); } else { REQUIRE(m_num_classes > 0, "The model needs to be trained before " "using it for prediction\n"); } int32_t dim = get_dim(); ASSERT(dim == w.vlen) // Find the class that gives the maximum score float64_t score = 0, ypred = 0; float64_t max_score = -CMath::INFTY; for ( int32_t c = 0 ; c < m_num_classes ; ++c ) { score = df->dense_dot(feat_idx, w.vector+c*feats_dim, feats_dim); if ( training ) score += delta_loss(feat_idx, c); if ( score > max_score ) { max_score = score; ypred = c; } } // Build the CResultSet object to return CResultSet* ret = new CResultSet(); SG_REF(ret); CRealNumber* y = new CRealNumber(ypred); SG_REF(y); ret->psi_pred = get_joint_feature_vector(feat_idx, y); ret->score = max_score; ret->argmax = y; if ( training ) { ret->delta = CStructuredModel::delta_loss(feat_idx, y); ret->psi_truth = CStructuredModel::get_joint_feature_vector( feat_idx, feat_idx); ret->score -= SGVector< float64_t >::dot(w.vector, ret->psi_truth.vector, dim); } return ret; }
int CMulticlassOCAS::msvm_full_add_new_cut(float64_t *new_col_H, uint32_t *new_cut, uint32_t nSel, void* user_data) { float64_t* full_A = ((mocas_data*)user_data)->full_A; float64_t* new_a = ((mocas_data*)user_data)->new_a; float64_t* data_y = ((mocas_data*)user_data)->data_y; uint32_t nY = ((mocas_data*)user_data)->nY; uint32_t nDim = ((mocas_data*)user_data)->nDim; uint32_t nData = ((mocas_data*)user_data)->nData; CDotFeatures* features = ((mocas_data*)user_data)->features; float64_t sq_norm_a; uint32_t i, j, y, y2; memset(new_a, 0, sizeof(float64_t)*nDim*nY); for(i=0; i < nData; i++) { y = (uint32_t)(data_y[i]); y2 = (uint32_t)new_cut[i]; if(y2 != y) { features->add_to_dense_vec(1.0,i,&new_a[nDim*y],nDim); features->add_to_dense_vec(-1.0,i,&new_a[nDim*y2],nDim); } } // compute new_a'*new_a and insert new_a to the last column of full_A sq_norm_a = CMath::dot(new_a,new_a,nDim*nY); for(j=0; j < nDim*nY; j++ ) full_A[LIBOCAS_INDEX(j,nSel,nDim*nY)] = new_a[j]; new_col_H[nSel] = sq_norm_a; for(i=0; i < nSel; i++) { float64_t tmp = 0; for(j=0; j < nDim*nY; j++ ) tmp += new_a[j]*full_A[LIBOCAS_INDEX(j,i,nDim*nY)]; new_col_H[i] = tmp; } return 0; }
void CCombinedDotFeatures::set_subfeature_weights( float64_t* weights, int32_t num_weights) { int32_t i=0 ; CListElement* current = NULL ; CDotFeatures* f = get_first_feature_obj(current); ASSERT(num_weights==get_num_feature_obj()); while(f) { f->set_combined_feature_weight(weights[i]); SG_UNREF(f); f = get_next_feature_obj(current); i++; } }
void CCombinedDotFeatures::get_subfeature_weights(float64_t** weights, int32_t* num_weights) { *num_weights = get_num_feature_obj(); ASSERT(*num_weights > 0); *weights=SG_MALLOC(float64_t, *num_weights); float64_t* w = *weights; CListElement* current = NULL; CDotFeatures* f = get_first_feature_obj(current); while (f) { *w++=f->get_combined_feature_weight(); SG_UNREF(f); f = get_next_feature_obj(current); } }
int CMulticlassOCAS::msvm_full_compute_output(float64_t *output, void* user_data) { float64_t* W = ((mocas_data*)user_data)->W; uint32_t nY = ((mocas_data*)user_data)->nY; uint32_t nDim = ((mocas_data*)user_data)->nDim; uint32_t nData = ((mocas_data*)user_data)->nData; float64_t* output_values = ((mocas_data*)user_data)->output_values; CDotFeatures* features = ((mocas_data*)user_data)->features; uint32_t i, y; for(y=0; y<nY; y++) { features->dense_dot_range(output_values,0,nData,NULL,&W[nDim*y],nDim,0.0); for (i=0; i<nData; i++) output[LIBOCAS_INDEX(y,i,nY)] = output_values[i]; } return 0; }
bool CGaussian::train(CFeatures* data) { // init features with data if necessary and assure type is correct if (data) { if (!data->has_property(FP_DOT)) SG_ERROR("Specified features are not of type CDotFeatures\n"); set_features(data); } CDotFeatures* dotdata = (CDotFeatures *) data; delete[] m_mean; delete[] m_cov; dotdata->get_mean(&m_mean, &m_mean_length); dotdata->get_cov(&m_cov, &m_cov_rows, &m_cov_cols); init(); return true; }
bool CGMM::train(CFeatures* data) { ASSERT(m_n != 0); if (m_components) cleanup(); /** init features with data if necessary and assure type is correct */ if (data) { if (!data->has_property(FP_DOT)) SG_ERROR("Specified features are not of type CDotFeatures\n"); set_features(data); } CDotFeatures* dotdata = (CDotFeatures *) data; int32_t num_vectors = dotdata->get_num_vectors(); int32_t num_dim = dotdata->get_dim_feature_space(); CEuclidianDistance* dist = new CEuclidianDistance(); CKMeans* init_k_means = new CKMeans(m_n, dist); init_k_means->train(dotdata); float64_t* init_means; int32_t init_mean_dim; int32_t init_mean_size; init_k_means->get_cluster_centers(&init_means, &init_mean_dim, &init_mean_size); float64_t* init_cov; int32_t init_cov_rows; int32_t init_cov_cols; dotdata->get_cov(&init_cov, &init_cov_rows, &init_cov_cols); m_coefficients = new float64_t[m_coef_size]; m_components = new CGaussian*[m_n]; for (int i=0; i<m_n; i++) { m_coefficients[i] = 1.0/m_coef_size; m_components[i] = new CGaussian(&(init_means[i*init_mean_dim]), init_mean_dim, init_cov, init_cov_rows, init_cov_cols); } /** question of faster vs. less memory using */ float64_t* pdfs = new float64_t[num_vectors*m_n]; float64_t* T = new float64_t[num_vectors*m_n]; int32_t iter = 0; float64_t e_log_likelihood_change = m_minimal_change + 1; float64_t e_log_likelihood_old = 0; float64_t e_log_likelihood_new = -FLT_MAX; while (iter<m_max_iter && e_log_likelihood_change>m_minimal_change) { e_log_likelihood_old = e_log_likelihood_new; e_log_likelihood_new = 0; /** Precomputing likelihoods */ float64_t* point; int32_t point_len; for (int i=0; i<num_vectors; i++) { dotdata->get_feature_vector(&point, &point_len, i); for (int j=0; j<m_n; j++) pdfs[i*m_n+j] = m_components[j]->compute_PDF(point, point_len); delete[] point; } for (int i=0; i<num_vectors; i++) { float64_t sum = 0; for (int j=0; j<m_n; j++) sum += m_coefficients[j]*pdfs[i*m_n+j]; for (int j=0; j<m_n; j++) { T[i*m_n+j] = (m_coefficients[j]*pdfs[i*m_n+j])/sum; e_log_likelihood_new += T[i*m_n+j]*CMath::log(m_coefficients[j]*pdfs[i*m_n+j]); } } /** Not sure if getting the abs value is a good idea */ e_log_likelihood_change = CMath::abs(e_log_likelihood_new - e_log_likelihood_old); /** Updates */ float64_t T_sum; float64_t* mean_sum; float64_t* cov_sum; for (int i=0; i<m_n; i++) { T_sum = 0; mean_sum = new float64_t[num_dim]; memset(mean_sum, 0, num_dim*sizeof(float64_t)); for (int j=0; j<num_vectors; j++) { T_sum += T[j*m_n+i]; dotdata->get_feature_vector(&point, &point_len, j); CMath::add<float64_t>(mean_sum, T[j*m_n+i], point, 1, mean_sum, point_len); delete[] point; } m_coefficients[i] = T_sum/num_vectors; for (int j=0; j<num_dim; j++) mean_sum[j] /= T_sum; m_components[i]->set_mean(mean_sum, num_dim); cov_sum = new float64_t[num_dim*num_dim]; memset(cov_sum, 0, num_dim*num_dim*sizeof(float64_t)); for (int j=0; j<num_vectors; j++) { dotdata->get_feature_vector(&point, &point_len, j); CMath::add<float64_t>(point, 1, point, -1, mean_sum, point_len); cblas_dger(CblasRowMajor, num_dim, num_dim, T[j*m_n+i], point, 1, point, 1, (double*) cov_sum, num_dim); delete[] point; } for (int j=0; j<num_dim*num_dim; j++) cov_sum[j] /= T_sum; m_components[i]->set_cov(cov_sum, num_dim, num_dim); delete[] mean_sum; delete[] cov_sum; } iter++; } delete[] pdfs; delete[] T; return true; }
CRegressionLabels* CGaussianProcessRegression::apply_regression(CFeatures* data) { if (data) { if(data->get_feature_class() == C_COMBINED) { CDotFeatures* feat = (CDotFeatures*)((CCombinedFeatures*)data)-> get_first_feature_obj(); if (!feat->has_property(FP_DOT)) SG_ERROR("Specified features are not of type CFeatures\n"); if (feat->get_feature_class() != C_DENSE) SG_ERROR("Expected Simple Features\n"); if (feat->get_feature_type() != F_DREAL) SG_ERROR("Expected Real Features\n"); SG_UNREF(feat); } else { if (!data->has_property(FP_DOT)) SG_ERROR("Specified features are not of type CFeatures\n"); if (data->get_feature_class() != C_DENSE) SG_ERROR("Expected Simple Features\n"); if (data->get_feature_type() != F_DREAL) SG_ERROR("Expected Real Features\n"); } SG_UNREF(m_data); SG_REF(data); m_data = (CFeatures*)data; update_kernel_matrices(); } else if (!m_data) SG_ERROR("No testing features!\n"); else if (update_parameter_hash()) update_kernel_matrices(); if (m_return == GP_RETURN_COV) { CRegressionLabels* result = new CRegressionLabels(getCovarianceVector()); return result; } if (m_return == GP_RETURN_MEANS) { CRegressionLabels* result = new CRegressionLabels(getMeanVector()); return result; } else { SGVector<float64_t> mean_vector = getMeanVector(); SGVector<float64_t> cov_vector = getCovarianceVector(); index_t size = mean_vector.vlen+cov_vector.vlen; SGVector<float64_t> result_vector(size); for (index_t i = 0; i < size; i++) { if (i < mean_vector.vlen) result_vector[i] = mean_vector[i]; else result_vector[i] = cov_vector[i-mean_vector.vlen]; } CRegressionLabels* result = new CRegressionLabels(result_vector); return result; } }
void CLibLinear::solve_l1r_lr( const problem *prob_col, double eps, double Cp, double Cn) { int l = prob_col->l; int w_size = prob_col->n; int j, s, iter = 0; int active_size = w_size; int max_num_linesearch = 20; double x_min = 0; double sigma = 0.01; double d, G, H; double Gmax_old = CMath::INFTY; double Gmax_new; double Gmax_init=0; double sum1, appxcond1; double sum2, appxcond2; double cond; int *index = SG_MALLOC(int, w_size); int32_t *y = SG_MALLOC(int32_t, l); double *exp_wTx = SG_MALLOC(double, l); double *exp_wTx_new = SG_MALLOC(double, l); double *xj_max = SG_MALLOC(double, w_size); double *C_sum = SG_MALLOC(double, w_size); double *xjneg_sum = SG_MALLOC(double, w_size); double *xjpos_sum = SG_MALLOC(double, w_size); CDotFeatures* x = prob_col->x; void* iterator; int ind; double val; double C[3] = {Cn,0,Cp}; int n = prob_col->n; if (prob_col->use_bias) n--; for(j=0; j<l; j++) { exp_wTx[j] = 1; if(prob_col->y[j] > 0) y[j] = 1; else y[j] = -1; } for(j=0; j<w_size; j++) { w.vector[j] = 0; index[j] = j; xj_max[j] = 0; C_sum[j] = 0; xjneg_sum[j] = 0; xjpos_sum[j] = 0; if (use_bias && j==n) { for (ind=0; ind<l; ind++) { x_min = CMath::min(x_min, 1.0); xj_max[j] = CMath::max(xj_max[j], 1.0); C_sum[j] += C[GETI(ind)]; if(y[ind] == -1) xjneg_sum[j] += C[GETI(ind)]; else xjpos_sum[j] += C[GETI(ind)]; } } else { iterator=x->get_feature_iterator(j); while (x->get_next_feature(ind, val, iterator)) { x_min = CMath::min(x_min, val); xj_max[j] = CMath::max(xj_max[j], val); C_sum[j] += C[GETI(ind)]; if(y[ind] == -1) xjneg_sum[j] += C[GETI(ind)]*val; else xjpos_sum[j] += C[GETI(ind)]*val; } x->free_feature_iterator(iterator); } } CTime start_time; while (iter < max_iterations && !CSignal::cancel_computations()) { if (m_max_train_time > 0 && start_time.cur_time_diff() > m_max_train_time) break; Gmax_new = 0; for(j=0; j<active_size; j++) { int i = j+rand()%(active_size-j); CMath::swap(index[i], index[j]); } for(s=0; s<active_size; s++) { j = index[s]; sum1 = 0; sum2 = 0; H = 0; if (use_bias && j==n) { for (ind=0; ind<l; ind++) { double exp_wTxind = exp_wTx[ind]; double tmp1 = 1.0/(1+exp_wTxind); double tmp2 = C[GETI(ind)]*tmp1; double tmp3 = tmp2*exp_wTxind; sum2 += tmp2; sum1 += tmp3; H += tmp1*tmp3; } } else { iterator=x->get_feature_iterator(j); while (x->get_next_feature(ind, val, iterator)) { double exp_wTxind = exp_wTx[ind]; double tmp1 = val/(1+exp_wTxind); double tmp2 = C[GETI(ind)]*tmp1; double tmp3 = tmp2*exp_wTxind; sum2 += tmp2; sum1 += tmp3; H += tmp1*tmp3; } x->free_feature_iterator(iterator); } G = -sum2 + xjneg_sum[j]; double Gp = G+1; double Gn = G-1; double violation = 0; if(w.vector[j] == 0) { if(Gp < 0) violation = -Gp; else if(Gn > 0) violation = Gn; else if(Gp>Gmax_old/l && Gn<-Gmax_old/l) { active_size--; CMath::swap(index[s], index[active_size]); s--; continue; } } else if(w.vector[j] > 0) violation = fabs(Gp); else violation = fabs(Gn); Gmax_new = CMath::max(Gmax_new, violation); // obtain Newton direction d if(Gp <= H*w.vector[j]) d = -Gp/H; else if(Gn >= H*w.vector[j]) d = -Gn/H; else d = -w.vector[j]; if(fabs(d) < 1.0e-12) continue; d = CMath::min(CMath::max(d,-10.0),10.0); double delta = fabs(w.vector[j]+d)-fabs(w.vector[j]) + G*d; int num_linesearch; for(num_linesearch=0; num_linesearch < max_num_linesearch; num_linesearch++) { cond = fabs(w.vector[j]+d)-fabs(w.vector[j]) - sigma*delta; if(x_min >= 0) { double tmp = exp(d*xj_max[j]); appxcond1 = log(1+sum1*(tmp-1)/xj_max[j]/C_sum[j])*C_sum[j] + cond - d*xjpos_sum[j]; appxcond2 = log(1+sum2*(1/tmp-1)/xj_max[j]/C_sum[j])*C_sum[j] + cond + d*xjneg_sum[j]; if(CMath::min(appxcond1,appxcond2) <= 0) { if (use_bias && j==n) { for (ind=0; ind<l; ind++) exp_wTx[ind] *= exp(d); } else { iterator=x->get_feature_iterator(j); while (x->get_next_feature(ind, val, iterator)) exp_wTx[ind] *= exp(d*val); x->free_feature_iterator(iterator); } break; } } cond += d*xjneg_sum[j]; int i = 0; if (use_bias && j==n) { for (ind=0; ind<l; ind++) { double exp_dx = exp(d); exp_wTx_new[i] = exp_wTx[ind]*exp_dx; cond += C[GETI(ind)]*log((1+exp_wTx_new[i])/(exp_dx+exp_wTx_new[i])); i++; } } else { iterator=x->get_feature_iterator(j); while (x->get_next_feature(ind, val, iterator)) { double exp_dx = exp(d*val); exp_wTx_new[i] = exp_wTx[ind]*exp_dx; cond += C[GETI(ind)]*log((1+exp_wTx_new[i])/(exp_dx+exp_wTx_new[i])); i++; } x->free_feature_iterator(iterator); } if(cond <= 0) { i = 0; if (use_bias && j==n) { for (ind=0; ind<l; ind++) { exp_wTx[ind] = exp_wTx_new[i]; i++; } } else { iterator=x->get_feature_iterator(j); while (x->get_next_feature(ind, val, iterator)) { exp_wTx[ind] = exp_wTx_new[i]; i++; } x->free_feature_iterator(iterator); } break; } else { d *= 0.5; delta *= 0.5; } } w.vector[j] += d; // recompute exp_wTx[] if line search takes too many steps if(num_linesearch >= max_num_linesearch) { SG_INFO("#"); for(int i=0; i<l; i++) exp_wTx[i] = 0; for(int i=0; i<w_size; i++) { if(w.vector[i]==0) continue; if (use_bias && i==n) { for (ind=0; ind<l; ind++) exp_wTx[ind] += w.vector[i]; } else { iterator=x->get_feature_iterator(i); while (x->get_next_feature(ind, val, iterator)) exp_wTx[ind] += w.vector[i]*val; x->free_feature_iterator(iterator); } } for(int i=0; i<l; i++) exp_wTx[i] = exp(exp_wTx[i]); } } if(iter == 0) Gmax_init = Gmax_new; iter++; SG_SABS_PROGRESS(Gmax_new, -CMath::log10(Gmax_new), -CMath::log10(Gmax_init), -CMath::log10(eps*Gmax_init), 6); if(Gmax_new <= eps*Gmax_init) { if(active_size == w_size) break; else { active_size = w_size; Gmax_old = CMath::INFTY; continue; } } Gmax_old = Gmax_new; } SG_DONE(); SG_INFO("optimization finished, #iter = %d\n", iter); if(iter >= max_iterations) SG_WARNING("\nWARNING: reaching max number of iterations\n"); // calculate objective value double v = 0; int nnz = 0; for(j=0; j<w_size; j++) if(w.vector[j] != 0) { v += fabs(w.vector[j]); nnz++; } for(j=0; j<l; j++) if(y[j] == 1) v += C[GETI(j)]*log(1+1/exp_wTx[j]); else v += C[GETI(j)]*log(1+exp_wTx[j]); SG_INFO("Objective value = %lf\n", v); SG_INFO("#nonzeros/#features = %d/%d\n", nnz, w_size); delete [] index; delete [] y; delete [] exp_wTx; delete [] exp_wTx_new; delete [] xj_max; delete [] C_sum; delete [] xjneg_sum; delete [] xjpos_sum; }
void CLibLinear::solve_l1r_l2_svc( problem *prob_col, double eps, double Cp, double Cn) { int l = prob_col->l; int w_size = prob_col->n; int j, s, iter = 0; int active_size = w_size; int max_num_linesearch = 20; double sigma = 0.01; double d, G_loss, G, H; double Gmax_old = CMath::INFTY; double Gmax_new; double Gmax_init=0; double d_old, d_diff; double loss_old=0, loss_new; double appxcond, cond; int *index = SG_MALLOC(int, w_size); int32_t *y = SG_MALLOC(int32_t, l); double *b = SG_MALLOC(double, l); // b = 1-ywTx double *xj_sq = SG_MALLOC(double, w_size); CDotFeatures* x = (CDotFeatures*) prob_col->x; void* iterator; int32_t ind; float64_t val; double C[3] = {Cn,0,Cp}; int n = prob_col->n; if (prob_col->use_bias) n--; for(j=0; j<l; j++) { b[j] = 1; if(prob_col->y[j] > 0) y[j] = 1; else y[j] = -1; } for(j=0; j<w_size; j++) { w.vector[j] = 0; index[j] = j; xj_sq[j] = 0; if (use_bias && j==n) { for (ind=0; ind<l; ind++) xj_sq[n] += C[GETI(ind)]; } else { iterator=x->get_feature_iterator(j); while (x->get_next_feature(ind, val, iterator)) xj_sq[j] += C[GETI(ind)]*val*val; x->free_feature_iterator(iterator); } } CTime start_time; while (iter < max_iterations && !CSignal::cancel_computations()) { if (m_max_train_time > 0 && start_time.cur_time_diff() > m_max_train_time) break; Gmax_new = 0; for(j=0; j<active_size; j++) { int i = j+rand()%(active_size-j); CMath::swap(index[i], index[j]); } for(s=0; s<active_size; s++) { j = index[s]; G_loss = 0; H = 0; if (use_bias && j==n) { for (ind=0; ind<l; ind++) { if(b[ind] > 0) { double tmp = C[GETI(ind)]*y[ind]; G_loss -= tmp*b[ind]; H += tmp*y[ind]; } } } else { iterator=x->get_feature_iterator(j); while (x->get_next_feature(ind, val, iterator)) { if(b[ind] > 0) { double tmp = C[GETI(ind)]*val*y[ind]; G_loss -= tmp*b[ind]; H += tmp*val*y[ind]; } } x->free_feature_iterator(iterator); } G_loss *= 2; G = G_loss; H *= 2; H = CMath::max(H, 1e-12); double Gp = G+1; double Gn = G-1; double violation = 0; if(w.vector[j] == 0) { if(Gp < 0) violation = -Gp; else if(Gn > 0) violation = Gn; else if(Gp>Gmax_old/l && Gn<-Gmax_old/l) { active_size--; CMath::swap(index[s], index[active_size]); s--; continue; } } else if(w.vector[j] > 0) violation = fabs(Gp); else violation = fabs(Gn); Gmax_new = CMath::max(Gmax_new, violation); // obtain Newton direction d if(Gp <= H*w.vector[j]) d = -Gp/H; else if(Gn >= H*w.vector[j]) d = -Gn/H; else d = -w.vector[j]; if(fabs(d) < 1.0e-12) continue; double delta = fabs(w.vector[j]+d)-fabs(w.vector[j]) + G*d; d_old = 0; int num_linesearch; for(num_linesearch=0; num_linesearch < max_num_linesearch; num_linesearch++) { d_diff = d_old - d; cond = fabs(w.vector[j]+d)-fabs(w.vector[j]) - sigma*delta; appxcond = xj_sq[j]*d*d + G_loss*d + cond; if(appxcond <= 0) { if (use_bias && j==n) { for (ind=0; ind<l; ind++) b[ind] += d_diff*y[ind]; break; } else { iterator=x->get_feature_iterator(j); while (x->get_next_feature(ind, val, iterator)) b[ind] += d_diff*val*y[ind]; x->free_feature_iterator(iterator); break; } } if(num_linesearch == 0) { loss_old = 0; loss_new = 0; if (use_bias && j==n) { for (ind=0; ind<l; ind++) { if(b[ind] > 0) loss_old += C[GETI(ind)]*b[ind]*b[ind]; double b_new = b[ind] + d_diff*y[ind]; b[ind] = b_new; if(b_new > 0) loss_new += C[GETI(ind)]*b_new*b_new; } } else { iterator=x->get_feature_iterator(j); while (x->get_next_feature(ind, val, iterator)) { if(b[ind] > 0) loss_old += C[GETI(ind)]*b[ind]*b[ind]; double b_new = b[ind] + d_diff*val*y[ind]; b[ind] = b_new; if(b_new > 0) loss_new += C[GETI(ind)]*b_new*b_new; } x->free_feature_iterator(iterator); } } else { loss_new = 0; if (use_bias && j==n) { for (ind=0; ind<l; ind++) { double b_new = b[ind] + d_diff*y[ind]; b[ind] = b_new; if(b_new > 0) loss_new += C[GETI(ind)]*b_new*b_new; } } else { iterator=x->get_feature_iterator(j); while (x->get_next_feature(ind, val, iterator)) { double b_new = b[ind] + d_diff*val*y[ind]; b[ind] = b_new; if(b_new > 0) loss_new += C[GETI(ind)]*b_new*b_new; } x->free_feature_iterator(iterator); } } cond = cond + loss_new - loss_old; if(cond <= 0) break; else { d_old = d; d *= 0.5; delta *= 0.5; } } w.vector[j] += d; // recompute b[] if line search takes too many steps if(num_linesearch >= max_num_linesearch) { SG_INFO("#"); for(int i=0; i<l; i++) b[i] = 1; for(int i=0; i<n; i++) { if(w.vector[i]==0) continue; iterator=x->get_feature_iterator(i); while (x->get_next_feature(ind, val, iterator)) b[ind] -= w.vector[i]*val*y[ind]; x->free_feature_iterator(iterator); } if (use_bias && w.vector[n]) { for (ind=0; ind<l; ind++) b[ind] -= w.vector[n]*y[ind]; } } } if(iter == 0) Gmax_init = Gmax_new; iter++; SG_SABS_PROGRESS(Gmax_new, -CMath::log10(Gmax_new), -CMath::log10(Gmax_init), -CMath::log10(eps*Gmax_init), 6); if(Gmax_new <= eps*Gmax_init) { if(active_size == w_size) break; else { active_size = w_size; Gmax_old = CMath::INFTY; continue; } } Gmax_old = Gmax_new; } SG_DONE(); SG_INFO("optimization finished, #iter = %d\n", iter); if(iter >= max_iterations) SG_WARNING("\nWARNING: reaching max number of iterations\n"); // calculate objective value double v = 0; int nnz = 0; for(j=0; j<w_size; j++) { if(w.vector[j] != 0) { v += fabs(w.vector[j]); nnz++; } } for(j=0; j<l; j++) if(b[j] > 0) v += C[GETI(j)]*b[j]*b[j]; SG_INFO("Objective value = %lf\n", v); SG_INFO("#nonzeros/#features = %d/%d\n", nnz, w_size); delete [] index; delete [] y; delete [] b; delete [] xj_sq; }
/*---------------------------------------------------------------------------------- sparse_add_new_cut( new_col_H, new_cut, cut_length, nSel ) does the following: new_a = sum(data_X(:,find(new_cut ~=0 )),2); new_col_H = [sparse_A(:,1:nSel)'*new_a ; new_a'*new_a]; sparse_A(:,nSel+1) = new_a; ---------------------------------------------------------------------------------*/ int CSVMOcas::add_new_cut( float64_t *new_col_H, uint32_t *new_cut, uint32_t cut_length, uint32_t nSel, void* ptr) { CSVMOcas* o = (CSVMOcas*) ptr; CDotFeatures* f = o->features; uint32_t nDim=(uint32_t) o->w_dim; float64_t* y = o->lab.vector; float64_t** c_val = o->cp_value; uint32_t** c_idx = o->cp_index; uint32_t* c_nzd = o->cp_nz_dims; float64_t* c_bias = o->cp_bias; float64_t sq_norm_a; uint32_t i, j, nz_dims; /* temporary vector */ float64_t* new_a = o->tmp_a_buf; memset(new_a, 0, sizeof(float64_t)*nDim); for(i=0; i < cut_length; i++) { f->add_to_dense_vec(y[new_cut[i]], new_cut[i], new_a, nDim); if (o->use_bias) c_bias[nSel]+=y[new_cut[i]]; } /* compute new_a'*new_a and count number of non-zerou dimensions */ nz_dims = 0; sq_norm_a = CMath::sq(c_bias[nSel]); for(j=0; j < nDim; j++ ) { if(new_a[j] != 0) { nz_dims++; sq_norm_a += new_a[j]*new_a[j]; } } /* sparsify new_a and insert it to the last column of sparse_A */ c_nzd[nSel] = nz_dims; c_idx[nSel]=NULL; c_val[nSel]=NULL; if(nz_dims > 0) { c_idx[nSel]=SG_MALLOC(uint32_t, nz_dims); c_val[nSel]=SG_MALLOC(float64_t, nz_dims); uint32_t idx=0; for(j=0; j < nDim; j++ ) { if(new_a[j] != 0) { c_idx[nSel][idx] = j; c_val[nSel][idx++] = new_a[j]; } } } new_col_H[nSel] = sq_norm_a; for(i=0; i < nSel; i++) { float64_t tmp = c_bias[nSel]*c_bias[i]; for(j=0; j < c_nzd[i]; j++) tmp += new_a[c_idx[i][j]]*c_val[i][j]; new_col_H[i] = tmp; } //CMath::display_vector(new_col_H, nSel+1, "new_col_H"); //CMath::display_vector((int32_t*) c_idx[nSel], (int32_t) nz_dims, "c_idx"); //CMath::display_vector((float64_t*) c_val[nSel], nz_dims, "c_val"); return 0; }