CLatentLabels* CLatentSVM::apply()
{
if (!m_model)
SG_ERROR("LatentModel is not set!\n");
if (!features)
return NULL;
index_t num_examples = m_model->get_num_vectors();
CLatentLabels* hs = new CLatentLabels(num_examples);
CBinaryLabels* ys = new CBinaryLabels(num_examples);
hs->set_labels(ys);
m_model->set_labels(hs);
for (index_t i = 0; i < num_examples; ++i)
{
/* find h for the example */
CData* h = m_model->infer_latent_variable(w, i);
hs->add_latent_label(h);
}
/* compute the y labels */
CDotFeatures* x = m_model->get_psi_feature_vectors();
x->dense_dot_range(ys->get_labels().vector, 0, num_examples, NULL, w.vector, w.vlen, 0.0);
return hs;
}
void CExactInferenceMethod::update_all()
{
if (m_labels)
m_label_vector =
((CRegressionLabels*) m_labels)->get_labels().clone();
if (m_features && m_features->has_property(FP_DOT) && m_features->get_num_vectors())
m_feature_matrix =
((CDotFeatures*)m_features)->get_computed_dot_feature_matrix();
else if (m_features && m_features->get_feature_class() == C_COMBINED)
{
CDotFeatures* feat =
(CDotFeatures*)((CCombinedFeatures*)m_features)->
get_first_feature_obj();
if (feat->get_num_vectors())
m_feature_matrix = feat->get_computed_dot_feature_matrix();
SG_UNREF(feat);
}
update_data_means();
if (m_kernel)
update_train_kernel();
if (m_ktrtr.num_cols*m_ktrtr.num_rows)
{
update_chol();
update_alpha();
}
}
void CExactInferenceMethod::check_members()
{
if (!m_labels)
SG_ERROR("No labels set\n")
if (m_labels->get_label_type() != LT_REGRESSION)
SG_ERROR("Expected RegressionLabels\n")
if (!m_features)
SG_ERROR("No features set!\n")
if (m_labels->get_num_labels() != m_features->get_num_vectors())
SG_ERROR("Number of training vectors does not match number of labels\n")
if(m_features->get_feature_class() == C_COMBINED)
{
CDotFeatures* feat =
(CDotFeatures*)((CCombinedFeatures*)m_features)->
get_first_feature_obj();
if (!feat->has_property(FP_DOT))
SG_ERROR("Specified features are not of type CFeatures\n")
if (feat->get_feature_class() != C_DENSE)
SG_ERROR("Expected Simple Features\n")
if (feat->get_feature_type() != F_DREAL)
SG_ERROR("Expected Real Features\n")
SG_UNREF(feat);
}
void CInferenceMethod::set_latent_features(CFeatures* feat)
{
SG_REF(feat);
SG_UNREF(m_latent_features);
m_latent_features=feat;
if (m_latent_features && m_latent_features->has_property(FP_DOT) && m_latent_features->get_num_vectors())
m_latent_matrix =
((CDotFeatures*)m_latent_features)->get_computed_dot_feature_matrix();
else if (m_latent_features && m_latent_features->get_feature_class() == C_COMBINED)
{
CDotFeatures* subfeat =
(CDotFeatures*)((CCombinedFeatures*)m_latent_features)->
get_first_feature_obj();
if (m_latent_features->get_num_vectors())
m_latent_matrix = subfeat->get_computed_dot_feature_matrix();
SG_UNREF(subfeat);
}
update_data_means();
update_train_kernel();
update_chol();
update_alpha();
}
CRegressionLabels* CGaussianProcessRegression::apply_regression(CFeatures* data)
{
if (data)
{
if(data->get_feature_class() == C_COMBINED)
{
CDotFeatures* feat =
(CDotFeatures*)((CCombinedFeatures*)data)->
get_first_feature_obj();
if (!feat->has_property(FP_DOT))
SG_ERROR("Specified features are not of type CFeatures\n")
if (feat->get_feature_class() != C_DENSE)
SG_ERROR("Expected Simple Features\n")
if (feat->get_feature_type() != F_DREAL)
SG_ERROR("Expected Real Features\n")
SG_UNREF(feat);
}
else
{
if (!data->has_property(FP_DOT))
CResultSet* CMulticlassModel::argmax(
SGVector< float64_t > w,
int32_t feat_idx,
bool const training)
{
CDotFeatures* df = (CDotFeatures*) m_features;
int32_t feats_dim = df->get_dim_feature_space();
if ( training )
{
CMulticlassSOLabels* ml = (CMulticlassSOLabels*) m_labels;
m_num_classes = ml->get_num_classes();
}
else
{
REQUIRE(m_num_classes > 0, "The model needs to be trained before "
"using it for prediction\n");
}
int32_t dim = get_dim();
ASSERT(dim == w.vlen)
// Find the class that gives the maximum score
float64_t score = 0, ypred = 0;
float64_t max_score = -CMath::INFTY;
for ( int32_t c = 0 ; c < m_num_classes ; ++c )
{
score = df->dense_dot(feat_idx, w.vector+c*feats_dim, feats_dim);
if ( training )
score += delta_loss(feat_idx, c);
if ( score > max_score )
{
max_score = score;
ypred = c;
}
}
// Build the CResultSet object to return
CResultSet* ret = new CResultSet();
SG_REF(ret);
CRealNumber* y = new CRealNumber(ypred);
SG_REF(y);
ret->psi_pred = get_joint_feature_vector(feat_idx, y);
ret->score = max_score;
ret->argmax = y;
if ( training )
{
ret->delta = CStructuredModel::delta_loss(feat_idx, y);
ret->psi_truth = CStructuredModel::get_joint_feature_vector(
feat_idx, feat_idx);
ret->score -= SGVector< float64_t >::dot(w.vector,
ret->psi_truth.vector, dim);
}
return ret;
}
int CMulticlassOCAS::msvm_full_add_new_cut(float64_t *new_col_H, uint32_t *new_cut,
uint32_t nSel, void* user_data)
{
float64_t* full_A = ((mocas_data*)user_data)->full_A;
float64_t* new_a = ((mocas_data*)user_data)->new_a;
float64_t* data_y = ((mocas_data*)user_data)->data_y;
uint32_t nY = ((mocas_data*)user_data)->nY;
uint32_t nDim = ((mocas_data*)user_data)->nDim;
uint32_t nData = ((mocas_data*)user_data)->nData;
CDotFeatures* features = ((mocas_data*)user_data)->features;
float64_t sq_norm_a;
uint32_t i, j, y, y2;
memset(new_a, 0, sizeof(float64_t)*nDim*nY);
for(i=0; i < nData; i++)
{
y = (uint32_t)(data_y[i]);
y2 = (uint32_t)new_cut[i];
if(y2 != y)
{
features->add_to_dense_vec(1.0,i,&new_a[nDim*y],nDim);
features->add_to_dense_vec(-1.0,i,&new_a[nDim*y2],nDim);
}
}
// compute new_a'*new_a and insert new_a to the last column of full_A
sq_norm_a = CMath::dot(new_a,new_a,nDim*nY);
for(j=0; j < nDim*nY; j++ )
full_A[LIBOCAS_INDEX(j,nSel,nDim*nY)] = new_a[j];
new_col_H[nSel] = sq_norm_a;
for(i=0; i < nSel; i++)
{
float64_t tmp = 0;
for(j=0; j < nDim*nY; j++ )
tmp += new_a[j]*full_A[LIBOCAS_INDEX(j,i,nDim*nY)];
new_col_H[i] = tmp;
}
return 0;
}
void CCombinedDotFeatures::set_subfeature_weights(
float64_t* weights, int32_t num_weights)
{
int32_t i=0 ;
CListElement* current = NULL ;
CDotFeatures* f = get_first_feature_obj(current);
ASSERT(num_weights==get_num_feature_obj());
while(f)
{
f->set_combined_feature_weight(weights[i]);
SG_UNREF(f);
f = get_next_feature_obj(current);
i++;
}
}
void CCombinedDotFeatures::get_subfeature_weights(float64_t** weights, int32_t* num_weights)
{
*num_weights = get_num_feature_obj();
ASSERT(*num_weights > 0);
*weights=SG_MALLOC(float64_t, *num_weights);
float64_t* w = *weights;
CListElement* current = NULL;
CDotFeatures* f = get_first_feature_obj(current);
while (f)
{
*w++=f->get_combined_feature_weight();
SG_UNREF(f);
f = get_next_feature_obj(current);
}
}
int CMulticlassOCAS::msvm_full_compute_output(float64_t *output, void* user_data)
{
float64_t* W = ((mocas_data*)user_data)->W;
uint32_t nY = ((mocas_data*)user_data)->nY;
uint32_t nDim = ((mocas_data*)user_data)->nDim;
uint32_t nData = ((mocas_data*)user_data)->nData;
float64_t* output_values = ((mocas_data*)user_data)->output_values;
CDotFeatures* features = ((mocas_data*)user_data)->features;
uint32_t i, y;
for(y=0; y<nY; y++)
{
features->dense_dot_range(output_values,0,nData,NULL,&W[nDim*y],nDim,0.0);
for (i=0; i<nData; i++)
output[LIBOCAS_INDEX(y,i,nY)] = output_values[i];
}
return 0;
}
bool CGaussian::train(CFeatures* data)
{
// init features with data if necessary and assure type is correct
if (data)
{
if (!data->has_property(FP_DOT))
SG_ERROR("Specified features are not of type CDotFeatures\n");
set_features(data);
}
CDotFeatures* dotdata = (CDotFeatures *) data;
delete[] m_mean;
delete[] m_cov;
dotdata->get_mean(&m_mean, &m_mean_length);
dotdata->get_cov(&m_cov, &m_cov_rows, &m_cov_cols);
init();
return true;
}
bool CGMM::train(CFeatures* data)
{
ASSERT(m_n != 0);
if (m_components)
cleanup();
/** init features with data if necessary and assure type is correct */
if (data)
{
if (!data->has_property(FP_DOT))
SG_ERROR("Specified features are not of type CDotFeatures\n");
set_features(data);
}
CDotFeatures* dotdata = (CDotFeatures *) data;
int32_t num_vectors = dotdata->get_num_vectors();
int32_t num_dim = dotdata->get_dim_feature_space();
CEuclidianDistance* dist = new CEuclidianDistance();
CKMeans* init_k_means = new CKMeans(m_n, dist);
init_k_means->train(dotdata);
float64_t* init_means;
int32_t init_mean_dim;
int32_t init_mean_size;
init_k_means->get_cluster_centers(&init_means, &init_mean_dim, &init_mean_size);
float64_t* init_cov;
int32_t init_cov_rows;
int32_t init_cov_cols;
dotdata->get_cov(&init_cov, &init_cov_rows, &init_cov_cols);
m_coefficients = new float64_t[m_coef_size];
m_components = new CGaussian*[m_n];
for (int i=0; i<m_n; i++)
{
m_coefficients[i] = 1.0/m_coef_size;
m_components[i] = new CGaussian(&(init_means[i*init_mean_dim]), init_mean_dim,
init_cov, init_cov_rows, init_cov_cols);
}
/** question of faster vs. less memory using */
float64_t* pdfs = new float64_t[num_vectors*m_n];
float64_t* T = new float64_t[num_vectors*m_n];
int32_t iter = 0;
float64_t e_log_likelihood_change = m_minimal_change + 1;
float64_t e_log_likelihood_old = 0;
float64_t e_log_likelihood_new = -FLT_MAX;
while (iter<m_max_iter && e_log_likelihood_change>m_minimal_change)
{
e_log_likelihood_old = e_log_likelihood_new;
e_log_likelihood_new = 0;
/** Precomputing likelihoods */
float64_t* point;
int32_t point_len;
for (int i=0; i<num_vectors; i++)
{
dotdata->get_feature_vector(&point, &point_len, i);
for (int j=0; j<m_n; j++)
pdfs[i*m_n+j] = m_components[j]->compute_PDF(point, point_len);
delete[] point;
}
for (int i=0; i<num_vectors; i++)
{
float64_t sum = 0;
for (int j=0; j<m_n; j++)
sum += m_coefficients[j]*pdfs[i*m_n+j];
for (int j=0; j<m_n; j++)
{
T[i*m_n+j] = (m_coefficients[j]*pdfs[i*m_n+j])/sum;
e_log_likelihood_new += T[i*m_n+j]*CMath::log(m_coefficients[j]*pdfs[i*m_n+j]);
}
}
/** Not sure if getting the abs value is a good idea */
e_log_likelihood_change = CMath::abs(e_log_likelihood_new - e_log_likelihood_old);
/** Updates */
float64_t T_sum;
float64_t* mean_sum;
float64_t* cov_sum;
for (int i=0; i<m_n; i++)
{
T_sum = 0;
mean_sum = new float64_t[num_dim];
memset(mean_sum, 0, num_dim*sizeof(float64_t));
for (int j=0; j<num_vectors; j++)
{
T_sum += T[j*m_n+i];
dotdata->get_feature_vector(&point, &point_len, j);
CMath::add<float64_t>(mean_sum, T[j*m_n+i], point, 1, mean_sum, point_len);
delete[] point;
}
m_coefficients[i] = T_sum/num_vectors;
for (int j=0; j<num_dim; j++)
mean_sum[j] /= T_sum;
m_components[i]->set_mean(mean_sum, num_dim);
cov_sum = new float64_t[num_dim*num_dim];
memset(cov_sum, 0, num_dim*num_dim*sizeof(float64_t));
for (int j=0; j<num_vectors; j++)
{
dotdata->get_feature_vector(&point, &point_len, j);
CMath::add<float64_t>(point, 1, point, -1, mean_sum, point_len);
cblas_dger(CblasRowMajor, num_dim, num_dim, T[j*m_n+i], point, 1, point,
1, (double*) cov_sum, num_dim);
delete[] point;
}
for (int j=0; j<num_dim*num_dim; j++)
cov_sum[j] /= T_sum;
m_components[i]->set_cov(cov_sum, num_dim, num_dim);
delete[] mean_sum;
delete[] cov_sum;
}
iter++;
}
delete[] pdfs;
delete[] T;
return true;
}
CRegressionLabels* CGaussianProcessRegression::apply_regression(CFeatures* data)
{
if (data)
{
if(data->get_feature_class() == C_COMBINED)
{
CDotFeatures* feat =
(CDotFeatures*)((CCombinedFeatures*)data)->
get_first_feature_obj();
if (!feat->has_property(FP_DOT))
SG_ERROR("Specified features are not of type CFeatures\n");
if (feat->get_feature_class() != C_DENSE)
SG_ERROR("Expected Simple Features\n");
if (feat->get_feature_type() != F_DREAL)
SG_ERROR("Expected Real Features\n");
SG_UNREF(feat);
}
else
{
if (!data->has_property(FP_DOT))
SG_ERROR("Specified features are not of type CFeatures\n");
if (data->get_feature_class() != C_DENSE)
SG_ERROR("Expected Simple Features\n");
if (data->get_feature_type() != F_DREAL)
SG_ERROR("Expected Real Features\n");
}
SG_UNREF(m_data);
SG_REF(data);
m_data = (CFeatures*)data;
update_kernel_matrices();
}
else if (!m_data)
SG_ERROR("No testing features!\n");
else if (update_parameter_hash())
update_kernel_matrices();
if (m_return == GP_RETURN_COV)
{
CRegressionLabels* result =
new CRegressionLabels(getCovarianceVector());
return result;
}
if (m_return == GP_RETURN_MEANS)
{
CRegressionLabels* result =
new CRegressionLabels(getMeanVector());
return result;
}
else
{
SGVector<float64_t> mean_vector = getMeanVector();
SGVector<float64_t> cov_vector = getCovarianceVector();
index_t size = mean_vector.vlen+cov_vector.vlen;
SGVector<float64_t> result_vector(size);
for (index_t i = 0; i < size; i++)
{
if (i < mean_vector.vlen)
result_vector[i] = mean_vector[i];
else
result_vector[i] = cov_vector[i-mean_vector.vlen];
}
CRegressionLabels* result =
new CRegressionLabels(result_vector);
return result;
}
}
void CLibLinear::solve_l1r_lr(
const problem *prob_col, double eps,
double Cp, double Cn)
{
int l = prob_col->l;
int w_size = prob_col->n;
int j, s, iter = 0;
int active_size = w_size;
int max_num_linesearch = 20;
double x_min = 0;
double sigma = 0.01;
double d, G, H;
double Gmax_old = CMath::INFTY;
double Gmax_new;
double Gmax_init=0;
double sum1, appxcond1;
double sum2, appxcond2;
double cond;
int *index = SG_MALLOC(int, w_size);
int32_t *y = SG_MALLOC(int32_t, l);
double *exp_wTx = SG_MALLOC(double, l);
double *exp_wTx_new = SG_MALLOC(double, l);
double *xj_max = SG_MALLOC(double, w_size);
double *C_sum = SG_MALLOC(double, w_size);
double *xjneg_sum = SG_MALLOC(double, w_size);
double *xjpos_sum = SG_MALLOC(double, w_size);
CDotFeatures* x = prob_col->x;
void* iterator;
int ind;
double val;
double C[3] = {Cn,0,Cp};
int n = prob_col->n;
if (prob_col->use_bias)
n--;
for(j=0; j<l; j++)
{
exp_wTx[j] = 1;
if(prob_col->y[j] > 0)
y[j] = 1;
else
y[j] = -1;
}
for(j=0; j<w_size; j++)
{
w.vector[j] = 0;
index[j] = j;
xj_max[j] = 0;
C_sum[j] = 0;
xjneg_sum[j] = 0;
xjpos_sum[j] = 0;
if (use_bias && j==n)
{
for (ind=0; ind<l; ind++)
{
x_min = CMath::min(x_min, 1.0);
xj_max[j] = CMath::max(xj_max[j], 1.0);
C_sum[j] += C[GETI(ind)];
if(y[ind] == -1)
xjneg_sum[j] += C[GETI(ind)];
else
xjpos_sum[j] += C[GETI(ind)];
}
}
else
{
iterator=x->get_feature_iterator(j);
while (x->get_next_feature(ind, val, iterator))
{
x_min = CMath::min(x_min, val);
xj_max[j] = CMath::max(xj_max[j], val);
C_sum[j] += C[GETI(ind)];
if(y[ind] == -1)
xjneg_sum[j] += C[GETI(ind)]*val;
else
xjpos_sum[j] += C[GETI(ind)]*val;
}
x->free_feature_iterator(iterator);
}
}
CTime start_time;
while (iter < max_iterations && !CSignal::cancel_computations())
{
if (m_max_train_time > 0 && start_time.cur_time_diff() > m_max_train_time)
break;
Gmax_new = 0;
for(j=0; j<active_size; j++)
{
int i = j+rand()%(active_size-j);
CMath::swap(index[i], index[j]);
}
for(s=0; s<active_size; s++)
{
j = index[s];
sum1 = 0;
sum2 = 0;
H = 0;
if (use_bias && j==n)
{
for (ind=0; ind<l; ind++)
{
double exp_wTxind = exp_wTx[ind];
double tmp1 = 1.0/(1+exp_wTxind);
double tmp2 = C[GETI(ind)]*tmp1;
double tmp3 = tmp2*exp_wTxind;
sum2 += tmp2;
sum1 += tmp3;
H += tmp1*tmp3;
}
}
else
{
iterator=x->get_feature_iterator(j);
while (x->get_next_feature(ind, val, iterator))
{
double exp_wTxind = exp_wTx[ind];
double tmp1 = val/(1+exp_wTxind);
double tmp2 = C[GETI(ind)]*tmp1;
double tmp3 = tmp2*exp_wTxind;
sum2 += tmp2;
sum1 += tmp3;
H += tmp1*tmp3;
}
x->free_feature_iterator(iterator);
}
G = -sum2 + xjneg_sum[j];
double Gp = G+1;
double Gn = G-1;
double violation = 0;
if(w.vector[j] == 0)
{
if(Gp < 0)
violation = -Gp;
else if(Gn > 0)
violation = Gn;
else if(Gp>Gmax_old/l && Gn<-Gmax_old/l)
{
active_size--;
CMath::swap(index[s], index[active_size]);
s--;
continue;
}
}
else if(w.vector[j] > 0)
violation = fabs(Gp);
else
violation = fabs(Gn);
Gmax_new = CMath::max(Gmax_new, violation);
// obtain Newton direction d
if(Gp <= H*w.vector[j])
d = -Gp/H;
else if(Gn >= H*w.vector[j])
d = -Gn/H;
else
d = -w.vector[j];
if(fabs(d) < 1.0e-12)
continue;
d = CMath::min(CMath::max(d,-10.0),10.0);
double delta = fabs(w.vector[j]+d)-fabs(w.vector[j]) + G*d;
int num_linesearch;
for(num_linesearch=0; num_linesearch < max_num_linesearch; num_linesearch++)
{
cond = fabs(w.vector[j]+d)-fabs(w.vector[j]) - sigma*delta;
if(x_min >= 0)
{
double tmp = exp(d*xj_max[j]);
appxcond1 = log(1+sum1*(tmp-1)/xj_max[j]/C_sum[j])*C_sum[j] + cond - d*xjpos_sum[j];
appxcond2 = log(1+sum2*(1/tmp-1)/xj_max[j]/C_sum[j])*C_sum[j] + cond + d*xjneg_sum[j];
if(CMath::min(appxcond1,appxcond2) <= 0)
{
if (use_bias && j==n)
{
for (ind=0; ind<l; ind++)
exp_wTx[ind] *= exp(d);
}
else
{
iterator=x->get_feature_iterator(j);
while (x->get_next_feature(ind, val, iterator))
exp_wTx[ind] *= exp(d*val);
x->free_feature_iterator(iterator);
}
break;
}
}
cond += d*xjneg_sum[j];
int i = 0;
if (use_bias && j==n)
{
for (ind=0; ind<l; ind++)
{
double exp_dx = exp(d);
exp_wTx_new[i] = exp_wTx[ind]*exp_dx;
cond += C[GETI(ind)]*log((1+exp_wTx_new[i])/(exp_dx+exp_wTx_new[i]));
i++;
}
}
else
{
iterator=x->get_feature_iterator(j);
while (x->get_next_feature(ind, val, iterator))
{
double exp_dx = exp(d*val);
exp_wTx_new[i] = exp_wTx[ind]*exp_dx;
cond += C[GETI(ind)]*log((1+exp_wTx_new[i])/(exp_dx+exp_wTx_new[i]));
i++;
}
x->free_feature_iterator(iterator);
}
if(cond <= 0)
{
i = 0;
if (use_bias && j==n)
{
for (ind=0; ind<l; ind++)
{
exp_wTx[ind] = exp_wTx_new[i];
i++;
}
}
else
{
iterator=x->get_feature_iterator(j);
while (x->get_next_feature(ind, val, iterator))
{
exp_wTx[ind] = exp_wTx_new[i];
i++;
}
x->free_feature_iterator(iterator);
}
break;
}
else
{
d *= 0.5;
delta *= 0.5;
}
}
w.vector[j] += d;
// recompute exp_wTx[] if line search takes too many steps
if(num_linesearch >= max_num_linesearch)
{
SG_INFO("#");
for(int i=0; i<l; i++)
exp_wTx[i] = 0;
for(int i=0; i<w_size; i++)
{
if(w.vector[i]==0) continue;
if (use_bias && i==n)
{
for (ind=0; ind<l; ind++)
exp_wTx[ind] += w.vector[i];
}
else
{
iterator=x->get_feature_iterator(i);
while (x->get_next_feature(ind, val, iterator))
exp_wTx[ind] += w.vector[i]*val;
x->free_feature_iterator(iterator);
}
}
for(int i=0; i<l; i++)
exp_wTx[i] = exp(exp_wTx[i]);
}
}
if(iter == 0)
Gmax_init = Gmax_new;
iter++;
SG_SABS_PROGRESS(Gmax_new, -CMath::log10(Gmax_new), -CMath::log10(Gmax_init), -CMath::log10(eps*Gmax_init), 6);
if(Gmax_new <= eps*Gmax_init)
{
if(active_size == w_size)
break;
else
{
active_size = w_size;
Gmax_old = CMath::INFTY;
continue;
}
}
Gmax_old = Gmax_new;
}
SG_DONE();
SG_INFO("optimization finished, #iter = %d\n", iter);
if(iter >= max_iterations)
SG_WARNING("\nWARNING: reaching max number of iterations\n");
// calculate objective value
double v = 0;
int nnz = 0;
for(j=0; j<w_size; j++)
if(w.vector[j] != 0)
{
v += fabs(w.vector[j]);
nnz++;
}
for(j=0; j<l; j++)
if(y[j] == 1)
v += C[GETI(j)]*log(1+1/exp_wTx[j]);
else
v += C[GETI(j)]*log(1+exp_wTx[j]);
SG_INFO("Objective value = %lf\n", v);
SG_INFO("#nonzeros/#features = %d/%d\n", nnz, w_size);
delete [] index;
delete [] y;
delete [] exp_wTx;
delete [] exp_wTx_new;
delete [] xj_max;
delete [] C_sum;
delete [] xjneg_sum;
delete [] xjpos_sum;
}
void CLibLinear::solve_l1r_l2_svc(
problem *prob_col, double eps, double Cp, double Cn)
{
int l = prob_col->l;
int w_size = prob_col->n;
int j, s, iter = 0;
int active_size = w_size;
int max_num_linesearch = 20;
double sigma = 0.01;
double d, G_loss, G, H;
double Gmax_old = CMath::INFTY;
double Gmax_new;
double Gmax_init=0;
double d_old, d_diff;
double loss_old=0, loss_new;
double appxcond, cond;
int *index = SG_MALLOC(int, w_size);
int32_t *y = SG_MALLOC(int32_t, l);
double *b = SG_MALLOC(double, l); // b = 1-ywTx
double *xj_sq = SG_MALLOC(double, w_size);
CDotFeatures* x = (CDotFeatures*) prob_col->x;
void* iterator;
int32_t ind;
float64_t val;
double C[3] = {Cn,0,Cp};
int n = prob_col->n;
if (prob_col->use_bias)
n--;
for(j=0; j<l; j++)
{
b[j] = 1;
if(prob_col->y[j] > 0)
y[j] = 1;
else
y[j] = -1;
}
for(j=0; j<w_size; j++)
{
w.vector[j] = 0;
index[j] = j;
xj_sq[j] = 0;
if (use_bias && j==n)
{
for (ind=0; ind<l; ind++)
xj_sq[n] += C[GETI(ind)];
}
else
{
iterator=x->get_feature_iterator(j);
while (x->get_next_feature(ind, val, iterator))
xj_sq[j] += C[GETI(ind)]*val*val;
x->free_feature_iterator(iterator);
}
}
CTime start_time;
while (iter < max_iterations && !CSignal::cancel_computations())
{
if (m_max_train_time > 0 && start_time.cur_time_diff() > m_max_train_time)
break;
Gmax_new = 0;
for(j=0; j<active_size; j++)
{
int i = j+rand()%(active_size-j);
CMath::swap(index[i], index[j]);
}
for(s=0; s<active_size; s++)
{
j = index[s];
G_loss = 0;
H = 0;
if (use_bias && j==n)
{
for (ind=0; ind<l; ind++)
{
if(b[ind] > 0)
{
double tmp = C[GETI(ind)]*y[ind];
G_loss -= tmp*b[ind];
H += tmp*y[ind];
}
}
}
else
{
iterator=x->get_feature_iterator(j);
while (x->get_next_feature(ind, val, iterator))
{
if(b[ind] > 0)
{
double tmp = C[GETI(ind)]*val*y[ind];
G_loss -= tmp*b[ind];
H += tmp*val*y[ind];
}
}
x->free_feature_iterator(iterator);
}
G_loss *= 2;
G = G_loss;
H *= 2;
H = CMath::max(H, 1e-12);
double Gp = G+1;
double Gn = G-1;
double violation = 0;
if(w.vector[j] == 0)
{
if(Gp < 0)
violation = -Gp;
else if(Gn > 0)
violation = Gn;
else if(Gp>Gmax_old/l && Gn<-Gmax_old/l)
{
active_size--;
CMath::swap(index[s], index[active_size]);
s--;
continue;
}
}
else if(w.vector[j] > 0)
violation = fabs(Gp);
else
violation = fabs(Gn);
Gmax_new = CMath::max(Gmax_new, violation);
// obtain Newton direction d
if(Gp <= H*w.vector[j])
d = -Gp/H;
else if(Gn >= H*w.vector[j])
d = -Gn/H;
else
d = -w.vector[j];
if(fabs(d) < 1.0e-12)
continue;
double delta = fabs(w.vector[j]+d)-fabs(w.vector[j]) + G*d;
d_old = 0;
int num_linesearch;
for(num_linesearch=0; num_linesearch < max_num_linesearch; num_linesearch++)
{
d_diff = d_old - d;
cond = fabs(w.vector[j]+d)-fabs(w.vector[j]) - sigma*delta;
appxcond = xj_sq[j]*d*d + G_loss*d + cond;
if(appxcond <= 0)
{
if (use_bias && j==n)
{
for (ind=0; ind<l; ind++)
b[ind] += d_diff*y[ind];
break;
}
else
{
iterator=x->get_feature_iterator(j);
while (x->get_next_feature(ind, val, iterator))
b[ind] += d_diff*val*y[ind];
x->free_feature_iterator(iterator);
break;
}
}
if(num_linesearch == 0)
{
loss_old = 0;
loss_new = 0;
if (use_bias && j==n)
{
for (ind=0; ind<l; ind++)
{
if(b[ind] > 0)
loss_old += C[GETI(ind)]*b[ind]*b[ind];
double b_new = b[ind] + d_diff*y[ind];
b[ind] = b_new;
if(b_new > 0)
loss_new += C[GETI(ind)]*b_new*b_new;
}
}
else
{
iterator=x->get_feature_iterator(j);
while (x->get_next_feature(ind, val, iterator))
{
if(b[ind] > 0)
loss_old += C[GETI(ind)]*b[ind]*b[ind];
double b_new = b[ind] + d_diff*val*y[ind];
b[ind] = b_new;
if(b_new > 0)
loss_new += C[GETI(ind)]*b_new*b_new;
}
x->free_feature_iterator(iterator);
}
}
else
{
loss_new = 0;
if (use_bias && j==n)
{
for (ind=0; ind<l; ind++)
{
double b_new = b[ind] + d_diff*y[ind];
b[ind] = b_new;
if(b_new > 0)
loss_new += C[GETI(ind)]*b_new*b_new;
}
}
else
{
iterator=x->get_feature_iterator(j);
while (x->get_next_feature(ind, val, iterator))
{
double b_new = b[ind] + d_diff*val*y[ind];
b[ind] = b_new;
if(b_new > 0)
loss_new += C[GETI(ind)]*b_new*b_new;
}
x->free_feature_iterator(iterator);
}
}
cond = cond + loss_new - loss_old;
if(cond <= 0)
break;
else
{
d_old = d;
d *= 0.5;
delta *= 0.5;
}
}
w.vector[j] += d;
// recompute b[] if line search takes too many steps
if(num_linesearch >= max_num_linesearch)
{
SG_INFO("#");
for(int i=0; i<l; i++)
b[i] = 1;
for(int i=0; i<n; i++)
{
if(w.vector[i]==0)
continue;
iterator=x->get_feature_iterator(i);
while (x->get_next_feature(ind, val, iterator))
b[ind] -= w.vector[i]*val*y[ind];
x->free_feature_iterator(iterator);
}
if (use_bias && w.vector[n])
{
for (ind=0; ind<l; ind++)
b[ind] -= w.vector[n]*y[ind];
}
}
}
if(iter == 0)
Gmax_init = Gmax_new;
iter++;
SG_SABS_PROGRESS(Gmax_new, -CMath::log10(Gmax_new), -CMath::log10(Gmax_init), -CMath::log10(eps*Gmax_init), 6);
if(Gmax_new <= eps*Gmax_init)
{
if(active_size == w_size)
break;
else
{
active_size = w_size;
Gmax_old = CMath::INFTY;
continue;
}
}
Gmax_old = Gmax_new;
}
SG_DONE();
SG_INFO("optimization finished, #iter = %d\n", iter);
if(iter >= max_iterations)
SG_WARNING("\nWARNING: reaching max number of iterations\n");
// calculate objective value
double v = 0;
int nnz = 0;
for(j=0; j<w_size; j++)
{
if(w.vector[j] != 0)
{
v += fabs(w.vector[j]);
nnz++;
}
}
for(j=0; j<l; j++)
if(b[j] > 0)
v += C[GETI(j)]*b[j]*b[j];
SG_INFO("Objective value = %lf\n", v);
SG_INFO("#nonzeros/#features = %d/%d\n", nnz, w_size);
delete [] index;
delete [] y;
delete [] b;
delete [] xj_sq;
}
/*----------------------------------------------------------------------------------
sparse_add_new_cut( new_col_H, new_cut, cut_length, nSel ) does the following:
new_a = sum(data_X(:,find(new_cut ~=0 )),2);
new_col_H = [sparse_A(:,1:nSel)'*new_a ; new_a'*new_a];
sparse_A(:,nSel+1) = new_a;
---------------------------------------------------------------------------------*/
int CSVMOcas::add_new_cut(
float64_t *new_col_H, uint32_t *new_cut, uint32_t cut_length,
uint32_t nSel, void* ptr)
{
CSVMOcas* o = (CSVMOcas*) ptr;
CDotFeatures* f = o->features;
uint32_t nDim=(uint32_t) o->w_dim;
float64_t* y = o->lab.vector;
float64_t** c_val = o->cp_value;
uint32_t** c_idx = o->cp_index;
uint32_t* c_nzd = o->cp_nz_dims;
float64_t* c_bias = o->cp_bias;
float64_t sq_norm_a;
uint32_t i, j, nz_dims;
/* temporary vector */
float64_t* new_a = o->tmp_a_buf;
memset(new_a, 0, sizeof(float64_t)*nDim);
for(i=0; i < cut_length; i++)
{
f->add_to_dense_vec(y[new_cut[i]], new_cut[i], new_a, nDim);
if (o->use_bias)
c_bias[nSel]+=y[new_cut[i]];
}
/* compute new_a'*new_a and count number of non-zerou dimensions */
nz_dims = 0;
sq_norm_a = CMath::sq(c_bias[nSel]);
for(j=0; j < nDim; j++ ) {
if(new_a[j] != 0) {
nz_dims++;
sq_norm_a += new_a[j]*new_a[j];
}
}
/* sparsify new_a and insert it to the last column of sparse_A */
c_nzd[nSel] = nz_dims;
c_idx[nSel]=NULL;
c_val[nSel]=NULL;
if(nz_dims > 0)
{
c_idx[nSel]=SG_MALLOC(uint32_t, nz_dims);
c_val[nSel]=SG_MALLOC(float64_t, nz_dims);
uint32_t idx=0;
for(j=0; j < nDim; j++ )
{
if(new_a[j] != 0)
{
c_idx[nSel][idx] = j;
c_val[nSel][idx++] = new_a[j];
}
}
}
new_col_H[nSel] = sq_norm_a;
for(i=0; i < nSel; i++)
{
float64_t tmp = c_bias[nSel]*c_bias[i];
for(j=0; j < c_nzd[i]; j++)
tmp += new_a[c_idx[i][j]]*c_val[i][j];
new_col_H[i] = tmp;
}
//CMath::display_vector(new_col_H, nSel+1, "new_col_H");
//CMath::display_vector((int32_t*) c_idx[nSel], (int32_t) nz_dims, "c_idx");
//CMath::display_vector((float64_t*) c_val[nSel], nz_dims, "c_val");
return 0;
}