bool Pivot::is_disjoint(const Pivot& rhs) const
{
if (rhs.is_upper_bound() == is_upper_bound()) return false;
if (bound_->equals(rhs.get_bound_value())) {
return !rhs.is_inclusive() || !is_inclusive();
}
if (is_upper_bound()) return bound_->is_smaller(rhs.get_bound_value());
else return bound_->is_greater(rhs.get_bound_value());
}
double Solver_NU::calculate_rho()
{
int nr_free1 = 0,nr_free2 = 0;
double ub1 = INF, ub2 = INF;
double lb1 = -INF, lb2 = -INF;
double sum_free1 = 0, sum_free2 = 0;
for(int i=0;i<active_size;i++)
{
if(y[i]==+1)
{
if(is_lower_bound(i))
ub1 = min(ub1,G[i]);
else if(is_upper_bound(i))
lb1 = max(lb1,G[i]);
else
{
++nr_free1;
sum_free1 += G[i];
}
}
else
{
if(is_lower_bound(i))
ub2 = min(ub2,G[i]);
else if(is_upper_bound(i))
lb2 = max(lb2,G[i]);
else
{
++nr_free2;
sum_free2 += G[i];
}
}
}
double r1,r2;
if(nr_free1 > 0)
r1 = sum_free1/nr_free1;
else
r1 = (ub1+lb1)/2;
if(nr_free2 > 0)
r2 = sum_free2/nr_free2;
else
r2 = (ub2+lb2)/2;
si->r = (r1+r2)/2;
return (r1-r2)/2;
}
void Pivot::debug() const
{
std::cerr << "LIMITER " << property_index_;
std::cerr << (is_upper_bound() ? "<" : ">");
std::cerr << (is_inclusive() ? "=" : "");
std::cerr << get_bound_value()->to_string() << "\n";
}
bool Solver<TQ>::be_shrunken(int i, float Gmax1, float Gmax2)
{
if(is_upper_bound(i))
{
if(y[i]==+1)
return(-G[i] > Gmax1);
else
return(-G[i] > Gmax2);
}
else if(is_lower_bound(i))
{
if(y[i]==+1)
return(G[i] > Gmax2);
else
return(G[i] > Gmax1);
}
else
return(false);
}
int
QP_Solver::check_inactive ()
{
// final check
fprintf (stdout, "\nChecking optimality of inactive variables ");
fflush (stdout);
// make dummy classifier
try {
Model *tmp_model = new Model (param);
tmp_model->b = -lambda_eq;
for (int i = 0; i < l; i++) {
if (! is_lower_bound (i)) tmp_model->add (alpha[i] * y[i], (feature_node *) x[i]);
}
int react_num = 0;
for (int k= l-1; k >= active_size; k--) {
double lambda_up = 1 - y[k] * tmp_model->classify (x[k]);
G[k] = y[k] * tmp_model->b - lambda_up;
// Oops!, must be added to the active example.
if ( (! is_lower_bound (k) && lambda_up < -eps) ||
(! is_upper_bound (k) && lambda_up > eps) ) {
swap_index (k, active_size);
active_size++;
++k;
++react_num;
}
}
delete tmp_model;
fprintf (stdout, " re-activated: %d\n", react_num);
return react_num;
}
catch (...) {
fprintf (stderr, "QP_Solver::check_inactive(): Out of memory\n");
exit (EXIT_FAILURE);
return 0;
}
}
double Solver::calculate_rho()
{
double r;
int nr_free = 0;
double ub = INF, lb = -INF, sum_free = 0;
for(int i=0;i<active_size;i++)
{
double yG = y[i]*G[i];
if(is_lower_bound(i))
{
if(y[i] > 0)
ub = min(ub,yG);
else
lb = max(lb,yG);
}
else if(is_upper_bound(i))
{
if(y[i] < 0)
ub = min(ub,yG);
else
lb = max(lb,yG);
}
else
{
++nr_free;
sum_free += yG;
}
}
if(nr_free>0)
r = sum_free/nr_free;
else
r = (ub+lb)/2;
return r;
}
float Solver<TQ>::calculate_rho()
{
float r;
int nr_free = 0;
float ub = INF, lb = -INF, sum_free = 0;
for(int i=0;i<active_size;i++)
{
float yG = y[i]*G[i];
if(is_upper_bound(i))
{
if(y[i]==-1)
ub = std::min(ub,yG);
else
lb = std::max(lb,yG);
}
else if(is_lower_bound(i))
{
if(y[i]==+1)
ub = std::min(ub,yG);
else
lb = std::max(lb,yG);
}
else
{
++nr_free;
sum_free += yG;
}
}
if(nr_free>0)
r = sum_free/nr_free;
else
r = (ub+lb)/2;
return r;
}
void
QP_Solver::learn_sub()
{
fprintf (stdout, "%6d examples, cache size: %d\n",
active_size, q_matrix->size);
while(++iter) {
/////////////////////////////////////////////////////////
// Select Working set
double Gmax1 = -INF;
int i = -1;
double Gmax2 = -INF;
int j = -1;
for (int k = 0; k < active_size; k++) {
if (y[k] > 0) {
if (!is_upper_bound (k)) {
if (-G[k] > Gmax1) {
Gmax1 = -G[k];
i = k;
}
}
if (!is_lower_bound (k)) {
if (G[k] > Gmax2) {
Gmax2 = G[k];
j = k;
}
}
} else {
if (!is_upper_bound (k)) {
if (-G[k] > Gmax2) {
Gmax2 = -G[k];
j = k;
}
}
if (!is_lower_bound (k)) {
if (G[k] > Gmax1) {
Gmax1 = G[k];
i = k;
}
}
}
}
/////////////////////////////////////////////////////////
//
// Solving QP sub problems
//
double old_alpha_i = alpha[i];
double old_alpha_j = alpha[j];
double *Q_i = q_matrix->getQ (i, active_size);
double *Q_j = q_matrix->getQ (j, active_size);
if (y[i] * y[j] < 0) {
double L = _max (0.0, alpha[j] - alpha[i]);
double H = _min (C, C + alpha[j] - alpha[i]);
alpha[j] += (-G[i] - G[j]) / (Q_i[i] + Q_j[j] + 2 * Q_i[j]);
if (alpha[j] >= H) alpha[j] = H;
else if (alpha[j] <= L) alpha[j] = L;
alpha[i] += (alpha[j] - old_alpha_j);
} else {
double L = _max (0.0, alpha[i] + alpha[j] - C);
double H = _min (C, alpha[i] + alpha[j]);
alpha[j] += (G[i] - G[j]) / (Q_i[i] + Q_j[j] - 2 * Q_i[j]);
if (alpha[j] >= H) alpha[j] = H;
else if (alpha[j] <= L) alpha[j] = L;
alpha[i] -= (alpha[j] - old_alpha_j);
}
/////////////////////////////////////////////////////////
//
// update status
//
status[i] = alpha2status (alpha[i]);
status[j] = alpha2status (alpha[j]);
double delta_alpha_i = alpha[i] - old_alpha_i;
double delta_alpha_j = alpha[j] - old_alpha_j;
/////////////////////////////////////////////////////////
//
// Update O and Calculate \lambda^eq for shrinking, Calculate lambda^eq,
// (c.f. Advances in Kernel Method pp.175)
// lambda_eq = 1/|FREE| \sum_{i \in FREE} y_i - \sum_{l} y_i \alpha_i k(x_i,x_j) (11.29)
//
lambda_eq = 0.0;
int size_A = 0;
for (int k = 0; k < active_size; k++) {
G[k] += Q_i[k] * delta_alpha_i + Q_j[k] * delta_alpha_j;
if (is_free (k)) {
lambda_eq -= G[k] * y[k];
size_A++;
}
}
/////////////////////////////////////////////////////////
//
// Select example for shrinking,
// (c.f. 11.5 Efficient Implementation in Advances in Kernel Method pp. 175)
//
lambda_eq = size_A ? (lambda_eq / size_A) : 0.0;
double kkt_violation = 0.0;
for (int k = 0; k < active_size; k++) {
double lambda_up = -(G[k] + y[k] * lambda_eq); // lambda_lo = -lambda_up
// termination criteria (11.32,11.33,11.34)
if (! is_lower_bound (k) && lambda_up < -kkt_violation) kkt_violation = -lambda_up;
if (! is_upper_bound (k) && lambda_up > kkt_violation) kkt_violation = lambda_up;
// "If the estimate (11.30) or (11.31) was positive (or above some threshold) at
// each of the last h iterations, it is likely that this will be true at the optimal solution"
// lambda^up (11.30) lambda^low = lambda^up * status[k]
if (lambda_up * status[k] > shrink_eps) {
if (shrink_iter[k]++ > shrink_size) {
active_size--;
swap_index (k, active_size); // remove this data from working set
q_matrix->swap_index(k, active_size);
q_matrix->update(active_size);
k--;
}
} else {
// reset iter, if current data dose not satisfy the condition (11.30), (11.31)
shrink_iter[k] = 0;
}
}
/////////////////////////////////////////////////////////
//
// Check terminal criteria, show information of iteration
//
if (kkt_violation < eps) break;
if ((iter % 50) == 0) { fprintf (stdout, "."); fflush (stdout); };
if ((iter % 1000) == 0) {
fprintf (stdout, " %6d %6d %5d %1.4f %5.1f%% %5.1f%%\n",
iter, active_size, q_matrix->size, kkt_violation,
100.0 * (q_matrix->hit - hit_old)/2000,
100.0 * q_matrix->hit/(q_matrix->hit + q_matrix->miss));
fflush (stdout);
// save old hit rate
hit_old = q_matrix->hit;
// This shrink eps rule is delivered from svm_light.
shrink_eps = shrink_eps * 0.7 + kkt_violation * 0.3;
}
}
}
void Solver_NU::do_shrinking()
{
double Gmax1 = -INF; // max { -grad(f)_i * d | y_i = +1, d = +1 }
double Gmax2 = -INF; // max { -grad(f)_i * d | y_i = +1, d = -1 }
double Gmax3 = -INF; // max { -grad(f)_i * d | y_i = -1, d = +1 }
double Gmax4 = -INF; // max { -grad(f)_i * d | y_i = -1, d = -1 }
int k;
for(k=0;k<active_size;k++)
{
if(!is_upper_bound(k))
{
if(y[k]==+1)
{
if(-G[k] > Gmax1) Gmax1 = -G[k];
}
else if(-G[k] > Gmax3) Gmax3 = -G[k];
}
if(!is_lower_bound(k))
{
if(y[k]==+1)
{
if(G[k] > Gmax2) Gmax2 = G[k];
}
else if(G[k] > Gmax4) Gmax4 = G[k];
}
}
double Gm1 = -Gmax2;
double Gm2 = -Gmax1;
double Gm3 = -Gmax4;
double Gm4 = -Gmax3;
for(k=0;k<active_size;k++)
{
if(is_lower_bound(k))
{
if(y[k]==+1)
{
if(-G[k] >= Gm1) continue;
}
else if(-G[k] >= Gm3) continue;
}
else if(is_upper_bound(k))
{
if(y[k]==+1)
{
if(G[k] >= Gm2) continue;
}
else if(G[k] >= Gm4) continue;
}
else continue;
--active_size;
swap_index(k,active_size);
--k; // look at the newcomer
}
// unshrink, check all variables again before final iterations
if(unshrinked || max(-(Gm1+Gm2),-(Gm3+Gm4)) > eps*10) return;
unshrinked = true;
reconstruct_gradient();
for(k=l-1;k>=active_size;k--)
{
if(is_lower_bound(k))
{
if(y[k]==+1)
{
if(-G[k] < Gm1) continue;
}
else if(-G[k] < Gm3) continue;
}
else if(is_upper_bound(k))
{
if(y[k]==+1)
{
if(G[k] < Gm2) continue;
}
else if(G[k] < Gm4) continue;
}
else continue;
swap_index(k,active_size);
active_size++;
++k; // look at the newcomer
}
}
int Solver_NU::select_working_set(int &out_i, int &out_j)
{
// return i,j which maximize -grad(f)^T d , under constraint
// if alpha_i == C, d != +1
// if alpha_i == 0, d != -1
double Gmax1 = -INF; // max { -grad(f)_i * d | y_i = +1, d = +1 }
int Gmax1_idx = -1;
double Gmax2 = -INF; // max { -grad(f)_i * d | y_i = +1, d = -1 }
int Gmax2_idx = -1;
double Gmax3 = -INF; // max { -grad(f)_i * d | y_i = -1, d = +1 }
int Gmax3_idx = -1;
double Gmax4 = -INF; // max { -grad(f)_i * d | y_i = -1, d = -1 }
int Gmax4_idx = -1;
for(int i=0;i<active_size;i++)
{
if(y[i]==+1) // y == +1
{
if(!is_upper_bound(i)) // d = +1
{
if(-G[i] > Gmax1)
{
Gmax1 = -G[i];
Gmax1_idx = i;
}
}
if(!is_lower_bound(i)) // d = -1
{
if(G[i] > Gmax2)
{
Gmax2 = G[i];
Gmax2_idx = i;
}
}
}
else // y == -1
{
if(!is_upper_bound(i)) // d = +1
{
if(-G[i] > Gmax3)
{
Gmax3 = -G[i];
Gmax3_idx = i;
}
}
if(!is_lower_bound(i)) // d = -1
{
if(G[i] > Gmax4)
{
Gmax4 = G[i];
Gmax4_idx = i;
}
}
}
}
if(max(Gmax1+Gmax2,Gmax3+Gmax4) < eps)
return 1;
if(Gmax1+Gmax2 > Gmax3+Gmax4)
{
out_i = Gmax1_idx;
out_j = Gmax2_idx;
}
else
{
out_i = Gmax3_idx;
out_j = Gmax4_idx;
}
return 0;
}
void Solver::do_shrinking()
{
int i,j,k;
if(select_working_set(i,j)!=0) return;
double Gm1 = -y[j]*G[j];
double Gm2 = y[i]*G[i];
// shrink
for(k=0;k<active_size;k++)
{
if(is_lower_bound(k))
{
if(y[k]==+1)
{
if(-G[k] >= Gm1) continue;
}
else if(-G[k] >= Gm2) continue;
}
else if(is_upper_bound(k))
{
if(y[k]==+1)
{
if(G[k] >= Gm2) continue;
}
else if(G[k] >= Gm1) continue;
}
else continue;
--active_size;
swap_index(k,active_size);
--k; // look at the newcomer
}
// unshrink, check all variables again before final iterations
if(unshrinked || -(Gm1 + Gm2) > eps*10) return;
unshrinked = true;
reconstruct_gradient();
for(k=l-1;k>=active_size;k--)
{
if(is_lower_bound(k))
{
if(y[k]==+1)
{
if(-G[k] < Gm1) continue;
}
else if(-G[k] < Gm2) continue;
}
else if(is_upper_bound(k))
{
if(y[k]==+1)
{
if(G[k] < Gm2) continue;
}
else if(G[k] < Gm1) continue;
}
else continue;
swap_index(k,active_size);
active_size++;
++k; // look at the newcomer
}
}
void Solver::Solve(int l, const Kernel& Q, const double *b_, const schar *y_,
double *alpha_, double Cp, double Cn, double eps,
SolutionInfo* si, int shrinking)
{
this->l = l;
this->Q = &Q;
clone(b, b_,l);
clone(y, y_,l);
clone(alpha,alpha_,l);
this->Cp = Cp;
this->Cn = Cn;
this->eps = eps;
unshrinked = false;
// initialize alpha_status
{
alpha_status = new char[l];
for(int i=0;i<l;i++)
update_alpha_status(i);
}
// initialize active set (for shrinking)
{
active_set = new int[l];
for(int i=0;i<l;i++)
active_set[i] = i;
active_size = l;
}
// initialize gradient
{
G = new double[l];
G_bar = new double[l];
int i;
for(i=0;i<l;i++)
{
G[i] = b[i];
G_bar[i] = 0;
}
for(i=0;i<l;i++)
if(!is_lower_bound(i))
{
Qfloat *Q_i = Q.get_Q(i,l);
double alpha_i = alpha[i];
int j;
for(j=0;j<l;j++)
G[j] += alpha_i*Q_i[j];
if(is_upper_bound(i))
for(j=0;j<l;j++)
G_bar[j] += get_C(i) * Q_i[j];
}
}
// optimization step
int iter = 0;
int counter = min(l,1000)+1;
while(1)
{
// show progress and do shrinking
if(--counter == 0)
{
counter = min(l,1000);
if(shrinking) do_shrinking();
info("."); info_flush();
}
int i,j;
if(select_working_set(i,j)!=0)
{
// reconstruct the whole gradient
reconstruct_gradient();
// reset active set size and check
active_size = l;
info("*"); info_flush();
if(select_working_set(i,j)!=0)
break;
else
counter = 1; // do shrinking next iteration
}
++iter;
// update alpha[i] and alpha[j], handle bounds carefully
const Qfloat *Q_i = Q.get_Q(i,active_size);
const Qfloat *Q_j = Q.get_Q(j,active_size);
double C_i = get_C(i);
double C_j = get_C(j);
double old_alpha_i = alpha[i];
double old_alpha_j = alpha[j];
if(y[i]!=y[j])
{
double delta = (-G[i]-G[j])/(Q_i[i]+Q_j[j]+2*Q_i[j]);
double diff = alpha[i] - alpha[j];
alpha[i] += delta;
alpha[j] += delta;
if(diff > 0)
{
if(alpha[j] < 0)
{
alpha[j] = 0;
alpha[i] = diff;
}
}
else
{
if(alpha[i] < 0)
{
alpha[i] = 0;
alpha[j] = -diff;
}
}
if(diff > C_i - C_j)
{
if(alpha[i] > C_i)
{
alpha[i] = C_i;
alpha[j] = C_i - diff;
}
}
else
{
if(alpha[j] > C_j)
{
alpha[j] = C_j;
alpha[i] = C_j + diff;
}
}
}
else
{
double delta = (G[i]-G[j])/(Q_i[i]+Q_j[j]-2*Q_i[j]);
double sum = alpha[i] + alpha[j];
alpha[i] -= delta;
alpha[j] += delta;
if(sum > C_i)
{
if(alpha[i] > C_i)
{
alpha[i] = C_i;
alpha[j] = sum - C_i;
}
}
else
{
if(alpha[j] < 0)
{
alpha[j] = 0;
alpha[i] = sum;
}
}
if(sum > C_j)
{
if(alpha[j] > C_j)
{
alpha[j] = C_j;
alpha[i] = sum - C_j;
}
}
else
{
if(alpha[i] < 0)
{
alpha[i] = 0;
alpha[j] = sum;
}
}
}
// update G
double delta_alpha_i = alpha[i] - old_alpha_i;
double delta_alpha_j = alpha[j] - old_alpha_j;
for(int k=0;k<active_size;k++)
{
G[k] += Q_i[k]*delta_alpha_i + Q_j[k]*delta_alpha_j;
}
// update alpha_status and G_bar
{
bool ui = is_upper_bound(i);
bool uj = is_upper_bound(j);
update_alpha_status(i);
update_alpha_status(j);
int k;
if(ui != is_upper_bound(i))
{
Q_i = Q.get_Q(i,l);
if(ui)
for(k=0;k<l;k++)
G_bar[k] -= C_i * Q_i[k];
else
for(k=0;k<l;k++)
G_bar[k] += C_i * Q_i[k];
}
if(uj != is_upper_bound(j))
{
Q_j = Q.get_Q(j,l);
if(uj)
for(k=0;k<l;k++)
G_bar[k] -= C_j * Q_j[k];
else
for(k=0;k<l;k++)
G_bar[k] += C_j * Q_j[k];
}
}
}
// calculate rho
si->rho = calculate_rho();
// calculate objective value
{
double v = 0;
int i;
for(i=0;i<l;i++)
v += alpha[i] * (G[i] + b[i]);
si->obj = v/2;
}
// put back the solution
{
for(int i=0;i<l;i++)
alpha_[active_set[i]] = alpha[i];
}
// juggle everything back
/*{
for(int i=0;i<l;i++)
while(active_set[i] != i)
swap_index(i,active_set[i]);
// or Q.swap_index(i,active_set[i]);
}*/
si->upper_bound_p = Cp;
si->upper_bound_n = Cn;
info("\noptimization finished, #iter = %d\n",iter);
delete[] b;
delete[] y;
delete[] alpha;
delete[] alpha_status;
delete[] active_set;
delete[] G;
delete[] G_bar;
}
float Solver<TQ>::solve(int l, TQ& Q, const signed char *y_,
float *alpha_, float C, float eps, int shrinking)
{
this->l = l;
this->Q = &Q;
this->QD = Q.get_QD();
this->C = C;
this->eps = eps;
unshrinked = false;
p = new float [l];
std::fill(p, p + l, float(-1.0));
y = new signed char [l];
std::copy(y_, y_ + l, y);
alpha = new float [l];
std::copy(alpha_, alpha_ + l, alpha);
// initialize alpha_status
{
alpha_status = new int[l];
for(int i=0;i<l;i++)
update_alpha_status(i);
}
// initialize active set (for shrinking)
{
active_set = new int[l];
for(int i=0;i<l;i++)
active_set[i] = i;
active_size = l;
}
// initialize gradient
{
G = new float[l];
G_bar = new float[l];
for(int i=0;i<l;i++)
{
G[i] = p[i];
G_bar[i] = 0;
}
for(int i=0;i<l;i++)
if(!is_lower_bound(i))
{
const float *Q_i = Q.get_Q(i,l);
float alpha_i = alpha[i];
for(int j=0;j<l;j++)
G[j] += alpha_i*Q_i[j];
if(is_upper_bound(i))
for(int j=0;j<l;j++)
G_bar[j] += get_C(i) * Q_i[j];
}
}
// optimization step
int iter = 0;
int counter = std::min(l,1000)+1;
while (1)
{
// show progress and do shrinking
if(--counter == 0)
{
counter = std::min(l,1000);
if(shrinking)
do_shrinking();
}
int i,j;
if (select_working_set(i, j) != 0)
{
// reconstruct the whole gradient
reconstruct_gradient();
// reset active set size and check
active_size = l;
if (select_working_set(i, j) != 0)
break;
else
counter = 1; // do shrinking next iteration
}
++iter;
// update alpha[i] and alpha[j], handle bounds carefully
const float *Q_i = Q.get_Q(i, active_size);
const float *Q_j = Q.get_Q(j, active_size);
NTA_ASSERT(Q_i != nullptr);
NTA_ASSERT(Q_j != nullptr);
float C_i = get_C(i);
float C_j = get_C(j);
float old_alpha_i = alpha[i];
float old_alpha_j = alpha[j];
if (y[i]!=y[j])
{
float quad_coef = Q_i[i]+Q_j[j]+2*Q_i[j];
if (quad_coef <= 0)
quad_coef = TAU;
NTA_ASSERT(quad_coef > 0);
float delta = (-G[i]-G[j])/quad_coef;
float diff = alpha[i] - alpha[j];
alpha[i] += delta;
alpha[j] += delta;
if(diff > 0)
{
if(alpha[j] < 0)
{
alpha[j] = 0;
alpha[i] = diff;
}
}
else
{
if(alpha[i] < 0)
{
alpha[i] = 0;
alpha[j] = -diff;
}
}
if(diff > C_i - C_j)
{
if(alpha[i] > C_i)
{
alpha[i] = C_i;
alpha[j] = C_i - diff;
}
}
else
{
if(alpha[j] > C_j)
{
alpha[j] = C_j;
alpha[i] = C_j + diff;
}
}
}
else
{
float quad_coef = Q_i[i]+Q_j[j]-2*Q_i[j];
if (quad_coef <= 0)
quad_coef = TAU;
NTA_ASSERT(quad_coef > 0);
float delta = (G[i]-G[j])/quad_coef;
float sum = alpha[i] + alpha[j];
alpha[i] -= delta;
alpha[j] += delta;
if(sum > C_i)
{
if(alpha[i] > C_i)
{
alpha[i] = C_i;
alpha[j] = sum - C_i;
}
}
else
{
if(alpha[j] < 0)
{
alpha[j] = 0;
alpha[i] = sum;
}
}
if(sum > C_j)
{
if(alpha[j] > C_j)
{
alpha[j] = C_j;
alpha[i] = sum - C_j;
}
}
else
{
if(alpha[i] < 0)
{
alpha[i] = 0;
alpha[j] = sum;
}
}
}
// update G
float delta_alpha_i = alpha[i] - old_alpha_i;
float delta_alpha_j = alpha[j] - old_alpha_j;
for(int k=0;k<active_size;k++) {
G[k] += Q_i[k]*delta_alpha_i + Q_j[k]*delta_alpha_j;
NTA_ASSERT(-HUGE_VAL <= G[k] && G[k] <= HUGE_VAL);
}
// update alpha_status and G_bar
{
bool ui = is_upper_bound(i);
bool uj = is_upper_bound(j);
update_alpha_status(i);
update_alpha_status(j);
if(ui != is_upper_bound(i))
{
Q_i = Q.get_Q(i,l);
if(ui)
for(int k=0;k<l;k++)
G_bar[k] -= C_i * Q_i[k];
else
for(int k=0;k<l;k++)
G_bar[k] += C_i * Q_i[k];
}
if(uj != is_upper_bound(j))
{
Q_j = Q.get_Q(j,l);
if(uj)
for(int k=0;k<l;k++)
G_bar[k] -= C_j * Q_j[k];
else
for(int k=0;k<l;k++)
G_bar[k] += C_j * Q_j[k];
}
}
}
float rho = calculate_rho();
// put back the solution
for(int i=0;i<l;i++)
alpha_[active_set[i]] = alpha[i];
delete[] p;
delete[] y;
delete[] alpha;
delete[] alpha_status;
delete[] active_set;
delete[] G;
delete[] G_bar;
return rho;
}
void Solver<TQ>::do_shrinking()
{
float Gmax1 = -INF; // max { -y_i * grad(f)_i | i in I_up(\alpha) }
float Gmax2 = -INF; // max { y_i * grad(f)_i | i in I_low(\alpha) }
// find maximal violating pair first
for(int i=0;i<active_size;i++)
{
if(y[i]==+1)
{
if(!is_upper_bound(i))
{
if(-G[i] >= Gmax1)
Gmax1 = -G[i];
}
if(!is_lower_bound(i))
{
if(G[i] >= Gmax2)
Gmax2 = G[i];
}
}
else
{
if(!is_upper_bound(i))
{
if(-G[i] >= Gmax2)
Gmax2 = -G[i];
}
if(!is_lower_bound(i))
{
if(G[i] >= Gmax1)
Gmax1 = G[i];
}
}
}
// shrink
for(int i=0;i<active_size;i++)
if (be_shrunken(i, Gmax1, Gmax2))
{
active_size--;
while (active_size > i)
{
if (!be_shrunken(active_size, Gmax1, Gmax2))
{
swap_index(i,active_size);
break;
}
active_size--;
}
}
// unshrink, check all variables again before final iterations
if(unshrinked || Gmax1 + Gmax2 > eps*10) return;
unshrinked = true;
reconstruct_gradient();
for(int i=l-1;i>=active_size;i--)
if (!be_shrunken(i, Gmax1, Gmax2))
{
while (active_size < i)
{
if (be_shrunken(active_size, Gmax1, Gmax2))
{
swap_index(i,active_size);
break;
}
active_size++;
}
active_size++;
}
}
int Solver<TQ>::select_working_set(int &out_i, int &out_j)
{
// return i,j such that
// i: maximizes -y_i * grad(f)_i, i in I_up(\alpha)
// j: minimizes the decrease of obj value
// (if quadratic coefficeint <= 0, replace it with tau)
// -y_j*grad(f)_j < -y_i*grad(f)_i, j in I_low(\alpha)
float Gmax = -HUGE_VAL; //std::numeric_limits<float>::max();
float Gmax2 = -HUGE_VAL; //std::numeric_limits<float>::max();
int Gmax_idx = -1;
int Gmin_idx = -1;
float obj_diff_min = HUGE_VAL; //std::numeric_limits<float>::max();
for (int t=0;t<active_size;t++) {
if (y[t] == +1)
{
if (!is_upper_bound(t))
if (-G[t] >= Gmax)
{
Gmax = -G[t];
Gmax_idx = t;
}
}
else
{
if (!is_lower_bound(t))
if (G[t] >= Gmax)
{
Gmax = G[t];
Gmax_idx = t;
}
}
}
int i = Gmax_idx;
const float *Q_i = nullptr;
if (i != -1) // NULL Q_i not accessed: Gmax=-INF if i=-1
Q_i = Q->get_Q(i,active_size);
NTA_ASSERT(0 <= i);
for(int j=0;j<active_size;j++)
{
if (y[j] == +1)
{
if (!is_lower_bound(j))
{
float grad_diff=Gmax+G[j];
if (G[j] >= Gmax2)
Gmax2 = G[j];
if (grad_diff > 0)
{
float obj_diff;
float quad_coef=Q_i[i]+QD[j]-2*y[i]*Q_i[j];
if (quad_coef > 0)
obj_diff = -(grad_diff*grad_diff)/quad_coef;
else
obj_diff = -(grad_diff*grad_diff)/TAU;
if (obj_diff <= obj_diff_min)
{
Gmin_idx=j;
obj_diff_min = obj_diff;
}
}
}
}
else
{
if (!is_upper_bound(j))
{
float grad_diff= Gmax - G[j];
if (-G[j] >= Gmax2)
Gmax2 = -G[j];
if (grad_diff > 0)
{
float obj_diff;
float quad_coef = Q_i[i]+QD[j]+2*y[i]*Q_i[j];
if (quad_coef > 0)
obj_diff = -(grad_diff*grad_diff)/quad_coef;
else
obj_diff = -(grad_diff*grad_diff)/TAU;
if (obj_diff <= obj_diff_min)
{
Gmin_idx = j;
obj_diff_min = obj_diff;
}
}
}
}
}
if (Gmax + Gmax2 < eps)
return 1;
out_i = Gmax_idx;
out_j = Gmin_idx;
NTA_ASSERT(0 <= out_i);
NTA_ASSERT(0 <= out_j);
return 0;
}