int QP_Solver::check_inactive () { // final check fprintf (stdout, "\nChecking optimality of inactive variables "); fflush (stdout); // make dummy classifier try { Model *tmp_model = new Model (param); tmp_model->b = -lambda_eq; for (int i = 0; i < l; i++) { if (! is_lower_bound (i)) tmp_model->add (alpha[i] * y[i], (feature_node *) x[i]); } int react_num = 0; for (int k= l-1; k >= active_size; k--) { double lambda_up = 1 - y[k] * tmp_model->classify (x[k]); G[k] = y[k] * tmp_model->b - lambda_up; // Oops!, must be added to the active example. if ( (! is_lower_bound (k) && lambda_up < -eps) || (! is_upper_bound (k) && lambda_up > eps) ) { swap_index (k, active_size); active_size++; ++k; ++react_num; } } delete tmp_model; fprintf (stdout, " re-activated: %d\n", react_num); return react_num; } catch (...) { fprintf (stderr, "QP_Solver::check_inactive(): Out of memory\n"); exit (EXIT_FAILURE); return 0; } }
void QP_Solver::learn_sub() { fprintf (stdout, "%6d examples, cache size: %d\n", active_size, q_matrix->size); while(++iter) { ///////////////////////////////////////////////////////// // Select Working set double Gmax1 = -INF; int i = -1; double Gmax2 = -INF; int j = -1; for (int k = 0; k < active_size; k++) { if (y[k] > 0) { if (!is_upper_bound (k)) { if (-G[k] > Gmax1) { Gmax1 = -G[k]; i = k; } } if (!is_lower_bound (k)) { if (G[k] > Gmax2) { Gmax2 = G[k]; j = k; } } } else { if (!is_upper_bound (k)) { if (-G[k] > Gmax2) { Gmax2 = -G[k]; j = k; } } if (!is_lower_bound (k)) { if (G[k] > Gmax1) { Gmax1 = G[k]; i = k; } } } } ///////////////////////////////////////////////////////// // // Solving QP sub problems // double old_alpha_i = alpha[i]; double old_alpha_j = alpha[j]; double *Q_i = q_matrix->getQ (i, active_size); double *Q_j = q_matrix->getQ (j, active_size); if (y[i] * y[j] < 0) { double L = _max (0.0, alpha[j] - alpha[i]); double H = _min (C, C + alpha[j] - alpha[i]); alpha[j] += (-G[i] - G[j]) / (Q_i[i] + Q_j[j] + 2 * Q_i[j]); if (alpha[j] >= H) alpha[j] = H; else if (alpha[j] <= L) alpha[j] = L; alpha[i] += (alpha[j] - old_alpha_j); } else { double L = _max (0.0, alpha[i] + alpha[j] - C); double H = _min (C, alpha[i] + alpha[j]); alpha[j] += (G[i] - G[j]) / (Q_i[i] + Q_j[j] - 2 * Q_i[j]); if (alpha[j] >= H) alpha[j] = H; else if (alpha[j] <= L) alpha[j] = L; alpha[i] -= (alpha[j] - old_alpha_j); } ///////////////////////////////////////////////////////// // // update status // status[i] = alpha2status (alpha[i]); status[j] = alpha2status (alpha[j]); double delta_alpha_i = alpha[i] - old_alpha_i; double delta_alpha_j = alpha[j] - old_alpha_j; ///////////////////////////////////////////////////////// // // Update O and Calculate \lambda^eq for shrinking, Calculate lambda^eq, // (c.f. Advances in Kernel Method pp.175) // lambda_eq = 1/|FREE| \sum_{i \in FREE} y_i - \sum_{l} y_i \alpha_i k(x_i,x_j) (11.29) // lambda_eq = 0.0; int size_A = 0; for (int k = 0; k < active_size; k++) { G[k] += Q_i[k] * delta_alpha_i + Q_j[k] * delta_alpha_j; if (is_free (k)) { lambda_eq -= G[k] * y[k]; size_A++; } } ///////////////////////////////////////////////////////// // // Select example for shrinking, // (c.f. 11.5 Efficient Implementation in Advances in Kernel Method pp. 175) // lambda_eq = size_A ? (lambda_eq / size_A) : 0.0; double kkt_violation = 0.0; for (int k = 0; k < active_size; k++) { double lambda_up = -(G[k] + y[k] * lambda_eq); // lambda_lo = -lambda_up // termination criteria (11.32,11.33,11.34) if (! is_lower_bound (k) && lambda_up < -kkt_violation) kkt_violation = -lambda_up; if (! is_upper_bound (k) && lambda_up > kkt_violation) kkt_violation = lambda_up; // "If the estimate (11.30) or (11.31) was positive (or above some threshold) at // each of the last h iterations, it is likely that this will be true at the optimal solution" // lambda^up (11.30) lambda^low = lambda^up * status[k] if (lambda_up * status[k] > shrink_eps) { if (shrink_iter[k]++ > shrink_size) { active_size--; swap_index (k, active_size); // remove this data from working set q_matrix->swap_index(k, active_size); q_matrix->update(active_size); k--; } } else { // reset iter, if current data dose not satisfy the condition (11.30), (11.31) shrink_iter[k] = 0; } } ///////////////////////////////////////////////////////// // // Check terminal criteria, show information of iteration // if (kkt_violation < eps) break; if ((iter % 50) == 0) { fprintf (stdout, "."); fflush (stdout); }; if ((iter % 1000) == 0) { fprintf (stdout, " %6d %6d %5d %1.4f %5.1f%% %5.1f%%\n", iter, active_size, q_matrix->size, kkt_violation, 100.0 * (q_matrix->hit - hit_old)/2000, 100.0 * q_matrix->hit/(q_matrix->hit + q_matrix->miss)); fflush (stdout); // save old hit rate hit_old = q_matrix->hit; // This shrink eps rule is delivered from svm_light. shrink_eps = shrink_eps * 0.7 + kkt_violation * 0.3; } } }
void Solver::do_shrinking() { int i,j,k; if(select_working_set(i,j)!=0) return; double Gm1 = -y[j]*G[j]; double Gm2 = y[i]*G[i]; // shrink for(k=0;k<active_size;k++) { if(is_lower_bound(k)) { if(y[k]==+1) { if(-G[k] >= Gm1) continue; } else if(-G[k] >= Gm2) continue; } else if(is_upper_bound(k)) { if(y[k]==+1) { if(G[k] >= Gm2) continue; } else if(G[k] >= Gm1) continue; } else continue; --active_size; swap_index(k,active_size); --k; // look at the newcomer } // unshrink, check all variables again before final iterations if(unshrinked || -(Gm1 + Gm2) > eps*10) return; unshrinked = true; reconstruct_gradient(); for(k=l-1;k>=active_size;k--) { if(is_lower_bound(k)) { if(y[k]==+1) { if(-G[k] < Gm1) continue; } else if(-G[k] < Gm2) continue; } else if(is_upper_bound(k)) { if(y[k]==+1) { if(G[k] < Gm2) continue; } else if(G[k] < Gm1) continue; } else continue; swap_index(k,active_size); active_size++; ++k; // look at the newcomer } }
void Solver_NU::do_shrinking() { double Gmax1 = -INF; // max { -grad(f)_i * d | y_i = +1, d = +1 } double Gmax2 = -INF; // max { -grad(f)_i * d | y_i = +1, d = -1 } double Gmax3 = -INF; // max { -grad(f)_i * d | y_i = -1, d = +1 } double Gmax4 = -INF; // max { -grad(f)_i * d | y_i = -1, d = -1 } int k; for(k=0;k<active_size;k++) { if(!is_upper_bound(k)) { if(y[k]==+1) { if(-G[k] > Gmax1) Gmax1 = -G[k]; } else if(-G[k] > Gmax3) Gmax3 = -G[k]; } if(!is_lower_bound(k)) { if(y[k]==+1) { if(G[k] > Gmax2) Gmax2 = G[k]; } else if(G[k] > Gmax4) Gmax4 = G[k]; } } double Gm1 = -Gmax2; double Gm2 = -Gmax1; double Gm3 = -Gmax4; double Gm4 = -Gmax3; for(k=0;k<active_size;k++) { if(is_lower_bound(k)) { if(y[k]==+1) { if(-G[k] >= Gm1) continue; } else if(-G[k] >= Gm3) continue; } else if(is_upper_bound(k)) { if(y[k]==+1) { if(G[k] >= Gm2) continue; } else if(G[k] >= Gm4) continue; } else continue; --active_size; swap_index(k,active_size); --k; // look at the newcomer } // unshrink, check all variables again before final iterations if(unshrinked || max(-(Gm1+Gm2),-(Gm3+Gm4)) > eps*10) return; unshrinked = true; reconstruct_gradient(); for(k=l-1;k>=active_size;k--) { if(is_lower_bound(k)) { if(y[k]==+1) { if(-G[k] < Gm1) continue; } else if(-G[k] < Gm3) continue; } else if(is_upper_bound(k)) { if(y[k]==+1) { if(G[k] < Gm2) continue; } else if(G[k] < Gm4) continue; } else continue; swap_index(k,active_size); active_size++; ++k; // look at the newcomer } }
void Solver<TQ>::do_shrinking() { float Gmax1 = -INF; // max { -y_i * grad(f)_i | i in I_up(\alpha) } float Gmax2 = -INF; // max { y_i * grad(f)_i | i in I_low(\alpha) } // find maximal violating pair first for(int i=0;i<active_size;i++) { if(y[i]==+1) { if(!is_upper_bound(i)) { if(-G[i] >= Gmax1) Gmax1 = -G[i]; } if(!is_lower_bound(i)) { if(G[i] >= Gmax2) Gmax2 = G[i]; } } else { if(!is_upper_bound(i)) { if(-G[i] >= Gmax2) Gmax2 = -G[i]; } if(!is_lower_bound(i)) { if(G[i] >= Gmax1) Gmax1 = G[i]; } } } // shrink for(int i=0;i<active_size;i++) if (be_shrunken(i, Gmax1, Gmax2)) { active_size--; while (active_size > i) { if (!be_shrunken(active_size, Gmax1, Gmax2)) { swap_index(i,active_size); break; } active_size--; } } // unshrink, check all variables again before final iterations if(unshrinked || Gmax1 + Gmax2 > eps*10) return; unshrinked = true; reconstruct_gradient(); for(int i=l-1;i>=active_size;i--) if (!be_shrunken(i, Gmax1, Gmax2)) { while (active_size < i) { if (be_shrunken(active_size, Gmax1, Gmax2)) { swap_index(i,active_size); break; } active_size++; } active_size++; } }