void learning() {
int data_sz = samples.size();
int data_dim = samples[0].size();
theta = random_double_list(data_dim);
vector<int> idx;
for(int i = 0; i < data_sz; ++i) {
idx.push_back(i);
}
double coff2 = 2. * beta * alpha;
// training loop
for(int rd = 0; rd < rounds; ++rd) {
random_shuffle(idx.begin(), idx.end());
vector<double> delta(data_dim, 0);
for(auto id : idx) {
double opt1 = alpha * (labels[id] - lr_hypothesis(samples[id]));
for(int i = 0; i < data_dim; ++i) {
double t = opt1 * samples[id][i] - coff2 * theta[i];
delta[i] += t;
}
} // traverse
for(int i = 0; i < data_dim; ++i) {
theta[i] += delta[i];
}
if(debug) {
loss_error.push_back(calc_loss());
}
std::cout << "round " << rd << " finished." << std::endl;
} // training loop
}
void logistic_regression::agd_learning() {
int data_sz = samples.size(), data_dim = samples[0].size();
int cnt = 0, read_batch = data_sz / 100, update_batch = data_sz / 100;
if(read_batch == 0) {
read_batch = 10;
}
if(update_batch == 0) {
update_batch = 10;
}
theta = paracel::random_double_list(data_dim);
paracel_write("theta", theta); // init push
vector<int> idx;
for(int i = 0; i < data_sz; ++i) {
idx.push_back(i);
}
paracel_register_bupdate("/mfs/user/wuhong/paracel/build/lib/liblg_update.so",
"lg_theta_update");
double coff2 = 2. * beta * alpha;
vector<double> delta(data_dim);
// main loop
for(int rd = 0; rd < rounds; ++rd) {
std::random_shuffle(idx.begin(), idx.end());
theta = paracel_read<vector<double> >("theta");
vector<double> theta_old(theta);
// traverse data
cnt = 0;
for(auto sample_id : idx) {
if( (cnt % read_batch == 0) || (cnt == (int)idx.size() - 1) ) {
theta = paracel_read<vector<double> >("theta");
theta_old = theta;
}
for(int i = 0; i < data_dim; ++i) {
double coff1 = alpha * (labels[sample_id] - lg_hypothesis(samples[sample_id]));
double t = coff1 * samples[sample_id][i] - coff2 * theta[i];
theta[i] += t;
}
if(debug) {
loss_error.push_back(calc_loss());
}
if( (cnt % update_batch == 0) || (cnt == (int)idx.size() - 1) ) {
for(int i = 0; i < data_dim; ++i) {
delta[i] = theta[i] - theta_old[i];
}
paracel_bupdate("theta", delta);
}
cnt += 1;
} // traverse
sync();
std::cout << "worker" << get_worker_id() << " at the end of rd" << rd << std::endl;
} // rounds
theta = paracel_read<vector<double> >("theta"); // last pull
}
void logistic_regression::ipm_learning() {
int data_sz = samples.size(), data_dim = samples[0].size();
theta = paracel::random_double_list(data_dim);
paracel_write("theta", theta); // init push
vector<int> idx;
for(int i = 0; i < data_sz; ++i) {
idx.push_back(i);
}
paracel_register_bupdate("/mfs/user/wuhong/paracel/build/lib/liblg_update.so",
"lg_theta_update");
double coff2 = 2. * beta * alpha;
double wgt = 1. / get_worker_size();
vector<double> delta(data_dim);
// main loop
for(int rd = 0; rd < rounds; ++rd) {
std::random_shuffle(idx.begin(), idx.end());
theta = paracel_read<vector<double> >("theta");
vector<double> theta_old(theta);
// traverse data
for(auto sample_id : idx) {
for(int i = 0; i < data_dim; ++i) {
double coff1 = alpha * (labels[sample_id] - lg_hypothesis(samples[sample_id]));
double t = coff1 * samples[sample_id][i] - coff2 * theta[i];
theta[i] += t;
}
if(debug) {
loss_error.push_back(calc_loss());
}
} // traverse
for(int i = 0; i < data_dim; ++i) {
delta[i] = wgt * (theta[i] - theta_old[i]);
}
sync(); // sync for map
paracel_bupdate("theta", delta); // update with delta
sync(); // sync for reduce
std::cout << "worker" << get_worker_id() << " at the end of rd" << rd << std::endl;
} // rounds
theta = paracel_read<vector<double> >("theta"); // last pull
}
void test(const std::string & test_fn) {
auto lines = pt->paracel_load(test_fn);
local_parser(lines);
std::cout << "loss in test dataset is:" << calc_loss() << std::endl;
}
void logistic_regression::predict(const std::string & pred_fn) {
auto lines = paracel_load(input);
local_parser(lines); // re-init samples, labels
std::cout << "mean loss" << calc_loss() << std::endl;
}
// Stochastic Gradient Descent (SGD) Optimization
int LR::train_sgd( int max_loop, double loss_thrd, float learn_rate, float lambda, int avg)
{
int id = 0;
double loss = 0.0;
double loss_pre = 0.0;
vector< vector<float> > omega_pre=omega;
float acc=0.0;
vector< vector<float> > omega_sum(omega);
//最多迭代(最大循环 * 训练样本类别个数)次
while (id <= max_loop*(int)samp_class_vec.size())
{
if (id%samp_class_vec.size() == 0) // 完成一次迭代，预处理工作。
{
int loop = id/(int)samp_class_vec.size(); //check loss
loss = 0.0;
acc = 0.0;
calc_loss(&loss, &acc); //计算损失
cout.setf(ios::left);
cout << "Iter: " << setw(8) << loop << "Loss: " << setw(18) << loss << "Acc: " << setw(8) << acc << endl;
//如果损失值小于阈值，停止迭代
if ((loss_pre - loss) < loss_thrd && loss_pre >= loss && id != 0)
{
cout << "Reaching the minimal loss decrease!" << endl;
break;
}
loss_pre = loss; //保留上次损失值
if (id) //表示第一次不做正则项计算
{
for (int i=0;i!=omega_pre.size();i++)
for (int j=0;j!=omega_pre[i].size();j++)
omega[i][j]+=omega_pre[i][j]*lambda; //lambda为权重衰减项
}
omega_pre=omega;
}
// update omega
//随机选择样本
int r = (int)(rand()%samp_class_vec.size());
sparse_feat samp_feat = samp_feat_vec[r]; //随机样本的特征
int samp_class = samp_class_vec[r]; //随机样本的类别
update_online(samp_class, samp_feat, learn_rate, lambda);
if (avg == 1 && id%samp_class_vec.size() == 0)
{
for (int i = 0; i < feat_set_size; i++)
{
for (int j = 0; j < class_set_size; j++)
{
omega_sum[i][j] += omega[i][j];
}
}
}
id++;
} //迭代结束
if (avg == 1)
{
for (int i = 0; i < feat_set_size; i++)
{
for (int j = 0; j < class_set_size; j++)
{
omega[i][j] = (float)omega_sum[i][j] / id;
}
}
}
return 1;
}
