void Blob<Dtype>::UpdatePSTable(const Dtype* update) { CHECK_EQ(blob_mode_, BlobProto_BlobMode_GLOBAL); int update_idx = 0; for (int r = 0; r < util::Context::num_rows_per_table(); ++r) { petuum::UpdateBatch<Dtype> update_batch(global_table_row_capacity_); for (int i = 0; i < global_table_row_capacity_; ++i) { update_batch.UpdateSet(i, i, Dtype(-1) * update[update_idx]); ++update_idx; if (update_idx >= count_) { break; } } global_table_ptr_->BatchInc(r, update_batch); if (update_idx >= count_) { break; } } }
void MMSBModel::SendZMsg(VIndex nbr_i, Vindex i, CIndex z) { #ifdef DEBUG CHECK(neighbor_worker_.find(nbr_i) != neighbor_worker_.end()); #endif // row id WIndex nbr_w = neighbor_worker_[nbr_i]; uint32 row_id = GetOMsgRowId(nbr_w); // msg content petuum::DenseUpdateBatch<float> update_batch( 0, kNumMsgPrfxCols + 1); update_batch[kColIdxMsgType] = kSetZ; update_batch[kColIdxMsgVId] = nbr_i; update_batch[kColIdxMsgNbrId] = i; update_batch[kColIdxMsgSt] = z; // send the msg msg_table_.DenseBatchInc(row_id, update_batch); }
//virtual void FillPSTable(Blob<Dtype>* blob) { // const int count = blob->count(); // const Dtype value = this->filler_param_.value(); // CHECK(count); // petuum::UpdateBatch<Dtype> update_batch(count); // for (int i = 0; i < count; ++i) { // update_batch.UpdateSet(i, i, value); // } // blob->table()->BatchInc(1, update_batch); // CHECK_EQ(this->filler_param_.sparse(), -1) // << "Sparsity not supported by this Filler."; //} virtual void FillPSTable(Blob<Dtype>* blob) { const int count = blob->count(); const int global_table_row_capacity = blob->global_table_row_capacity(); const Dtype value = this->filler_param_.value(); int update_idx = 0; for (int r = 0; r < util::Context::num_rows_per_table(); ++r) { petuum::UpdateBatch<Dtype> update_batch(global_table_row_capacity); for (int i = 0; i < global_table_row_capacity; ++i) { update_batch.UpdateSet(i, i, value); ++update_idx; if (update_idx >= count) { break; } } blob->table()->BatchInc(r, update_batch); if (update_idx >= count) { break; } } CHECK_EQ(this->filler_param_.sparse(), -1) << "Sparsity not supported by this Filler."; }
//virtual void FillPSTable(Blob<Dtype>* blob) { // const int count = blob->count(); // CHECK(count); // Dtype* rn = new Dtype[count]; // GenerateSparseGaussianRN(blob, rn); // petuum::UpdateBatch<Dtype> update_batch(count); // for (int i = 0; i < count; ++i) { // update_batch.UpdateSet(i, i, rn[i]); // } // blob->table()->BatchInc(1, update_batch); // delete rn; //} virtual void FillPSTable(Blob<Dtype>* blob) { const int count = blob->count(); const int global_table_row_capacity = blob->global_table_row_capacity(); Dtype* rn = new Dtype[count]; GenerateSparseGaussianRN(blob, rn); int update_idx = 0; for (int r = 0; r < util::Context::num_rows_per_table(); ++r) { petuum::UpdateBatch<Dtype> update_batch(global_table_row_capacity); for (int i = 0; i < global_table_row_capacity; ++i) { update_batch.UpdateSet(i, i, rn[update_idx]); ++update_idx; if (update_idx >= count) { break; } } blob->table()->BatchInc(r, update_batch); if (update_idx >= count) { break; } } delete rn; }
float MMSBModel::ComputeLinkLikelihoodRemote( const VIndex i, const VIndex j, const bool positive) { #ifdef DEBUG CHECK(test_neighbor_worker_.find(j) != neighbor_worker__.end()); #endif Vertex* v_i = vertices_[i]; const auto& z_cnts = v_i->z_cnts(); WIndex nbr_w = test_neighbor_worker_[j]; const uint32 rid = GetOMsgRowId(nbr_w); // msg content petuum::DenseUpdateBatch<float> update_batch( 0, kNumMsgPrfxCols + z_cnts.size() * 2); update_batch[kColIdxMsgType] = kSetZ; update_batch[kColIdxMsgVId] = nbr_i; update_batch[kColIdxMsgNbrId] = i; update_batch[kColIdxMsgSt] = z; // send the msg msg_table_.DenseBatchInc(row_id, update_batch); }
// CHECK_EQ(this->filler_param_.sparse(), -1) // << "Sparsity not supported by this Filler."; //} virtual void FillPSTable(Blob<Dtype>* blob) { const int count = blob->count(); const int global_table_row_capacity = blob->global_table_row_capacity(); int fan_in = blob->count() / blob->num(); Dtype scale = sqrt(Dtype(3) / fan_in); Dtype* rn = new Dtype[count]; caffe_rng_uniform<Dtype>(blob->count(), -scale, scale, rn); int update_idx = 0; for (int r = 0; r < util::Context::num_rows_per_table(); ++r) { petuum::UpdateBatch<Dtype> update_batch(global_table_row_capacity); for (int i = 0; i < global_table_row_capacity; ++i) { update_batch.UpdateSet(i, i, rn[update_idx]); ++update_idx; if (update_idx >= count) { break; } } //LOG(INFO) << "test get " << r; //petuum::RowAccessor row_acc; //blob->table()->template Get<petuum::DenseRow<Dtype> >( // r, &row_acc, 0); //LOG(INFO) << "batch inc " << r; blob->table()->BatchInc(r, update_batch); //LOG(INFO) << "batch inc done " << r; if (update_idx >= count) { break; } } // //for (int r = 0; r < util::Context::num_rows_per_table(); ++r) { // petuum::DenseUpdateBatch<Dtype> update_batch(0, global_table_row_capacity); // for (int i = 0; i < global_table_row_capacity; ++i) { // update_batch[i] = rn[update_idx]; // ++update_idx; // if (update_idx >= count) { break; } // } // blob->table()->DenseBatchInc(r, update_batch); // if (update_idx >= count) { break; } //} delete rn; CHECK_EQ(this->filler_param_.sparse(), -1) << "Sparsity not supported by this Filler."; }
void MMSBModel::MHSampleLinkRemote( const VIndex i, const VIndex j, const CIndex z_prev) { #ifdef DEBUG CHECK(vertices_.find(i) != vertices_.end()); CHECK(neighbor_worker_.find(j) != neighbor_worker_.end()); #endif CIndex s; /// beta proposal s = z_prev; CIndex betap = beta_alias_table_.Propose(); /// prefetch 1 i-proposal Vertex* v_a = vertices_[i]; CIndex ip; float nak_or_alpha = Context::rand() * (alpha_ * K_ + v_a->degree()); if (nak_or_alpha < v_a->degree()) { // propose by n_ak uint32 nidx = Context::randUInt64() % v_a->degree(); ip = v_a->z_by_neighbor_idx(nidx); } else { // propose by alpha (uniform) ip = Context::randUInt64() % K_; } /// store the 2 proposals locally pair<CIndex, CIndex>& betav_prpsls = v_a->nbr_betav_prpsls(j); betav_prpsls.first = betap; betav_prpsls.second = ip; /// send the 2 proposals to vertex j // row id WIndex nbr_w = neighbor_worker_[j]; uint32 row_id = GetOMsgRowId(nbr_w); // msg content petuum::DenseUpdateBatch<float> update_batch( 0, kNumMsgPrfxCols + 2); update_batch[kColIdxMsgType] = kProposal; update_batch[kColIdxMsgVId] = j; update_batch[kColIdxMsgNbrId] = i; update_batch[kColIdxMsgSt] = betap; update_batch[kColIdxMsgSt + 1] = ip; // send the msg msg_table_.DenseBatchInc(row_id, update_batch); }
// CHECK_EQ(this->filler_param_.sparse(), -1) // << "Sparsity not supported by this Filler."; //} virtual void FillPSTable(Blob<Dtype>* blob) { const int count = blob->count(); const int global_table_row_capacity = blob->global_table_row_capacity(); Dtype* rn = new Dtype[count]; GeneratePositiveUnitballRN(blob, rn); int update_idx = 0; for (int r = 0; r < util::Context::num_rows_per_table(); ++r) { petuum::UpdateBatch<Dtype> update_batch(global_table_row_capacity); for (int i = 0; i < global_table_row_capacity; ++i) { update_batch.UpdateSet(i, i, rn[update_idx]); ++update_idx; if (update_idx >= count) { break; } } blob->table()->BatchInc(r, update_batch); if (update_idx >= count) { break; } } delete rn; CHECK_EQ(this->filler_param_.sparse(), -1) << "Sparsity not supported by this Filler."; }
// Implement the worker thread function void LRApp::WorkerThread(int client_id, int thread_id) { // Step 3. Gain Table Access petuum::Table<float> W = GetTable<float>("w"); // Step 4. Initialize parameters if (thread_id == 0) { petuum::DenseUpdateBatch<float> update_batch(0, feat_dim_); for (int i = 0; i < feat_dim_; ++i) { update_batch[i] = (rand() % 1001 - 500) / 500.0 * 1000.0; } W.DenseBatchInc(0, update_batch); } // Step 5. Sync after initialization using process_barrier_ // from the parent class process_barrier_->wait(); if (client_id == 0 && thread_id == 0) { LOG(INFO) << "training starts"; } // Step 6-8. Read & Update parameters for (int epoch = 0; epoch < num_epochs_; ++epoch) { // Step 6. Get weights from Parameter Server petuum::RowAccessor row_acc; const petuum::DenseRow<float>& r = W.Get<petuum::DenseRow<float>>( 0, &row_acc); for (int i = 0; i < feat_dim_; ++i) { paras_[i] = r[i]; } // Reset gradients std::fill(grad_.begin(), grad_.end(), 0.0); // Calculate gradients CalGrad(); // Step 7. Update weights petuum::DenseUpdateBatch<float> update_batch(0, feat_dim_); learning_rate_ /= 1.005; for (int i = 0; i < feat_dim_; ++i) { update_batch[i] = 0.0 - learning_rate_ * (grad_[i] / batch_size_ + lambda_ * paras_[i]); } W.DenseBatchInc(0, update_batch); // Evaluate on training set if (epoch % eval_epochs_ == 0 && client_id == 0 && thread_id == 0) { PrintAcc(epoch); } // Step 8. Don't forget the Clock Tick petuum::PSTableGroup::Clock(); } // Evaluate on test set if (client_id == 0 && thread_id == 0) { Eval(); } }
void DML::Learn(float learn_rate, int epochs, const char * model_file) { // tmp buffers float *vec_buf_1 = new float[src_feat_dim]; // src_feat_dim float *vec_buf_2 = new float[dst_feat_dim]; // dst_feat_dim // assign id to threads if (!thread_id.get()) { thread_id.reset(new int(thread_counter++)); } // get access to tables petuum::PSTableGroup::RegisterThread(); mat L = petuum::PSTableGroup::GetTableOrDie<float>(0); // Run additional iterations to let stale values finish propagating for (int iter = 0; iter < staleness; ++iter) { petuum::PSTableGroup::Clock(); } // initialize parameters if (client_id == 0 && (*thread_id) == 0) { std::cout << "init parameters" << std::endl; for (int i = 0; i < dst_feat_dim; i++) { petuum::DenseUpdateBatch<float> update_batch(0, src_feat_dim); for (int j = 0; j < src_feat_dim; j++) { float a = rand()%1000/1000.0/2000; update_batch[j]=a; } L.DenseBatchInc(i, update_batch); } std::cout << "init parameters done" << std::endl; } process_barrier->wait(); if (client_id == 0 && (*thread_id) == 0) std::cout << "training starts" << std::endl; sleep((client_id+(*thread_id))*2); std::vector<int> idx_perm_simi_pairs; for (int i = 0; i < num_simi_pairs; i++) idx_perm_simi_pairs.push_back(i); std::random_shuffle(idx_perm_simi_pairs.begin(), \ idx_perm_simi_pairs.end(), myrandom2); int * idx_perm_arr_simi_pairs = new int[num_simi_pairs]; for (int i = 0; i < num_simi_pairs; i++) idx_perm_arr_simi_pairs[i] = idx_perm_simi_pairs[i]; std::vector<int> idx_perm_diff_pairs; for (int i = 0; i < num_diff_pairs; i++) idx_perm_diff_pairs.push_back(i); std::random_shuffle(idx_perm_diff_pairs.begin(), \ idx_perm_diff_pairs.end(), myrandom2); int * idx_perm_arr_diff_pairs = new int[num_diff_pairs]; for (int i = 0; i < num_diff_pairs; i++) idx_perm_arr_diff_pairs[i] = idx_perm_diff_pairs[i]; std::vector<int> idx_perm; for (int i = 0; i < dst_feat_dim; i++) idx_perm.push_back(i); std::random_shuffle(idx_perm.begin(), idx_perm.end(), myrandom2); int * idx_perm_arr = new int[dst_feat_dim]; for (int i = 0; i < dst_feat_dim; i++) idx_perm_arr[i] = idx_perm[i]; //local buffer of parameter float ** local_paras = new float *[dst_feat_dim]; for (int i = 0; i < dst_feat_dim; i++) local_paras[i] = new float[src_feat_dim]; float ** grad=new float *[dst_feat_dim]; for ( int i=0;i<dst_feat_dim;i++) grad[i]=new float[src_feat_dim]; int inner_iters=(num_simi_pairs+num_diff_pairs)/size_mb/num_clients/num_worker_threads; int * mb_idx=new int[size_mb/2]; for (int e = 0; e < epochs; e++) { for(int it=0;it<inner_iters;it++){ //copy parameters petuum::RowAccessor row_acc; for (int i = 0; i < dst_feat_dim; i++) { const petuum::DenseRow<float>& r = L.Get<petuum::DenseRow<float> >(i, &row_acc); for (int j = 0; j < src_feat_dim; j++) { local_paras[i][j] = r[j]; } } //evaluate if (client_id == 0 && (*thread_id) == 0 && it%num_iters_evaluate==0) { // evaluate float simi_loss = 0, diff_loss = 0, total_loss = 0; Evaluate(local_paras, simi_loss, diff_loss, total_loss, vec_buf_1, vec_buf_2); //std::cout << "epoch:\t" << e << "\tsimi_loss:\t" << simi_loss \ //<< "\tdiff_loss:\t" << diff_loss << "\ttotal_loss:\t" \ //<< total_loss << std::endl; std::cout << "epoch: " << e << " iter: " << it << " loss: " << total_loss <<std::endl; } //set gradient to zero for(int i=0;i<dst_feat_dim;i++) memset(grad[i], 0, src_feat_dim*sizeof(float)); rand_init_vec_int(mb_idx, size_mb/2,num_simi_pairs); for(int i=0;i<size_mb/2;i++){ int idx = idx_perm_arr_simi_pairs[mb_idx[i]]; Update(local_paras, grad, data[simi_pairs[idx].x], data[simi_pairs[idx].y], \ 1, vec_buf_1, vec_buf_2); } rand_init_vec_int(mb_idx, size_mb/2,num_diff_pairs); for(int i=0;i<size_mb/2;i++){ int idx = idx_perm_arr_diff_pairs[mb_idx[i]]; Update(local_paras, grad, data[diff_pairs[idx].x], data[diff_pairs[idx].y], \ 0, vec_buf_1, vec_buf_2); } //update parameters float coeff =- learn_rate*2/size_mb; for (int i = 0; i < dst_feat_dim; i++) { petuum::DenseUpdateBatch<float> update_batch(0,src_feat_dim); for (int j = 0; j < src_feat_dim; j++) update_batch[j]= coeff*grad[i][j]; L.DenseBatchInc(i, update_batch); } petuum::PSTableGroup::Clock(); } } if (client_id == 0 && (*thread_id) == 0) SaveModel(L, model_file); delete[] mb_idx; delete[] vec_buf_1; delete[] vec_buf_2; for(int i=0;i< dst_feat_dim;i++) delete[]local_paras[i]; delete[] local_paras; for(int i=0;i<dst_feat_dim;i++) delete[]grad[i]; delete[]grad; petuum::PSTableGroup::DeregisterThread(); }
void MMSBModel::MHSampleLinkFeedback(const VIndex j, const VIndex nbr_j, const CIndex z_prev, const CIndex betap, const CIndex nbrp) { #ifdef DEBUG CHECK(vertices_.find(j) != vertices_.end()); CHECK(neighbor_worker_.find(nbr_j) != neighbor_worker_.end()); #endif Vertex* v_a = vertices_[j]; /* /// prefetch 4 j-proposals CIndex jp[4]; float jp_prob[4]; for (int p=0; p<4; ++p) { float nak_or_alpha = Context::rand() * (alpha_ * K_ + v_a->degree()); if (nak_or_alpha < v_a->degree()) { // propose by n_ak uint32 nidx = Context::randUInt64() % v_a->degree(); jp[p] = v_a->z_by_neighbor_idx(nidx); } else { // propose by alpha (uniform) jp[p] = Context::randUInt64() % K_; } } */ /// prefetch *1* j-proposal CIndex jp; float nak_or_alpha = Context::rand() * (alpha_ * K_ + v_a->degree()); if (nak_or_alpha < v_a->degree()) { // propose by n_ak uint32 nidx = Context::randUInt64() % v_a->degree(); jp = v_a->z_by_neighbor_idx(nidx); } else { // propose by alpha (uniform) jp = Context::randUInt64() % K_; } /// compute j-related terms of the reject-accept ratios float jp_prob[kNumMHStepStates]; // *: z_prev -> betap (ratio for beta-proposal) jp_prob[kB] = ComputeMHRatioTerm(v_a, z_prev, betap, z_prev, kBetaPrpsl); // 0*: z_prev -> nbrp (ratio for nrbj-proposal) jp_prob[kI0] = ComputeMHRatioTerm(v_a, z_prev, nbrp, z_prev, kViPrpsl); // 1*: betap -> nbrp jp_prob[kI1] = ComputeMHRatioTerm(v_a, betap, nbrp, z_prev, kViPrpsl); // 00*: z_prev -> jp (ratio for j-proposal) jp_prob[kJ00] = ComputeMHRatioTerm(v_a, z_prev, jp, z_prev, kVjPrpsl); // 10*: betap -> jp jp_prob[kJ10] = ComputeMHRatioTerm(v_a, betap, jp, z_prev, kVjPrpsl); // [01]1*: nbrp -> jp jp_prob[kJw1] = ComputeMHRatioTerm(v_a, nbrp, jp, z_prev, kVjPrpsl); /// send the msg // row id WIndex nbr_w = neighbor_worker_[nbr_j]; uint32 row_id = GetOMsgRowId(nbr_w); // msg content petuum::DenseUpdateBatch<float> update_batch( 0, kNumMsgPrfxCols + 7); update_batch[kColIdxMsgType] = kFeedback; update_batch[kColIdxMsgVId] = nbr_j; update_batch[kColIdxMsgNbrId] = j; update_batch[kColIdxMsgSt] = jp; for (int p=0; p<kNumMHStepStates; ++p) { update_batch[kColIdxMsgSt + 1 + p] = jp_prob[p]; } msg_table_.DenseBatchInc(row_id, update_batch); }