void Blob<Dtype>::UpdatePSTable(const Dtype* update) {
CHECK_EQ(blob_mode_, BlobProto_BlobMode_GLOBAL);
int update_idx = 0;
for (int r = 0; r < util::Context::num_rows_per_table(); ++r) {
petuum::UpdateBatch<Dtype> update_batch(global_table_row_capacity_);
for (int i = 0; i < global_table_row_capacity_; ++i) {
update_batch.UpdateSet(i, i, Dtype(-1) * update[update_idx]);
++update_idx;
if (update_idx >= count_) { break; }
}
global_table_ptr_->BatchInc(r, update_batch);
if (update_idx >= count_) { break; }
}
}
void MMSBModel::SendZMsg(VIndex nbr_i, Vindex i, CIndex z) {
#ifdef DEBUG
CHECK(neighbor_worker_.find(nbr_i) != neighbor_worker_.end());
#endif
// row id
WIndex nbr_w = neighbor_worker_[nbr_i];
uint32 row_id = GetOMsgRowId(nbr_w);
// msg content
petuum::DenseUpdateBatch<float> update_batch(
0, kNumMsgPrfxCols + 1);
update_batch[kColIdxMsgType] = kSetZ;
update_batch[kColIdxMsgVId] = nbr_i;
update_batch[kColIdxMsgNbrId] = i;
update_batch[kColIdxMsgSt] = z;
// send the msg
msg_table_.DenseBatchInc(row_id, update_batch);
}
//virtual void FillPSTable(Blob<Dtype>* blob) {
// const int count = blob->count();
// const Dtype value = this->filler_param_.value();
// CHECK(count);
// petuum::UpdateBatch<Dtype> update_batch(count);
// for (int i = 0; i < count; ++i) {
// update_batch.UpdateSet(i, i, value);
// }
// blob->table()->BatchInc(1, update_batch);
// CHECK_EQ(this->filler_param_.sparse(), -1)
// << "Sparsity not supported by this Filler.";
//}
virtual void FillPSTable(Blob<Dtype>* blob) {
const int count = blob->count();
const int global_table_row_capacity = blob->global_table_row_capacity();
const Dtype value = this->filler_param_.value();
int update_idx = 0;
for (int r = 0; r < util::Context::num_rows_per_table(); ++r) {
petuum::UpdateBatch<Dtype> update_batch(global_table_row_capacity);
for (int i = 0; i < global_table_row_capacity; ++i) {
update_batch.UpdateSet(i, i, value);
++update_idx;
if (update_idx >= count) { break; }
}
blob->table()->BatchInc(r, update_batch);
if (update_idx >= count) { break; }
}
CHECK_EQ(this->filler_param_.sparse(), -1)
<< "Sparsity not supported by this Filler.";
}
//virtual void FillPSTable(Blob<Dtype>* blob) {
// const int count = blob->count();
// CHECK(count);
// Dtype* rn = new Dtype[count];
// GenerateSparseGaussianRN(blob, rn);
// petuum::UpdateBatch<Dtype> update_batch(count);
// for (int i = 0; i < count; ++i) {
// update_batch.UpdateSet(i, i, rn[i]);
// }
// blob->table()->BatchInc(1, update_batch);
// delete rn;
//}
virtual void FillPSTable(Blob<Dtype>* blob) {
const int count = blob->count();
const int global_table_row_capacity = blob->global_table_row_capacity();
Dtype* rn = new Dtype[count];
GenerateSparseGaussianRN(blob, rn);
int update_idx = 0;
for (int r = 0; r < util::Context::num_rows_per_table(); ++r) {
petuum::UpdateBatch<Dtype> update_batch(global_table_row_capacity);
for (int i = 0; i < global_table_row_capacity; ++i) {
update_batch.UpdateSet(i, i, rn[update_idx]);
++update_idx;
if (update_idx >= count) { break; }
}
blob->table()->BatchInc(r, update_batch);
if (update_idx >= count) { break; }
}
delete rn;
}
float MMSBModel::ComputeLinkLikelihoodRemote(
const VIndex i, const VIndex j, const bool positive) {
#ifdef DEBUG
CHECK(test_neighbor_worker_.find(j) != neighbor_worker__.end());
#endif
Vertex* v_i = vertices_[i];
const auto& z_cnts = v_i->z_cnts();
WIndex nbr_w = test_neighbor_worker_[j];
const uint32 rid = GetOMsgRowId(nbr_w);
// msg content
petuum::DenseUpdateBatch<float> update_batch(
0, kNumMsgPrfxCols + z_cnts.size() * 2);
update_batch[kColIdxMsgType] = kSetZ;
update_batch[kColIdxMsgVId] = nbr_i;
update_batch[kColIdxMsgNbrId] = i;
update_batch[kColIdxMsgSt] = z;
// send the msg
msg_table_.DenseBatchInc(row_id, update_batch);
}
// CHECK_EQ(this->filler_param_.sparse(), -1)
// << "Sparsity not supported by this Filler.";
//}
virtual void FillPSTable(Blob<Dtype>* blob) {
const int count = blob->count();
const int global_table_row_capacity = blob->global_table_row_capacity();
int fan_in = blob->count() / blob->num();
Dtype scale = sqrt(Dtype(3) / fan_in);
Dtype* rn = new Dtype[count];
caffe_rng_uniform<Dtype>(blob->count(), -scale, scale, rn);
int update_idx = 0;
for (int r = 0; r < util::Context::num_rows_per_table(); ++r) {
petuum::UpdateBatch<Dtype> update_batch(global_table_row_capacity);
for (int i = 0; i < global_table_row_capacity; ++i) {
update_batch.UpdateSet(i, i, rn[update_idx]);
++update_idx;
if (update_idx >= count) { break; }
}
//LOG(INFO) << "test get " << r;
//petuum::RowAccessor row_acc;
//blob->table()->template Get<petuum::DenseRow<Dtype> >(
// r, &row_acc, 0);
//LOG(INFO) << "batch inc " << r;
blob->table()->BatchInc(r, update_batch);
//LOG(INFO) << "batch inc done " << r;
if (update_idx >= count) { break; }
}
//
//for (int r = 0; r < util::Context::num_rows_per_table(); ++r) {
// petuum::DenseUpdateBatch<Dtype> update_batch(0, global_table_row_capacity);
// for (int i = 0; i < global_table_row_capacity; ++i) {
// update_batch[i] = rn[update_idx];
// ++update_idx;
// if (update_idx >= count) { break; }
// }
// blob->table()->DenseBatchInc(r, update_batch);
// if (update_idx >= count) { break; }
//}
delete rn;
CHECK_EQ(this->filler_param_.sparse(), -1)
<< "Sparsity not supported by this Filler.";
}
void MMSBModel::MHSampleLinkRemote(
const VIndex i, const VIndex j, const CIndex z_prev) {
#ifdef DEBUG
CHECK(vertices_.find(i) != vertices_.end());
CHECK(neighbor_worker_.find(j) != neighbor_worker_.end());
#endif
CIndex s;
/// beta proposal
s = z_prev;
CIndex betap = beta_alias_table_.Propose();
/// prefetch 1 i-proposal
Vertex* v_a = vertices_[i];
CIndex ip;
float nak_or_alpha = Context::rand() * (alpha_ * K_ + v_a->degree());
if (nak_or_alpha < v_a->degree()) { // propose by n_ak
uint32 nidx = Context::randUInt64() % v_a->degree();
ip = v_a->z_by_neighbor_idx(nidx);
} else { // propose by alpha (uniform)
ip = Context::randUInt64() % K_;
}
/// store the 2 proposals locally
pair<CIndex, CIndex>& betav_prpsls = v_a->nbr_betav_prpsls(j);
betav_prpsls.first = betap;
betav_prpsls.second = ip;
/// send the 2 proposals to vertex j
// row id
WIndex nbr_w = neighbor_worker_[j];
uint32 row_id = GetOMsgRowId(nbr_w);
// msg content
petuum::DenseUpdateBatch<float> update_batch(
0, kNumMsgPrfxCols + 2);
update_batch[kColIdxMsgType] = kProposal;
update_batch[kColIdxMsgVId] = j;
update_batch[kColIdxMsgNbrId] = i;
update_batch[kColIdxMsgSt] = betap;
update_batch[kColIdxMsgSt + 1] = ip;
// send the msg
msg_table_.DenseBatchInc(row_id, update_batch);
}
// CHECK_EQ(this->filler_param_.sparse(), -1)
// << "Sparsity not supported by this Filler.";
//}
virtual void FillPSTable(Blob<Dtype>* blob) {
const int count = blob->count();
const int global_table_row_capacity = blob->global_table_row_capacity();
Dtype* rn = new Dtype[count];
GeneratePositiveUnitballRN(blob, rn);
int update_idx = 0;
for (int r = 0; r < util::Context::num_rows_per_table(); ++r) {
petuum::UpdateBatch<Dtype> update_batch(global_table_row_capacity);
for (int i = 0; i < global_table_row_capacity; ++i) {
update_batch.UpdateSet(i, i, rn[update_idx]);
++update_idx;
if (update_idx >= count) { break; }
}
blob->table()->BatchInc(r, update_batch);
if (update_idx >= count) { break; }
}
delete rn;
CHECK_EQ(this->filler_param_.sparse(), -1)
<< "Sparsity not supported by this Filler.";
}
// Implement the worker thread function
void LRApp::WorkerThread(int client_id, int thread_id) {
// Step 3. Gain Table Access
petuum::Table<float> W = GetTable<float>("w");
// Step 4. Initialize parameters
if (thread_id == 0) {
petuum::DenseUpdateBatch<float> update_batch(0, feat_dim_);
for (int i = 0; i < feat_dim_; ++i) {
update_batch[i] = (rand() % 1001 - 500) / 500.0 * 1000.0;
}
W.DenseBatchInc(0, update_batch);
}
// Step 5. Sync after initialization using process_barrier_
// from the parent class
process_barrier_->wait();
if (client_id == 0
&& thread_id == 0) {
LOG(INFO) << "training starts";
}
// Step 6-8. Read & Update parameters
for (int epoch = 0; epoch < num_epochs_; ++epoch) {
// Step 6. Get weights from Parameter Server
petuum::RowAccessor row_acc;
const petuum::DenseRow<float>& r = W.Get<petuum::DenseRow<float>>(
0, &row_acc);
for (int i = 0; i < feat_dim_; ++i) {
paras_[i] = r[i];
}
// Reset gradients
std::fill(grad_.begin(), grad_.end(), 0.0);
// Calculate gradients
CalGrad();
// Step 7. Update weights
petuum::DenseUpdateBatch<float> update_batch(0, feat_dim_);
learning_rate_ /= 1.005;
for (int i = 0; i < feat_dim_; ++i) {
update_batch[i] = 0.0 - learning_rate_ *
(grad_[i] / batch_size_ + lambda_ * paras_[i]);
}
W.DenseBatchInc(0, update_batch);
// Evaluate on training set
if (epoch % eval_epochs_ == 0
&& client_id == 0
&& thread_id == 0) {
PrintAcc(epoch);
}
// Step 8. Don't forget the Clock Tick
petuum::PSTableGroup::Clock();
}
// Evaluate on test set
if (client_id == 0
&& thread_id == 0) {
Eval();
}
}
void DML::Learn(float learn_rate, int epochs, const char * model_file) {
// tmp buffers
float *vec_buf_1 = new float[src_feat_dim]; // src_feat_dim
float *vec_buf_2 = new float[dst_feat_dim]; // dst_feat_dim
// assign id to threads
if (!thread_id.get()) {
thread_id.reset(new int(thread_counter++));
}
// get access to tables
petuum::PSTableGroup::RegisterThread();
mat L = petuum::PSTableGroup::GetTableOrDie<float>(0);
// Run additional iterations to let stale values finish propagating
for (int iter = 0; iter < staleness; ++iter) {
petuum::PSTableGroup::Clock();
}
// initialize parameters
if (client_id == 0 && (*thread_id) == 0) {
std::cout << "init parameters" << std::endl;
for (int i = 0; i < dst_feat_dim; i++) {
petuum::DenseUpdateBatch<float> update_batch(0, src_feat_dim);
for (int j = 0; j < src_feat_dim; j++) {
float a = rand()%1000/1000.0/2000;
update_batch[j]=a;
}
L.DenseBatchInc(i, update_batch);
}
std::cout << "init parameters done" << std::endl;
}
process_barrier->wait();
if (client_id == 0 && (*thread_id) == 0)
std::cout << "training starts" << std::endl;
sleep((client_id+(*thread_id))*2);
std::vector<int> idx_perm_simi_pairs;
for (int i = 0; i < num_simi_pairs; i++)
idx_perm_simi_pairs.push_back(i);
std::random_shuffle(idx_perm_simi_pairs.begin(), \
idx_perm_simi_pairs.end(), myrandom2);
int * idx_perm_arr_simi_pairs = new int[num_simi_pairs];
for (int i = 0; i < num_simi_pairs; i++)
idx_perm_arr_simi_pairs[i] = idx_perm_simi_pairs[i];
std::vector<int> idx_perm_diff_pairs;
for (int i = 0; i < num_diff_pairs; i++)
idx_perm_diff_pairs.push_back(i);
std::random_shuffle(idx_perm_diff_pairs.begin(), \
idx_perm_diff_pairs.end(), myrandom2);
int * idx_perm_arr_diff_pairs = new int[num_diff_pairs];
for (int i = 0; i < num_diff_pairs; i++)
idx_perm_arr_diff_pairs[i] = idx_perm_diff_pairs[i];
std::vector<int> idx_perm;
for (int i = 0; i < dst_feat_dim; i++)
idx_perm.push_back(i);
std::random_shuffle(idx_perm.begin(), idx_perm.end(), myrandom2);
int * idx_perm_arr = new int[dst_feat_dim];
for (int i = 0; i < dst_feat_dim; i++)
idx_perm_arr[i] = idx_perm[i];
//local buffer of parameter
float ** local_paras = new float *[dst_feat_dim];
for (int i = 0; i < dst_feat_dim; i++)
local_paras[i] = new float[src_feat_dim];
float ** grad=new float *[dst_feat_dim];
for ( int i=0;i<dst_feat_dim;i++)
grad[i]=new float[src_feat_dim];
int inner_iters=(num_simi_pairs+num_diff_pairs)/size_mb/num_clients/num_worker_threads;
int * mb_idx=new int[size_mb/2];
for (int e = 0; e < epochs; e++) {
for(int it=0;it<inner_iters;it++){
//copy parameters
petuum::RowAccessor row_acc;
for (int i = 0; i < dst_feat_dim; i++) {
const petuum::DenseRow<float>& r = L.Get<petuum::DenseRow<float> >(i, &row_acc);
for (int j = 0; j < src_feat_dim; j++) {
local_paras[i][j] = r[j];
}
}
//evaluate
if (client_id == 0 && (*thread_id) == 0 && it%num_iters_evaluate==0) {
// evaluate
float simi_loss = 0, diff_loss = 0, total_loss = 0;
Evaluate(local_paras, simi_loss, diff_loss, total_loss, vec_buf_1, vec_buf_2);
//std::cout << "epoch:\t" << e << "\tsimi_loss:\t" << simi_loss \
//<< "\tdiff_loss:\t" << diff_loss << "\ttotal_loss:\t" \
//<< total_loss << std::endl;
std::cout << "epoch: " << e << " iter: " << it << " loss: " << total_loss <<std::endl;
}
//set gradient to zero
for(int i=0;i<dst_feat_dim;i++)
memset(grad[i], 0, src_feat_dim*sizeof(float));
rand_init_vec_int(mb_idx, size_mb/2,num_simi_pairs);
for(int i=0;i<size_mb/2;i++){
int idx = idx_perm_arr_simi_pairs[mb_idx[i]];
Update(local_paras, grad, data[simi_pairs[idx].x], data[simi_pairs[idx].y], \
1, vec_buf_1, vec_buf_2);
}
rand_init_vec_int(mb_idx, size_mb/2,num_diff_pairs);
for(int i=0;i<size_mb/2;i++){
int idx = idx_perm_arr_diff_pairs[mb_idx[i]];
Update(local_paras, grad, data[diff_pairs[idx].x], data[diff_pairs[idx].y], \
0, vec_buf_1, vec_buf_2);
}
//update parameters
float coeff =- learn_rate*2/size_mb;
for (int i = 0; i < dst_feat_dim; i++) {
petuum::DenseUpdateBatch<float> update_batch(0,src_feat_dim);
for (int j = 0; j < src_feat_dim; j++)
update_batch[j]= coeff*grad[i][j];
L.DenseBatchInc(i, update_batch);
}
petuum::PSTableGroup::Clock();
}
}
if (client_id == 0 && (*thread_id) == 0)
SaveModel(L, model_file);
delete[] mb_idx;
delete[] vec_buf_1;
delete[] vec_buf_2;
for(int i=0;i< dst_feat_dim;i++)
delete[]local_paras[i];
delete[] local_paras;
for(int i=0;i<dst_feat_dim;i++)
delete[]grad[i];
delete[]grad;
petuum::PSTableGroup::DeregisterThread();
}
void MMSBModel::MHSampleLinkFeedback(const VIndex j, const VIndex nbr_j,
const CIndex z_prev, const CIndex betap, const CIndex nbrp) {
#ifdef DEBUG
CHECK(vertices_.find(j) != vertices_.end());
CHECK(neighbor_worker_.find(nbr_j) != neighbor_worker_.end());
#endif
Vertex* v_a = vertices_[j];
/*
/// prefetch 4 j-proposals
CIndex jp[4];
float jp_prob[4];
for (int p=0; p<4; ++p) {
float nak_or_alpha = Context::rand() * (alpha_ * K_ + v_a->degree());
if (nak_or_alpha < v_a->degree()) { // propose by n_ak
uint32 nidx = Context::randUInt64() % v_a->degree();
jp[p] = v_a->z_by_neighbor_idx(nidx);
} else { // propose by alpha (uniform)
jp[p] = Context::randUInt64() % K_;
}
}
*/
/// prefetch *1* j-proposal
CIndex jp;
float nak_or_alpha = Context::rand() * (alpha_ * K_ + v_a->degree());
if (nak_or_alpha < v_a->degree()) { // propose by n_ak
uint32 nidx = Context::randUInt64() % v_a->degree();
jp = v_a->z_by_neighbor_idx(nidx);
} else { // propose by alpha (uniform)
jp = Context::randUInt64() % K_;
}
/// compute j-related terms of the reject-accept ratios
float jp_prob[kNumMHStepStates];
// *: z_prev -> betap (ratio for beta-proposal)
jp_prob[kB] = ComputeMHRatioTerm(v_a, z_prev, betap, z_prev, kBetaPrpsl);
// 0*: z_prev -> nbrp (ratio for nrbj-proposal)
jp_prob[kI0] = ComputeMHRatioTerm(v_a, z_prev, nbrp, z_prev, kViPrpsl);
// 1*: betap -> nbrp
jp_prob[kI1] = ComputeMHRatioTerm(v_a, betap, nbrp, z_prev, kViPrpsl);
// 00*: z_prev -> jp (ratio for j-proposal)
jp_prob[kJ00] = ComputeMHRatioTerm(v_a, z_prev, jp, z_prev, kVjPrpsl);
// 10*: betap -> jp
jp_prob[kJ10] = ComputeMHRatioTerm(v_a, betap, jp, z_prev, kVjPrpsl);
// [01]1*: nbrp -> jp
jp_prob[kJw1] = ComputeMHRatioTerm(v_a, nbrp, jp, z_prev, kVjPrpsl);
/// send the msg
// row id
WIndex nbr_w = neighbor_worker_[nbr_j];
uint32 row_id = GetOMsgRowId(nbr_w);
// msg content
petuum::DenseUpdateBatch<float> update_batch(
0, kNumMsgPrfxCols + 7);
update_batch[kColIdxMsgType] = kFeedback;
update_batch[kColIdxMsgVId] = nbr_j;
update_batch[kColIdxMsgNbrId] = j;
update_batch[kColIdxMsgSt] = jp;
for (int p=0; p<kNumMHStepStates; ++p) {
update_batch[kColIdxMsgSt + 1 + p] = jp_prob[p];
}
msg_table_.DenseBatchInc(row_id, update_batch);
}