void MatrixWorkerTable<T>::Add(integer_t row_id, T* data, size_t size,
const AddOption* option) {
if (row_id >= 0) CHECK(size == num_col_);
Blob ids_blob(&row_id, sizeof(integer_t));
Blob data_blob(data, size * sizeof(T));
WorkerTable::Add(ids_blob, data_blob, option);
Log::Debug("[Add] worker = %d, #row = %d\n", MV_Rank(), row_id);
}
void ArrayWorker<T>::Get(T* data, size_t size) {
CHECK(size == size_);
data_ = data;
integer_t all_key = -1;
Blob whole_table(&all_key, sizeof(integer_t));
WorkerTable::Get(whole_table);
Log::Debug("worker %d getting all parameters.\n", MV_Rank());
}
void ArrayWorker<T>::Add(T* data, size_t size, const AddOption* option) {
CHECK(size == size_);
integer_t all_key = -1;
Blob key(&all_key, sizeof(integer_t));
Blob val(data, sizeof(T) * size);
WorkerTable::Add(key, val, option);
Log::Debug("worker %d adding parameters with size of %d.\n", MV_Rank(), size);
}
ArrayWorker<T>::ArrayWorker(size_t size) : WorkerTable(), size_(size) {
num_server_ = MV_NumServers();
server_offsets_.push_back(0);
CHECK(size_ > MV_NumServers());
integer_t length = static_cast<integer_t>(size_) / MV_NumServers();
for (auto i = 1; i < MV_NumServers(); ++i) {
server_offsets_.push_back(i * length); // may not balance
}
server_offsets_.push_back(size_);
Log::Debug("worker %d create arrayTable with %d elements.\n", MV_Rank(), size);
}
ArrayServer<T>::ArrayServer(size_t size) : ServerTable() {
server_id_ = MV_Rank();
size_ = size / MV_NumServers();
if (server_id_ == MV_NumServers() - 1) { // last server
size_ += size % MV_NumServers();
}
storage_.resize(size_);
updater_ = Updater<T>::GetUpdater(size_);
Log::Debug("server %d create arrayTable with %d elements of %d elements.\n",
server_id_, size_, size);
}
void MatrixWorkerTable<T>::Get(integer_t row_id, T* data, size_t size) {
if (row_id >= 0) CHECK(size == num_col_);
for (auto i = 0; i < num_row_ + 1; ++i) row_index_[i] = nullptr;
if (row_id == -1) {
row_index_[num_row_] = data;
} else {
row_index_[row_id] = data; // data_ = data;
}
WorkerTable::Get(Blob(&row_id, sizeof(integer_t)));
Log::Debug("[Get] worker = %d, #row = %d\n", MV_Rank(), row_id);
}
void MatrixWorkerTable<T>::Get(const std::vector<integer_t>& row_ids,
const std::vector<T*>& data_vec,
size_t size) {
CHECK(size == num_col_);
CHECK(row_ids.size() == data_vec.size());
for (auto i = 0; i < num_row_ + 1; ++i) row_index_[i] = nullptr;
for (auto i = 0; i < row_ids.size(); ++i){
row_index_[row_ids[i]] = data_vec[i];
}
WorkerTable::Get(Blob(row_ids.data(), sizeof(integer_t)* row_ids.size()));
Log::Debug("[Get] worker = %d, #rows_set = %d\n", MV_Rank(), row_ids.size());
}
void MatrixWorkerTable<T>::Add(const std::vector<integer_t>& row_ids,
const std::vector<T*>& data_vec,
size_t size,
const AddOption* option) {
CHECK(size == num_col_);
Blob ids_blob(&row_ids[0], sizeof(integer_t)* row_ids.size());
Blob data_blob(row_ids.size() * row_size_);
//copy each row
for (auto i = 0; i < row_ids.size(); ++i){
memcpy(data_blob.data() + i * row_size_, data_vec[i], row_size_);
}
WorkerTable::Add(ids_blob, data_blob, option);
Log::Debug("[Add] worker = %d, #rows_set = %d\n", MV_Rank(), row_ids.size());
}
void TestArray(int argc, char* argv[]) {
Log::Info("Test Array \n");
multiverso::SetCMDFlag("sync", true);
MV_Init(&argc, argv);
size_t array_size = 500;
auto shared_array = MV_CreateTable(ArrayTableOption<int>(array_size));
Log::Info("Create tables OK. Rank = %d, worker_id = %d\n",
MV_Rank(), MV_WorkerId());
std::vector<int> delta(array_size);
for (int i = 0; i < array_size; ++i)
delta[i] = static_cast<int>(i);
int* data = new int[array_size];
int iter = 10 * (MV_Rank() + 10);
for (int i = 0; i < iter; ++i) {
shared_array->Add(delta.data(), array_size);
shared_array->Add(delta.data(), array_size);
shared_array->Add(delta.data(), array_size);
shared_array->Get(data, array_size);
shared_array->Get(data, array_size);
shared_array->Get(data, array_size);
if (iter < 100) {
for (int k = 0; k < array_size; ++k) {
CHECK (data[k] != delta[k] * (i + 1) * MV_NumWorkers()) ;
}
}
}
delete[] data;
MV_ShutDown();
}
MatrixWorkerTable<T>::MatrixWorkerTable(integer_t num_row, integer_t num_col) :
WorkerTable(), num_row_(num_row), num_col_(num_col) {
row_size_ = num_col * sizeof(T);
get_reply_count_ = 0;
num_server_ = MV_NumServers();
//compute row offsets in all servers
server_offsets_.push_back(0);
integer_t length = num_row / num_server_;
integer_t offset = length;
int i = 0;
while (length > 0 && offset < num_row && ++i < num_server_) {
server_offsets_.push_back(offset);
offset += length;
}
server_offsets_.push_back(num_row);
Log::Debug("[Init] worker = %d, type = matrixTable, size = [ %d x %d ].\n",
MV_Rank(), num_row, num_col);
row_index_ = new T*[num_row_ + 1];
}
void SparseMatrixWorkerTable<T>::Get(integer_t row_id, T* data, size_t size,
const GetOption* option) {
if (row_id >= 0) CHECK(size == this->num_col_);
for (auto i = 0; i < this->num_row_ + 1; ++i) this->row_index_[i] = nullptr;
if (row_id == -1) {
this->row_index_[this->num_row_] = data;
} else {
this->row_index_[row_id] = data; // data_ = data;
}
Blob keys(&row_id, sizeof(integer_t) * 1);
bool is_option_mine = false;
if (option == nullptr){
is_option_mine = true;
option = new GetOption();
}
WorkerTable::Get(keys, option);
Log::Debug("[Get] worker = %d, #row = %d\n", MV_Rank(), row_id);
if (is_option_mine) delete option;
}
void SparseMatrixWorkerTable<T>::Get(const std::vector<integer_t>& row_ids,
const std::vector<T*>& data_vec, size_t size,
const GetOption* option) {
for (auto i = 0; i < this->num_row_ + 1; ++i) this->row_index_[i] = nullptr;
CHECK(size == this->num_col_);
CHECK(row_ids.size() == data_vec.size());
for (integer_t i = 0; i < row_ids.size(); ++i) {
this->row_index_[row_ids[i]] = data_vec[i];
}
Blob keys(row_ids.data(), sizeof(integer_t) * row_ids.size());
bool is_option_mine = false;
if (option == nullptr){
is_option_mine = true;
option = new GetOption();
}
WorkerTable::Get(keys, option);
Log::Debug("[Get] worker = %d, #rows_set = %d\n", MV_Rank(),
row_ids.size());
if (is_option_mine) delete option;
}
void TestmatrixPerformance(int argc, char* argv[],
std::function<std::shared_ptr<WT>(int num_row, int num_col)>CreateWorkerTable,
std::function<std::shared_ptr<ST>(int num_row, int num_col)>CreateServerTable,
std::function<void(const std::shared_ptr<WT>& worker_table, const std::vector<int>& row_ids, const std::vector<float*>& data_vec, size_t size, const AddOption* option, int worker_id)> Add,
std::function<void(const std::shared_ptr<WT>& worker_table, float* data, size_t size, int worker_id)> Get) {
Log::ResetLogLevel(LogLevel::Info);
Log::Info("Test Matrix\n");
Timer timmer;
//multiverso::SetCMDFlag("sync", true);
MV_Init(&argc, argv);
int num_row = 1000000, num_col = 50;
if (argc == 3){
num_row = atoi(argv[2]);
}
int size = num_row * num_col;
int worker_id = MV_Rank();
int worker_num = MV_Size();
// test data
float* data = new float[size];
float* delta = new float[size];
for (auto row = 0; row < num_row; ++row) {
for (auto col = 0; col < num_col; ++col) {
delta[row * num_col + col] = static_cast<float>(row * num_col + col);
}
}
AddOption option;
option.set_worker_id(worker_id);
for (auto percent = 0; percent < 10; ++percent)
for (auto turn = 0; turn < 10; ++turn)
{
//std::shuffle(unique_index.begin(), unique_index.end(), eng);
if (worker_id == 0) {
std::cout << "\nTesting: Get All Rows => Add "
<< percent + 1 << "0% Rows to Server => Get All Rows" << std::endl;
}
auto worker_table = CreateWorkerTable(num_row, num_col);
auto server_table = CreateServerTable(num_row, num_col);
MV_Barrier();
timmer.Start();
Get(worker_table, data, size, worker_id);
std::cout << " " << 1.0 * timmer.elapse() / 1000 << "s:\t" << "get all rows first time, worker id: " << worker_id << std::endl;
MV_Barrier();
std::vector<int> row_ids;
std::vector<float*> data_vec;
for (auto i = 0; i < num_row; ++i) {
if (i % 10 <= percent && i % worker_num == worker_id) {
row_ids.push_back(i);
data_vec.push_back(delta + i * num_col);
}
}
if (worker_id == 0) {
std::cout << "adding " << percent + 1 << " /10 rows to matrix server" << std::endl;
}
if (row_ids.size() > 0) {
Add(worker_table, row_ids, data_vec, num_col, &option, worker_id);
}
Get(worker_table, data, size, -1);
MV_Barrier();
timmer.Start();
Get(worker_table, data, size, worker_id);
std::cout << " " << 1.0 * timmer.elapse() / 1000 << "s:\t" << "get all rows after adding to rows, worker id: " << worker_id << std::endl;
for (auto i = 0; i < num_row; ++i) {
auto row_start = data + i * num_col;
for (auto col = 0; col < num_col; ++col) {
float expected = (float)i * num_col + col;
float actual = *(row_start + col);
if (i % 10 <= percent) {
CHECK(expected == actual);
}
else {
CHECK(0 == *(row_start + col));
}
}
}
}
MV_Barrier();
Log::ResetLogLevel(LogLevel::Info);
Dashboard::Display();
Log::ResetLogLevel(LogLevel::Error);
MV_ShutDown();
}
void SparseMatrixWorkerTable<T>::ProcessReplyGet(
std::vector<Blob>& reply_data) {
// replace row_index when original key == -1
if (this->row_index_[this->num_row_] != nullptr) {
size_t keys_size = reply_data[0].size<integer_t>();
Log::Debug("[SparseMatrixWorkerTable:ProcessReplyGet] worker = %d, #keys_size = %d\n", MV_Rank(),
keys_size);
integer_t* keys = reinterpret_cast<integer_t*>(reply_data[0].data());
for (auto i = 0; i < keys_size; ++i) {
this->row_index_[keys[i]] = this->row_index_[this->num_row_] + keys[i] * this->num_col_;
}
}
MatrixWorkerTable<T>::ProcessReplyGet(reply_data);
}