Esempio n. 1
0
void MatrixWorkerTable<T>::Add(integer_t row_id, T* data, size_t size, 
                               const AddOption* option) {
  if (row_id >= 0) CHECK(size == num_col_);
  Blob ids_blob(&row_id, sizeof(integer_t));
  Blob data_blob(data, size * sizeof(T));
  WorkerTable::Add(ids_blob, data_blob, option);
  Log::Debug("[Add] worker = %d, #row = %d\n", MV_Rank(), row_id);
}
Esempio n. 2
0
void ArrayWorker<T>::Get(T* data, size_t size) {
  CHECK(size == size_);
  data_ = data;
  integer_t all_key = -1;
  Blob whole_table(&all_key, sizeof(integer_t));
  WorkerTable::Get(whole_table);
  Log::Debug("worker %d getting all parameters.\n", MV_Rank());
}
Esempio n. 3
0
void ArrayWorker<T>::Add(T* data, size_t size, const AddOption* option) {
  CHECK(size == size_);
  integer_t all_key = -1;

  Blob key(&all_key, sizeof(integer_t));
  Blob val(data, sizeof(T) * size);
  WorkerTable::Add(key, val, option);
  Log::Debug("worker %d adding parameters with size of %d.\n", MV_Rank(), size);
}
Esempio n. 4
0
ArrayWorker<T>::ArrayWorker(size_t size) : WorkerTable(), size_(size) {
  num_server_ = MV_NumServers();
  server_offsets_.push_back(0);
  CHECK(size_ > MV_NumServers());
  integer_t length = static_cast<integer_t>(size_) / MV_NumServers();
  for (auto i = 1; i < MV_NumServers(); ++i) {
    server_offsets_.push_back(i * length); // may not balance
  }
  server_offsets_.push_back(size_);
  Log::Debug("worker %d create arrayTable with %d elements.\n", MV_Rank(), size);
}
Esempio n. 5
0
ArrayServer<T>::ArrayServer(size_t size) : ServerTable() {
  server_id_ = MV_Rank();
  size_ = size / MV_NumServers();
  if (server_id_ == MV_NumServers() - 1) { // last server 
    size_ += size % MV_NumServers();
  }
  storage_.resize(size_);
  updater_ = Updater<T>::GetUpdater(size_);
  Log::Debug("server %d create arrayTable with %d elements of %d elements.\n", 
             server_id_, size_, size);
}
Esempio n. 6
0
void MatrixWorkerTable<T>::Get(integer_t row_id, T* data, size_t size) {
  if (row_id >= 0) CHECK(size == num_col_);
  for (auto i = 0; i < num_row_ + 1; ++i) row_index_[i] = nullptr;
  if (row_id == -1) {
    row_index_[num_row_] = data;
  } else {
    row_index_[row_id] = data; // data_ = data;
  }
  WorkerTable::Get(Blob(&row_id, sizeof(integer_t)));
  Log::Debug("[Get] worker = %d, #row = %d\n", MV_Rank(), row_id);
}
Esempio n. 7
0
void MatrixWorkerTable<T>::Get(const std::vector<integer_t>& row_ids,
  const std::vector<T*>& data_vec,
  size_t size) {
  CHECK(size == num_col_);
  CHECK(row_ids.size() == data_vec.size());
  for (auto i = 0; i < num_row_ + 1; ++i) row_index_[i] = nullptr;
  for (auto i = 0; i < row_ids.size(); ++i){
    row_index_[row_ids[i]] = data_vec[i];
  }
  WorkerTable::Get(Blob(row_ids.data(), sizeof(integer_t)* row_ids.size()));
  Log::Debug("[Get] worker = %d, #rows_set = %d\n", MV_Rank(), row_ids.size());
}
Esempio n. 8
0
void MatrixWorkerTable<T>::Add(const std::vector<integer_t>& row_ids,
                               const std::vector<T*>& data_vec,
                               size_t size,
                               const AddOption* option) {
  CHECK(size == num_col_);
  Blob ids_blob(&row_ids[0], sizeof(integer_t)* row_ids.size());
  Blob data_blob(row_ids.size() * row_size_);
  //copy each row
  for (auto i = 0; i < row_ids.size(); ++i){
    memcpy(data_blob.data() + i * row_size_, data_vec[i], row_size_);
  }
  WorkerTable::Add(ids_blob, data_blob, option);
  Log::Debug("[Add] worker = %d, #rows_set = %d\n", MV_Rank(), row_ids.size());
}
Esempio n. 9
0
void TestArray(int argc, char* argv[]) {
  Log::Info("Test Array \n");

  multiverso::SetCMDFlag("sync", true);
  MV_Init(&argc, argv);

  size_t array_size = 500;

  auto shared_array = MV_CreateTable(ArrayTableOption<int>(array_size));

  Log::Info("Create tables OK. Rank = %d, worker_id = %d\n",
    MV_Rank(), MV_WorkerId());

  std::vector<int> delta(array_size);
  for (int i = 0; i < array_size; ++i)
    delta[i] = static_cast<int>(i);

  int* data = new int[array_size];

  int iter = 10 * (MV_Rank() + 10);
  for (int i = 0; i < iter; ++i) {
    shared_array->Add(delta.data(), array_size);
    shared_array->Add(delta.data(), array_size);
    shared_array->Add(delta.data(), array_size);
    shared_array->Get(data, array_size);
    shared_array->Get(data, array_size);
    shared_array->Get(data, array_size);
    if (iter < 100) {
      for (int k = 0; k < array_size; ++k) {
        CHECK (data[k] != delta[k] * (i + 1) * MV_NumWorkers()) ;
      }
    }
  }
  delete[] data;

  MV_ShutDown();
}
Esempio n. 10
0
MatrixWorkerTable<T>::MatrixWorkerTable(integer_t num_row, integer_t num_col) :
  WorkerTable(), num_row_(num_row), num_col_(num_col) {
  row_size_ = num_col * sizeof(T);
  get_reply_count_ = 0;

  num_server_ = MV_NumServers();
  //compute row offsets in all servers
  server_offsets_.push_back(0);
  integer_t length = num_row / num_server_;
  integer_t offset = length;
  int i = 0;
  while (length > 0 && offset < num_row && ++i < num_server_) {
    server_offsets_.push_back(offset);
    offset += length;
  }
  server_offsets_.push_back(num_row);

  Log::Debug("[Init] worker =  %d, type = matrixTable, size =  [ %d x %d ].\n",
    MV_Rank(), num_row, num_col);
  row_index_ = new T*[num_row_ + 1];
}
Esempio n. 11
0
void SparseMatrixWorkerTable<T>::Get(integer_t row_id, T* data, size_t size,
  const GetOption* option) {
  if (row_id >= 0) CHECK(size == this->num_col_);
  for (auto i = 0; i < this->num_row_ + 1; ++i) this->row_index_[i] = nullptr;
  if (row_id == -1) {
    this->row_index_[this->num_row_] = data;
  } else {
    this->row_index_[row_id] = data; // data_ = data;
  }
  Blob keys(&row_id, sizeof(integer_t) * 1);

  bool is_option_mine = false;
  if (option == nullptr){
    is_option_mine = true;
    option = new GetOption();
  }

  WorkerTable::Get(keys, option);
  Log::Debug("[Get] worker = %d, #row = %d\n", MV_Rank(), row_id);
  if (is_option_mine) delete option;
}
Esempio n. 12
0
void SparseMatrixWorkerTable<T>::Get(const std::vector<integer_t>& row_ids,
  const std::vector<T*>& data_vec, size_t size, 
  const GetOption* option) {
  for (auto i = 0; i < this->num_row_ + 1; ++i) this->row_index_[i] = nullptr;
  CHECK(size == this->num_col_);
  CHECK(row_ids.size() == data_vec.size());
  for (integer_t i = 0; i < row_ids.size(); ++i) {
    this->row_index_[row_ids[i]] = data_vec[i];
  }
  Blob keys(row_ids.data(), sizeof(integer_t) * row_ids.size());

  bool is_option_mine = false;
  if (option == nullptr){
    is_option_mine = true;
    option = new GetOption();
  }

  WorkerTable::Get(keys, option);
  Log::Debug("[Get] worker = %d, #rows_set = %d\n", MV_Rank(),
    row_ids.size());
  if (is_option_mine) delete option;
}
Esempio n. 13
0
void TestmatrixPerformance(int argc, char* argv[],
  std::function<std::shared_ptr<WT>(int num_row, int num_col)>CreateWorkerTable,
  std::function<std::shared_ptr<ST>(int num_row, int num_col)>CreateServerTable,
  std::function<void(const std::shared_ptr<WT>& worker_table, const std::vector<int>& row_ids, const std::vector<float*>& data_vec, size_t size, const AddOption* option, int worker_id)> Add,
  std::function<void(const std::shared_ptr<WT>& worker_table, float* data, size_t size, int worker_id)> Get) {

  Log::ResetLogLevel(LogLevel::Info);
  Log::Info("Test Matrix\n");
  Timer timmer;

  //multiverso::SetCMDFlag("sync", true);
  MV_Init(&argc, argv);
  int num_row = 1000000, num_col = 50;
  if (argc == 3){
    num_row = atoi(argv[2]);
  }

  int size = num_row * num_col;
  int worker_id = MV_Rank();
  int worker_num = MV_Size();

  // test data
  float* data = new float[size];
  float* delta = new float[size];
  for (auto row = 0; row < num_row; ++row) {
    for (auto col = 0; col < num_col; ++col) {
      delta[row * num_col + col] = static_cast<float>(row * num_col + col);
    }
  }

  AddOption option;
  option.set_worker_id(worker_id);

  for (auto percent = 0; percent < 10; ++percent)
    for (auto turn = 0; turn < 10; ++turn)
    {
      //std::shuffle(unique_index.begin(), unique_index.end(), eng);
      if (worker_id == 0) {
        std::cout << "\nTesting: Get All Rows => Add "
          << percent + 1 << "0% Rows to Server => Get All Rows" << std::endl;
      }


      auto worker_table = CreateWorkerTable(num_row, num_col);
      auto server_table = CreateServerTable(num_row, num_col);
      MV_Barrier();

      timmer.Start();
      Get(worker_table, data, size, worker_id);
      std::cout << " " << 1.0 * timmer.elapse() / 1000 << "s:\t" << "get all rows first time, worker id: " << worker_id << std::endl;
      MV_Barrier();

      std::vector<int> row_ids;
      std::vector<float*> data_vec;
      for (auto i = 0; i < num_row; ++i) {
        if (i % 10 <= percent && i % worker_num == worker_id) {
          row_ids.push_back(i);
          data_vec.push_back(delta + i * num_col);
        }
      }

      if (worker_id == 0) {
        std::cout << "adding " << percent + 1 << " /10 rows to matrix server" << std::endl;
      }

      if (row_ids.size() > 0) {
        Add(worker_table, row_ids, data_vec, num_col, &option, worker_id);
      }
      Get(worker_table, data, size, -1);
      MV_Barrier();

      timmer.Start();
      Get(worker_table, data, size, worker_id);
      std::cout << " " << 1.0 * timmer.elapse() / 1000 << "s:\t" << "get all rows after adding to rows, worker id: " << worker_id << std::endl;

      for (auto i = 0; i < num_row; ++i) {
        auto row_start = data + i * num_col;
        for (auto col = 0; col < num_col; ++col) {
          float expected = (float)i * num_col + col;
          float actual = *(row_start + col);
          if (i % 10 <= percent) {
            CHECK(expected == actual); 
          }
          else {
            CHECK(0 == *(row_start + col)); 
          }
        }
      }
    }

  MV_Barrier();
  Log::ResetLogLevel(LogLevel::Info);
  Dashboard::Display();
  Log::ResetLogLevel(LogLevel::Error);
  MV_ShutDown();
}
Esempio n. 14
0
void SparseMatrixWorkerTable<T>::ProcessReplyGet(
  std::vector<Blob>& reply_data) {
  // replace row_index when original key == -1
  if (this->row_index_[this->num_row_] != nullptr) {
    size_t keys_size = reply_data[0].size<integer_t>();
    Log::Debug("[SparseMatrixWorkerTable:ProcessReplyGet] worker = %d, #keys_size = %d\n", MV_Rank(),
      keys_size);
    integer_t* keys = reinterpret_cast<integer_t*>(reply_data[0].data());
    for (auto i = 0; i < keys_size; ++i) {
      this->row_index_[keys[i]] = this->row_index_[this->num_row_] + keys[i] * this->num_col_;
    }
  }

  MatrixWorkerTable<T>::ProcessReplyGet(reply_data);
}