/* main program */
int main(void)
{
int c;
if (allegro_init() != 0)
return 1;
install_keyboard();
install_mouse();
install_timer();
if (set_gfx_mode(GFX_AUTODETECT, 640, 480, 0, 0) != 0) {
if (set_gfx_mode(GFX_SAFE, 640, 480, 0, 0) != 0) {
set_gfx_mode(GFX_TEXT, 0, 0, 0, 0);
allegro_message("Unable to set any graphic mode\n%s\n",
allegro_error);
return 1;
}
}
set_palette(desktop_palette);
input_nodes();
calc_tangents();
curviness = ftofix(0.25);
show_tangents = FALSE;
show_control_points = FALSE;
draw_splines();
for (;;) {
if (keypressed()) {
c = readkey() >> 8;
if (c == KEY_ESC)
break;
else if (c == KEY_UP) {
curviness += ftofix(0.05);
draw_splines();
}
else if (c == KEY_DOWN) {
curviness -= ftofix(0.05);
draw_splines();
}
else if (c == KEY_SPACE) {
walk();
draw_splines();
}
else if (c == KEY_T) {
show_tangents = !show_tangents;
draw_splines();
}
else if (c == KEY_C) {
show_control_points = !show_control_points;
draw_splines();
}
}
}
return 0;
}
//Add delta to local buffer and send it to the parameter sever
void Communicator::AddDeltaParameter(DataBlock *data_block){
if (data_block == nullptr) {
multiverso::Log::Info("Rank %d has null DataBlcok.\n", process_id_);
return;
}
clock_t start = clock();
std::vector<real*> blocks;
std::vector<real*> recycle_blocks;
std::vector<int> input_nodes(data_block->input_nodes.begin(), data_block->input_nodes.end());
std::vector<int> output_nodes(data_block->output_nodes.begin(), data_block->output_nodes.end());
std::vector<real*> input_blocks;
std::vector<real*> output_blocks;
//Request blocks to store parameters
memory_mamanger_->RequestBlocks(input_nodes.size(), input_blocks);
memory_mamanger_->RequestBlocks(output_nodes.size(), output_blocks);
assert(input_blocks.size() == input_nodes.size());
assert(output_blocks.size() == output_nodes.size());
AddParameterByTableId(data_block, kInputEmbeddingTableId, input_nodes, input_blocks, recycle_blocks);
AddParameterByTableId(data_block, kEmbeddingOutputTableId, output_nodes, output_blocks, recycle_blocks);
memory_mamanger_->ReturnBlocks(input_blocks);
memory_mamanger_->ReturnBlocks(output_blocks);
if (option_->use_adagrad) {
std::vector<real*> input_gradient_blocks;
std::vector<real*> output_gradient_blocks;
memory_mamanger_->RequestBlocks(input_nodes.size(), input_gradient_blocks);
memory_mamanger_->RequestBlocks(output_nodes.size(), output_gradient_blocks);
assert(input_gradient_blocks.size() == input_nodes.size());
assert(output_gradient_blocks.size() == output_nodes.size());
AddParameterByTableId(data_block, kSumGradient2IETableId, input_nodes, input_gradient_blocks, recycle_blocks);
AddParameterByTableId(data_block, kSumGradient2EOTableId, output_nodes, output_gradient_blocks, recycle_blocks);
memory_mamanger_->ReturnBlocks(input_gradient_blocks);
memory_mamanger_->ReturnBlocks(output_gradient_blocks);
}
memory_mamanger_->ReturnBlocks(recycle_blocks);
multiverso::Log::Info("Rank %d Add Parameters time:%lfs\n", process_id_, (clock() - start) / static_cast<double>(CLOCKS_PER_SEC));
}
void Trainer::TrainIteration(multiverso::DataBlockBase *data_block)
{
if (process_id_ == -1)
process_id_ = multiverso::Multiverso::ProcessRank();
if (trainer_id_ == 0)
//Record the starting time of the Trainiteration
{
fprintf(log_file_, "%lf\n", (clock()) / (double)CLOCKS_PER_SEC);
fflush(log_file_);
}
multiverso::Log::Info("Rank %d [Trainer]------Train %d Begin TrainIteration%d ...\n",
process_id_, trainer_id_, train_count_);
++train_count_;
//Compute the total number of processes
if (process_count_ == -1)
process_count_ = multiverso::Multiverso::TotalProcessCount();
DataBlock *data = reinterpret_cast<DataBlock*>(data_block);
std::vector<int> input_nodes(data->input_nodes.begin(), data->input_nodes.end());
std::vector<int> output_nodes(data->output_nodes.begin(), data->output_nodes.end());
//A trainer only copy or add apart of parameters
//This trainer should copy or add the parameters according to
//local_input_nodes and local_output_nodes
std::vector<int> local_input_nodes;
std::vector<int> local_output_nodes;
for (int i = trainer_id_; i < input_nodes.size(); i += option_->thread_cnt)
local_input_nodes.push_back(input_nodes[i]);
for (int i = trainer_id_; i < output_nodes.size(); i += option_->thread_cnt)
local_output_nodes.push_back(output_nodes[i]);
if (trainer_id_ == 0)
{
multiverso::Log::Info("Rank %d [Trainer]------ input_size=%d, output_size=%d, negativesample_size=%d \n",
process_id_, input_nodes.size(), output_nodes.size(), data->negativesample_pools.size());
}
//Step 1, Copy the parameter from multiverso to WordEmbedding_
//One trainer only copy a part of parameters
multiverso::Log::Info("Rank %d [Trainer]------Train %d Copyparameter Begin TrainIteration%d ...\n",
process_id_, trainer_id_, train_count_);
CopyParameter(local_input_nodes, local_output_nodes);
if (trainer_id_ == 0)
{
multiverso::Row<int64> ©_row = GetRow<int64>(kWordCountActualTableId, 0);
WordEmbedding_->word_count_actual = copy_row.At(0);
WordEmbedding_->UpdateLearningRate();
}
multiverso::Log::Info("Rank %d [Trainer]------Train %d Copyparameter end TrainIteration%d ...\n",
process_id_, trainer_id_, train_count_);
//Wait for all the trainers to finish copying parameter
barrier_->Wait();
//Step 2, After finishing copying parameter,
//Use WordEmbedding_ to train a part of data_block
int64 last_word_count = word_count;
clock_t start = clock();
multiverso::Log::Info("Rank %d [Trainer]------Train %d TrainNN Begin TrainIteration%d ...\n",
process_id_, trainer_id_, train_count_);
if (trainer_id_ == 0)
multiverso::Log::Info("Rank %d [Trainer]------Train %d Datablock's sentence number:%d ...\n",
process_id_, trainer_id_, data->Size());
WordEmbedding_->Train(data, trainer_id_, option_->thread_cnt,
word_count, hidden_act_, hidden_err_);
if (word_count > last_word_count)
{
multiverso::Log::Info("[Trainer]------TrainNNSpeed: Words/thread/second %lfk\n",
((double)word_count - last_word_count) /
(clock() - start) * (double)CLOCKS_PER_SEC / 1000);
}
multiverso::Log::Info("Rank %d [Trainer]------Train %d TrainNN end TrainIteration%d ...\n",
process_id_, trainer_id_, train_count_);
//Wait for all the trainers to finish training
barrier_->Wait();
multiverso::Log::Info("Rank %d [Trainer]------Train %d AddDeltaParameter Begin TrainIteration%d ...\n",
process_id_, trainer_id_, train_count_);
//Step 3, After finishing training, add the delta of parameters to multiverso
AddDeltaParameter(local_input_nodes, local_output_nodes);
if (trainer_id_ == 0)
{
multiverso::Row<int64> ©_row = GetRow<int64>(kWordCountActualTableId, 0);
Add<int64>(kWordCountActualTableId, 0, 0, WordEmbedding_->word_count_actual - copy_row.At(0));
}
multiverso::Log::Info("Rank %d [Trainer]------Train %d AddDeltaParameter end TrainIteration%d ...\n",
process_id_, trainer_id_, train_count_);
//If the data_block is the last one,Dump the input-embedding weights
if (data->Type() == DataBlockType::Test && trainer_id_ == 0)
{
SaveEmbedding(option_->output_file, option_->output_binary);
}
if (data->Type() == DataBlockType::Test && process_id_ == 0 && trainer_id_ == 0)
{
SaveEmbedding("tmp.bin", 1);
char s[128] = { 0 };
sprintf_s(s, "check.py tmp.bin %d >> records.txt", clock() / CLOCKS_PER_SEC);
system(s);
}
if (trainer_id_ == 0)
{
fprintf(log_file_, "%lf\n",
(clock()) / (double)CLOCKS_PER_SEC);
fflush(log_file_);
}
}
