void clean_snapshots_network_data_pusher::push(
const training_task_state& task_state,
const network_schema& schema)
{
unsigned int previous_epoch = task_state.get_current_epoch() - 1;
if (previous_epoch % keep_frequency != 0)
{
unsigned int current_index = task_state.index_peeked;
std::string snapshot_folder_name = (boost::format("ann_trained_%|1$03d|_epoch_%|2$05d|") % current_index % previous_epoch).str();
boost::filesystem::path folder_path_to_clean = folder_path / snapshot_folder_name;
if (boost::filesystem::exists(folder_path_to_clean))
boost::filesystem::remove_all(folder_path_to_clean);
}
}
void validate_progress_network_data_pusher::push(const training_task_state& task_state)
{
tester->set_data(task_state.data);
testing_complete_result_set testing_res(ef, actual_output_neuron_value_set);
tester->test(
*reader,
testing_res);
unsigned int last_index = static_cast<unsigned int>(task_state.history.size()) - 1;
std::cout << "# " << task_state.index_peeked
<< ", Epoch " << task_state.get_current_epoch()
<< ", Validating ";
visualizer->dump(std::cout, testing_res);
std::cout << std::endl;
}
void validate_progress_network_data_pusher::push(
const training_task_state& task_state,
const network_schema& schema)
{
if ((task_state.get_current_epoch() % report_frequency) == 0)
{
forward_prop->set_data(*task_state.data);
neuron_value_set_data_bunch_writer writer;
forward_propagation::stat st = forward_prop->run(*reader, writer);
forward_prop->clear_data();
unsigned int last_index = static_cast<unsigned int>(task_state.history.size()) - 1;
std::cout << "----- Validating -----" << std::endl;
std::cout << st << std::endl;
for(std::map<std::string, std::pair<layer_configuration_specific, neuron_value_set::ptr> >::const_iterator it = writer.layer_name_to_config_and_value_set_map.begin(); it != writer.layer_name_to_config_and_value_set_map.end(); ++it)
std::cout << schema.get_layer(it->first)->get_string_for_average_data(it->second.first, *it->second.second->get_average()) << std::endl;
}
}
void network_trainer_sgd::train_step(
structured_data_bunch_reader& reader,
training_task_state& task)
{
boost::chrono::steady_clock::time_point start = boost::chrono::high_resolution_clock::now();
std::pair<std::map<std::string, std::vector<float> >, std::string> lr_and_comment = prepare_learning_rates(task.get_current_epoch(), task.data);
task.comments.push_back(lr_and_comment.second);
neuron_value_set_data_bunch_writer writer;
backward_propagation::stat training_stat = backprop->run(
reader,
writer,
*task.data,
task.momentum_data,
lr_and_comment.first,
batch_size,
weight_decay,
momentum);
std::map<std::string, std::pair<layer_configuration_specific, nnforge_shared_ptr<std::vector<float> > > > output_data_average_results;
for(std::map<std::string, std::pair<layer_configuration_specific, neuron_value_set::ptr> >::const_iterator it = writer.layer_name_to_config_and_value_set_map.begin(); it != writer.layer_name_to_config_and_value_set_map.end(); ++it)
output_data_average_results.insert(std::make_pair(it->first, std::make_pair(it->second.first, it->second.second->get_average())));
task.history.push_back(std::make_pair(training_stat, output_data_average_results));
}
void network_trainer_sgd::train_step(
supervised_data_reader& reader,
training_task_state& task)
{
boost::chrono::steady_clock::time_point start = boost::chrono::high_resolution_clock::now();
std::pair<std::vector<std::vector<float> >, std::string> lr_and_comment = prepare_learning_rates(task.get_current_epoch(), task.data);
task.comments.push_back(lr_and_comment.second);
std::pair<testing_result_smart_ptr, training_stat_smart_ptr> train_result = updater->update(
reader,
lr_and_comment.first,
task.data,
batch_size,
weight_decay,
momentum,
layer_to_dropout_rate_map);
boost::chrono::duration<float> sec = (boost::chrono::high_resolution_clock::now() - start);
float flops = updater->get_flops_for_single_entry();
train_result.first->time_to_complete_seconds = sec.count();
train_result.first->flops = static_cast<float>(train_result.first->get_entry_count()) * flops;
task.history.push_back(train_result);
}
bool network_trainer::is_last_epoch(const training_task_state& state) const
{
return (state.get_current_epoch() >= epoch_count);
}
void save_resume_network_data_pusher::push(const training_task_state& task_state)
{
unsigned int index = task_state.index_peeked;
std::string data_filename = (boost::format("ann_trained_%|1$03d|_epoch_%|2$05d|.data") % index % task_state.get_current_epoch()).str();
save_data_to_file(task_state.data, data_filename);
std::string momentum_data_filename = (boost::format("momentum_%|1$03d|.data") % index).str();
if (task_state.momentum_data)
save_data_to_file(task_state.momentum_data, momentum_data_filename);
else
boost::filesystem::remove(momentum_data_filename);
}
