void clean_snapshots_network_data_pusher::push( const training_task_state& task_state, const network_schema& schema) { unsigned int previous_epoch = task_state.get_current_epoch() - 1; if (previous_epoch % keep_frequency != 0) { unsigned int current_index = task_state.index_peeked; std::string snapshot_folder_name = (boost::format("ann_trained_%|1$03d|_epoch_%|2$05d|") % current_index % previous_epoch).str(); boost::filesystem::path folder_path_to_clean = folder_path / snapshot_folder_name; if (boost::filesystem::exists(folder_path_to_clean)) boost::filesystem::remove_all(folder_path_to_clean); } }
void validate_progress_network_data_pusher::push(const training_task_state& task_state) { tester->set_data(task_state.data); testing_complete_result_set testing_res(ef, actual_output_neuron_value_set); tester->test( *reader, testing_res); unsigned int last_index = static_cast<unsigned int>(task_state.history.size()) - 1; std::cout << "# " << task_state.index_peeked << ", Epoch " << task_state.get_current_epoch() << ", Validating "; visualizer->dump(std::cout, testing_res); std::cout << std::endl; }
void validate_progress_network_data_pusher::push( const training_task_state& task_state, const network_schema& schema) { if ((task_state.get_current_epoch() % report_frequency) == 0) { forward_prop->set_data(*task_state.data); neuron_value_set_data_bunch_writer writer; forward_propagation::stat st = forward_prop->run(*reader, writer); forward_prop->clear_data(); unsigned int last_index = static_cast<unsigned int>(task_state.history.size()) - 1; std::cout << "----- Validating -----" << std::endl; std::cout << st << std::endl; for(std::map<std::string, std::pair<layer_configuration_specific, neuron_value_set::ptr> >::const_iterator it = writer.layer_name_to_config_and_value_set_map.begin(); it != writer.layer_name_to_config_and_value_set_map.end(); ++it) std::cout << schema.get_layer(it->first)->get_string_for_average_data(it->second.first, *it->second.second->get_average()) << std::endl; } }
void network_trainer_sgd::train_step( structured_data_bunch_reader& reader, training_task_state& task) { boost::chrono::steady_clock::time_point start = boost::chrono::high_resolution_clock::now(); std::pair<std::map<std::string, std::vector<float> >, std::string> lr_and_comment = prepare_learning_rates(task.get_current_epoch(), task.data); task.comments.push_back(lr_and_comment.second); neuron_value_set_data_bunch_writer writer; backward_propagation::stat training_stat = backprop->run( reader, writer, *task.data, task.momentum_data, lr_and_comment.first, batch_size, weight_decay, momentum); std::map<std::string, std::pair<layer_configuration_specific, nnforge_shared_ptr<std::vector<float> > > > output_data_average_results; for(std::map<std::string, std::pair<layer_configuration_specific, neuron_value_set::ptr> >::const_iterator it = writer.layer_name_to_config_and_value_set_map.begin(); it != writer.layer_name_to_config_and_value_set_map.end(); ++it) output_data_average_results.insert(std::make_pair(it->first, std::make_pair(it->second.first, it->second.second->get_average()))); task.history.push_back(std::make_pair(training_stat, output_data_average_results)); }
void network_trainer_sgd::train_step( supervised_data_reader& reader, training_task_state& task) { boost::chrono::steady_clock::time_point start = boost::chrono::high_resolution_clock::now(); std::pair<std::vector<std::vector<float> >, std::string> lr_and_comment = prepare_learning_rates(task.get_current_epoch(), task.data); task.comments.push_back(lr_and_comment.second); std::pair<testing_result_smart_ptr, training_stat_smart_ptr> train_result = updater->update( reader, lr_and_comment.first, task.data, batch_size, weight_decay, momentum, layer_to_dropout_rate_map); boost::chrono::duration<float> sec = (boost::chrono::high_resolution_clock::now() - start); float flops = updater->get_flops_for_single_entry(); train_result.first->time_to_complete_seconds = sec.count(); train_result.first->flops = static_cast<float>(train_result.first->get_entry_count()) * flops; task.history.push_back(train_result); }
bool network_trainer::is_last_epoch(const training_task_state& state) const { return (state.get_current_epoch() >= epoch_count); }
void save_resume_network_data_pusher::push(const training_task_state& task_state) { unsigned int index = task_state.index_peeked; std::string data_filename = (boost::format("ann_trained_%|1$03d|_epoch_%|2$05d|.data") % index % task_state.get_current_epoch()).str(); save_data_to_file(task_state.data, data_filename); std::string momentum_data_filename = (boost::format("momentum_%|1$03d|.data") % index).str(); if (task_state.momentum_data) save_data_to_file(task_state.momentum_data, momentum_data_filename); else boost::filesystem::remove(momentum_data_filename); }