void Solver<Dtype>::Solve(const char* resume_file) {
Caffe::set_mode(Caffe::Brew(param_.solver_mode()));
if (param_.solver_mode() && param_.has_device_id()) {
Caffe::SetDevice(param_.device_id());
}
Caffe::set_phase(Caffe::TRAIN);
LOG(INFO) << "Solving " << net_->name();
PreSolve();
iter_ = 0;
if (resume_file) {
LOG(INFO) << "Restoring previous solver status from " << resume_file;
Restore(resume_file);
}
// For a network that is trained by the solver, no bottom or top vecs
// should be given, and we will just provide dummy vecs.
vector<Blob<Dtype>*> bottom_vec;
while (iter_++ < param_.max_iter()) {
Dtype loss = net_->ForwardBackward(bottom_vec);
ComputeUpdateValue();
net_->Update();
// Notify the info monitors by its interval
for (int i = 0; i < info_.size(); ++i) {
info_[i].get()->Iter(loss, iter_);
}
// Display
// if (param_.display() && iter_ % param_.display() == 0) {
// Handle the loss printing to info monitor installed.
// LOG(INFO) << "Iteration " << iter_ << ", loss = " << loss;
//}
if (param_.test_interval() && iter_ % param_.test_interval() == 0) {
// We need to set phase to test before running.
Caffe::set_phase(Caffe::TEST);
Test();
Caffe::set_phase(Caffe::TRAIN);
}
// Check if we need to do snapshot
if (param_.snapshot() && iter_ % param_.snapshot() == 0) {
Snapshot();
}
}
// After the optimization is done, always do a snapshot.
iter_--;
Snapshot();
LOG(INFO) << "Optimization Done.";
}
void FeedbackSolver<Dtype>::Solve(const char* resume_file) {
Caffe::set_mode(Caffe::Brew(param_.solver_mode()));
if (param_.solver_mode() == SolverParameter_SolverMode_GPU &&
param_.has_device_id()) {
Caffe::SetDevice(param_.device_id());
}
Caffe::set_phase(Caffe::TRAIN);
LOG(INFO) << "Solving " << net_->name();
PreSolve();
iter_ = 0;
resume_file = NULL;
if (resume_file ) {
LOG(INFO) << "Restoring previous solver status from " << resume_file;
Restore(resume_file);
}
// Run a test pass before doing any training to avoid waiting a potentially
// very long time (param_.test_interval() training iterations) to report that
// there's not enough memory to run the test net and crash, etc.; and to gauge
// the effect of the first training iterations.
if (param_.test_interval()) {
Test();
}
// For a network that is trained by the solver, no bottom or top vecs
// should be given, and we will just provide dummy vecs.
vector<Blob<Dtype>*> bottom_vec;
while (iter_++ < param_.max_iter()) {
Dtype loss = net_->FeedbackForwardBackward(bottom_vec, param_.top_k());
ComputeUpdateValue();
net_->Update();
if (param_.display() && iter_ % param_.display() == 0) {
LOG(INFO) << "Iteration " << iter_ << ", loss = " << loss;
}
if (param_.test_interval() && iter_ % param_.test_interval() == 0) {
Test();
}
// Check if we need to do snapshot
if (param_.snapshot() && iter_ % param_.snapshot() == 0) {
Snapshot();
}
}
// After the optimization is done, always do a snapshot.
iter_--;
Snapshot();
LOG(INFO) << "Optimization Done.";
}
void Solver<Dtype>::Solve(Net<Dtype>* net) {
net_ = net;
LOG(INFO) << "Solving " << net_->name();
PreSolve();
iter_ = 0;
// For a network that is trained by the solver, no bottom or top vecs
// should be given, and we will just provide dummy vecs.
vector<Blob<Dtype>*> bottom_vec;
while (iter_++ < param_.max_iter()) {
Dtype loss = net_->ForwardBackward(bottom_vec);
ComputeUpdateValue();
net_->Update();
// Check if we need to do snapshot
if (param_.snapshot() > 0 && iter_ % param_.snapshot() == 0) {
Snapshot(false);
}
if (param_.display()) {
LOG(ERROR) << "Iteration " << iter_ << ", loss = " << loss;
}
}
LOG(INFO) << "Optimization Done.";
}
explicit SGDSolver(const string& param_file)
: Solver<Dtype>(param_file) { PreSolve(); }
explicit SGDSolver(const SolverParameter& param)
: Solver<Dtype>(param) { PreSolve(); }
void Solver<Dtype>::Solve(const char* resume_file) {
Caffe::set_phase(Caffe::TRAIN);
LOG(INFO) << "Solving " << net_->name();
PreSolve();
iter_ = 0;
if (resume_file) {
LOG(INFO) << "Restoring previous solver status from " << resume_file;
Restore(resume_file);
}
// Remember the initial iter_ value; will be non-zero if we loaded from a
// resume_file above.
const int start_iter = iter_;
// For a network that is trained by the solver, no bottom or top vecs
// should be given, and we will just provide dummy vecs.
vector<Blob<Dtype>*> bottom_vec;
for (; iter_ < param_.max_iter(); ++iter_) {
// Save a snapshot if needed.
if (param_.snapshot() && iter_ > start_iter &&
iter_ % param_.snapshot() == 0) {
Snapshot();
}
if (param_.test_interval() && iter_ % param_.test_interval() == 0
&& (iter_ > 0 || param_.test_initialization())) {
TestAll();
}
const bool display = param_.display() && iter_ % param_.display() == 0;
net_->set_debug_info(display && param_.debug_info());
net_->set_sample_print(display && param_.debug_info() && param_.sample_print());
Dtype loss = net_->ForwardBackward(bottom_vec);
if (display) {
LOG(INFO) << "Iteration " << iter_ << ", loss = " << loss;
const vector<Blob<Dtype>*>& result = net_->output_blobs();
int score_index = 0;
for (int j = 0; j < result.size(); ++j) {
const Dtype* result_vec = result[j]->cpu_data();
const string& output_name =
net_->blob_names()[net_->output_blob_indices()[j]];
const Dtype loss_weight =
net_->blob_loss_weights()[net_->output_blob_indices()[j]];
for (int k = 0; k < result[j]->count(); ++k) {
ostringstream loss_msg_stream;
if (loss_weight) {
loss_msg_stream << " (* " << loss_weight
<< " = " << loss_weight * result_vec[k] << " loss)";
}
LOG(INFO) << " Train net output #"
<< score_index++ << ": " << output_name << " = "
<< result_vec[k] << loss_msg_stream.str();
}
}
}
ComputeUpdateValue();
net_->Update();
}
// Always save a snapshot after optimization, unless overridden by setting
// snapshot_after_train := false.
if (param_.snapshot_after_train()) { Snapshot(); }
// After the optimization is done, run an additional train and test pass to
// display the train and test loss/outputs if appropriate (based on the
// display and test_interval settings, respectively). Unlike in the rest of
// training, for the train net we only run a forward pass as we've already
// updated the parameters "max_iter" times -- this final pass is only done to
// display the loss, which is computed in the forward pass.
if (param_.display() && iter_ % param_.display() == 0) {
Dtype loss;
net_->Forward(bottom_vec, &loss);
LOG(INFO) << "Iteration " << iter_ << ", loss = " << loss;
}
if (param_.test_interval() && iter_ % param_.test_interval() == 0) {
TestAll();
}
LOG(INFO) << "Optimization Done.";
}
void Solver<Dtype>::Solve(const char* resume_file) {
Caffe::set_mode(Caffe::Brew(param_.solver_mode()));
if (param_.solver_mode() && param_.has_device_id()) {
Caffe::SetDevice(param_.device_id());
}
Caffe::set_phase(Caffe::TRAIN);
LOG(INFO)<< "Solving " << net_->name();
PreSolve();
iter_ = 0;
if (resume_file) {
LOG(INFO)<< "Restoring previous solver status from " << resume_file;
Restore(resume_file);
}
// For a network that is trained by the solver, no bottom or top vecs
// should be given, and we will just provide dummy vecs.
vector<Blob<Dtype>*> bottom_vec;
timeval start_t, finish_t, tmp_t;
gettimeofday(&start_t, NULL);
gettimeofday(&tmp_t, NULL);
int pic_counts = 0;
int pos_triplets = 0;
int triplets_count = 0;
while (iter_++ < param_.max_iter()) {
Dtype loss = net_->ForwardBackward(bottom_vec);
ComputeUpdateValue();
net_->Update();
pic_counts += Caffe::mutable_name2id().size();
pos_triplets += Caffe::mutable_pos_triplets();
triplets_count += Caffe::mutable_triplets().size();
if (param_.display() && iter_ % param_.display() == 0) {
gettimeofday(&finish_t, NULL);
long int time_cost = (finish_t.tv_sec - tmp_t.tv_sec) * 1000000
+ (finish_t.tv_usec - tmp_t.tv_usec);
LOG(INFO)<< "Iteration " << iter_ << ", loss = " << loss
<< ", image counts: " << (pic_counts * 1.0 / param_.display())
<< ", triplets count: " << (triplets_count * 1.0 / param_.display())
<< ", positive triplet: " << (pos_triplets * 1.0 / param_.display())
<< ", cost time = " << (time_cost / 1000.0) << "ms";
gettimeofday(&tmp_t, NULL);
pic_counts = 0;
pos_triplets = 0;
triplets_count = 0;
}
if (param_.test_interval() && iter_ % param_.test_interval() == 0) {
// We need to set phase to test before running.
Caffe::set_phase(Caffe::TEST);
Test();
Caffe::set_phase(Caffe::TRAIN);
}
// Check if we need to do snapshot
if (param_.snapshot() && iter_ % param_.snapshot() == 0) {
Snapshot();
}
}
// After the optimization is done, always do a snapshot.
iter_--;
Snapshot();
LOG(INFO)<< "Optimization Done.";
}
