TYPED_TEST(CyclicPoolLayerTest, TestMAXBackward) { typedef typename TypeParam::Dtype Dtype; LayerParameter layer_param; layer_param.mutable_cyclic_pool_param()-> set_pool(CyclicPoolParameter_PoolMethod_MAX); CyclicPoolLayer<Dtype> layer(layer_param); layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_); for (int i = 0; i < this->blob_top_->count(); ++i) { this->blob_top_->mutable_cpu_diff()[i] = -1; } vector<bool> propagate_down(1, true); layer.Backward(this->blob_top_vec_, propagate_down, this->blob_bottom_vec_); // expected output Dtype output[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, 0, 0, 0, -1, -1, -1, 0, 0, 0, -1, -1, -1, 0, 0, 0, }; for (int i = 0; i < this->blob_bottom_->count(); ++i) { EXPECT_EQ(this->blob_bottom_->cpu_diff()[i], output[i]); } }
TYPED_TEST(SPPLayerTest, TestForwardBackward) { typedef typename TypeParam::Dtype Dtype; LayerParameter layer_param; layer_param.mutable_spp_param()->set_pyramid_height(3); SPPLayer<Dtype> layer(layer_param); layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_); vector<bool> propagate_down(this->blob_bottom_vec_.size(), true); layer.Backward(this->blob_top_vec_, propagate_down, this->blob_bottom_vec_); }
TYPED_TEST(ClusteringLayerTest, TestForwardBackward) { typedef typename TypeParam::Dtype Dtype; LayerParameter layer_param; ClusteringParameter* clustering_layer_param = layer_param.mutable_clustering_param(); clustering_layer_param->set_num_output(NUM_OUT); clustering_layer_param->set_total_class(2); ClusteringLayer<Dtype> layer(layer_param); layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_); vector<bool> propagate_down(this->blob_bottom_vec_.size(), true); layer.Backward(this->blob_top_vec_, propagate_down, this->blob_bottom_vec_); }
static void propagate_down(Data_Obj *dp,uint32_t flagbit) { Node *np; if( dp->dt_children != NULL ){ np=QLIST_HEAD(dp->dt_children); while(np!=NULL){ Data_Obj *child_dp; child_dp = (Data_Obj *)np->n_data; xfer_dobj_flag(child_dp,dp,flagbit); propagate_down(child_dp,flagbit); np = np->n_next; } } }
static void propagate_down(Data_Obj *dp,uint32_t flagbit) { Node *np; if( dp->dt_children != NULL ){ np=dp->dt_children->l_head; while(np!=NULL){ Data_Obj *child_dp; child_dp = (Data_Obj *)np->n_data; xfer_cuda_flag(child_dp,dp,flagbit); propagate_down(child_dp,flagbit); np = np->n_next; } } }
TYPED_TEST(CuDNNLRNLayerTest, TestGradientAcrossChannelsCuDNN) { typedef TypeParam Dtype; LayerParameter layer_param; CuDNNLRNLayer<Dtype> layer(layer_param); GradientChecker<Dtype> checker(1e-2, 1e-2); layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_); for (int i = 0; i < this->blob_top_->count(); ++i) { this->blob_top_->mutable_cpu_diff()[i] = 1.; } vector<bool> propagate_down(this->blob_bottom_vec_.size(), true); layer.Backward(this->blob_top_vec_, propagate_down, this->blob_bottom_vec_); checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_, this->blob_top_vec_); }
TYPED_TEST(ScaleLayerTest, TestBackwardBroadcastMiddleInPlace) { typedef typename TypeParam::Dtype Dtype; TBlob<Dtype> orig_bottom(this->blob_bottom_->shape()); orig_bottom.CopyFrom(*this->blob_bottom_); this->blob_bottom_vec_.push_back(this->blob_bottom_broadcast_1_); LayerParameter layer_param; layer_param.mutable_scale_param()->set_axis(1); shared_ptr<ScaleLayer<Dtype, Dtype> > layer(new ScaleLayer<Dtype, Dtype>(layer_param)); TBlob<Dtype> top_diff(this->blob_bottom_->shape()); FillerParameter filler_param; filler_param.set_type("gaussian"); filler_param.set_std(1); GaussianFiller<Dtype> filler(filler_param); filler.Fill(&top_diff); vector<bool> propagate_down(2, true); // Run forward + backward without in-place computation; // save resulting bottom diffs. layer->SetUp(this->blob_bottom_vec_, this->blob_top_vec_); layer->Forward(this->blob_bottom_vec_, this->blob_top_vec_); caffe_copy<Dtype>(top_diff.count(), top_diff.cpu_data(), this->blob_top_->mutable_cpu_diff()); layer->Backward(this->blob_top_vec_, propagate_down, this->blob_bottom_vec_); const bool kReshape = true; const bool kCopyDiff = true; TBlob<Dtype> orig_bottom_diff; orig_bottom_diff.CopyFrom(*this->blob_bottom_, kCopyDiff, kReshape); TBlob<Dtype> orig_scale_diff; orig_scale_diff.CopyFrom(*this->blob_bottom_broadcast_1_, kCopyDiff, kReshape); // Rerun forward + backward with in-place computation; // check that resulting bottom diffs are the same. this->blob_top_vec_[0] = this->blob_bottom_; // in-place computation layer->Forward(this->blob_bottom_vec_, this->blob_top_vec_); caffe_copy<Dtype>(top_diff.count(), top_diff.cpu_data(), this->blob_bottom_->mutable_cpu_diff()); layer->Backward(this->blob_top_vec_, propagate_down, this->blob_bottom_vec_); for (int i = 0; i < this->blob_bottom_->count(); ++i) { EXPECT_NEAR(orig_bottom_diff.cpu_diff()[i], this->blob_bottom_->cpu_diff()[i], 1e-5); } for (int i = 0; i < this->blob_bottom_broadcast_1_->count(); ++i) { EXPECT_NEAR(orig_scale_diff.cpu_diff()[i], this->blob_bottom_broadcast_1_->cpu_diff()[i], tol<Dtype>(1e-5, 4e-2)); } }
// If we average to per image, the forward pass is 0.34ms and // backward pass is 0.43ms(when two bottoms are different). TYPED_TEST(BilinearLayerTest, TestSpeed) { typedef typename TypeParam::Dtype Dtype; LayerParameter layer_param; BilinearLayer<Dtype> layer(layer_param); layer.SetUp(this->bottom_vec_large, this->blob_top_vec_); layer.Forward(this->bottom_vec_large, this->blob_top_vec_); if (Caffe::mode() == Caffe::GPU) { caffe_copy(this->blob_top_->count(), this->blob_top_->gpu_data(), this->blob_top_->mutable_gpu_diff()); } else { caffe_copy(this->blob_top_->count(), this->blob_top_->cpu_data(), this->blob_top_->mutable_cpu_diff()); } vector<bool> propagate_down(2, true); layer.Backward(this->blob_top_vec_, propagate_down, this->bottom_vec_large); }
TYPED_TEST(LRNLayerTest, TestGradientAcrossChannels) { typedef typename TypeParam::Dtype Dtype; LayerParameter layer_param; LRNLayer<Dtype> layer(layer_param); GradientChecker<Dtype> checker(1e-2, 1e-2); layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_); for (int i = 0; i < this->blob_top_->count(); ++i) { this->blob_top_->mutable_cpu_diff()[i] = 1.; } vector<bool> propagate_down(this->blob_bottom_vec_.size(), true); layer.Backward(this->blob_top_vec_, propagate_down, this->blob_bottom_vec_); // for (int i = 0; i < this->blob_bottom_->count(); ++i) { // std::cout << "CPU diff " << this->blob_bottom_->cpu_diff()[i] // << std::endl; // } checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_, this->blob_top_vec_); }
TYPED_TEST(LRNLayerTest, TestGPUGradientAcrossChannels) { LayerParameter layer_param; LRNLayer<TypeParam> layer(layer_param); GradientChecker<TypeParam> checker(1e-2, 1e-2); Caffe::set_mode(Caffe::GPU); layer.SetUp(this->blob_bottom_vec_, &(this->blob_top_vec_)); layer.Forward(this->blob_bottom_vec_, &(this->blob_top_vec_)); for (int i = 0; i < this->blob_top_->count(); ++i) { this->blob_top_->mutable_cpu_diff()[i] = 1.; } vector<bool> propagate_down(this->blob_bottom_vec_.size(), true); layer.Backward(this->blob_top_vec_, propagate_down, &(this->blob_bottom_vec_)); // for (int i = 0; i < this->blob_bottom_->count(); ++i) { // std::cout << "GPU diff " << this->blob_bottom_->cpu_diff()[i] // << std::endl; // } checker.CheckGradientExhaustive(&layer, &(this->blob_bottom_vec_), &(this->blob_top_vec_)); }
TYPED_TEST(MaxPoolingDropoutTest, TestBackward) { typedef typename TypeParam::Dtype Dtype; LayerParameter layer_param; layer_param.set_phase(TRAIN); PoolingParameter* pooling_param = layer_param.mutable_pooling_param(); pooling_param->clear_kernel_size(); pooling_param->add_kernel_size(3); pooling_param->clear_stride(); pooling_param->add_stride(2); PoolingLayer<Dtype> layer(layer_param); layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_); for (int i = 0; i < this->blob_top_->count(); ++i) { this->blob_top_->mutable_cpu_diff()[i] = 1.; } vector<bool> propagate_down(this->blob_bottom_vec_.size(), true); layer.Backward(this->blob_top_vec_, propagate_down, this->blob_bottom_vec_); const Dtype* bottom_diff = this->blob_bottom_->cpu_diff(); Dtype sum = 0.; for (int i = 0; i < this->blob_bottom_->count(); ++i) { sum += bottom_diff[i]; } EXPECT_EQ(sum, this->blob_top_->count()); // Dropout in-place DropoutLayer<Dtype> dropout_layer(layer_param); dropout_layer.SetUp(this->blob_top_vec_, this->blob_top_vec_); dropout_layer.Forward(this->blob_top_vec_, this->blob_top_vec_); dropout_layer.Backward(this->blob_top_vec_, propagate_down, this->blob_top_vec_); layer.Backward(this->blob_top_vec_, propagate_down, this->blob_bottom_vec_); Dtype sum_with_dropout = 0.; bottom_diff = this->blob_bottom_->cpu_diff(); for (int i = 0; i < this->blob_bottom_->count(); ++i) { sum_with_dropout += bottom_diff[i]; } EXPECT_GE(sum_with_dropout, sum); }
void TestBackward(MergeCropParameter_MergeOp op) { TypeParam eps = 0.0; if (std::is_same<TypeParam, half_fp>::value) { eps = EPS_HALF; } if (std::is_same<TypeParam, float>::value) { eps = EPS_FLOAT; } if (std::is_same<TypeParam, double>::value) { eps = EPS_DOUBLE; } vector<int_tp> shape_a = blob_bottom_a_->shape(); vector<int_tp> shape_b = blob_bottom_b_->shape(); vector<int_tp> shape_top = blob_top_->shape(); for (int_tp i = 0; i < blob_bottom_a_->count(); ++i) { int val = i; int out = 0; int dec = 1; for (int_tp d = shape_a.size() - 1; d >= 0; --d) { out += (val % shape_a[d]) * dec; val /= shape_a[d]; dec *= 10; } blob_bottom_a_->mutable_cpu_data()[i] = out; } for (int_tp i = 0; i < blob_bottom_b_->count(); ++i) { int val = i; int out = 0; int dec = 1; for (int_tp d = shape_b.size() - 1; d >= 0; --d) { out += (val % shape_b[d]) * dec; val /= shape_b[d]; dec *= 10; } blob_bottom_b_->mutable_cpu_data()[i] = out; } LayerParameter layer_param; MergeCropParameter *merge_param = layer_param.mutable_mergecrop_param(); merge_param->set_operation(op); MergeCropLayer<TypeParam, TypeParam, TypeParam> layer(layer_param); layer.SetUp(blob_bottom_vec_, blob_top_vec_); layer.Forward(blob_bottom_vec_, blob_top_vec_); caffe_copy<TypeParam>(blob_top_->count(), blob_top_->cpu_data(), blob_top_->mutable_cpu_diff()); vector<bool> propagate_down(blob_bottom_vec_.size(), true); layer.Backward(blob_top_vec_, propagate_down, blob_bottom_vec_); // Test copy to a for (int_tp i = 0; i < blob_bottom_a_->count(); ++i) { int val = i; int out = 0; int dec = 1; for (int_tp d = shape_a.size() - 1; d >= 0; --d) { out += (val % shape_a[d]) * dec; val /= shape_a[d]; dec *= 10; } EXPECT_NEAR(out, blob_bottom_a_->mutable_cpu_data()[i], eps); } }
void propagate_flag(Data_Obj *dp,uint32_t flagbit) { propagate_up(dp,flagbit); propagate_down(dp,flagbit); }
TYPED_TEST(DownPoolingLayerTest, TestSimpleForwardBackward) { typedef typename TypeParam::Dtype Dtype; LayerParameter layer_param; TransParameter *t0 = layer_param.add_transformations(); t0->set_interp(NN); TransParameter *t1 = layer_param.add_transformations(); t1->set_scale(2.); TransParameter *t2 = layer_param.add_transformations(); t2->set_scale(4); TransParameter *t3 = layer_param.add_transformations(); t3->set_rotation(-90); // 2x2, 4x4, 8x8, and 2 x 2 with: // [0 1; [0 0; [0 0; [0 4; // 1 1] 2 2] 0 3]; 0 0]; // when all are transforemd into canonical shape of 2x2 // bottom 0, the canonical one: this->blob_bottom_->Reshape(2, 3, 2, 2); Dtype img[4] = { 1, 1, 1, 1 }; int width = this->blob_bottom_->width(); for (int n = 0; n < this->blob_bottom_->num(); ++n) { for (int c = 0; c < this->blob_bottom_->channels(); ++c) { for (int i = 0; i < 4; ++i) { this->blob_bottom_->set_data_at(img[i], n, c, i / width, i % width); // printf("%.1f ", this->blob_bottom_->data_at( n, c, i / width, i % // width)); // if (i % 2) printf("\n"); } } } // bottom 1, 4x4: Blob<Dtype> *blob_bottom_1 = new Blob<Dtype>(2, 3, 4, 4); Dtype img1[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2 }; width = blob_bottom_1->width(); for (int n = 0; n < blob_bottom_1->num(); ++n) { for (int c = 0; c < blob_bottom_1->channels(); ++c) { for (int i = 0; i < 16; ++i) { blob_bottom_1->set_data_at(img1[i], n, c, i / width, i % width); // printf("%.1f ", this->blob_bottom_1->data_at( n, c, i / width, i % // width)); // if (i % 2) printf("\n"); } } } // bottom 2, 8x8 Blob<Dtype> *blob_bottom_2 = new Blob<Dtype>(2, 3, 8, 8); Dtype img2[64] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0, 0, 0, 0, 3, 3, 3, 3, 0, 0, 0, 0, 3, 3, 3, 3, 0, 0, 0, 0, 3, 3, 3, 3 }; width = blob_bottom_2->width(); for (int n = 0; n < blob_bottom_2->num(); ++n) { for (int c = 0; c < blob_bottom_2->channels(); ++c) { for (int i = 0; i < 64; ++i) { blob_bottom_2->set_data_at(img2[i], n, c, i / width, i % width); // printf("%.1f ", this->blob_bottom_2->data_at( n, c, i / width, i % // width)); // if (i % 2) printf("\n"); } } } // bottom3 2x2 Blob<Dtype> *blob_bottom_3 = new Blob<Dtype>(2, 3, 2, 2); Dtype img3[4] = { 0, 0, 0, 4 }; for (int n = 0; n < blob_bottom_3->num(); ++n) { for (int c = 0; c < blob_bottom_3->channels(); ++c) { for (int i = 0; i < 4; ++i) { blob_bottom_3->set_data_at(img3[i], n, c, i / 2, i % 2); // printf("%.1f ", this->blob_bottom_3->data_at( n, c, i / width, i % // width)); // if (i % 2) printf("\n"); } } } this->blob_bottom_vec_.push_back(blob_bottom_1); this->blob_bottom_vec_.push_back(blob_bottom_2); this->blob_bottom_vec_.push_back(blob_bottom_3); // want for top: Dtype want[4] = { 1, 4, 2, 3 }; //----------Start---------- DownPoolingLayer<Dtype> layer(layer_param); layer.SetUp(this->blob_bottom_vec_, &(this->blob_top_vec_)); //----------Forward---------- layer.Forward(this->blob_bottom_vec_, &(this->blob_top_vec_)); EXPECT_EQ(this->blob_top_->count(), this->blob_bottom_->count()); // this->printMat(this->blob_top_->cpu_data(), this->blob_top_->height(), // this->blob_top_->width()); // Top should be same as want for (int n = 0; n < this->blob_bottom_->num(); ++n) { for (int c = 0; c < this->blob_bottom_->channels(); ++c) { for (int i = 0; i < 4; ++i) { // printf("%.1f vs %.1f (n%d c%d i%d)\t", // this->blob_top_->data_at( n, c, i / 2, i % 2), want[i], // n, c, i); // if (i % 2) printf("\n"); ASSERT_EQ(this->blob_top_->data_at(n, c, i / 2, i % 2), want[i]); } } } // -- setup for bkwd -- Dtype top_diff[4] = { 100, 10, 20, 30 }; for (int n = 0; n < this->blob_top_->num(); ++n) { for (int c = 0; c < this->blob_top_->channels(); ++c) { for (int i = 0; i < 4; ++i) { this->blob_top_->set_diff_at(top_diff[i], n, c, i / 2, i % 2); // printf("%.1f ", this->blob_top_->data_at( n, c, i / 2, i % 2)); // if (i % 2) printf("\n"); } } } Dtype want_diff[4] = { top_diff[0], 0, 0, 0 }; Dtype want_diff_1[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, top_diff[2], 0, 0 }; Dtype want_diff_2[64] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, top_diff[3], 0, 0, 0, 0, 0, 0, 0, 0, 0 }; Dtype want_diff_3[4] = { 0, 0, 0, top_diff[1] }; // print switch_idx_ // this->printMat(layer.max_switch().cpu_data(), layer.max_switch().height(), // layer.max_switch().width()); //----------Backward---------- vector<bool> propagate_down(1, true); layer.Backward(this->blob_top_vec_, propagate_down, &(this->blob_bottom_vec_)); // bottom 0: // this->printMat(this->blob_bottom_->cpu_diff(), 2, 2); for (int n = 0; n < this->blob_bottom_->num(); ++n) { for (int c = 0; c < this->blob_bottom_->channels(); ++c) { for (int i = 0; i < 4; ++i) { ASSERT_EQ(this->blob_bottom_->diff_at(n, c, i / 2, i % 2), want_diff[i]); } } } for (int n = 0; n < this->blob_bottom_->num(); ++n) { for (int c = 0; c < this->blob_bottom_->channels(); ++c) { for (int i = 0; i < 16; ++i) { ASSERT_EQ(blob_bottom_1->diff_at(n, c, i / 4, i % 4), want_diff_1[i]); } } } for (int n = 0; n < this->blob_bottom_->num(); ++n) { for (int c = 0; c < this->blob_bottom_->channels(); ++c) { for (int i = 0; i < 64; ++i) { ASSERT_EQ(blob_bottom_2->diff_at(n, c, i / 8, i % 8), want_diff_2[i]); } } } // this->printMat(this->blob_bottom_->cpu_diff(), 2, 2); // this->printMat(blob_bottom_1->cpu_diff(), 4, 4); // this->printMat(blob_bottom_3->cpu_diff(), 2, 2); for (int n = 0; n < this->blob_bottom_->num(); ++n) { for (int c = 0; c < this->blob_bottom_->channels(); ++c) { for (int i = 0; i < 4; ++i) { ASSERT_EQ(blob_bottom_3->diff_at(n, c, i / 2, i % 2), want_diff_3[i]); } } } delete blob_bottom_1; delete blob_bottom_2; delete blob_bottom_3; }
void GradientChecker<Dtype>::CheckGradientSingle( Layer<Dtype>* layer, const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top, int_tp check_bottom, int_tp top_id, int_tp top_data_id, bool element_wise) { if (element_wise) { CHECK_EQ(0, layer->blobs().size()); CHECK_LE(0, top_id); CHECK_LE(0, top_data_id); const int_tp top_count = top[top_id]->count(); for (int_tp blob_id = 0; blob_id < bottom.size(); ++blob_id) { CHECK_EQ(top_count, bottom[blob_id]->count()); } } // First, figure out what blobs we need to check against, and zero init // parameter blobs. vector<Blob<Dtype>*> blobs_to_check; vector<bool> propagate_down(bottom.size(), check_bottom == -1); for (int_tp i = 0; i < layer->blobs().size(); ++i) { Blob<Dtype>* blob = layer->blobs()[i].get(); caffe_set(blob->count(), static_cast<Dtype>(0), blob->mutable_cpu_diff()); blobs_to_check.push_back(blob); } if (check_bottom == -1) { for (int_tp i = 0; i < bottom.size(); ++i) { blobs_to_check.push_back(bottom[i]); } } else if (check_bottom >= 0) { CHECK_LT(check_bottom, bottom.size()); blobs_to_check.push_back(bottom[check_bottom]); propagate_down[check_bottom] = true; } CHECK_GT(blobs_to_check.size(), 0)<< "No blobs to check."; // Compute the gradient analytically using Backward Caffe::set_random_seed(seed_, Caffe::GetDefaultDevice()); // Ignore the loss from the layer (it's just the weighted sum of the losses // from the top blobs, whose gradients we may want to test individually). layer->Forward(bottom, top); // Get additional loss from the objective GetObjAndGradient(*layer, top, top_id, top_data_id); layer->Backward(top, propagate_down, bottom); // Store computed gradients for all checked blobs vector<shared_ptr<Blob<Dtype> > > computed_gradient_blobs( blobs_to_check.size()); for (int_tp blob_id = 0; blob_id < blobs_to_check.size(); ++blob_id) { Blob<Dtype>* current_blob = blobs_to_check[blob_id]; computed_gradient_blobs[blob_id].reset(new Blob<Dtype>()); computed_gradient_blobs[blob_id]->ReshapeLike(*current_blob); const int_tp count = blobs_to_check[blob_id]->count(); const Dtype* diff = blobs_to_check[blob_id]->cpu_diff(); Dtype* computed_gradients = computed_gradient_blobs[blob_id] ->mutable_cpu_data(); caffe_cpu_copy(count, diff, computed_gradients); } // Compute derivative of top w.r.t. each bottom and parameter input using // finite differencing. // LOG(ERROR) << "Checking " << blobs_to_check.size() << " blobs."; for (int_tp blob_id = 0; blob_id < blobs_to_check.size(); ++blob_id) { Blob<Dtype>* current_blob = blobs_to_check[blob_id]; const Dtype* computed_gradients = computed_gradient_blobs[blob_id]->cpu_data(); // LOG(ERROR) << "Blob " << blob_id << ": checking " // << current_blob->count() << " parameters."; for (int_tp feat_id = 0; feat_id < current_blob->count(); ++feat_id) { // For an element-wise layer, we only need to do finite differencing to // compute the derivative of top[top_id][top_data_id] w.r.t. // bottom[blob_id][i] only for i == top_data_id. For any other // i != top_data_id, we know the derivative is 0 by definition, and simply // check that that's true. Dtype estimated_gradient = 0; Dtype positive_objective = 0; Dtype negative_objective = 0; if (!element_wise || (feat_id == top_data_id)) { // Do finite differencing. // Compute loss with stepsize_ added to input. current_blob->mutable_cpu_data()[feat_id] += stepsize_; Caffe::set_random_seed(seed_, Caffe::GetDefaultDevice()); layer->Forward(bottom, top); positive_objective = GetObjAndGradient(*layer, top, top_id, top_data_id); // Compute loss with stepsize_ subtracted from input. current_blob->mutable_cpu_data()[feat_id] -= stepsize_ * 2; Caffe::set_random_seed(seed_, Caffe::GetDefaultDevice()); layer->Forward(bottom, top); negative_objective = GetObjAndGradient(*layer, top, top_id, top_data_id); // Recover original input value. current_blob->mutable_cpu_data()[feat_id] += stepsize_; estimated_gradient = (positive_objective - negative_objective) / stepsize_ / 2.; } Dtype computed_gradient = computed_gradients[feat_id]; Dtype feature = current_blob->cpu_data()[feat_id]; // LOG(ERROR) << "debug: " << current_blob->cpu_data()[feat_id] << " " // << current_blob->cpu_diff()[feat_id]; if (kink_ - kink_range_ > fabs(feature) || fabs(feature) > kink_ + kink_range_) { // We check relative accuracy, but for too small values, we threshold // the scale factor by 1. Dtype scale = std::max<Dtype>( std::max(fabs(computed_gradient), fabs(estimated_gradient)), Dtype(1.)); EXPECT_NEAR(computed_gradient, estimated_gradient, threshold_ * scale) << "debug: (top_id, top_data_id, blob_id, feat_id)=" << top_id << "," << top_data_id << "," << blob_id << "," << feat_id << "; feat = " << feature << "; objective+ = " << positive_objective << "; objective- = " << negative_objective; } // LOG(ERROR) << "Feature: " << current_blob->cpu_data()[feat_id]; // LOG(ERROR) << "computed gradient: " << computed_gradient // << " estimated_gradient: " << estimated_gradient; } } }
TYPED_TEST(DeconvolutionLayerTest, TestNDAgainst2D) { typedef typename TypeParam::Dtype Dtype; const int kernel_h = 11; const int kernel_w = 13; vector<int> bottom_shape(4); bottom_shape[0] = 15; bottom_shape[1] = 12; bottom_shape[2] = kernel_h * 2; bottom_shape[3] = kernel_w * 2; FillerParameter filler_param; GaussianFiller<Dtype> filler(filler_param); for (int i = 0; i < this->blob_bottom_vec_.size(); ++i) { this->blob_bottom_vec_[i]->Reshape(bottom_shape); filler.Fill(this->blob_bottom_vec_[i]); } LayerParameter layer_param; ConvolutionParameter* convolution_param = layer_param.mutable_convolution_param(); convolution_param->set_num_output(18); convolution_param->set_bias_term(false); convolution_param->set_group(6); convolution_param->set_kernel_h(kernel_h); convolution_param->set_kernel_w(kernel_w); convolution_param->mutable_weight_filler()->set_type("gaussian"); TBlob<Dtype> weights; TBlob<Dtype> top_diff; // Shape and fill weights and top_diff. bool copy_diff; bool reshape; { DeconvolutionLayer<Dtype, Dtype> layer(layer_param); layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); top_diff.ReshapeLike(*this->blob_top_); filler.Fill(&top_diff); ASSERT_EQ(1, layer.blobs().size()); copy_diff = false; reshape = true; weights.CopyFrom(*layer.blobs()[0], copy_diff, reshape); } vector<bool> propagate_down(1, true); TBlob<Dtype> result_2d; TBlob<Dtype> backward_result_2d; TBlob<Dtype> backward_weight_result_2d; // Test with 2D im2col { caffe_set<Dtype>(this->blob_top_->count(), TypedConsts<Dtype>::zero, this->blob_top_->mutable_cpu_data()); caffe_set<Dtype>(this->blob_bottom_->count(), TypedConsts<Dtype>::zero, this->blob_bottom_->mutable_cpu_diff()); caffe_set<Dtype>(weights.count(), TypedConsts<Dtype>::zero, weights.mutable_cpu_diff()); // Do SetUp and Forward; save Forward result in result_2d. convolution_param->set_force_nd_im2col(false); DeconvolutionLayer<Dtype, Dtype> layer_2d(layer_param); layer_2d.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); ASSERT_EQ(1, layer_2d.blobs().size()); copy_diff = false; reshape = false; layer_2d.blobs()[0]->CopyFrom(weights, copy_diff, reshape); layer_2d.Forward(this->blob_bottom_vec_, this->blob_top_vec_); copy_diff = false; reshape = true; result_2d.CopyFrom(*this->blob_top_, copy_diff, reshape); // Copy pre-generated top diff into actual top diff; // do Backward and save result in backward_result_2d. ASSERT_EQ(this->blob_top_->shape(), top_diff.shape()); caffe_copy<Dtype>(top_diff.count(), top_diff.cpu_data(), this->blob_top_->mutable_cpu_diff()); layer_2d.Backward(this->blob_top_vec_, propagate_down, this->blob_bottom_vec_); copy_diff = true; reshape = true; backward_result_2d.CopyFrom(*this->blob_bottom_, copy_diff, reshape); backward_weight_result_2d.CopyFrom(weights, copy_diff, reshape); } TBlob<Dtype> result_nd; TBlob<Dtype> backward_result_nd; TBlob<Dtype> backward_weight_result_nd; // Test with ND im2col { caffe_set<Dtype>(this->blob_top_->count(), TypedConsts<Dtype>::zero, this->blob_top_->mutable_cpu_data()); caffe_set<Dtype>(this->blob_bottom_->count(), TypedConsts<Dtype>::zero, this->blob_bottom_->mutable_cpu_diff()); caffe_set<Dtype>(weights.count(), TypedConsts<Dtype>::zero, weights.mutable_cpu_diff()); // Do SetUp and Forward; save Forward result in result_nd. convolution_param->set_force_nd_im2col(true); DeconvolutionLayer<Dtype, Dtype> layer_nd(layer_param); layer_nd.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); ASSERT_EQ(1, layer_nd.blobs().size()); copy_diff = false; reshape = false; layer_nd.blobs()[0]->CopyFrom(weights, copy_diff, reshape); layer_nd.Forward(this->blob_bottom_vec_, this->blob_top_vec_); copy_diff = false; reshape = true; result_nd.CopyFrom(*this->blob_top_, copy_diff, reshape); // Copy pre-generated top diff into actual top diff; // do Backward and save result in backward_result_nd. ASSERT_EQ(this->blob_top_->shape(), top_diff.shape()); caffe_copy<Dtype>(top_diff.count(), top_diff.cpu_data(), this->blob_top_->mutable_cpu_diff()); layer_nd.Backward(this->blob_top_vec_, propagate_down, this->blob_bottom_vec_); copy_diff = true; reshape = true; backward_result_nd.CopyFrom(*this->blob_bottom_, copy_diff, reshape); backward_weight_result_nd.CopyFrom(weights, copy_diff, reshape); } ASSERT_EQ(result_nd.count(), result_2d.count()); for (int i = 0; i < result_2d.count(); ++i) { if (is_type<Dtype>(FLOAT16)) EXPECT_NEAR(result_2d.cpu_data()[i], result_nd.cpu_data()[i], 0.5F); else EXPECT_EQ(result_2d.cpu_data()[i], result_nd.cpu_data()[i]); } ASSERT_EQ(backward_result_nd.count(), backward_result_2d.count()); for (int i = 0; i < backward_result_2d.count(); ++i) { EXPECT_EQ(backward_result_2d.cpu_diff()[i], backward_result_nd.cpu_diff()[i]); } ASSERT_EQ(backward_weight_result_nd.count(), backward_weight_result_2d.count()); for (int i = 0; i < backward_weight_result_2d.count(); ++i) { EXPECT_EQ(backward_weight_result_2d.cpu_diff()[i], backward_weight_result_nd.cpu_diff()[i]); } }
TYPED_TEST(InnerProductLayerTest, TestBackwardTranspose) { typedef typename TypeParam::Dtype Dtype; this->blob_bottom_vec_.push_back(this->blob_bottom_); bool IS_VALID_CUDA = false; #ifndef CPU_ONLY IS_VALID_CUDA = CAFFE_TEST_CUDA_PROP.major >= 2; #endif if (Caffe::mode() == Caffe::CPU || sizeof(Dtype) == 4 || IS_VALID_CUDA) { LayerParameter layer_param; InnerProductParameter* inner_product_param = layer_param.mutable_inner_product_param(); inner_product_param->set_num_output(10); inner_product_param->mutable_weight_filler()->set_type("uniform"); inner_product_param->mutable_bias_filler()->set_type("uniform"); inner_product_param->mutable_bias_filler()->set_min(1); inner_product_param->mutable_bias_filler()->set_max(2); inner_product_param->set_transpose(false); shared_ptr<InnerProductLayer<Dtype> > layer( new InnerProductLayer<Dtype>(layer_param)); layer->SetUp(this->blob_bottom_vec_, this->blob_top_vec_); layer->Forward(this->blob_bottom_vec_, this->blob_top_vec_); // copy top blob Blob<Dtype>* const top = new Blob<Dtype>(); top->CopyFrom(*this->blob_top_, false, true); // fake top diff Blob<Dtype>* const diff = new Blob<Dtype>(); diff->ReshapeLike(*this->blob_top_); { FillerParameter filler_param; UniformFiller<Dtype> filler(filler_param); filler.Fill(diff); } caffe_copy(this->blob_top_vec_[0]->count(), diff->cpu_data(), this->blob_top_vec_[0]->mutable_cpu_diff()); vector<bool> propagate_down(1, true); layer->Backward(this->blob_top_vec_, propagate_down, this->blob_bottom_vec_); // copy first ip's weights and their diffs Blob<Dtype>* const w = new Blob<Dtype>(); w->CopyFrom(*layer->blobs()[0], false, true); w->CopyFrom(*layer->blobs()[0], true, true); // copy bottom diffs Blob<Dtype>* const bottom_diff = new Blob<Dtype>(); bottom_diff->CopyFrom(*this->blob_bottom_vec_[0], true, true); // repeat original top with tranposed ip std::for_each(this->blob_top_vec_.begin(), this->blob_top_vec_.end(), [](Blob<Dtype>* pPtr) {delete pPtr; }); this->blob_top_vec_.clear(); this->blob_top_vec_.push_back(new Blob<Dtype>()); inner_product_param->set_transpose(true); shared_ptr<InnerProductLayer<Dtype> > ip_t( new InnerProductLayer<Dtype>(layer_param)); ip_t->SetUp(this->blob_bottom_vec_, this->blob_top_vec_); // manually copy and transpose the weights from 1st IP layer into 2nd { const Dtype* w_src = w->cpu_data(); Dtype* w_t = ip_t->blobs()[0]->mutable_cpu_data(); const int width = layer->blobs()[0]->shape(1); const int width_t = ip_t->blobs()[0]->shape(1); for (int i = 0; i < layer->blobs()[0]->count(); ++i) { int r = i / width; int c = i % width; w_t[c*width_t+r] = w_src[r*width+c]; // copy while transposing } // copy bias from 1st IP layer to 2nd IP layer ASSERT_EQ(layer->blobs()[1]->count(), ip_t->blobs()[1]->count()); caffe_copy(layer->blobs()[1]->count(), layer->blobs()[1]->cpu_data(), ip_t->blobs()[1]->mutable_cpu_data()); } ip_t->Forward(this->blob_bottom_vec_, this->blob_top_vec_); caffe_copy(this->blob_top_vec_[0]->count(), diff->cpu_data(), this->blob_top_vec_[0]->mutable_cpu_diff()); ip_t->Backward(this->blob_top_vec_, propagate_down, this->blob_bottom_vec_); const Dtype* data = w->cpu_diff(); const Dtype* data_t = ip_t->blobs()[0]->cpu_diff(); const int WIDTH = layer->blobs()[0]->shape(1); const int WIDTH_T = ip_t->blobs()[0]->shape(1); for (int i = 0; i < layer->blobs()[0]->count(); ++i) { int r = i / WIDTH; int c = i % WIDTH; EXPECT_NE(Dtype(0.), data[r*WIDTH+c]); EXPECT_FLOAT_EQ(data[r*WIDTH+c], data_t[c*WIDTH_T+r]); } data = bottom_diff->cpu_diff(); data_t = this->blob_bottom_vec_[0]->cpu_diff(); for (int i = 0; i < this->blob_bottom_vec_[0]->count(); ++i) { EXPECT_NE(Dtype(0.), data[i]); EXPECT_FLOAT_EQ(data[i], data_t[i]); } delete bottom_diff; delete diff; delete w; delete top; } else { LOG(ERROR) << "Skipping test due to old architecture."; } }
void GradientChecker<Dtype>::CheckGradientSingle(Layer<Dtype>* layer, vector<Blob<Dtype>*>* bottom, vector<Blob<Dtype>*>* top, int check_bottom, int top_id, int top_data_id, bool element_wise) { if (element_wise) { CHECK_EQ(0, layer->blobs().size()); CHECK_LE(0, top_id); CHECK_LE(0, top_data_id); const int top_count = (*top)[top_id]->count(); for (int blob_id = 0; blob_id < bottom->size(); ++blob_id) { CHECK_EQ(top_count, (*bottom)[blob_id]->count()); } } // First, figure out what blobs we need to check against. vector<Blob<Dtype>*> blobs_to_check; vector<bool> propagate_down(bottom->size(), check_bottom < 0); for (int i = 0; i < layer->blobs().size(); ++i) { blobs_to_check.push_back(layer->blobs()[i].get()); } if (check_bottom < 0) { for (int i = 0; i < bottom->size(); ++i) { blobs_to_check.push_back((*bottom)[i]); } } else { //printf("TestGradientUtil: setting propogat down to true\n"); CHECK_LT(check_bottom, bottom->size()); blobs_to_check.push_back((*bottom)[check_bottom]); propagate_down[check_bottom] = true; } // Compute the gradient analytically using Backward Caffe::set_random_seed(seed_); // Ignore the loss from the layer (it's just the weighted sum of the losses // from the top blobs, whose gradients we may want to test individually). layer->Forward(*bottom, top); // Get additional loss from the objective GetObjAndGradient(*layer, top, top_id, top_data_id); layer->Backward(*top, propagate_down, bottom); // Store computed gradients for all checked blobs vector<shared_ptr<Blob<Dtype> > > computed_gradient_blobs(blobs_to_check.size()); for (int blob_id = 0; blob_id < blobs_to_check.size(); ++blob_id) { Blob<Dtype>* current_blob = blobs_to_check[blob_id]; computed_gradient_blobs[blob_id].reset(new Blob<Dtype>()); computed_gradient_blobs[blob_id]->ReshapeLike(*current_blob); const int count = blobs_to_check[blob_id]->count(); const Dtype* diff = blobs_to_check[blob_id]->cpu_diff(); Dtype* computed_gradients = computed_gradient_blobs[blob_id]->mutable_cpu_data(); caffe_copy(count, diff, computed_gradients); } // Compute derivative of top w.r.t. each bottom and parameter input using // finite differencing. // LOG(ERROR) << "Checking " << blobs_to_check.size() << " blobs."; for (int blob_id = 0; blob_id < blobs_to_check.size(); ++blob_id) { Blob<Dtype>* current_blob = blobs_to_check[blob_id]; const Dtype* computed_gradients = computed_gradient_blobs[blob_id]->cpu_data(); // LOG(ERROR) << "Blob " << blob_id << ": checking " // << current_blob->count() << " parameters."; /* printf("TestGradientUtil: current_blob->count() : %d \n", current_blob->count()); printf("TestGradientUtil: start printing current_blob data \n"); for (int feat_id = 0; feat_id < current_blob->count(); ++feat_id) { printf("%f \t", (float) current_blob->mutable_cpu_data()[feat_id]); } */ //printf("TestGradientUtil: end printing current_blob data \n \n"); for (int feat_id = 0; feat_id < current_blob->count(); ++feat_id) { // For an element-wise layer, we only need to do finite differencing to // compute the derivative of (*top)[top_id][top_data_id] w.r.t. // (*bottom)[blob_id][i] only for i == top_data_id. For any other // i != top_data_id, we know the derivative is 0 by definition, and simply // check that that's true. Dtype estimated_gradient = 0; Dtype positive_objective = 0; Dtype negative_objective = 0; if (!element_wise || (feat_id == top_data_id)) { // Do finite differencing. // Compute loss with stepsize_ added to input. /* printf("TestGradientUtil: feature_id %d , feature_value %f \n",feat_id , (float) current_blob->mutable_cpu_data()[feat_id]); */ current_blob->mutable_cpu_data()[feat_id] += stepsize_; Caffe::set_random_seed(seed_); layer->Forward(*bottom, top); positive_objective = GetObjAndGradient(*layer, top, top_id, top_data_id); // printf("TestGradientUtil: positive_objective : %f \n", (float) positive_objective ); //printf("TestGradientUtil: stepsize_ : %f \n", (float) stepsize_); // Compute loss with stepsize_ subtracted from input. current_blob->mutable_cpu_data()[feat_id] -= stepsize_ * 2; Caffe::set_random_seed(seed_); layer->Forward(*bottom, top); negative_objective = GetObjAndGradient(*layer, top, top_id, top_data_id); //printf("TestGradientUtil: negative_objective : %f \n", (float) negative_objective ); // Recover original input value. current_blob->mutable_cpu_data()[feat_id] += stepsize_; estimated_gradient = (positive_objective - negative_objective) / stepsize_ / 2.; } Dtype computed_gradient = computed_gradients[feat_id]; Dtype feature = current_blob->cpu_data()[feat_id]; // LOG(ERROR) << "debug: " << current_blob->cpu_data()[feat_id] << " " // << current_blob->cpu_diff()[feat_id]; if (kink_ - kink_range_ > fabs(feature) || fabs(feature) > kink_ + kink_range_) { // We check relative accuracy, but for too small values, we threshold // the scale factor by 1. Dtype scale = std::max( std::max(fabs(computed_gradient), fabs(estimated_gradient)), 1.); /* printf("TestGradientUtil: computed_gradient : %f \n", (float) computed_gradient); printf("TestGradientUtil: estimated_gradient : %f \n", (float) estimated_gradient ); printf("TestGradientUtil: Diff between computed and estimate : %f \n", (float) (computed_gradient - estimated_gradient)); printf("TestGradientUtil: threshold_ : %f \n", (float) threshold_ ); printf("TestGradientUtil: scale : %f \n", (float) scale ); */ EXPECT_NEAR(computed_gradient, estimated_gradient, threshold_ * scale) << "debug: (top_id, top_data_id, blob_id, feat_id)=" << top_id << "," << top_data_id << "," << blob_id << "," << feat_id << "; feat = " << feature << "; objective+ = " << positive_objective << "; objective- = " << negative_objective; } // LOG(ERROR) << "Feature: " << current_blob->cpu_data()[feat_id]; // LOG(ERROR) << "computed gradient: " << computed_gradient // << " estimated_gradient: " << estimated_gradient; } } }