TYPED_TEST(FilterMapImplTest, TestUnpaddingSimple) {
typedef typename TypeParam::Dtype Dtype;
FillerParameter filler_param;
filler_param.set_value(1);
ConstantFiller<Dtype> filler(filler_param);
vector<int> testshape(3);
testshape[0] = 3;
testshape[1] = 5;
testshape[2] = 5;
Blob<Dtype> testBlob(testshape), ref_out, impl_out, diff;
filler.Fill(&testBlob);
ref_unpadding(&testBlob, &ref_out, 1);
impl_unpadding(&testBlob, &impl_out, 1);
diff.ReshapeLike(ref_out);
caffe_sub(
ref_out.count(),
ref_out.cpu_data(),
impl_out.cpu_data(),
diff.mutable_cpu_data());
EXPECT_EQ(diff.asum_data(), 0);
}
void DepthLossLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
int count = bottom[0]->count();
Dtype* bottom0_log = new Dtype(count);
Dtype* bootom1_log = new Dtype(count);
//compute log for Y and Y*
for (int i = 0; i < count; i++)
{
bottom0_log[i] = log(bottom[0]->cpu_data()[i]);
bootom1_log[i] = log(bottom[0]->cpu_data()[i]);
}
//compute logY - logY*
caffe_sub(
count,
bottom0_log,
bootom1_log,
diff_.mutable_cpu_data());
//part1 of Loss
Dtype dot = caffe_cpu_dot(count, diff_.cpu_data(), diff_.cpu_data());
//part2 of Loss
Dtype log_sum = Dtype(0);
Dtype* diff_data = diff_.mutable_cpu_data();
for (int i = 0; i < count; i++) {
log_sum += diff_data[i];
}
//double gamma = 0.5;
Dtype loss = dot / bottom[0]->num() - gamma * log_sum * log_sum / bottom[0]->num() / bottom[0]->num();
top[0]->mutable_cpu_data()[0] = loss;
//free memory space
delete bottom0_log;
delete bootom1_log;
}
TYPED_TEST(FilterMapImplTest, TestUnpadding) {
typedef typename TypeParam::Dtype Dtype;
vector<int> dim1, dim2;
dim1.push_back(40); dim2.push_back(50);
dim1.push_back(50); dim2.push_back(40);
vector<int> padding_size;
padding_size.push_back(1);
padding_size.push_back(2);
padding_size.push_back(3);
padding_size.push_back(4);
padding_size.push_back(5);
FillerParameter filler_param;
GaussianFiller<Dtype> filler(filler_param);
for (int ishape = 0; ishape < dim1.size(); ++ishape) {
for (int ips = 0; ips < padding_size.size(); ++ips) {
vector<int> testshape(3);
testshape[0] = 3;
testshape[1] = dim1[ishape];
testshape[2] = dim2[ishape];
Blob<Dtype> testBlob(testshape), ref_out, impl_out, diff;
filler.Fill(&testBlob);
ref_unpadding(&testBlob, &ref_out, padding_size[ips]);
impl_unpadding(&testBlob, &impl_out, padding_size[ips]);
diff.ReshapeLike(ref_out);
caffe_sub(
ref_out.count(),
ref_out.cpu_data(),
impl_out.cpu_data(),
diff.mutable_cpu_data());
EXPECT_NEAR(diff.asum_data(), 0, 1e-7*diff.count());
}
}
}
Dtype VoxelCustomLossLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
vector<Blob<Dtype>*>* top) {
int count = bottom[0]->count();
caffe_sub(
count,
bottom[0]->cpu_data(),
bottom[1]->cpu_data(),
diff_.mutable_cpu_data());
Dtype loss = Dtype(0);
const Dtype *truth = bottom[1]->cpu_data();
Dtype *diff = diff_.mutable_cpu_data();
for (int i=0; i < count; i++) {
if (diff[i]<Dtype(-1))
diff[i] = Dtype(-1);
if (diff[i]>Dtype(1))
diff[i] = Dtype(1);
if (fabs(diff[i])>1)
loss += fabs(diff[i]);
else
loss += diff[i]*diff[i];
}
return (loss / count);
}
void SigmoidCrossEntropyLossLayer<Dtype>::Backward_cpu(
const vector<Blob<Dtype>*>& top, const vector<bool>& propagate_down,
const vector<Blob<Dtype>*>& bottom) {
if (propagate_down[1]) {
LOG(FATAL) << this->type()
<< " Layer cannot backpropagate to label inputs.";
}
if (propagate_down[0]) {
// First, compute the diff
const int_tp count = bottom[0]->count();
const int_tp num = bottom[0]->num();
const Dtype* sigmoid_output_data = sigmoid_output_->cpu_data();
const Dtype* target = bottom[1]->cpu_data();
Dtype* bottom_diff = bottom[0]->mutable_cpu_diff();
caffe_sub(count, sigmoid_output_data, target, bottom_diff);
if (bottom.size()==3) {
const Dtype* mask = bottom[2]->cpu_data();
caffe_mul(count, bottom_diff, mask, bottom_diff);
}
// Scale down gradient
const Dtype loss_weight = top[0]->cpu_diff()[0];
caffe_scal(count, loss_weight / num, bottom_diff);
}
}
void ContrastiveLossLayer<Dtype>::Forward_cpu(
const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
int count = bottom[0]->count();
caffe_sub(
count,
bottom[0]->cpu_data(), // a
bottom[1]->cpu_data(), // b
diff_.mutable_cpu_data()); // a_i-b_i
const int channels = bottom[0]->channels();
Dtype margin = this->layer_param_.contrastive_loss_param().margin();
Dtype loss(0.0);
for (int i = 0; i < bottom[0]->num(); ++i) {
dist_sq_.mutable_cpu_data()[i] = caffe_cpu_dot(channels,
diff_.cpu_data() + (i*channels), diff_.cpu_data() + (i*channels));
if (static_cast<int>(bottom[2]->cpu_data()[i])) { // similar pairs
loss += dist_sq_.cpu_data()[i];
} else { // dissimilar pairs
loss += std::max(margin-dist_sq_.cpu_data()[i], Dtype(0.0));
}
}
loss = loss / static_cast<Dtype>(bottom[0]->num()) / Dtype(2);
top[0]->mutable_cpu_data()[0] = loss;
}
void DataTransformer<Dtype>::Transform(Blob<Dtype>* input_blob,
Blob<Dtype>* transformed_blob) {
const int crop_size = param_.crop_size();
const int input_num = input_blob->num();
const int input_channels = input_blob->channels();
const int input_height = input_blob->height();
const int input_width = input_blob->width();
if (transformed_blob->count() == 0) {
// Initialize transformed_blob with the right shape.
if (crop_size) {
transformed_blob->Reshape(input_num, input_channels,
crop_size, crop_size);
} else {
transformed_blob->Reshape(input_num, input_channels,
input_height, input_width);
}
}
const int num = transformed_blob->num();
const int channels = transformed_blob->channels();
const int height = transformed_blob->height();
const int width = transformed_blob->width();
const int size = transformed_blob->count();
CHECK_LE(input_num, num);
CHECK_EQ(input_channels, channels);
CHECK_GE(input_height, height);
CHECK_GE(input_width, width);
const Dtype scale = param_.scale();
const bool do_mirror = param_.mirror() && Rand(2);
const bool has_mean_file = param_.has_mean_file();
const bool has_mean_values = mean_values_.size() > 0;
int h_off = 0;
int w_off = 0;
if (crop_size) {
CHECK_EQ(crop_size, height);
CHECK_EQ(crop_size, width);
// We only do random crop when we do training.
if (phase_ == TRAIN) {
h_off = Rand(input_height - crop_size + 1);
w_off = Rand(input_width - crop_size + 1);
} else {
h_off = (input_height - crop_size) / 2;
w_off = (input_width - crop_size) / 2;
}
} else {
CHECK_EQ(input_height, height);
CHECK_EQ(input_width, width);
}
Dtype* input_data = input_blob->mutable_cpu_data();
if (has_mean_file) {
CHECK_EQ(input_channels, data_mean_.channels());
CHECK_EQ(input_height, data_mean_.height());
CHECK_EQ(input_width, data_mean_.width());
for (int n = 0; n < input_num; ++n) {
int offset = input_blob->offset(n);
caffe_sub(data_mean_.count(), input_data + offset,
data_mean_.cpu_data(), input_data + offset);
}
}
if (has_mean_values) {
CHECK(mean_values_.size() == 1 || mean_values_.size() == input_channels) <<
"Specify either 1 mean_value or as many as channels: " << input_channels;
if (mean_values_.size() == 1) {
caffe_add_scalar(input_blob->count(), -(mean_values_[0]), input_data);
} else {
for (int n = 0; n < input_num; ++n) {
for (int c = 0; c < input_channels; ++c) {
int offset = input_blob->offset(n, c);
caffe_add_scalar(input_height * input_width, -(mean_values_[c]),
input_data + offset);
}
}
}
}
Dtype* transformed_data = transformed_blob->mutable_cpu_data();
for (int n = 0; n < input_num; ++n) {
int top_index_n = n * channels;
int data_index_n = n * channels;
for (int c = 0; c < channels; ++c) {
int top_index_c = (top_index_n + c) * height;
int data_index_c = (data_index_n + c) * input_height + h_off;
for (int h = 0; h < height; ++h) {
int top_index_h = (top_index_c + h) * width;
int data_index_h = (data_index_c + h) * input_width + w_off;
if (do_mirror) {
int top_index_w = top_index_h + width - 1;
for (int w = 0; w < width; ++w) {
transformed_data[top_index_w-w] = input_data[data_index_h + w];
}
} else {
for (int w = 0; w < width; ++w) {
transformed_data[top_index_h + w] = input_data[data_index_h + w];
}
}
}
}
}
if (scale != Dtype(1)) {
DLOG(INFO) << "Scale: " << scale;
caffe_scal(size, scale, transformed_data);
}
}
void DeconvNormLayer<Dtype>::Backward_cpu(const vector<Blob<Dtype>*>& top,
const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom)
{
const Dtype* top_diff = top[0]->cpu_diff();
Dtype* deconv1_top_vec_diff = deconv1_top_vec[0]->mutable_cpu_diff();
Dtype* deconv2_top_vec_diff = deconv2_top_vec[0]->mutable_cpu_diff();
const Dtype* deconv2_top_vec_data = deconv2_top_vec[0]->cpu_data();
const Dtype* deconv1_top_vec_data = deconv1_top_vec[0]->cpu_data();
caffe_set(deconv2_top_vec[0]->count(), (Dtype)0, deconv2_top_vec_diff);
caffe_set(deconv1_top_vec[0]->count(), (Dtype)0, deconv1_top_vec_diff);
caffe_set(exp_top_vec[0]->count(), (Dtype)0, exp_top_vec[0]->mutable_cpu_diff());
//caffe_set(exp_bottom_vec[0]->count(), (Dtype)0, exp_bottom_vec[0]->mutable_cpu_diff());
caffe_set(deconv1_layer->blobs()[0]->count(), (Dtype)0, deconv1_layer->blobs()[0]->mutable_cpu_diff());
caffe_set(deconv2_layer->blobs()[0]->count(), (Dtype)0, deconv2_layer->blobs()[0]->mutable_cpu_diff());
//bias gradient, if necessary
if (this->bias_term_ && this->param_propagate_down_[2])
{
Dtype* bias_diff = this->blobs_[2]->mutable_cpu_diff();
for (int n = 0; n < top[0]->num(); ++n)
{
caffe_cpu_gemv<Dtype>(CblasNoTrans, top[0]->channels(), top[0]->height() * top[0]->width(),
1., top_diff+top[0]->offset(n), bias_multiplier.cpu_data(), 1., bias_diff);
}
}
// weights and alpha gradient, propagate down to bottom
if (param_propagate_down_[0] || param_propagate_down_[1] || propagate_down[0])
{
vector<bool> no_propagate_down;
no_propagate_down.push_back(false);
vector<bool> yes_propagate_down;
yes_propagate_down.push_back(true);
// top_diff backward to deconv2_top_vec_diff
for (int n = 0; n < top[0]->num(); ++n)
{
caffe_div(deconv1_top_vec[0]->count(), top_diff + top[0]->offset(n),
deconv1_top_vec_data, deconv2_top_vec_diff + deconv2_top_vec[0]->offset(n));
}
// backward throud deconv2_layer
deconv2_layer->Backward(deconv2_top_vec, propagate_down, bottom);
const Dtype* wa_diff = weights_alphas->cpu_diff();
// weight gradient
if (param_propagate_down_[0])
{
Dtype* weight_diff = this->blobs_[0]->mutable_cpu_diff();
const Dtype* alpha = alphas->cpu_data();
for (int ch_in = 0; ch_in < weights_alphas->num(); ++ch_in)
{
caffe_mul(alphas->count(), wa_diff + weights_alphas->offset(ch_in),
alpha, weight_diff + this->blobs_[0]->offset(ch_in));
}
}
// alpha gradient
if (param_propagate_down_[1] && average_train)
{
//alpha_diff1
Dtype* alpha_cache_diff = alpha_cache.mutable_cpu_diff();
Dtype* alpha_cache_diff2 = alpha_cache2.mutable_cpu_diff();
caffe_set(alpha_cache.count(), (Dtype)0, alpha_cache_diff);
caffe_set(alpha_cache2.count(), (Dtype)0, alpha_cache_diff2);
const Dtype* weight = this->blobs_[0]->cpu_data();
for (int ch_in = 0; ch_in < weights_alphas->num(); ++ch_in)
{
caffe_mul(alpha_cache.count(), wa_diff + weights_alphas->offset(ch_in),
weight + this->blobs_[0]->offset(ch_in), alpha_cache_diff);
caffe_add(alpha_cache2.count(), alpha_cache_diff, alpha_cache_diff2, alpha_cache_diff2);
}
// top_diff backward to deonv1_top_vec_diff
Dtype* deconv1_top_cache_diff = deconv1_top_cache.mutable_cpu_diff();
caffe_set(deconv1_top_cache.count(), (Dtype)0, deconv1_top_cache_diff);
for (int n = 0; n < top[0]->num(); ++n)
{
caffe_mul(deconv1_top_cache.count(), top_diff + top[0]->offset(n),
deconv2_top_vec_data + deconv2_top_vec[0]->offset(n), deconv1_top_cache_diff);
caffe_add(deconv1_top_cache.count(), deconv1_top_cache_diff, deconv1_top_vec_diff, deconv1_top_vec_diff);
}
caffe_div(deconv1_top_cache.count(), deconv1_top_vec_diff,
deconv1_top_vec_data, deconv1_top_vec_diff);
caffe_div(deconv1_top_cache.count(), deconv1_top_vec_diff,
deconv1_top_vec_data, deconv1_top_vec_diff);
// backward through deconv1_layer
deconv1_layer->Backward(deconv1_top_vec, no_propagate_down, deconv1_bottom_vec);
// alpha_diff2
Dtype* alpha_diff = alphas->mutable_cpu_diff();
//fuse alpha_diff1 and alpha_diff2
caffe_sub(alpha_cache.count(), alpha_cache_diff2, alpha_diff, alpha_diff);
exp_layer->Backward(exp_top_vec, yes_propagate_down, exp_bottom_vec);
}
}
}
void DataTransformer<Dtype>::Transform(Blob<Dtype>* input_blob,
Blob<Dtype>* transformed_blob) {
const int crop_size = param_.crop_size();
const int input_num = input_blob->num();
const int input_channels = input_blob->channels();
const int input_height = input_blob->height();
const int input_width = input_blob->width();
if (transformed_blob->count() == 0) {
// Initialize transformed_blob with the right shape.
if (crop_size) {
transformed_blob->Reshape(input_num, input_channels,
crop_size, crop_size);
} else {
transformed_blob->Reshape(input_num, input_channels,
input_height, input_width);
}
}
const int num = transformed_blob->num();
const int channels = transformed_blob->channels();
const int height = transformed_blob->height();
const int width = transformed_blob->width();
const int size = transformed_blob->count();
CHECK_LE(input_num, num);
CHECK_EQ(input_channels, channels);
CHECK_GE(input_height, height);
CHECK_GE(input_width, width);
const Dtype scale = param_.scale();
const bool do_mirror = param_.mirror() && Rand(2);
const bool has_mean_file = param_.has_mean_file();
const bool has_mean_values = mean_values_.size() > 0;
// mask_size is defaulted to 0 in caffe/proto/caffe.proto
const int mask_size = param_.mask_size();
// mask_freq is defaulted to 1 in 3 in caffe/proto/caffe.proto
const int mask_freq = param_.mask_freq();
int h_off = 0;
int w_off = 0;
if (crop_size) {
CHECK_EQ(crop_size, height);
CHECK_EQ(crop_size, width);
// We only do random crop when we do training.
if (phase_ == TRAIN) {
h_off = Rand(input_height - crop_size + 1);
w_off = Rand(input_width - crop_size + 1);
} else {
h_off = (input_height - crop_size) / 2;
w_off = (input_width - crop_size) / 2;
}
} else {
CHECK_EQ(input_height, height);
CHECK_EQ(input_width, width);
}
// initialize masking offsets to be same as cropping offsets
// so that there is no conflict
bool masking = (phase_ == TRAIN) && (mask_size > 0) && (Rand(mask_freq) == 0);
int h_mask_start = h_off;
int w_mask_start = w_off;
if (masking) {
int h_effective = input_height;
int w_effective = input_width;
if (crop_size) { h_effective = w_effective = crop_size; }
CHECK_GE(h_effective, mask_size);
CHECK_GE(w_effective, mask_size);
h_mask_start += Rand(h_effective-mask_size+1);
w_mask_start += Rand(w_effective-mask_size+1);
}
int h_mask_end = h_mask_start + mask_size;
int w_mask_end = w_mask_start + mask_size;
Dtype* input_data = input_blob->mutable_cpu_data();
if (has_mean_file) {
CHECK_EQ(input_channels, data_mean_.channels());
CHECK_EQ(input_height, data_mean_.height());
CHECK_EQ(input_width, data_mean_.width());
for (int n = 0; n < input_num; ++n) {
int offset = input_blob->offset(n);
caffe_sub(data_mean_.count(), input_data + offset,
data_mean_.cpu_data(), input_data + offset);
}
}
if (has_mean_values) {
CHECK(mean_values_.size() == 1 || mean_values_.size() == input_channels) <<
"Specify either 1 mean_value or as many as channels: " << input_channels;
if (mean_values_.size() == 1) {
caffe_add_scalar(input_blob->count(), -(mean_values_[0]), input_data);
} else {
for (int n = 0; n < input_num; ++n) {
for (int c = 0; c < input_channels; ++c) {
int offset = input_blob->offset(n, c);
caffe_add_scalar(input_height * input_width, -(mean_values_[c]),
input_data + offset);
}
}
}
}
Dtype* transformed_data = transformed_blob->mutable_cpu_data();
for (int n = 0; n < input_num; ++n) {
int top_index_n = n * channels;
int data_index_n = n * channels;
for (int c = 0; c < channels; ++c) {
int top_index_c = (top_index_n + c) * height;
int data_index_c = (data_index_n + c) * input_height + h_off;
for (int h = 0; h < height; ++h) {
int top_index_h = (top_index_c + h) * width;
int data_index_h = (data_index_c + h) * input_width + w_off;
if (do_mirror) {
int top_index_w = top_index_h + width - 1;
for (int w = 0; w < width; ++w) {
if (masking && (h > h_mask_start) && (w > w_mask_start) &&
(h < h_mask_end) && (w < w_mask_end)) {
transformed_data[top_index_w-w] = 0;
} else {
transformed_data[top_index_w-w] = input_data[data_index_h + w];
}
}
} else {
for (int w = 0; w < width; ++w) {
if (masking && (h > h_mask_start) && (w > w_mask_start) &&
(h < h_mask_end) && (w < w_mask_end)) {
transformed_data[top_index_h + w] = 0;
} else {
transformed_data[top_index_h + w] = input_data[data_index_h + w];
}
}
}
}
}
}
if (scale != Dtype(1)) {
DLOG(INFO) << "Scale: " << scale;
caffe_scal(size, scale, transformed_data);
}
}
void TripletClipHingeLossLayer<Dtype>::Backward_cpu(
const vector<Blob<Dtype>*>& top, const vector<bool> &propagate_down,
const vector<Blob<Dtype>*>& bottom){
const Dtype* orignalcode;
const Dtype* similarcode;
const Dtype* diffrcode;
if (propagate_down[0]) {
for (int i = 0; i < 3; ++i) {// for each stream need to get a loss
int num = bottom[i]->num();
int channels = bottom[i]->channels();
for (int j = 0; j < num; ++j){
Dtype* bout = bottom[i]->mutable_cpu_diff();// get the 3 bottoms' address, the i th bottom's address
orignalcode = ave_or.cpu_data() + (j / frame_num)*dim;
similarcode = ave_si.cpu_data() + (j / frame_num)*dim;
diffrcode = ave_di.cpu_data() + (j / frame_num)*dim;
if (i == 0){
if (dist_sq_.cpu_data()[j / frame_num]>Dtype(FLT_MIN)){
caffe_sub(dim, diffrcode, similarcode,
gradient_triplet.mutable_cpu_data());// the distance of F- and F+
caffe_scal(dim, Dtype(2) / Dtype(num),
gradient_triplet.mutable_cpu_data());
}
else
caffe_set(dim, Dtype(FLT_MIN),
gradient_triplet.mutable_cpu_data());
compute_gradient_structure(i, j);
caffe_scal(dim, lamda, gradient_triplet.mutable_cpu_data());
caffe_scal(dim, Dtype(1.0) - lamda, gradient_structure.mutable_cpu_data());
caffe_add(dim, gradient_triplet.cpu_data(),
gradient_structure.cpu_data(), gradient.mutable_cpu_data());
}
if (i == 1){
if (dist_sq_.cpu_data()[j / frame_num] > Dtype(FLT_MIN)){
caffe_sub(dim, similarcode, orignalcode,
gradient_triplet.mutable_cpu_data());// the distance of F+ and F
caffe_scal(dim, Dtype(2) / Dtype(num),
gradient_triplet.mutable_cpu_data());
}
else
caffe_set(dim, Dtype(FLT_MIN),
gradient_triplet.mutable_cpu_data());
compute_gradient_structure(i, j);
caffe_scal(dim, lamda, gradient_triplet.mutable_cpu_data());
caffe_scal(dim, Dtype(1.0) - lamda, gradient_structure.mutable_cpu_data());
caffe_add(dim, gradient_triplet.cpu_data(),
gradient_structure.cpu_data(), gradient.mutable_cpu_data());
}
if (i == 2){
if (dist_sq_.cpu_data()[j / frame_num] > Dtype(FLT_MIN)){
caffe_sub(dim, orignalcode, diffrcode,
gradient_triplet.mutable_cpu_data());
caffe_scal(dim, Dtype(2) / Dtype(num),
gradient_triplet.mutable_cpu_data());
}
else
caffe_set(dim, Dtype(FLT_MIN),
gradient_triplet.mutable_cpu_data());
compute_gradient_structure(i, j);
caffe_scal(dim, lamda, gradient_triplet.mutable_cpu_data());
caffe_scal(dim, Dtype(1.0) - lamda, gradient_structure.mutable_cpu_data());
caffe_add(dim, gradient_triplet.cpu_data(),
gradient_structure.cpu_data(), gradient.mutable_cpu_data());
}
caffe_scal(dim, Dtype(2.0), gradient.mutable_cpu_data());
caffe_copy(channels, gradient.cpu_data(), bout + (j*channels));//return the BP vector to the j th batch's bottom
}
}
}
}
void BNLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
const Dtype* bottom_data = bottom[0]->cpu_data();
Dtype* top_data = top[0]->mutable_cpu_data();
const Dtype* const_top_data = top[0]->cpu_data();
const Dtype* scale_data = this->blobs_[0]->cpu_data();
const Dtype* shift_data = this->blobs_[1]->cpu_data();
switch (this->layer_param_.bn_param().bn_mode()) {
case BNParameter_BNMode_LEARN:
// put the squares of bottom into buffer_blob_
caffe_powx(bottom[0]->count(), bottom_data, Dtype(2),
buffer_blob_.mutable_cpu_data());
// computes variance using var(X) = E(X^2) - (EX)^2
// EX across spatial
caffe_cpu_gemv<Dtype>(CblasNoTrans, N_ * C_, H_ * W_,
Dtype(1. / (H_ * W_)), bottom_data,
spatial_sum_multiplier_.cpu_data(), Dtype(0),
spatial_mean_.mutable_cpu_data());
// EX across batch
caffe_cpu_gemv<Dtype>(CblasTrans, N_, C_, Dtype(1. / N_),
spatial_mean_.cpu_data(),
batch_sum_multiplier_.cpu_data(), Dtype(0),
batch_mean_.mutable_cpu_data());
// E(X^2) across spatial
caffe_cpu_gemv<Dtype>(CblasNoTrans, N_ * C_, H_ * W_,
Dtype(1. / (H_ * W_)), buffer_blob_.cpu_data(),
spatial_sum_multiplier_.cpu_data(), Dtype(0),
spatial_variance_.mutable_cpu_data());
// E(X^2) across batch
caffe_cpu_gemv<Dtype>(CblasTrans, N_, C_, Dtype(1. / N_),
spatial_variance_.cpu_data(),
batch_sum_multiplier_.cpu_data(), Dtype(0),
batch_variance_.mutable_cpu_data());
caffe_powx(batch_mean_.count(), batch_mean_.cpu_data(), Dtype(2),
buffer_blob_.mutable_cpu_data()); // (EX)^2
caffe_sub(batch_mean_.count(), batch_variance_.cpu_data(),
buffer_blob_.cpu_data(),
batch_variance_.mutable_cpu_data()); // variance
// save top[1] (batch_mean) and top[2] (batch_variance)
if (top.size() > 1) {
caffe_copy(batch_mean_.count(), batch_mean_.cpu_data(),
top[1]->mutable_cpu_data());
}
if (top.size() > 2) {
caffe_copy(batch_variance_.count(), batch_variance_.cpu_data(),
top[2]->mutable_cpu_data());
}
// do mean and variance normalization
// subtract mean
caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, N_,
C_, 1, Dtype(1),
batch_sum_multiplier_.cpu_data(),
batch_mean_.cpu_data(), Dtype(0),
spatial_mean_.mutable_cpu_data());
caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, N_ * C_,
H_ * W_, 1, Dtype(-1),
spatial_mean_.cpu_data(),
spatial_sum_multiplier_.cpu_data(), Dtype(0),
buffer_blob_.mutable_cpu_data());
caffe_add(buffer_blob_.count(), bottom_data,
buffer_blob_.cpu_data(), top_data);
// normalize variance
caffe_add_scalar(batch_variance_.count(), var_eps_,
batch_variance_.mutable_cpu_data());
caffe_powx(batch_variance_.count(),
batch_variance_.cpu_data(), Dtype(0.5),
batch_variance_.mutable_cpu_data());
caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, N_,
C_, 1, Dtype(1),
batch_sum_multiplier_.cpu_data(),
batch_variance_.cpu_data(), Dtype(0),
spatial_variance_.mutable_cpu_data());
caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans,
N_ * C_, H_ * W_, 1, Dtype(1),
spatial_variance_.cpu_data(),
spatial_sum_multiplier_.cpu_data(), Dtype(0),
buffer_blob_.mutable_cpu_data());
caffe_div(buffer_blob_.count(), const_top_data,
buffer_blob_.cpu_data(), top_data);
// Saving x_norm
caffe_copy(buffer_blob_.count(), const_top_data,
x_norm_.mutable_cpu_data());
// scale
caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, N_, C_, 1, Dtype(1),
batch_sum_multiplier_.cpu_data(), scale_data, Dtype(0),
spatial_variance_.mutable_cpu_data());
caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, N_ * C_,
H_ * W_, 1, Dtype(1),
spatial_variance_.cpu_data(),
spatial_sum_multiplier_.cpu_data(), Dtype(0),
buffer_blob_.mutable_cpu_data());
caffe_mul(buffer_blob_.count(), top_data,
buffer_blob_.cpu_data(), top_data);
// shift
caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, N_, C_, 1, Dtype(1),
batch_sum_multiplier_.cpu_data(), shift_data, Dtype(0),
spatial_mean_.mutable_cpu_data());
caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans,
N_ * C_, H_ * W_, 1, Dtype(1),
spatial_mean_.cpu_data(),
spatial_sum_multiplier_.cpu_data(), Dtype(0),
buffer_blob_.mutable_cpu_data());
caffe_add(buffer_blob_.count(), const_top_data,
buffer_blob_.cpu_data(), top_data);
break;
case BNParameter_BNMode_INFERENCE:
// scale
caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, N_, C_, 1, Dtype(1),
batch_sum_multiplier_.cpu_data(), scale_data, Dtype(0),
spatial_variance_.mutable_cpu_data());
caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, N_ * C_,
H_ * W_, 1, Dtype(1),
spatial_variance_.cpu_data(),
spatial_sum_multiplier_.cpu_data(), Dtype(0),
buffer_blob_.mutable_cpu_data());
caffe_mul(buffer_blob_.count(), bottom_data,
buffer_blob_.cpu_data(), top_data);
// shift
caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, N_, C_, 1, Dtype(1),
batch_sum_multiplier_.cpu_data(), shift_data, Dtype(0),
spatial_mean_.mutable_cpu_data());
caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans,
N_ * C_, H_ * W_, 1, Dtype(1),
spatial_mean_.cpu_data(),
spatial_sum_multiplier_.cpu_data(), Dtype(0),
buffer_blob_.mutable_cpu_data());
caffe_add(buffer_blob_.count(), const_top_data,
buffer_blob_.cpu_data(), top_data);
break;
default:
LOG(FATAL) << "Unknown BN mode.";
}
}
void MVNLayer<Dtype>::Backward_cpu(const vector<Blob<Dtype>*>& top,
const vector<bool>& propagate_down,
vector<Blob<Dtype>*>* bottom) {
const Dtype* top_diff = top[0]->cpu_diff();
const Dtype* top_data = top[0]->cpu_data();
const Dtype* bottom_data = (*bottom)[0]->cpu_data();
Dtype* bottom_diff = (*bottom)[0]->mutable_cpu_diff();
int num;
if (this->layer_param_.mvn_param().across_channels())
num = (*bottom)[0]->num();
else
num = (*bottom)[0]->num() * (*bottom)[0]->channels();
int dim = (*bottom)[0]->count() / num;
Dtype eps = 1e-10;
if (this->layer_param_.mvn_param().normalize_variance()) {
caffe_mul(temp_.count(), top_data, top_diff, bottom_diff);
caffe_cpu_gemv<Dtype>(CblasNoTrans, num, dim, 1., bottom_diff,
sum_multiplier_.cpu_data(), 0., mean_.mutable_cpu_data());
caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num, dim, 1, 1.,
mean_.cpu_data(), sum_multiplier_.cpu_data(), 0.,
bottom_diff);
caffe_mul(temp_.count(), top_data, bottom_diff, bottom_diff);
caffe_cpu_gemv<Dtype>(CblasNoTrans, num, dim, 1., top_diff,
sum_multiplier_.cpu_data(), 0., mean_.mutable_cpu_data());
caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num, dim, 1, 1.,
mean_.cpu_data(), sum_multiplier_.cpu_data(), 1.,
bottom_diff);
caffe_cpu_axpby(temp_.count(), Dtype(1), top_diff, Dtype(-1. / dim),
bottom_diff);
// put the squares of bottom into temp_
caffe_powx(temp_.count(), bottom_data, Dtype(2),
temp_.mutable_cpu_data());
// computes variance using var(X) = E(X^2) - (EX)^2
caffe_cpu_gemv<Dtype>(CblasNoTrans, num, dim, 1. / dim, bottom_data,
sum_multiplier_.cpu_data(), 0., mean_.mutable_cpu_data()); // EX
caffe_cpu_gemv<Dtype>(CblasNoTrans, num, dim, 1. / dim, temp_.cpu_data(),
sum_multiplier_.cpu_data(), 0.,
variance_.mutable_cpu_data()); // E(X^2)
caffe_powx(mean_.count(), mean_.cpu_data(), Dtype(2),
temp_.mutable_cpu_data()); // (EX)^2
caffe_sub(mean_.count(), variance_.cpu_data(), temp_.cpu_data(),
variance_.mutable_cpu_data()); // variance
// normalize variance
caffe_powx(variance_.count(), variance_.cpu_data(), Dtype(0.5),
variance_.mutable_cpu_data());
caffe_add_scalar(variance_.count(), eps, variance_.mutable_cpu_data());
caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num, dim, 1, 1.,
variance_.cpu_data(), sum_multiplier_.cpu_data(), 0.,
temp_.mutable_cpu_data());
caffe_div(temp_.count(), bottom_diff, temp_.cpu_data(), bottom_diff);
} else {
caffe_copy(temp_.count(), top_diff, bottom_diff);
}
}
void SparseDepthMahalanobisLossLayer<Dtype>::Forward_cpu(
const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
int count = bottom[0]->count();
int num = bottom[0]->num();
int height = bottom[0]->height();
int width = bottom[0]->width();
int spatial_count = height*width;
Dtype* diff = diff_.mutable_cpu_data();
const Dtype* label = bottom[1]->cpu_data();
// compute the difference
caffe_sub(
count,
bottom[0]->cpu_data(), // a
bottom[1]->cpu_data(), // b
diff_.mutable_cpu_data()); // a_i-b_i
// set diff_ = 0 if groundtruth data is missing
// the channel in the bottom labels blob == -10.0
// implies that the groundtruth data is missing
for (int n = 0; n < num; ++n)
{
for (int i = 0; i < spatial_count; ++i)
{
Dtype mask = *(label + bottom[1]->offset(n) + i);
if (mask == Dtype(MASK_VAL))
{
*(diff + bottom[1]->offset(n) + i) = Dtype(0);
}
}
}
if (bottom.size() >= 3) { // weighted distance
// TODO(NCB) is there a more efficient way to do this?
Dtype reg(0);
Dtype U;
for (int n = 0; n < num; ++n)
{
for (int h = 0; h < height; ++h)
{
for (int w = 0; w < width; ++w)
{
int offset = n*spatial_count + h*width + w;
Dtype mask = *(label + bottom[1]->offset(n,0,h,w));
if (mask == Dtype(MASK_VAL))
{
U = Dtype(1.0);
}
else U = fabs(bottom[2]->cpu_data()[offset]);
// Udiff
Udiff_.mutable_cpu_data()[offset] = U*diff_.cpu_data()[offset];
// UtUdiff
UtUdiff_.mutable_cpu_data()[offset] = U*Udiff_.cpu_data()[offset];
// compute regularizer
reg += log(U + Dtype(EPS));
} //for w
} //for h
} //for n
// difftUtUdiff
Dtype dot = caffe_cpu_dot(count, diff_.cpu_data(), UtUdiff_.cpu_data());
Dtype loss = dot / bottom[0]->num() / Dtype(2);
top[0]->mutable_cpu_data()[0] = loss - reg / bottom[0]->num();
} else { // unweighted distance
Dtype dot = caffe_cpu_dot(count, diff_.cpu_data(), diff_.cpu_data());
Dtype loss = dot / bottom[0]->num() / Dtype(2);
top[0]->mutable_cpu_data()[0] = loss;
}
}
void ContrastiveLossLayer<Dtype>::Forward_cpu(
const vector<Blob<Dtype>*>& bottom,
vector<Blob<Dtype>*>* top) {
int count = bottom[0]->count();
caffe_sub(
count,
bottom[0]->cpu_data(), // a
bottom[1]->cpu_data(), // b
diff_.mutable_cpu_data()); // a_i-b_i
const int channels = bottom[0]->channels();
/*
* margin refers to the maximum value of energy -- parameter Q in the paper
*/
printf("CLL : the values of a_i are \n");
for (int i = 0; i < bottom[0]->num(); ++i) {
for (int j = 0; j < channels; ++j) {
printf("%f \t ",(float) bottom[0]->cpu_data()[i*channels+j] );
}
}
printf("CLL : End printing values of a_i\n");
printf("CLL : the values of b_i are \n");
for (int i = 0; i < bottom[1]->num(); ++i) {
for (int j = 0; j < channels; ++j) {
printf("%f \t ",(float) bottom[1]->cpu_data()[i*channels+j] );
}
}
printf("CLL : End printing values of b_i\n");
printf("CLL : the diff values for the input vector are \n");
for(int temp = 0 ; temp < count ; temp++){
printf("%f \t ",(float) diff_.mutable_cpu_data()[temp] );
}
printf("CLL : End printing the diff values\n");
Dtype margin = this->layer_param_.contrastive_loss_param().margin();
//margin = Dtype(1000);
Dtype loss(0.0);
for (int i = 0; i < bottom[0]->num(); ++i) {
dist_sq_.mutable_cpu_data()[i] = caffe_cpu_asum(channels,
diff_.cpu_data() + (i*channels));
printf("CLL : values of L1 norm are , %f \n", (float) dist_sq_.mutable_cpu_data()[i] );
/*
* 1 is similar pair, 0 is impostor pair.
* The paper follows opposite notation
*/
printf("CLL: label : %d \n", bottom[2]->cpu_data()[i]);
if (static_cast<int>(bottom[2]->cpu_data()[i])) { // similar pairs
loss += Dtype(2) / margin * dist_sq_.cpu_data()[i] * dist_sq_.cpu_data()[i];
printf(" CLL: loss computed : %f\n", dist_sq_.cpu_data()[i]);
} else { // dissimilar pairs
//loss += std::max(margin-dist_sq_.cpu_data()[i], Dtype(0.0));
printf("CLL : the exponent of 1 is : %f \n",exp(Dtype(1)));
printf("CLL : the exponent of -1 is : %f \n", exp(Dtype(-1)));
loss += Dtype(2) * margin * exp(-Dtype(2.77) / margin * dist_sq_.cpu_data()[i]);
printf(" CLL: loss computed : %f\n", dist_sq_.cpu_data()[i]);
}
printf("CLL: value of label : %d \n", static_cast<int>(bottom[2]->cpu_data()[i]));
printf("CLL: value of margin : %f \n", (float) margin);
}
loss = loss / static_cast<Dtype>(bottom[0]->num());
printf("CLL: value of loss : %f \n", loss);
(*top)[0]->mutable_cpu_data()[0] = loss;
}
void CoupledClusterLossLayer<Dtype>::Forward_cpu(
const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
pos_ids = std::vector<std::vector<int> >(group_num, std::vector<int>());
neg_ids = std::vector<std::vector<int> >(group_num, std::vector<int>());
pos_backward = std::vector<bool>(group_num*N, false);
neg_backward = std::vector<bool>(group_num*N, false);
const Dtype *feat_ptr = bottom[0]->cpu_data();
const Dtype *label_ptr = bottom[1]->cpu_data();
Dtype *diff_ptr_ = diff_.mutable_cpu_data();
Dtype loss(0);
caffe_set(feat_len*group_num, Dtype(0), pos_center_.mutable_cpu_data());
int cnt = 0;
/* i -> group index */
for(int i=0; i<group_num; ++i) {
/* search for the positive id */
std::set<Dtype> labels;
Dtype anchor_id = -1;
for(int j=0; j<N; ++j) {
Dtype tmp = label_ptr[N*i+j];
if(labels.count(tmp)>0) {
anchor_id = tmp;
break;
}
else
labels.insert(tmp);
}
// CHECK_NE(anchor_id, -1);
/* collect for positive and negative ids, compute the center of positive samples */
for(int j=0; j<N; ++j) {
if(label_ptr[i*N+j]==anchor_id){
pos_ids[i].push_back(j);
caffe_add(feat_len, feat_ptr+feat_len*(i*N+j), pos_center_.mutable_cpu_data()+feat_len*i, pos_center_.mutable_cpu_data()+feat_len*i);
}
else neg_ids[i].push_back(j);
}
caffe_cpu_scale(feat_len, Dtype(1)/pos_ids[i].size(), pos_center_.mutable_cpu_data()+feat_len*i, pos_center_.mutable_cpu_data()+feat_len*i);
if(neg_ids[i].size()==0 || pos_ids[i].size()<=1) continue;
Dtype pos_mdist = Dtype(0);
Dtype neg_min_val = -1;
Dtype pos_max_val = -1;
for(int j=0; j<N; ++j) {
// f[j]-center
caffe_sub(feat_len, feat_ptr+feat_len*(i*N+j), pos_center_.cpu_data()+feat_len*i, diff_ptr_+feat_len*(i*N+j));
if(scale!=1)
caffe_cpu_scale(feat_len, scale, diff_ptr_+feat_len*(i*N+j), diff_ptr_+feat_len*(i*N+j));
Dtype d = caffe_cpu_dot(feat_len, diff_ptr_+feat_len*(i*N+j), diff_ptr_+feat_len*(i*N+j));
if(log_flag)
LOG(INFO) << "i " << i << ", j " << j << ", d " << d;
dist_sq_.mutable_cpu_data()[i*N+j] = d;
if(std::count(neg_ids[i].begin(), neg_ids[i].end(), j)>0 && (neg_min_val==-1 || d<neg_min_val)) neg_min_val = d;
else if(std::count(neg_ids[i].begin(), neg_ids[i].end(), j)==0 && (pos_max_val==-1 || d>pos_max_val)) pos_max_val = d;
}
for(int j=0; j<N; ++j) {
if(std::count(neg_ids[i].begin(), neg_ids[i].end(), j)>0) {
Dtype d = dist_sq_.cpu_data()[i*N+j];
Dtype mdist = std::max(-d+margin+pos_max_val, Dtype(0));
if(log_flag)
LOG(INFO) << "j=" << j << ", d=" << d << ", pos_max_val=" << pos_max_val << ", mdist=" << mdist;
if(mdist>0) neg_backward[i*N+j] = true;
}
else {
Dtype d = dist_sq_.cpu_data()[i*N+j];
Dtype mdist = std::max(d+margin-neg_min_val, Dtype(0));
if(log_flag)
LOG(INFO) << "j=" << j << ", d=" << d << ", neg_min_val=" << neg_min_val << ", mdist=" << mdist;
if(mdist>0) pos_backward[i*N+j] = true;
pos_mdist += mdist;
}
}
/* average punishment */
pos_mdist /= pos_ids[i].size();
// pos_mdist *= 2;
if(log_flag)
LOG(INFO) << "pos_mdist " << pos_mdist << ", neg_min_val " << neg_min_val;
CHECK_GE(pos_ids[i].size(), 2);
CHECK_GE(neg_ids[i].size(), 1);
loss += pos_mdist;
++cnt;
}
loss = loss / cnt;
top[0]->mutable_cpu_data()[0] = loss;
}
