void SampleBBox(const Sampler& sampler, NormalizedBBox* sampled_bbox) {
// Get random scale.
CHECK_GE(sampler.max_scale(), sampler.min_scale());
CHECK_GT(sampler.min_scale(), 0.);
CHECK_LE(sampler.max_scale(), 1.);
float scale;
caffe_rng_uniform(1, sampler.min_scale(), sampler.max_scale(), &scale);
// Get random aspect ratio.
CHECK_GE(sampler.max_aspect_ratio(), sampler.min_aspect_ratio());
CHECK_GT(sampler.min_aspect_ratio(), 0.);
CHECK_LT(sampler.max_aspect_ratio(), FLT_MAX);
float aspect_ratio;
float min_aspect_ratio = std::max<float>(sampler.min_aspect_ratio(),
std::pow(scale, 2.));
float max_aspect_ratio = std::min<float>(sampler.max_aspect_ratio(),
1 / std::pow(scale, 2.));
caffe_rng_uniform(1, min_aspect_ratio, max_aspect_ratio, &aspect_ratio);
// Figure out bbox dimension.
float bbox_width = scale * sqrt(aspect_ratio);
float bbox_height = scale / sqrt(aspect_ratio);
// Figure out top left coordinates.
float w_off, h_off;
caffe_rng_uniform(1, 0.f, 1 - bbox_width, &w_off);
caffe_rng_uniform(1, 0.f, 1 - bbox_height, &h_off);
sampled_bbox->set_xmin(w_off);
sampled_bbox->set_ymin(h_off);
sampled_bbox->set_xmax(w_off + bbox_width);
sampled_bbox->set_ymax(h_off + bbox_height);
}
EntropyTLossLayerTest()
: blob_bottom_data_(new Blob<Dtype>(M_, 1, 1, K_)),
blob_bottom_label_(new Blob<Dtype>(M_, 1, 1, N_)),
blob_top_loss_(new Blob<Dtype>()),
blob_top_std_(new Blob<Dtype>()),
blob_top_ind_(new Blob<Dtype>()),
blob_top_dist_(new Blob<Dtype>()) {
// fill the values
FillerParameter filler_param;
filler_param.set_value(0);
GaussianFiller<Dtype> filler(filler_param);
filler.Fill(this->blob_bottom_data_);
Dtype *cpu_label = blob_bottom_label_->mutable_cpu_data();
caffe_rng_uniform(blob_bottom_label_->count(), Dtype(0), Dtype(1), cpu_label);
for (int i = 0; i < M_; i++) {
Dtype norm = Dtype(0);
for (int j = 0; j < N_; j++) {
norm += cpu_label[i*N_+j];
}
for (int j = 0; j < N_; j++) {
cpu_label[i*N_+j] /= norm;
}
}
blob_bottom_vec_.push_back(blob_bottom_data_);
blob_bottom_vec_.push_back(blob_bottom_label_);
blob_top_vec_.push_back(blob_top_loss_);
blob_top_vec_.push_back(blob_top_std_);
blob_top_vec_.push_back(blob_top_ind_);
blob_top_vec_.push_back(blob_top_dist_);
}
void TestForward() {
// Get the loss without a specified objective weight -- should be
// equivalent to explicitly specifiying a weight of 1.
LayerParameter layer_param;
layer_param.mutable_multi_t_loss_param()->set_num_center(N_);
EntropyTLossLayer<Dtype> layer_weight_1(layer_param);
layer_weight_1.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_);
layer_weight_1.Forward(this->blob_bottom_vec_, &this->blob_top_vec_);
FillerParameter filler_param;
GaussianFiller<Dtype> filler2(filler_param);
filler2.Fill(layer_weight_1.blobs()[0].get());
caffe_rng_uniform(layer_weight_1.blobs()[1]->count(), Dtype(0.9), Dtype(1.1), layer_weight_1.blobs()[1]->mutable_cpu_data());
layer_weight_1.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_);
const Dtype loss_weight_1 =
layer_weight_1.Forward(this->blob_bottom_vec_, &this->blob_top_vec_);
// Get the loss again with a different objective weight; check that it is
// scaled appropriately.
const Dtype kLossWeight = 3.7;
LayerParameter layer_param2;
layer_param2.mutable_multi_t_loss_param()->set_num_center(N_);
layer_param2.add_loss_weight(kLossWeight);
EntropyTLossLayer<Dtype> layer_weight_2(layer_param2);
layer_weight_2.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_);
layer_weight_2.Forward(this->blob_bottom_vec_, &this->blob_top_vec_);
caffe_copy(layer_weight_2.blobs()[0]->count(), layer_weight_1.blobs()[0]->cpu_data(),
layer_weight_2.blobs()[0]->mutable_cpu_data());
caffe_copy(layer_weight_2.blobs()[1]->count(), layer_weight_1.blobs()[1]->cpu_data(),
layer_weight_2.blobs()[1]->mutable_cpu_data());
layer_weight_2.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_);
const Dtype loss_weight_2 =
layer_weight_2.Forward(this->blob_bottom_vec_, &this->blob_top_vec_);
const Dtype kErrorMargin = 1e-3;
EXPECT_NEAR(loss_weight_1 * kLossWeight, loss_weight_2, kErrorMargin);
// Make sure the loss is non-trivial.
const Dtype kNonTrivialAbsThresh = 1e-1;
EXPECT_GE(fabs(loss_weight_1), kNonTrivialAbsThresh);
int m = M_, n = layer_param.multi_t_loss_param().num_center(), p = K_;
Blob<Dtype> *distance = layer_weight_1.distance();
const Dtype *cpu_data = blob_bottom_data_->cpu_data();
const Dtype *cpu_dist = distance->cpu_data();
const Dtype *cpu_center = layer_weight_1.blobs()[0]->cpu_data();
const Dtype *cpu_sigma = layer_weight_1.blobs()[1]->cpu_data();
for (int i = 0; i < m; ++i) {
for (int j = 0; j < n; ++j) {
Dtype acc = Dtype(0);
for (int k = 0; k < p; ++k) {
acc += (cpu_data[i*p + k] - cpu_center[k*n + j])*(cpu_data[i*p + k] - cpu_center[k*n + j])*cpu_sigma[k*n+j];
}
EXPECT_NEAR(acc, cpu_dist[i*n + j], kErrorMargin) << i << " " << j;
}
}
}
void DropoutLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
const Dtype* bottom_data = bottom[0]->cpu_data();
Dtype* top_data = top[0]->mutable_cpu_data();
Dtype* mask = rand_vec_->mutable_cpu_data();
const int count = rand_vec_->count();
if (this->phase_ == TRAIN) {
switch (drop_type_){
case DropoutParameter_DropType_BERNOULLI:
{
// Create random numbers
caffe_rng_bernoulli(count, Dtype(1. - threshold_), mask);
break;
}
case DropoutParameter_DropType_GAUSSIAN:
{
caffe_rng_gaussian(count, Dtype(mu_), Dtype(sigma_), mask);
// clip to be in [0,1]
for (int i = 0; i < rand_vec_->count(); ++i){
Dtype m = mask[i];
mask[i] = m > 1 ? 1 : (m < 0 ? 0 : m);
}
break;
}
case DropoutParameter_DropType_UNIFORM:
{
caffe_rng_uniform(count, Dtype(a_), Dtype(b_), mask);
break;
}
}
if (drop_batch_){
Dtype drop = mask[0];
caffe_copy(top[0]->count(), bottom_data, top_data);
caffe_scal(top[0]->count(), Dtype(scale_ * drop), top_data);
}
else{
vector<Blob<Dtype>*> scale_bottom(2, NULL);
scale_bottom[0] = bottom[0];
scale_bottom[1] = rand_vec_;
const vector<Blob<Dtype>*> scale_top(1, top[0]);
scale_layer_->Forward(scale_bottom, scale_top);
caffe_scal(top[0]->count(), scale_, top_data);
}
} else {
caffe_copy(bottom[0]->count(), bottom_data, top_data);
}
}
void DataTransformer<Dtype>::Transform(const cv::Mat& img,
Blob<Dtype>* transformed_blob) {
const int min_side = param_.min_side();
const int min_side_min = param_.min_side_min();
const int min_side_max = param_.min_side_max();
const int crop_size = param_.crop_size();
const int rotation_angle = param_.max_rotation_angle();
const float min_contrast = param_.min_contrast();
const float max_contrast = param_.max_contrast();
const int max_brightness_shift = param_.max_brightness_shift();
const float max_smooth = param_.max_smooth();
const int max_color_shift = param_.max_color_shift();
const float apply_prob = 1.f - param_.apply_probability();
const bool debug_params = param_.debug_params();
// Check dimensions.
const int channels = transformed_blob->channels();
const int height = transformed_blob->height();
const int width = transformed_blob->width();
const int num = transformed_blob->num();
const Dtype scale = param_.scale();
const bool has_mean_file = param_.has_mean_file();
const bool has_mean_values = mean_values_.size() > 0;
float current_prob;
const bool do_rotation = rotation_angle > 0 && phase_ == TRAIN;
const bool do_resize_to_min_side = min_side > 0;
const bool do_resize_to_min_side_min = min_side_min > 0;
const bool do_resize_to_min_side_max = min_side_max > 0;
const bool do_mirror = param_.mirror() && phase_ == TRAIN && Rand(2);
caffe_rng_uniform(1, 0.f, 1.f, ¤t_prob);
const bool do_brightness = param_.contrast_brightness_adjustment() && phase_ == TRAIN && current_prob > apply_prob;
caffe_rng_uniform(1, 0.f, 1.f, ¤t_prob);
const bool do_smooth = param_.smooth_filtering() && phase_ == TRAIN && max_smooth > 1 && current_prob > apply_prob;
caffe_rng_uniform(1, 0.f, 1.f, ¤t_prob);
const bool do_color_shift = max_color_shift > 0 && phase_ == TRAIN && current_prob > apply_prob;
cv::Mat cv_img = img;
int current_angle = 0;
if (do_rotation) {
current_angle = Rand(rotation_angle*2 + 1) - rotation_angle;
if (current_angle)
rotate(cv_img, current_angle);
}
// resizing and crop according to min side, preserving aspect ratio
if (do_resize_to_min_side) {
resize(cv_img, min_side);
//random_crop(cv_img, min_side);
}
if (do_resize_to_min_side_min && do_resize_to_min_side_max) {
//std::cout << min_side_min << " "<<min_side_max<<std::endl;
int min_side_length = min_side_min + Rand(min_side_max - min_side_min + 1);
resize(cv_img, min_side_length);
//crop_center(cv_img, min_side, min_side);
//random_crop(cv_img, min_side_length);
}
// apply color shift
if (do_color_shift) {
int b = Rand(max_color_shift + 1);
int g = Rand(max_color_shift + 1);
int r = Rand(max_color_shift + 1);
int sign = Rand(2);
cv::Mat shiftArr = cv_img.clone();
shiftArr.setTo(cv::Scalar(b,g,r));
if (sign == 1) {
cv_img -= shiftArr;
} else {
cv_img += shiftArr;
}
}
// set contrast and brightness
float alpha;
int beta;
if (do_brightness){
caffe_rng_uniform(1, min_contrast, max_contrast, &alpha);
beta = Rand(max_brightness_shift * 2 + 1) - max_brightness_shift;
cv_img.convertTo(cv_img, -1, alpha, beta);
}
// set smoothness
int smooth_param = 0;
int smooth_type = 0;
if (do_smooth) {
smooth_type = Rand(4);
smooth_param = 1 + 2 * Rand(max_smooth/2);
switch (smooth_type) {
case 0:
cv::GaussianBlur(cv_img, cv_img, cv::Size(smooth_param, smooth_param), 0);
break;
case 1:
cv::blur(cv_img, cv_img, cv::Size(smooth_param, smooth_param));
break;
case 2:
cv::medianBlur(cv_img, cv_img, smooth_param);
break;
case 3:
cv::boxFilter(cv_img, cv_img, -1, cv::Size(smooth_param * 2, smooth_param * 2));
break;
default:
break;
}
}
if (debug_params && phase_ == TRAIN) {
LOG(INFO) << "----------------------------------------";
if (do_rotation) {
LOG(INFO) << "* parameter for rotation: ";
LOG(INFO) << " current rotation angle: " << current_angle;
}
if (do_brightness) {
LOG(INFO) << "* parameter for contrast adjustment: ";
LOG(INFO) << " alpha: " << alpha << ", beta: " << beta;
}
if (do_smooth) {
LOG(INFO) << "* parameter for smooth filtering: ";
LOG(INFO) << " smooth type: " << smooth_type << ", smooth param: " << smooth_param;
}
}
const int img_channels = cv_img.channels();
const int img_height = cv_img.rows;
const int img_width = cv_img.cols;
CHECK_GT(img_channels, 0);
CHECK_GE(img_height, crop_size);
CHECK_GE(img_width, crop_size);
CHECK_EQ(channels, img_channels);
CHECK_LE(height, img_height);
CHECK_LE(width, img_width);
CHECK_GE(num, 1);
CHECK(cv_img.depth() == CV_8U) << "Image data type must be unsigned byte";
Dtype* mean = NULL;
if (has_mean_file) {
CHECK_EQ(img_channels, data_mean_.channels());
CHECK_EQ(img_height, data_mean_.height());
CHECK_EQ(img_width, data_mean_.width());
mean = data_mean_.mutable_cpu_data();
}
if (has_mean_values) {
CHECK(mean_values_.size() == 1 || mean_values_.size() == img_channels) <<
"Specify either 1 mean_value or as many as channels: " << img_channels;
if (img_channels > 1 && mean_values_.size() == 1) {
// Replicate the mean_value for simplicity
for (int c = 1; c < img_channels; ++c) {
mean_values_.push_back(mean_values_[0]);
}
}
}
int h_off = 0;
int w_off = 0;
cv::Mat cv_cropped_img = cv_img;
if (crop_size) {
CHECK_EQ(crop_size, height);
CHECK_EQ(crop_size, width);
// We only do random crop when we do training.
if (phase_ == TRAIN) {
h_off = Rand(img_height - crop_size + 1);
w_off = Rand(img_width - crop_size + 1);
} else {
h_off = (img_height - crop_size) / 2;
w_off = (img_width - crop_size) / 2;
}
cv::Rect roi(w_off, h_off, crop_size, crop_size);
cv_cropped_img = cv_img(roi);
} else {
//CHECK_EQ(img_height, height);
//CHECK_EQ(img_width, width);
}
CHECK(cv_cropped_img.data);
Dtype* transformed_data = transformed_blob->mutable_cpu_data();
int top_index;
for (int h = 0; h < height; ++h) {
const uchar* ptr = cv_cropped_img.ptr<uchar>(h);
int img_index = 0;
for (int w = 0; w < width; ++w) {
for (int c = 0; c < img_channels; ++c) {
if (do_mirror) {
top_index = (c * height + h) * width + (width - 1 - w);
} else {
top_index = (c * height + h) * width + w;
}
// int top_index = (c * height + h) * width + w;
Dtype pixel = static_cast<Dtype>(ptr[img_index++]);
if (has_mean_file) {
int mean_index = (c * img_height + h_off + h) * img_width + w_off + w;
transformed_data[top_index] =
(pixel - mean[mean_index]) * scale;
} else {
if (has_mean_values) {
transformed_data[top_index] =
(pixel - mean_values_[c]) * scale;
} else {
transformed_data[top_index] = pixel * scale;
}
}
}
}
}
}
void RngUniformFill(const Dtype lower, const Dtype upper, void* cpu_data) {
CHECK_GE(upper, lower);
Dtype* rng_data = static_cast<Dtype*>(cpu_data);
caffe_rng_uniform(sample_size_, lower, upper, rng_data);
}
void Device::rng_uniform(const uint_tp n, vptr<uint64_t> r) {
vector<uint64_t> random(n); //NOLINT
caffe_rng_uniform(n, &random[0]);
this->memcpy(sizeof(uint64_t) * n, &random[0], vptr<void>(r));
}
void Device::rng_uniform_double(const uint_tp n, const double a,
const double b, vptr<double> r) {
vector<double> random(n); // NOLINT
caffe_rng_uniform(n, a, b, &random[0]);
this->memcpy(sizeof(double) * n, &random[0], vptr<void>(r));
}
void Device::rng_uniform_float(const uint_tp n, const float a,
const float b, vptr<float> r) {
vector<float> random(n); // NOLINT
caffe_rng_uniform(n, a, b, &random[0]);
this->memcpy(sizeof(float) * n, &random[0], vptr<void>(r));
}
void Device::rng_uniform_half(const uint_tp n, const half_fp a,
const half_fp b, vptr<half_fp> r) {
vector<half_fp> random(n); // NOLINT
caffe_rng_uniform(n, a, b, &random[0]);
this->memcpy(sizeof(half_fp) * n, &random[0], vptr<void>(r));
}
