void AnnotatedDataLayer<Dtype>::DataLayerSetUp(
const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
data_transformer_.reset(new SSDTransformer<Dtype>(this->transform_param_, this->phase_));
data_transformer_->InitRand();
const int batch_size = this->layer_param_.data_param().batch_size();
const AnnotatedDataParameter& anno_data_param =
this->layer_param_.annotated_data_param();
for (int i = 0; i < anno_data_param.batch_sampler_size(); ++i) {
batch_samplers_.push_back(anno_data_param.batch_sampler(i));
}
label_map_file_ = anno_data_param.label_map_file();
// Make sure dimension is consistent within batch.
const TransformationParameter& transform_param =
this->layer_param_.transform_param();
if (transform_param.has_resize_param()) {
if (transform_param.resize_param().resize_mode() ==
ResizeParameter_Resize_mode_FIT_SMALL_SIZE) {
CHECK_EQ(batch_size, 1)
<< "Only support batch size of 1 for FIT_SMALL_SIZE.";
}
}
// Read a data point, and use it to initialize the top blob.
//AnnotatedDatum& anno_datum = *(reader_.full().peek());
AnnotatedDatum anno_datum;
anno_datum.ParseFromString(cursor_->value());
// Use data_transformer to infer the expected blob shape from anno_datum.
vector<int> top_shape =
this->data_transformer_->InferBlobShape(anno_datum.datum());
this->transformed_data_.Reshape(top_shape);
// Reshape top[0] and prefetch_data according to the batch_size.
top_shape[0] = batch_size;
top[0]->Reshape(top_shape);
for (int i = 0; i < this->prefetch_.size(); ++i) {
this->prefetch_[i]->data_.Reshape(top_shape);
}
//LOG(INFO) << "output data size: " << top[0]->num() << ","
LOG_IF(INFO, Caffe::root_solver())
<< "output data size: " << top[0]->num() << ","
<< top[0]->channels() << "," << top[0]->height() << ","
<< top[0]->width();
// label
if (this->output_labels_) {
has_anno_type_ = anno_datum.has_type() || anno_data_param.has_anno_type();
vector<int> label_shape(4, 1);
if (has_anno_type_) {
anno_type_ = anno_datum.type();
if (anno_data_param.has_anno_type()) {
// If anno_type is provided in AnnotatedDataParameter, replace
// the type stored in each individual AnnotatedDatum.
LOG(WARNING) << "type stored in AnnotatedDatum is shadowed.";
anno_type_ = anno_data_param.anno_type();
}
// Infer the label shape from anno_datum.AnnotationGroup().
int num_bboxes = 0;
if (anno_type_ == AnnotatedDatum_AnnotationType_BBOX) {
// Since the number of bboxes can be different for each image,
// we store the bbox information in a specific format. In specific:
// All bboxes are stored in one spatial plane (num and channels are 1)
// And each row contains one and only one box in the following format:
// [item_id, group_label, instance_id, xmin, ymin, xmax, ymax, diff]
// Note: Refer to caffe.proto for details about group_label and
// instance_id.
for (int g = 0; g < anno_datum.annotation_group_size(); ++g) {
num_bboxes += anno_datum.annotation_group(g).annotation_size();
}
label_shape[0] = 1;
label_shape[1] = 1;
// BasePrefetchingDataLayer<Dtype>::LayerSetUp() requires to call
// cpu_data and gpu_data for consistent prefetch thread. Thus we make
// sure there is at least one bbox.
label_shape[2] = std::max(num_bboxes, 1);
label_shape[3] = 8;
} else {
LOG(FATAL) << "Unknown annotation type.";
}
} else {
label_shape[0] = batch_size;
}
top[1]->Reshape(label_shape);
for (int i = 0; i < this->prefetch_.size(); ++i) {
this->prefetch_[i]->label_.Reshape(label_shape);
}
}
}
void AnnotatedDataLayer<Dtype>::load_batch(Batch<Dtype>* batch) {
CPUTimer batch_timer;
batch_timer.Start();
double read_time = 0;
double trans_time = 0;
CPUTimer timer;
CHECK(batch->data_.count());
CHECK(this->transformed_data_.count());
// Reshape according to the first anno_datum of each batch
// on single input batches allows for inputs of varying dimension.
const int batch_size = this->layer_param_.data_param().batch_size();
const AnnotatedDataParameter& anno_data_param =
this->layer_param_.annotated_data_param();
const TransformationParameter& transform_param =
this->layer_param_.transform_param();
AnnotatedDatum anno_datum;
//AnnotatedDatum& anno_datum = *(reader_.full().peek());
// Use data_transformer to infer the expected blob shape from anno_datum.
//vector<int> top_shape =
// this->data_transformer_->InferBlobShape(anno_datum.datum());
//this->transformed_data_.Reshape(top_shape);
// Reshape batch according to the batch_size.
//top_shape[0] = batch_size;
//batch->data_.Reshape(top_shape);
//Dtype* top_data = batch->data_.mutable_cpu_data();
//Dtype* top_label = NULL; // suppress warnings about uninitialized variables
//if (this->output_labels_ && !has_anno_type_) {
// top_label = batch->label_.mutable_cpu_data();
//}
// Store transformed annotation.
map<int, vector<AnnotationGroup> > all_anno;
int num_bboxes = 0;
for (int item_id = 0; item_id < batch_size; ++item_id) {
timer.Start();
// get a anno_datum
//AnnotatedDatum& anno_datum = *(reader_.full().pop("Waiting for data"));
while (Skip()) {
Next();
}
anno_datum.ParseFromString(cursor_->value());
read_time += timer.MicroSeconds();
//timer.Start();
if (item_id == 0) {
// Reshape according to the first datum of each batch
// on single input batches allows for inputs of varying dimension.
// Use data_transformer to infer the expected blob shape from datum.
vector<int> top_shape = this->data_transformer_->InferBlobShape(anno_datum.datum());
this->transformed_data_.Reshape(top_shape);
// Reshape batch according to the batch_size.
top_shape[0] = batch_size;
batch->data_.Reshape(top_shape);
}
AnnotatedDatum distort_datum;
AnnotatedDatum* expand_datum = NULL;
if (transform_param.has_distort_param()) {
distort_datum.CopyFrom(anno_datum);
this->data_transformer_->DistortImage(anno_datum.datum(),
distort_datum.mutable_datum());
if (transform_param.has_expand_param()) {
expand_datum = new AnnotatedDatum();
this->data_transformer_->ExpandImage(distort_datum, expand_datum);
} else {
expand_datum = &distort_datum;
}
} else {
if (transform_param.has_expand_param()) {
expand_datum = new AnnotatedDatum();
this->data_transformer_->ExpandImage(anno_datum, expand_datum);
} else {
expand_datum = &anno_datum;
}
}
AnnotatedDatum* sampled_datum = NULL;
bool has_sampled = false;
if (batch_samplers_.size() > 0) {
// Generate sampled bboxes from expand_datum.
vector<NormalizedBBox> sampled_bboxes;
GenerateBatchSamples(*expand_datum, batch_samplers_, &sampled_bboxes);
if (sampled_bboxes.size() > 0) {
// Randomly pick a sampled bbox and crop the expand_datum.
int rand_idx = caffe_rng_rand() % sampled_bboxes.size();
sampled_datum = new AnnotatedDatum();
this->data_transformer_->CropImage(*expand_datum,
sampled_bboxes[rand_idx],
sampled_datum);
has_sampled = true;
} else {
sampled_datum = expand_datum;
}
} else {
sampled_datum = expand_datum;
}
CHECK(sampled_datum != NULL);
timer.Start();
vector<int> top_shape =
this->data_transformer_->InferBlobShape(anno_datum.datum());
vector<int> shape =
this->data_transformer_->InferBlobShape(sampled_datum->datum());
if (transform_param.has_resize_param()) {
if (transform_param.resize_param().resize_mode() ==
ResizeParameter_Resize_mode_FIT_SMALL_SIZE) {
this->transformed_data_.Reshape(shape);
batch->data_.Reshape(shape);
//top_data = batch->data_.mutable_cpu_data();
} else {
CHECK(std::equal(top_shape.begin() + 1, top_shape.begin() + 4,
shape.begin() + 1));
}
} else {
CHECK(std::equal(top_shape.begin() + 1, top_shape.begin() + 4,
shape.begin() + 1));
}
// Apply data transformations (mirror, scale, crop...)
timer.Start();
int offset = batch->data_.offset(item_id);
Dtype* top_data = batch->data_.mutable_cpu_data();
this->transformed_data_.set_cpu_data(top_data + offset);
vector<AnnotationGroup> transformed_anno_vec;
if (this->output_labels_) {
if (has_anno_type_) {
// Make sure all data have same annotation type.
CHECK(sampled_datum->has_type()) << "Some datum misses AnnotationType.";
if (anno_data_param.has_anno_type()) {
sampled_datum->set_type(anno_type_);
} else {
CHECK_EQ(anno_type_, sampled_datum->type()) <<
"Different AnnotationType.";
}
// Transform datum and annotation_group at the same time
transformed_anno_vec.clear();
this->data_transformer_->Transform(*sampled_datum,
&(this->transformed_data_),
&transformed_anno_vec);
if (anno_type_ == AnnotatedDatum_AnnotationType_BBOX) {
// Count the number of bboxes.
for (int g = 0; g < transformed_anno_vec.size(); ++g) {
num_bboxes += transformed_anno_vec[g].annotation_size();
}
} else {
LOG(FATAL) << "Unknown annotation type.";
}
all_anno[item_id] = transformed_anno_vec;
} else {
this->data_transformer_->Transform(sampled_datum->datum(),
&(this->transformed_data_));
// Otherwise, store the label from datum.
CHECK(sampled_datum->datum().has_label()) << "Cannot find any label.";
Dtype* top_label = batch->label_.mutable_cpu_data();
top_label[item_id] = sampled_datum->datum().label();
}
} else {
this->data_transformer_->Transform(sampled_datum->datum(),
&(this->transformed_data_));
}
// clear memory
if (has_sampled) {
delete sampled_datum;
}
if (transform_param.has_expand_param()) {
delete expand_datum;
}
trans_time += timer.MicroSeconds();
//reader_.free().push(const_cast<AnnotatedDatum*>(&anno_datum));
Next();
}
// Store "rich" annotation if needed.
if (this->output_labels_ && has_anno_type_) {
vector<int> label_shape(4);
if (anno_type_ == AnnotatedDatum_AnnotationType_BBOX) {
label_shape[0] = 1;
label_shape[1] = 1;
label_shape[3] = 8;
if (num_bboxes == 0) {
// Store all -1 in the label.
label_shape[2] = 1;
batch->label_.Reshape(label_shape);
caffe_set<Dtype>(8, -1, batch->label_.mutable_cpu_data());
} else {
// Reshape the label and store the annotation.
label_shape[2] = num_bboxes;
batch->label_.Reshape(label_shape);
Dtype* top_label = batch->label_.mutable_cpu_data();
int idx = 0;
for (int item_id = 0; item_id < batch_size; ++item_id) {
const vector<AnnotationGroup>& anno_vec = all_anno[item_id];
for (int g = 0; g < anno_vec.size(); ++g) {
const AnnotationGroup& anno_group = anno_vec[g];
for (int a = 0; a < anno_group.annotation_size(); ++a) {
const Annotation& anno = anno_group.annotation(a);
const NormalizedBBox& bbox = anno.bbox();
top_label[idx++] = item_id;
top_label[idx++] = anno_group.group_label();
top_label[idx++] = anno.instance_id();
top_label[idx++] = bbox.xmin();
top_label[idx++] = bbox.ymin();
top_label[idx++] = bbox.xmax();
top_label[idx++] = bbox.ymax();
top_label[idx++] = bbox.difficult();
}
}
}
}
} else {
LOG(FATAL) << "Unknown annotation type.";
}
}
timer.Stop();
batch_timer.Stop();
DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
DLOG(INFO) << " Read time: " << read_time / 1000 << " ms.";
DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
}
void MultiLabelImageDataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
MultiLabelImageDataParameter image_data_param = this->layer_param_.multilabel_image_data_param();
const int new_height = image_data_param.new_height();
const int new_width = image_data_param.new_width();
const bool is_color = image_data_param.is_color();
const int max_labels = image_data_param.max_labels();
const int references = image_data_param.references();
const int expected_labels = max_labels*references;
string root_folder = this->layer_param_.multilabel_image_data_param().root_folder();
CHECK((new_height == 0 && new_width == 0) ||
(new_height > 0 && new_width > 0)) << "Current implementation requires "
"new_height and new_width to be set at the same time.";
// Read the file with filenames and labels
const string& source = this->layer_param_.multilabel_image_data_param().source();
LOG(INFO) << "Opening file " << source;
std::ifstream infile(source.c_str());
string filename;
int label;
std::string line;
int line_num = 0;
LOG(INFO) << "Expecting " << expected_labels << " labels.";
while (std::getline(infile, line)) {
int label;
std::istringstream iss(line);
iss >> filename;
std::vector<int> labels;
while(iss >> label) {
labels.push_back(label);
}
if( labels.size() != expected_labels ){
LOG(INFO) << "Unexpected number of labels on line " << line_num
<< " of file " << source.c_str() << ": (" << labels.size()
<< " vs. " << expected_labels << ". Skipping...";
}
else {
lines_.push_back(std::make_pair(filename, labels));
}
line_num += 1;
}
//while (infile >> filename >> labels) {
// lines_.push_back(std::make_pair(filename, label));
// }
if (this->layer_param_.multilabel_image_data_param().shuffle()) {
// randomly shuffle data
LOG(INFO) << "Shuffling data";
const unsigned int prefetch_rng_seed = caffe_rng_rand();
prefetch_rng_.reset(new Caffe::RNG(prefetch_rng_seed));
ShuffleImages();
}
LOG(INFO) << "*A total of " << lines_.size() << " images.";
lines_id_ = 0;
// Check if we would need to randomly skip a few data points
if (this->layer_param_.multilabel_image_data_param().rand_skip()) {
unsigned int skip = caffe_rng_rand() %
this->layer_param_.multilabel_image_data_param().rand_skip();
LOG(INFO) << "Skipping first " << skip << " data points.";
CHECK_GT(lines_.size(), skip) << "Not enough points to skip";
lines_id_ = skip;
}
// Read an image, and use it to initialize the top blob.
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color);
const int channels = cv_img.channels();
const int height = cv_img.rows;
const int width = cv_img.cols;
// image
const int crop_size = this->layer_param_.transform_param().crop_size();
const int batch_size = this->layer_param_.multilabel_image_data_param().batch_size();
if (crop_size > 0) {
top[0]->Reshape(batch_size, channels, crop_size, crop_size);
this->prefetch_data_.Reshape(batch_size, channels, crop_size, crop_size);
this->transformed_data_.Reshape(1, channels, crop_size, crop_size);
} else {
top[0]->Reshape(batch_size, channels, height, width);
this->prefetch_data_.Reshape(batch_size, channels, height, width);
this->transformed_data_.Reshape(1, channels, height, width);
}
LOG(INFO) << "output data size: " << top[0]->num() << ","
<< top[0]->channels() << "," << top[0]->height() << ","
<< top[0]->width();
// label
vector<int> label_shape(expected_labels, batch_size);
top[1]->Reshape(batch_size, 1, 1, expected_labels);
this->prefetch_label_.Reshape(batch_size, 1, 1, expected_labels);
}
void ImageDataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
const int_tp new_height = this->layer_param_.image_data_param().new_height();
const int_tp new_width = this->layer_param_.image_data_param().new_width();
const bool is_color = this->layer_param_.image_data_param().is_color();
string root_folder = this->layer_param_.image_data_param().root_folder();
CHECK((new_height == 0 && new_width == 0) ||
(new_height > 0 && new_width > 0)) << "Current implementation requires "
"new_height and new_width to be set at the same time.";
// Read the file with filenames and labels
const string& source = this->layer_param_.image_data_param().source();
LOG(INFO) << "Opening file " << source;
std::ifstream infile(source.c_str());
string filename;
int_tp label;
while (infile >> filename >> label) {
lines_.push_back(std::make_pair(filename, label));
}
if (this->layer_param_.image_data_param().shuffle()) {
// randomly shuffle data
LOG(INFO) << "Shuffling data";
const uint_tp prefetch_rng_seed = caffe_rng_rand();
prefetch_rng_.reset(new Caffe::RNG(prefetch_rng_seed));
ShuffleImages();
}
LOG(INFO) << "A total of " << lines_.size() << " images.";
lines_id_ = 0;
// Check if we would need to randomly skip a few data points
if (this->layer_param_.image_data_param().rand_skip()) {
uint_tp skip = caffe_rng_rand() %
this->layer_param_.image_data_param().rand_skip();
LOG(INFO) << "Skipping first " << skip << " data points.";
CHECK_GT(lines_.size(), skip) << "Not enough points to skip";
lines_id_ = skip;
}
// Read an image, and use it to initialize the top blob.
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color);
CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
// Use data_transformer to infer the expected blob shape from a cv_image.
vector<int_tp> top_shape = this->data_transformer_->InferBlobShape(cv_img);
this->transformed_data_.Reshape(top_shape);
// Reshape prefetch_data and top[0] according to the batch_size.
const int_tp batch_size = this->layer_param_.image_data_param().batch_size();
CHECK_GT(batch_size, 0) << "Positive batch size required";
top_shape[0] = batch_size;
for (int_tp i = 0; i < this->PREFETCH_COUNT; ++i) {
this->prefetch_[i].data_.Reshape(top_shape);
}
top[0]->Reshape(top_shape);
LOG(INFO) << "output data size: " << top[0]->num() << ","
<< top[0]->channels() << "," << top[0]->height() << ","
<< top[0]->width();
// label
vector<int_tp> label_shape(1, batch_size);
top[1]->Reshape(label_shape);
for (int_tp i = 0; i < this->PREFETCH_COUNT; ++i) {
this->prefetch_[i].label_.Reshape(label_shape);
}
}
void ImageDataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
const int new_height = this->layer_param_.image_data_param().new_height();
const int new_width = this->layer_param_.image_data_param().new_width();
const bool is_color = this->layer_param_.image_data_param().is_color();
string root_folder = this->layer_param_.image_data_param().root_folder();
CHECK((new_height == 0 && new_width == 0) ||
(new_height > 0 && new_width > 0)) << "Current implementation requires "
"new_height and new_width to be set at the same time.";
// Read the file with filenames and labels
const string& source = this->layer_param_.image_data_param().source();
LOG(INFO) << "Opening file " << source;
std::ifstream infile(source.c_str());
string filename;
int label;
while (infile >> filename >> label) {
lines_.push_back(std::make_pair(filename, label));
}
if (this->layer_param_.image_data_param().shuffle()) {
// randomly shuffle data
LOG(INFO) << "Shuffling data";
const unsigned int prefetch_rng_seed = caffe_rng_rand();
prefetch_rng_.reset(new Engine::RNG(prefetch_rng_seed));
ShuffleImages();
}
LOG(INFO) << "A total of " << lines_.size() << " images.";
lines_id_ = 0;
// Check if we would need to randomly skip a few data points
if (this->layer_param_.image_data_param().rand_skip()) {
unsigned int skip = caffe_rng_rand() %
this->layer_param_.image_data_param().rand_skip();
LOG(INFO) << "Skipping first " << skip << " data points.";
CHECK_GT(lines_.size(), skip) << "Not enough points to skip";
lines_id_ = skip;
}
// Read an image, and use it to initialize the top blob.
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color);
const int channels = cv_img.channels();
const int height = cv_img.rows;
const int width = cv_img.cols;
// image
const int crop_size = this->layer_param_.transform_param().crop_size();
const int batch_size = this->layer_param_.image_data_param().batch_size();
if (crop_size > 0) {
top[0]->Reshape(batch_size, channels, crop_size, crop_size);
this->prefetch_data_.Reshape(batch_size, channels, crop_size, crop_size);
this->transformed_data_.Reshape(1, channels, crop_size, crop_size);
} else {
top[0]->Reshape(batch_size, channels, height, width);
this->prefetch_data_.Reshape(batch_size, channels, height, width);
this->transformed_data_.Reshape(1, channels, height, width);
}
LOG(INFO) << "output data size: " << top[0]->num() << ","
<< top[0]->channels() << "," << top[0]->height() << ","
<< top[0]->width();
// label
vector<int> label_shape(1, batch_size);
top[1]->Reshape(label_shape);
this->prefetch_label_.Reshape(label_shape);
}
