void DataLayer<Dtype>::InternalThreadEntry() {
CPUTimer batch_timer;
batch_timer.Start();
double read_time = 0;
double trans_time = 0;
CPUTimer timer;
CHECK(this->prefetch_data_.count());
CHECK(this->transformed_data_.count());
// Reshape according to the first datum of each batch
// on single input batches allows for inputs of varying dimension.
const int batch_size = this->layer_param_.data_param().batch_size();
Datum datum;
datum.ParseFromString(cursor_->value());
// Use data_transformer to infer the expected blob shape from datum.
vector<int> top_shape = this->data_transformer_->InferBlobShape(datum);
this->transformed_data_.Reshape(top_shape);
// Reshape prefetch_data according to the batch_size.
top_shape[0] = batch_size;
this->prefetch_data_.Reshape(top_shape);
Dtype* top_data = this->prefetch_data_.mutable_cpu_data();
Dtype* top_label = NULL; // suppress warnings about uninitialized variables
if (this->output_labels_) {
top_label = this->prefetch_label_.mutable_cpu_data();
}
timer.Start();
for (int item_id = 0; item_id < batch_size; ++item_id) {
// get a datum
Datum datum;
datum.ParseFromString(cursor_->value());
read_time += timer.MicroSeconds();
timer.Start();
// Apply data transformations (mirror, scale, crop...)
int offset = this->prefetch_data_.offset(item_id);
this->transformed_data_.set_cpu_data(top_data + offset);
this->data_transformer_->Transform(datum, &(this->transformed_data_));
// Copy label.
if (this->output_labels_) {
top_label[item_id] = datum.label();
}
trans_time += timer.MicroSeconds();
timer.Start();
// go to the next item.
cursor_->Next();
if (!cursor_->valid()) {
DLOG(INFO) << "Restarting data prefetching from start.";
cursor_->SeekToFirst();
}
}
timer.Stop();
batch_timer.Stop();
DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
DLOG(INFO) << " Read time: " << read_time / 1000 << " ms.";
DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
}
void FlowDataLayer<Dtype>::load_batch(Batch<Dtype>* batch) {
CPUTimer batch_timer;
batch_timer.Start();
double read_time = 0;
double trans_time = 0;
CPUTimer timer;
CHECK(batch->data_.count());
CHECK(this->transformed_data_.count());
// Reshape according to the first datum of each batch
// on single input batches allows for inputs of varying dimension.
const int batch_size = this->layer_param_.flow_data_param().batch_size();
Datum& datum = *(reader_.full().peek());
// Use data_transformer to infer the expected blob shape from datum.
vector<int> top_shape = this->data_transformer_->InferBlobShape(datum);
top_shape[0] = num_test_views_;
this->transformed_data_.Reshape(top_shape);
// Reshape batch according to the batch_size.
top_shape[0] = batch_size * num_test_views_;
batch->data_.Reshape(top_shape);
Dtype* top_data = batch->data_.mutable_cpu_data();
Dtype* top_label = NULL; // suppress warnings about uninitialized variables
if (this->output_labels_) {
top_label = batch->label_.mutable_cpu_data();
}
for (int item_id = 0; item_id < batch_size; ++item_id) {
timer.Start();
// get a datum
Datum& datum = *(reader_.full().pop("Waiting for flow data"));
read_time += timer.MicroSeconds();
// DLOG(INFO) << "number of data in full queue: " << reader_.full().size();
timer.Start();
// Apply data transformations (mirror, scale, crop...)
int offset = batch->data_.offset(item_id * num_test_views_);
this->transformed_data_.set_cpu_data(top_data + offset);
if (this->phase_ == TRAIN)
this->data_transformer_->TransformVariedSizeDatum(datum, &(this->transformed_data_));
else if (this->phase_ == TEST)
this->data_transformer_->TransformVariedSizeTestDatum(datum, &(this->transformed_data_), num_test_views_);
// Copy label.
if (this->output_labels_) {
top_label[item_id] = datum.label();
}
trans_time += timer.MicroSeconds();
reader_.free().push(const_cast<Datum*>(&datum));
}
timer.Stop();
batch_timer.Stop();
DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
DLOG(INFO) << " Read time: " << read_time / 1000 << " ms.";
DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
}
void SegDataLayer<Dtype>::load_batch(Batch<Dtype>* batch) {
CPUTimer batch_timer;
batch_timer.Start();
double deque_time = 0;
double trans_time = 0;
CPUTimer timer;
CHECK(batch->data_.count());
CHECK(this->transformed_data_.count());
// Reshape on single input batches for inputs of varying dimension.
const int batch_size = this->layer_param_.data_param().batch_size();
Dtype* top_data = batch->data_.mutable_cpu_data();
Dtype* top_label = NULL; // suppress warnings about uninitialized variables
if (this->output_labels_) {
top_label = batch->label_.mutable_cpu_data();
}
for (int item_id = 0; item_id < batch_size; ++item_id) {
// get a blob
timer.Start();
Datum& datum = *(reader_.full().pop("Waiting for data"));
deque_time += timer.MicroSeconds();
// Apply data transformations (mirror, scale, crop...)
timer.Start();
const int offset_data = batch->data_.offset(item_id);
const int offset_label = batch->label_.offset(item_id);
this->transformed_data_.set_cpu_data(top_data + offset_data);
this->transformed_label_.set_cpu_data(top_label + offset_label);
this->data_transformer_->SegTransform(datum,
&(this->transformed_data_),
&(this->transformed_label_));
trans_time += timer.MicroSeconds();
reader_.free().push(const_cast<Datum*>(&datum));
}
timer.Stop();
batch_timer.Stop();
#ifdef BENCHMARK_DATA
LOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
LOG(INFO) << " Dequeue time: " << deque_time / 1000 << " ms.";
LOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
#endif
}
void AnnotatedDataLayer<Dtype>::load_batch(Batch<Dtype>* batch) {
CPUTimer batch_timer;
batch_timer.Start();
double read_time = 0;
double trans_time = 0;
CPUTimer timer;
CHECK(batch->data_.count());
CHECK(this->transformed_data_.count());
// Reshape according to the first anno_datum of each batch
// on single input batches allows for inputs of varying dimension.
const int batch_size = this->layer_param_.data_param().batch_size();
const AnnotatedDataParameter& anno_data_param =
this->layer_param_.annotated_data_param();
const TransformationParameter& transform_param =
this->layer_param_.transform_param();
AnnotatedDatum& anno_datum = *(reader_.full().peek());
// Use data_transformer to infer the expected blob shape from anno_datum.
vector<int> top_shape =
this->data_transformer_->InferBlobShape(anno_datum.datum());
this->transformed_data_.Reshape(top_shape);
// Reshape batch according to the batch_size.
top_shape[0] = batch_size;
batch->data_.Reshape(top_shape);// 图像数据域=======================
Dtype* top_data = batch->data_.mutable_cpu_data();
Dtype* top_label = NULL; // suppress warnings about uninitialized variables
if (this->output_labels_ && !has_anno_type_) {
top_label = batch->label_.mutable_cpu_data();
}
// Store transformed annotation.
map<int, vector<AnnotationGroup> > all_anno;
int num_bboxes = 0;
// 数据预处理=============================================================
for (int item_id = 0; item_id < batch_size; ++item_id)
{
timer.Start();
// get a anno_datum
AnnotatedDatum& anno_datum = *(reader_.full().pop("Waiting for data"));
read_time += timer.MicroSeconds();
timer.Start();
AnnotatedDatum distort_datum;
AnnotatedDatum* expand_datum = NULL;// expand_datum 成后的数据
// distort （变形亮度变化） 处理==================================
// 图像的HSL（ 色相 Hue 、 饱和度 Saturation 、 明亮度 Lightness） 对比度 Contrast
// random_order_prob 重新排布 图像通道 rgb 顺序
// expand 0像素扩展处理
if (transform_param.has_distort_param())
{
distort_datum.CopyFrom(anno_datum);// 复制 anno_datum 到 distort_datum
// （变形亮度变化） 处理
this->data_transformer_->DistortImage(anno_datum.datum(),
distort_datum.mutable_datum());
if (transform_param.has_expand_param())
{
expand_datum = new AnnotatedDatum();
// 这个主要是将DistortImage的图片用像素0进行扩展，标签bbox此时肯定会改变，
// 就重新以黑边的左上角为原点计算[0,1]的bbox的左上角和右下角两个点坐标。
this->data_transformer_->ExpandImage(distort_datum, expand_datum);
}
else
{
expand_datum = &distort_datum;
}
}
// 这个主要是将DistortImage的图片用像素0进行扩展，标签bbox此时肯定会改变，
// 就重新以黑边的左上角为原点计算[0,1]的bbox的左上角和右下角两个点坐标。
else
{
if (transform_param.has_expand_param()) {
expand_datum = new AnnotatedDatum();
this->data_transformer_->ExpandImage(anno_datum, expand_datum);
} else {
expand_datum = &anno_datum;
}
}
// 标签裁剪也很好理解首先要，
// 通过ProjectBBox将原坐标系标签投影到裁剪后图片的新坐标系的坐标，
// 然后再ClipBBox到通过ProjectBBox将原坐标系标签投影到裁剪后图片的新坐标系的坐标，
// 然后再ClipBBox到[0,1]之间。
AnnotatedDatum* sampled_datum = NULL;
bool has_sampled = false;
if (batch_samplers_.size() > 0)
{
// Generate sampled bboxes from expand_datum.
vector<NormalizedBBox> sampled_bboxes;
GenerateBatchSamples(*expand_datum, batch_samplers_, &sampled_bboxes);
if (sampled_bboxes.size() > 0) {
// Randomly pick a sampled bbox and crop the expand_datum.
int rand_idx = caffe_rng_rand() % sampled_bboxes.size();
sampled_datum = new AnnotatedDatum();
this->data_transformer_->CropImage(*expand_datum,
sampled_bboxes[rand_idx],
sampled_datum);
has_sampled = true;
} else {
sampled_datum = expand_datum;
}
}
else
{
sampled_datum = expand_datum;
}
CHECK(sampled_datum != NULL);
timer.Start();
vector<int> shape =
this->data_transformer_->InferBlobShape(sampled_datum->datum());
///////////////////////////////////////////////////////////////////////////////
// Resize：最后将图片放缩到某个尺寸(300x300、416*416)，标签框也是线性放缩坐标而已。
if (transform_param.has_resize_param())
{
if (transform_param.resize_param().resize_mode() ==
ResizeParameter_Resize_mode_FIT_SMALL_SIZE) {
this->transformed_data_.Reshape(shape);
batch->data_.Reshape(shape);
top_data = batch->data_.mutable_cpu_data();
} else {
CHECK(std::equal(top_shape.begin() + 1, top_shape.begin() + 4,
shape.begin() + 1));
}
}
else
{
CHECK(std::equal(top_shape.begin() + 1, top_shape.begin() + 4,
shape.begin() + 1));
}
// 图片镜像、缩放、剪裁等===========================================
// Apply data transformations (mirror, scale, crop...)
int offset = batch->data_.offset(item_id);
this->transformed_data_.set_cpu_data(top_data + offset);
vector<AnnotationGroup> transformed_anno_vec;
if (this->output_labels_)
{
if (has_anno_type_) {
// Make sure all data have same annotation type.
CHECK(sampled_datum->has_type()) << "Some datum misses AnnotationType.";
if (anno_data_param.has_anno_type())
{
sampled_datum->set_type(anno_type_);
}
else
{
CHECK_EQ(anno_type_, sampled_datum->type()) <<
"Different AnnotationType.";
}
// Transform datum and annotation_group at the same time
transformed_anno_vec.clear();
this->data_transformer_->Transform(*sampled_datum,
&(this->transformed_data_),
&transformed_anno_vec);
if (anno_type_ == AnnotatedDatum_AnnotationType_BBOX) {
// Count the number of bboxes.
for (int g = 0; g < transformed_anno_vec.size(); ++g) {
num_bboxes += transformed_anno_vec[g].annotation_size();
}
} else {
LOG(FATAL) << "Unknown annotation type.";
}
all_anno[item_id] = transformed_anno_vec;
} else {
this->data_transformer_->Transform(sampled_datum->datum(),
&(this->transformed_data_));
// Otherwise, store the label from datum.
CHECK(sampled_datum->datum().has_label()) << "Cannot find any label.";
top_label[item_id] = sampled_datum->datum().label();
}
}
else
{
this->data_transformer_->Transform(sampled_datum->datum(),
&(this->transformed_data_));
}
// clear memory=====================================================
if (has_sampled)
{
delete sampled_datum;
}
if (transform_param.has_expand_param())
{
delete expand_datum;
}
trans_time += timer.MicroSeconds();
reader_.free().push(const_cast<AnnotatedDatum*>(&anno_datum));
}
// 转换检测数据AnnotatedDatum
// 图像数据
// datum = anno_datum->mutable_datum()
// datum->set_data(); // 所有8位像素数据
// datum->set_channels(cv_img.channels()); // 通道数量
// datum->set_height(cv_img.rows); // 行
// datum->set_width(cv_img.cols); // 列
// datum->set_encoded(true); // 编码?
// 标签
// anno_datum->mutable_annotation_group(g)->set_group_label(label) 标签
// ->add_annotation()->mutable_bbox() 边框数据
// Store "rich" annotation if needed.
if (this->output_labels_ && has_anno_type_) {
vector<int> label_shape(4);
if (anno_type_ == AnnotatedDatum_AnnotationType_BBOX) {
label_shape[0] = 1;
label_shape[1] = 1;
label_shape[3] = 8;
// 无边框 标签label
if (num_bboxes == 0)
{
// Store all -1 in the label.
label_shape[2] = 1;
batch->label_.Reshape(label_shape);
caffe_set<Dtype>(8, -1, batch->label_.mutable_cpu_data());
}
else
{
// Reshape the label and store the annotation.
label_shape[2] = num_bboxes;// 多个边框数据
batch->label_.Reshape(label_shape);
top_label = batch->label_.mutable_cpu_data();
int idx = 0;
for (int item_id = 0; item_id < batch_size; ++item_id) {
const vector<AnnotationGroup>& anno_vec = all_anno[item_id];
for (int g = 0; g < anno_vec.size(); ++g)
{
const AnnotationGroup& anno_group = anno_vec[g];
for (int a = 0; a < anno_group.annotation_size(); ++a)
{
const Annotation& anno = anno_group.annotation(a);
const NormalizedBBox& bbox = anno.bbox();// 边框数据
top_label[idx++] = item_id;// batch id 0~3 / 0~8 / 0~15
top_label[idx++] = anno_group.group_label();// 类比标签 id
top_label[idx++] = anno.instance_id();
top_label[idx++] = bbox.xmin();// 左上角
top_label[idx++] = bbox.ymin();
top_label[idx++] = bbox.xmax();// 右下角
top_label[idx++] = bbox.ymax();
top_label[idx++] = bbox.difficult();
}
}
}
}
}
else
{
LOG(FATAL) << "Unknown annotation type.";
}
}
timer.Stop();
batch_timer.Stop();
DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
DLOG(INFO) << " Read time: " << read_time / 1000 << " ms.";
DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
}
void MILDataLayer<Dtype>::load_batch(Batch<Dtype>* batch) {
CPUTimer timer;
timer.Start();
CHECK(batch->data_.count());
//Dtype* top_data = this->prefetch_data_.mutable_cpu_data();
//Dtype* top_label = this->prefetch_label_.mutable_cpu_data();
Dtype* top_data = batch->data_.mutable_cpu_data();
Dtype* top_label = batch->label_.mutable_cpu_data();
const int img_size = this->transform_param_.crop_size();
const int channels = this->layer_param_.mil_data_param().channels();
const int scale = this->transform_param_.scale();
const bool mirror = this->transform_param_.mirror();
const int images_per_batch = this->layer_param_.mil_data_param().images_per_batch();
const int n_classes = this->layer_param_.mil_data_param().n_classes();
const int num_scales = this->layer_param_.mil_data_param().num_scales();
const float scale_factor = this->layer_param_.mil_data_param().scale_factor();
// zero out batch
//caffe_set(this->prefetch_data_.count(), Dtype(0), top_data);
caffe_set(batch->data_.count(), Dtype(0), top_data);
int item_id;
for(int i_image = 0; i_image < images_per_batch; i_image++){
// Sample which image to read
unsigned int index = counter_; counter_ = counter_ + 1;
const unsigned int rand_index = this->PrefetchRand();
if(this->layer_param_.mil_data_param().randomize())
index = rand_index;
// LOG(INFO) << index % this->num_images_ << ", " << this->num_images_;
pair<string, string> p = this->image_database_[index % this->num_images_];
string im_name = p.first;
string full_im_name = p.second;
cv::Mat cv_img = cv::imread(full_im_name, CV_LOAD_IMAGE_COLOR);
if (!cv_img.data) {
LOG(ERROR) << "Could not open or find file " << full_im_name;
return;
}
//REVIEW ktran: do not hardcode dataset name (or its prefix "/labels-")
//REVIEW ktran: also do not use deep dataset name so that we don't have to modify the core caffe code
//(ref: https://github.com/BVLC/caffe/commit/a0787631a27ca6478f70341462aafdcf35dabb19)
hdf5_load_nd_dataset(this->label_file_id_, string("/labels-"+im_name).c_str(), 4, 4, &this->label_blob_);
const Dtype* label = label_blob_.mutable_cpu_data();
CHECK_EQ(label_blob_.width(), 1) << "Expected width of label to be 1." ;
CHECK_EQ(label_blob_.height(), n_classes) << "Expected height of label to be " << n_classes;
CHECK_EQ(label_blob_.channels(), 1) << "Expected channels of label to be 1." ;
CHECK_EQ(label_blob_.num(), 1) << "Expected num of label to be 1." ;
float img_size_i = img_size;
for(int i_scales = 0; i_scales < num_scales; i_scales++){
// Resize such that the image is of size img_size, img_size
item_id = i_image*num_scales + i_scales;
// LOG(INFO) << "MIL Data Layer: scale: " << (int) round(img_size_i);
cv::Mat cv_cropped_img = Transform_IDL(cv_img, static_cast<int>(round(img_size_i)), mirror);
for (int c = 0; c < channels; ++c) {
for (int h = 0; h < cv_cropped_img.rows; ++h) {
for (int w = 0; w < cv_cropped_img.cols; ++w) {
Dtype pixel =
static_cast<Dtype>(cv_cropped_img.at<cv::Vec3b>(h, w)[c]);
top_data[((item_id * channels + c) * img_size + h)
* img_size + w]
= (pixel - static_cast<Dtype>(mean_value_[c]))*scale;
}
}
}
img_size_i = std::max(static_cast<float>(1.), img_size_i*scale_factor);
}
for(int i_label = 0; i_label < n_classes; i_label++){
top_label[i_image*n_classes + i_label] =
label[i_label];
}
}
timer.Stop();
DLOG(INFO) << "Prefetch batch: " << timer.MilliSeconds() << " ms.";
}
void DataLayer<Dtype>::load_batch(Batch<Dtype>* batch) {
CPUTimer batch_timer;
batch_timer.Start();
double read_time = 0;
double trans_time = 0;
CPUTimer timer;
// LOG(INFO)<<"load_batch";
CHECK(batch->data_.count());
CHECK(this->transformed_data_.count());
// Reshape according to the first datum of each batch
// on single input batches allows for inputs of varying dimension.
const int batch_size = this->layer_param_.data_param().batch_size();
// Datum& datum = *(reader_.full().peek());
// // Use data_transformer to infer the expected blob shape from datum.
Datum datum;
datum.ParseFromString(cursor_->value());
// Use data_transformer to infer the expected blob shape from datum.
vector<int> top_shape = this->data_transformer_->InferBlobShape(datum);
this->transformed_data_.Reshape(top_shape);
// Reshape batch according to the batch_size.
top_shape[0] = batch_size;
batch->data_.Reshape(top_shape);
Dtype* top_data = batch->data_.mutable_cpu_data();
// Dtype* top_data = this->prefetch_data_.mutable_cpu_data();
Dtype* top_label = NULL; // suppress warnings about uninitialized variables
// LOG(INFO)<<" output_labels_:"<<this->output_labels_;
if (this->output_labels_) {
top_label = batch->label_.mutable_cpu_data();
// top_label = this->prefetch_label_.mutable_cpu_data();
}
Dtype* use_data=this->use_data_.mutable_cpu_data();
// LOG(INFO)<<" use_data[0]:"<<use_data[0];
if (use_data[0]==0.0){
// LOG(INFO)<<"visit in order";
for (int item_id = 0; item_id < batch_size; item_id++) {
Datum datum;
datum.ParseFromString(cursor_->value());
// Apply data transformations (mirror, scale, crop...)
// LOG(INFO)<<"jq enter data_layers"<< item_id;
int offset = batch->data_.offset(item_id);
// LOG(INFO)<<"jq enter data_layers";
this->transformed_data_.set_cpu_data(top_data + offset);
this->data_transformer_->Transform(datum, &(this->transformed_data_));
// Copy label.
if (this->output_labels_) {
top_label[item_id] = datum.label();
// std::cout<<" cursor_:"<<datum.label();
}
// use_data[item_id +5] = start;
// trans_time += timer.MicroSeconds();
cursor_->Next();
// start +=1.0;
// std::cout<<" output_labels_:"<<this->output_labels_;
if (!cursor_->valid()) {
DLOG(INFO) << "Restarting data prefetching from start.";
cursor_->SeekToFirst();
}
// reader_.free().push(const_cast<Datum*>(&datum));
}
}else if (use_data[0]!=0.0){
// forward-backward using semi supervised with false label
// 0, sami-super-unsuper, 1, label_kinds, 2, step over,
// 3, datanum, 4, start index
// LOG(INFO)<<"visit in Key/value";
// LOG(INFO)<<"this->PREFETCH_COUNT:"<<this->PREFETCH_COUNT;
int step_over = batch_size+1;
// std::cout<<std::endl;
scoped_ptr<db::Transaction> txn(db_->NewTransaction());
// std::cout<<"key:";
int kCIFARImageNBytes=3072;
for (int item_id = 0; item_id < batch_size; item_id++) {
char str_buffer[kCIFARImageNBytes];
int id= static_cast<int>(use_data[item_id+ 1]);
// std::cout<<" "<<id<<":";
int length = snprintf(str_buffer, kCIFARImageNBytes, "%05d", id);
string value;
string str=string(str_buffer, length);
txn->Get(str, value);
Datum datum;
datum.ParseFromString(value);
int offset = batch->data_.offset(item_id);
// LOG(INFO)<<"jq enter data_layers";
this->transformed_data_.set_cpu_data(top_data + offset);
this->data_transformer_->Transform(datum, &(this->transformed_data_));
// std::cout<<" output_labels_:"<<this->output_labels_;
if (this->output_labels_) {
// top_label[item_id] = datum.label();
top_label[item_id] = use_data[item_id+ step_over];
// std::cout<<" KV:"<<datum.label();
// top_label[item_id]= static_cast<int>(use_data[item_id + batch_size +3]);
}
if( use_data[item_id+ step_over]!=(datum.label()%1000))
LOG(INFO)<<"image id:"<<id<<" not correctly fetch: "<<datum.label()
<<" vs "<<use_data[item_id+ step_over];
// std::cout<<top_label[item_id];
// std::cout<<" key:"<<id;
}
// std::cout<<std::endl;
// for (int item_id = 0; item_id < 50000; item_id++) {
// char str_buffer[kCIFARImageNBytes];
// // int id= static_cast<int>(use_data[item_id+ 1]);
// int length = snprintf(str_buffer, kCIFARImageNBytes, "%05d", item_id);
// string value;
// string str=string(str_buffer, length);
// txn->Get(str, value);
// // Datum datum;
// // datum.ParseFromString(value);
// // int offset = batch->data_.offset(item_id);
// // // LOG(INFO)<<"jq enter data_layers";
// // this->transformed_data_.set_cpu_data(top_data + offset);
// // this->data_transformer_->Transform(datum, &(this->transformed_data_));
// // if (this->output_labels_) {
// // top_label[item_id] = datum.label();
// // // top_label[item_id]= static_cast<int>(use_data[item_id + batch_size +3]);
// // }
// // std::cout<<" "<<item_id;
// }
// std::cout<<std::endl;
txn->Commit();
}
timer.Stop();
batch_timer.Stop();
DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
DLOG(INFO) << " Read time: " << read_time / 1000 << " ms.";
DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
}
void AnnotatedDataLayer<Dtype>::load_batch(Batch<Dtype>* batch) {
CPUTimer batch_timer;
batch_timer.Start();
double read_time = 0;
double trans_time = 0;
CPUTimer timer;
CHECK(batch->data_.count());
CHECK(this->transformed_data_.count());
// Reshape according to the first anno_datum of each batch
// on single input batches allows for inputs of varying dimension.
const int batch_size = this->layer_param_.data_param().batch_size();
const AnnotatedDataParameter& anno_data_param =
this->layer_param_.annotated_data_param();
const TransformationParameter& transform_param =
this->layer_param_.transform_param();
AnnotatedDatum anno_datum;
//AnnotatedDatum& anno_datum = *(reader_.full().peek());
// Use data_transformer to infer the expected blob shape from anno_datum.
//vector<int> top_shape =
// this->data_transformer_->InferBlobShape(anno_datum.datum());
//this->transformed_data_.Reshape(top_shape);
// Reshape batch according to the batch_size.
//top_shape[0] = batch_size;
//batch->data_.Reshape(top_shape);
//Dtype* top_data = batch->data_.mutable_cpu_data();
//Dtype* top_label = NULL; // suppress warnings about uninitialized variables
//if (this->output_labels_ && !has_anno_type_) {
// top_label = batch->label_.mutable_cpu_data();
//}
// Store transformed annotation.
map<int, vector<AnnotationGroup> > all_anno;
int num_bboxes = 0;
for (int item_id = 0; item_id < batch_size; ++item_id) {
timer.Start();
// get a anno_datum
//AnnotatedDatum& anno_datum = *(reader_.full().pop("Waiting for data"));
while (Skip()) {
Next();
}
anno_datum.ParseFromString(cursor_->value());
read_time += timer.MicroSeconds();
//timer.Start();
if (item_id == 0) {
// Reshape according to the first datum of each batch
// on single input batches allows for inputs of varying dimension.
// Use data_transformer to infer the expected blob shape from datum.
vector<int> top_shape = this->data_transformer_->InferBlobShape(anno_datum.datum());
this->transformed_data_.Reshape(top_shape);
// Reshape batch according to the batch_size.
top_shape[0] = batch_size;
batch->data_.Reshape(top_shape);
}
AnnotatedDatum distort_datum;
AnnotatedDatum* expand_datum = NULL;
if (transform_param.has_distort_param()) {
distort_datum.CopyFrom(anno_datum);
this->data_transformer_->DistortImage(anno_datum.datum(),
distort_datum.mutable_datum());
if (transform_param.has_expand_param()) {
expand_datum = new AnnotatedDatum();
this->data_transformer_->ExpandImage(distort_datum, expand_datum);
} else {
expand_datum = &distort_datum;
}
} else {
if (transform_param.has_expand_param()) {
expand_datum = new AnnotatedDatum();
this->data_transformer_->ExpandImage(anno_datum, expand_datum);
} else {
expand_datum = &anno_datum;
}
}
AnnotatedDatum* sampled_datum = NULL;
bool has_sampled = false;
if (batch_samplers_.size() > 0) {
// Generate sampled bboxes from expand_datum.
vector<NormalizedBBox> sampled_bboxes;
GenerateBatchSamples(*expand_datum, batch_samplers_, &sampled_bboxes);
if (sampled_bboxes.size() > 0) {
// Randomly pick a sampled bbox and crop the expand_datum.
int rand_idx = caffe_rng_rand() % sampled_bboxes.size();
sampled_datum = new AnnotatedDatum();
this->data_transformer_->CropImage(*expand_datum,
sampled_bboxes[rand_idx],
sampled_datum);
has_sampled = true;
} else {
sampled_datum = expand_datum;
}
} else {
sampled_datum = expand_datum;
}
CHECK(sampled_datum != NULL);
timer.Start();
vector<int> top_shape =
this->data_transformer_->InferBlobShape(anno_datum.datum());
vector<int> shape =
this->data_transformer_->InferBlobShape(sampled_datum->datum());
if (transform_param.has_resize_param()) {
if (transform_param.resize_param().resize_mode() ==
ResizeParameter_Resize_mode_FIT_SMALL_SIZE) {
this->transformed_data_.Reshape(shape);
batch->data_.Reshape(shape);
//top_data = batch->data_.mutable_cpu_data();
} else {
CHECK(std::equal(top_shape.begin() + 1, top_shape.begin() + 4,
shape.begin() + 1));
}
} else {
CHECK(std::equal(top_shape.begin() + 1, top_shape.begin() + 4,
shape.begin() + 1));
}
// Apply data transformations (mirror, scale, crop...)
timer.Start();
int offset = batch->data_.offset(item_id);
Dtype* top_data = batch->data_.mutable_cpu_data();
this->transformed_data_.set_cpu_data(top_data + offset);
vector<AnnotationGroup> transformed_anno_vec;
if (this->output_labels_) {
if (has_anno_type_) {
// Make sure all data have same annotation type.
CHECK(sampled_datum->has_type()) << "Some datum misses AnnotationType.";
if (anno_data_param.has_anno_type()) {
sampled_datum->set_type(anno_type_);
} else {
CHECK_EQ(anno_type_, sampled_datum->type()) <<
"Different AnnotationType.";
}
// Transform datum and annotation_group at the same time
transformed_anno_vec.clear();
this->data_transformer_->Transform(*sampled_datum,
&(this->transformed_data_),
&transformed_anno_vec);
if (anno_type_ == AnnotatedDatum_AnnotationType_BBOX) {
// Count the number of bboxes.
for (int g = 0; g < transformed_anno_vec.size(); ++g) {
num_bboxes += transformed_anno_vec[g].annotation_size();
}
} else {
LOG(FATAL) << "Unknown annotation type.";
}
all_anno[item_id] = transformed_anno_vec;
} else {
this->data_transformer_->Transform(sampled_datum->datum(),
&(this->transformed_data_));
// Otherwise, store the label from datum.
CHECK(sampled_datum->datum().has_label()) << "Cannot find any label.";
Dtype* top_label = batch->label_.mutable_cpu_data();
top_label[item_id] = sampled_datum->datum().label();
}
} else {
this->data_transformer_->Transform(sampled_datum->datum(),
&(this->transformed_data_));
}
// clear memory
if (has_sampled) {
delete sampled_datum;
}
if (transform_param.has_expand_param()) {
delete expand_datum;
}
trans_time += timer.MicroSeconds();
//reader_.free().push(const_cast<AnnotatedDatum*>(&anno_datum));
Next();
}
// Store "rich" annotation if needed.
if (this->output_labels_ && has_anno_type_) {
vector<int> label_shape(4);
if (anno_type_ == AnnotatedDatum_AnnotationType_BBOX) {
label_shape[0] = 1;
label_shape[1] = 1;
label_shape[3] = 8;
if (num_bboxes == 0) {
// Store all -1 in the label.
label_shape[2] = 1;
batch->label_.Reshape(label_shape);
caffe_set<Dtype>(8, -1, batch->label_.mutable_cpu_data());
} else {
// Reshape the label and store the annotation.
label_shape[2] = num_bboxes;
batch->label_.Reshape(label_shape);
Dtype* top_label = batch->label_.mutable_cpu_data();
int idx = 0;
for (int item_id = 0; item_id < batch_size; ++item_id) {
const vector<AnnotationGroup>& anno_vec = all_anno[item_id];
for (int g = 0; g < anno_vec.size(); ++g) {
const AnnotationGroup& anno_group = anno_vec[g];
for (int a = 0; a < anno_group.annotation_size(); ++a) {
const Annotation& anno = anno_group.annotation(a);
const NormalizedBBox& bbox = anno.bbox();
top_label[idx++] = item_id;
top_label[idx++] = anno_group.group_label();
top_label[idx++] = anno.instance_id();
top_label[idx++] = bbox.xmin();
top_label[idx++] = bbox.ymin();
top_label[idx++] = bbox.xmax();
top_label[idx++] = bbox.ymax();
top_label[idx++] = bbox.difficult();
}
}
}
}
} else {
LOG(FATAL) << "Unknown annotation type.";
}
}
timer.Stop();
batch_timer.Stop();
DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
DLOG(INFO) << " Read time: " << read_time / 1000 << " ms.";
DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
}
