void AnnotatedDataLayer<Dtype>::DataLayerSetUp( const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) { data_transformer_.reset(new SSDTransformer<Dtype>(this->transform_param_, this->phase_)); data_transformer_->InitRand(); const int batch_size = this->layer_param_.data_param().batch_size(); const AnnotatedDataParameter& anno_data_param = this->layer_param_.annotated_data_param(); for (int i = 0; i < anno_data_param.batch_sampler_size(); ++i) { batch_samplers_.push_back(anno_data_param.batch_sampler(i)); } label_map_file_ = anno_data_param.label_map_file(); // Make sure dimension is consistent within batch. const TransformationParameter& transform_param = this->layer_param_.transform_param(); if (transform_param.has_resize_param()) { if (transform_param.resize_param().resize_mode() == ResizeParameter_Resize_mode_FIT_SMALL_SIZE) { CHECK_EQ(batch_size, 1) << "Only support batch size of 1 for FIT_SMALL_SIZE."; } } // Read a data point, and use it to initialize the top blob. //AnnotatedDatum& anno_datum = *(reader_.full().peek()); AnnotatedDatum anno_datum; anno_datum.ParseFromString(cursor_->value()); // Use data_transformer to infer the expected blob shape from anno_datum. vector<int> top_shape = this->data_transformer_->InferBlobShape(anno_datum.datum()); this->transformed_data_.Reshape(top_shape); // Reshape top[0] and prefetch_data according to the batch_size. top_shape[0] = batch_size; top[0]->Reshape(top_shape); for (int i = 0; i < this->prefetch_.size(); ++i) { this->prefetch_[i]->data_.Reshape(top_shape); } //LOG(INFO) << "output data size: " << top[0]->num() << "," LOG_IF(INFO, Caffe::root_solver()) << "output data size: " << top[0]->num() << "," << top[0]->channels() << "," << top[0]->height() << "," << top[0]->width(); // label if (this->output_labels_) { has_anno_type_ = anno_datum.has_type() || anno_data_param.has_anno_type(); vector<int> label_shape(4, 1); if (has_anno_type_) { anno_type_ = anno_datum.type(); if (anno_data_param.has_anno_type()) { // If anno_type is provided in AnnotatedDataParameter, replace // the type stored in each individual AnnotatedDatum. LOG(WARNING) << "type stored in AnnotatedDatum is shadowed."; anno_type_ = anno_data_param.anno_type(); } // Infer the label shape from anno_datum.AnnotationGroup(). int num_bboxes = 0; if (anno_type_ == AnnotatedDatum_AnnotationType_BBOX) { // Since the number of bboxes can be different for each image, // we store the bbox information in a specific format. In specific: // All bboxes are stored in one spatial plane (num and channels are 1) // And each row contains one and only one box in the following format: // [item_id, group_label, instance_id, xmin, ymin, xmax, ymax, diff] // Note: Refer to caffe.proto for details about group_label and // instance_id. for (int g = 0; g < anno_datum.annotation_group_size(); ++g) { num_bboxes += anno_datum.annotation_group(g).annotation_size(); } label_shape[0] = 1; label_shape[1] = 1; // BasePrefetchingDataLayer<Dtype>::LayerSetUp() requires to call // cpu_data and gpu_data for consistent prefetch thread. Thus we make // sure there is at least one bbox. label_shape[2] = std::max(num_bboxes, 1); label_shape[3] = 8; } else { LOG(FATAL) << "Unknown annotation type."; } } else { label_shape[0] = batch_size; } top[1]->Reshape(label_shape); for (int i = 0; i < this->prefetch_.size(); ++i) { this->prefetch_[i]->label_.Reshape(label_shape); } } }
void AnnotatedDataLayer<Dtype>::load_batch(Batch<Dtype>* batch) { CPUTimer batch_timer; batch_timer.Start(); double read_time = 0; double trans_time = 0; CPUTimer timer; CHECK(batch->data_.count()); CHECK(this->transformed_data_.count()); // Reshape according to the first anno_datum of each batch // on single input batches allows for inputs of varying dimension. const int batch_size = this->layer_param_.data_param().batch_size(); const AnnotatedDataParameter& anno_data_param = this->layer_param_.annotated_data_param(); const TransformationParameter& transform_param = this->layer_param_.transform_param(); AnnotatedDatum anno_datum; //AnnotatedDatum& anno_datum = *(reader_.full().peek()); // Use data_transformer to infer the expected blob shape from anno_datum. //vector<int> top_shape = // this->data_transformer_->InferBlobShape(anno_datum.datum()); //this->transformed_data_.Reshape(top_shape); // Reshape batch according to the batch_size. //top_shape[0] = batch_size; //batch->data_.Reshape(top_shape); //Dtype* top_data = batch->data_.mutable_cpu_data(); //Dtype* top_label = NULL; // suppress warnings about uninitialized variables //if (this->output_labels_ && !has_anno_type_) { // top_label = batch->label_.mutable_cpu_data(); //} // Store transformed annotation. map<int, vector<AnnotationGroup> > all_anno; int num_bboxes = 0; for (int item_id = 0; item_id < batch_size; ++item_id) { timer.Start(); // get a anno_datum //AnnotatedDatum& anno_datum = *(reader_.full().pop("Waiting for data")); while (Skip()) { Next(); } anno_datum.ParseFromString(cursor_->value()); read_time += timer.MicroSeconds(); //timer.Start(); if (item_id == 0) { // Reshape according to the first datum of each batch // on single input batches allows for inputs of varying dimension. // Use data_transformer to infer the expected blob shape from datum. vector<int> top_shape = this->data_transformer_->InferBlobShape(anno_datum.datum()); this->transformed_data_.Reshape(top_shape); // Reshape batch according to the batch_size. top_shape[0] = batch_size; batch->data_.Reshape(top_shape); } AnnotatedDatum distort_datum; AnnotatedDatum* expand_datum = NULL; if (transform_param.has_distort_param()) { distort_datum.CopyFrom(anno_datum); this->data_transformer_->DistortImage(anno_datum.datum(), distort_datum.mutable_datum()); if (transform_param.has_expand_param()) { expand_datum = new AnnotatedDatum(); this->data_transformer_->ExpandImage(distort_datum, expand_datum); } else { expand_datum = &distort_datum; } } else { if (transform_param.has_expand_param()) { expand_datum = new AnnotatedDatum(); this->data_transformer_->ExpandImage(anno_datum, expand_datum); } else { expand_datum = &anno_datum; } } AnnotatedDatum* sampled_datum = NULL; bool has_sampled = false; if (batch_samplers_.size() > 0) { // Generate sampled bboxes from expand_datum. vector<NormalizedBBox> sampled_bboxes; GenerateBatchSamples(*expand_datum, batch_samplers_, &sampled_bboxes); if (sampled_bboxes.size() > 0) { // Randomly pick a sampled bbox and crop the expand_datum. int rand_idx = caffe_rng_rand() % sampled_bboxes.size(); sampled_datum = new AnnotatedDatum(); this->data_transformer_->CropImage(*expand_datum, sampled_bboxes[rand_idx], sampled_datum); has_sampled = true; } else { sampled_datum = expand_datum; } } else { sampled_datum = expand_datum; } CHECK(sampled_datum != NULL); timer.Start(); vector<int> top_shape = this->data_transformer_->InferBlobShape(anno_datum.datum()); vector<int> shape = this->data_transformer_->InferBlobShape(sampled_datum->datum()); if (transform_param.has_resize_param()) { if (transform_param.resize_param().resize_mode() == ResizeParameter_Resize_mode_FIT_SMALL_SIZE) { this->transformed_data_.Reshape(shape); batch->data_.Reshape(shape); //top_data = batch->data_.mutable_cpu_data(); } else { CHECK(std::equal(top_shape.begin() + 1, top_shape.begin() + 4, shape.begin() + 1)); } } else { CHECK(std::equal(top_shape.begin() + 1, top_shape.begin() + 4, shape.begin() + 1)); } // Apply data transformations (mirror, scale, crop...) timer.Start(); int offset = batch->data_.offset(item_id); Dtype* top_data = batch->data_.mutable_cpu_data(); this->transformed_data_.set_cpu_data(top_data + offset); vector<AnnotationGroup> transformed_anno_vec; if (this->output_labels_) { if (has_anno_type_) { // Make sure all data have same annotation type. CHECK(sampled_datum->has_type()) << "Some datum misses AnnotationType."; if (anno_data_param.has_anno_type()) { sampled_datum->set_type(anno_type_); } else { CHECK_EQ(anno_type_, sampled_datum->type()) << "Different AnnotationType."; } // Transform datum and annotation_group at the same time transformed_anno_vec.clear(); this->data_transformer_->Transform(*sampled_datum, &(this->transformed_data_), &transformed_anno_vec); if (anno_type_ == AnnotatedDatum_AnnotationType_BBOX) { // Count the number of bboxes. for (int g = 0; g < transformed_anno_vec.size(); ++g) { num_bboxes += transformed_anno_vec[g].annotation_size(); } } else { LOG(FATAL) << "Unknown annotation type."; } all_anno[item_id] = transformed_anno_vec; } else { this->data_transformer_->Transform(sampled_datum->datum(), &(this->transformed_data_)); // Otherwise, store the label from datum. CHECK(sampled_datum->datum().has_label()) << "Cannot find any label."; Dtype* top_label = batch->label_.mutable_cpu_data(); top_label[item_id] = sampled_datum->datum().label(); } } else { this->data_transformer_->Transform(sampled_datum->datum(), &(this->transformed_data_)); } // clear memory if (has_sampled) { delete sampled_datum; } if (transform_param.has_expand_param()) { delete expand_datum; } trans_time += timer.MicroSeconds(); //reader_.free().push(const_cast<AnnotatedDatum*>(&anno_datum)); Next(); } // Store "rich" annotation if needed. if (this->output_labels_ && has_anno_type_) { vector<int> label_shape(4); if (anno_type_ == AnnotatedDatum_AnnotationType_BBOX) { label_shape[0] = 1; label_shape[1] = 1; label_shape[3] = 8; if (num_bboxes == 0) { // Store all -1 in the label. label_shape[2] = 1; batch->label_.Reshape(label_shape); caffe_set<Dtype>(8, -1, batch->label_.mutable_cpu_data()); } else { // Reshape the label and store the annotation. label_shape[2] = num_bboxes; batch->label_.Reshape(label_shape); Dtype* top_label = batch->label_.mutable_cpu_data(); int idx = 0; for (int item_id = 0; item_id < batch_size; ++item_id) { const vector<AnnotationGroup>& anno_vec = all_anno[item_id]; for (int g = 0; g < anno_vec.size(); ++g) { const AnnotationGroup& anno_group = anno_vec[g]; for (int a = 0; a < anno_group.annotation_size(); ++a) { const Annotation& anno = anno_group.annotation(a); const NormalizedBBox& bbox = anno.bbox(); top_label[idx++] = item_id; top_label[idx++] = anno_group.group_label(); top_label[idx++] = anno.instance_id(); top_label[idx++] = bbox.xmin(); top_label[idx++] = bbox.ymin(); top_label[idx++] = bbox.xmax(); top_label[idx++] = bbox.ymax(); top_label[idx++] = bbox.difficult(); } } } } } else { LOG(FATAL) << "Unknown annotation type."; } } timer.Stop(); batch_timer.Stop(); DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms."; DLOG(INFO) << " Read time: " << read_time / 1000 << " ms."; DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms."; }
void MultiLabelImageDataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) { MultiLabelImageDataParameter image_data_param = this->layer_param_.multilabel_image_data_param(); const int new_height = image_data_param.new_height(); const int new_width = image_data_param.new_width(); const bool is_color = image_data_param.is_color(); const int max_labels = image_data_param.max_labels(); const int references = image_data_param.references(); const int expected_labels = max_labels*references; string root_folder = this->layer_param_.multilabel_image_data_param().root_folder(); CHECK((new_height == 0 && new_width == 0) || (new_height > 0 && new_width > 0)) << "Current implementation requires " "new_height and new_width to be set at the same time."; // Read the file with filenames and labels const string& source = this->layer_param_.multilabel_image_data_param().source(); LOG(INFO) << "Opening file " << source; std::ifstream infile(source.c_str()); string filename; int label; std::string line; int line_num = 0; LOG(INFO) << "Expecting " << expected_labels << " labels."; while (std::getline(infile, line)) { int label; std::istringstream iss(line); iss >> filename; std::vector<int> labels; while(iss >> label) { labels.push_back(label); } if( labels.size() != expected_labels ){ LOG(INFO) << "Unexpected number of labels on line " << line_num << " of file " << source.c_str() << ": (" << labels.size() << " vs. " << expected_labels << ". Skipping..."; } else { lines_.push_back(std::make_pair(filename, labels)); } line_num += 1; } //while (infile >> filename >> labels) { // lines_.push_back(std::make_pair(filename, label)); // } if (this->layer_param_.multilabel_image_data_param().shuffle()) { // randomly shuffle data LOG(INFO) << "Shuffling data"; const unsigned int prefetch_rng_seed = caffe_rng_rand(); prefetch_rng_.reset(new Caffe::RNG(prefetch_rng_seed)); ShuffleImages(); } LOG(INFO) << "*A total of " << lines_.size() << " images."; lines_id_ = 0; // Check if we would need to randomly skip a few data points if (this->layer_param_.multilabel_image_data_param().rand_skip()) { unsigned int skip = caffe_rng_rand() % this->layer_param_.multilabel_image_data_param().rand_skip(); LOG(INFO) << "Skipping first " << skip << " data points."; CHECK_GT(lines_.size(), skip) << "Not enough points to skip"; lines_id_ = skip; } // Read an image, and use it to initialize the top blob. cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first, new_height, new_width, is_color); const int channels = cv_img.channels(); const int height = cv_img.rows; const int width = cv_img.cols; // image const int crop_size = this->layer_param_.transform_param().crop_size(); const int batch_size = this->layer_param_.multilabel_image_data_param().batch_size(); if (crop_size > 0) { top[0]->Reshape(batch_size, channels, crop_size, crop_size); this->prefetch_data_.Reshape(batch_size, channels, crop_size, crop_size); this->transformed_data_.Reshape(1, channels, crop_size, crop_size); } else { top[0]->Reshape(batch_size, channels, height, width); this->prefetch_data_.Reshape(batch_size, channels, height, width); this->transformed_data_.Reshape(1, channels, height, width); } LOG(INFO) << "output data size: " << top[0]->num() << "," << top[0]->channels() << "," << top[0]->height() << "," << top[0]->width(); // label vector<int> label_shape(expected_labels, batch_size); top[1]->Reshape(batch_size, 1, 1, expected_labels); this->prefetch_label_.Reshape(batch_size, 1, 1, expected_labels); }
void ImageDataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) { const int_tp new_height = this->layer_param_.image_data_param().new_height(); const int_tp new_width = this->layer_param_.image_data_param().new_width(); const bool is_color = this->layer_param_.image_data_param().is_color(); string root_folder = this->layer_param_.image_data_param().root_folder(); CHECK((new_height == 0 && new_width == 0) || (new_height > 0 && new_width > 0)) << "Current implementation requires " "new_height and new_width to be set at the same time."; // Read the file with filenames and labels const string& source = this->layer_param_.image_data_param().source(); LOG(INFO) << "Opening file " << source; std::ifstream infile(source.c_str()); string filename; int_tp label; while (infile >> filename >> label) { lines_.push_back(std::make_pair(filename, label)); } if (this->layer_param_.image_data_param().shuffle()) { // randomly shuffle data LOG(INFO) << "Shuffling data"; const uint_tp prefetch_rng_seed = caffe_rng_rand(); prefetch_rng_.reset(new Caffe::RNG(prefetch_rng_seed)); ShuffleImages(); } LOG(INFO) << "A total of " << lines_.size() << " images."; lines_id_ = 0; // Check if we would need to randomly skip a few data points if (this->layer_param_.image_data_param().rand_skip()) { uint_tp skip = caffe_rng_rand() % this->layer_param_.image_data_param().rand_skip(); LOG(INFO) << "Skipping first " << skip << " data points."; CHECK_GT(lines_.size(), skip) << "Not enough points to skip"; lines_id_ = skip; } // Read an image, and use it to initialize the top blob. cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first, new_height, new_width, is_color); CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first; // Use data_transformer to infer the expected blob shape from a cv_image. vector<int_tp> top_shape = this->data_transformer_->InferBlobShape(cv_img); this->transformed_data_.Reshape(top_shape); // Reshape prefetch_data and top[0] according to the batch_size. const int_tp batch_size = this->layer_param_.image_data_param().batch_size(); CHECK_GT(batch_size, 0) << "Positive batch size required"; top_shape[0] = batch_size; for (int_tp i = 0; i < this->PREFETCH_COUNT; ++i) { this->prefetch_[i].data_.Reshape(top_shape); } top[0]->Reshape(top_shape); LOG(INFO) << "output data size: " << top[0]->num() << "," << top[0]->channels() << "," << top[0]->height() << "," << top[0]->width(); // label vector<int_tp> label_shape(1, batch_size); top[1]->Reshape(label_shape); for (int_tp i = 0; i < this->PREFETCH_COUNT; ++i) { this->prefetch_[i].label_.Reshape(label_shape); } }
void ImageDataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) { const int new_height = this->layer_param_.image_data_param().new_height(); const int new_width = this->layer_param_.image_data_param().new_width(); const bool is_color = this->layer_param_.image_data_param().is_color(); string root_folder = this->layer_param_.image_data_param().root_folder(); CHECK((new_height == 0 && new_width == 0) || (new_height > 0 && new_width > 0)) << "Current implementation requires " "new_height and new_width to be set at the same time."; // Read the file with filenames and labels const string& source = this->layer_param_.image_data_param().source(); LOG(INFO) << "Opening file " << source; std::ifstream infile(source.c_str()); string filename; int label; while (infile >> filename >> label) { lines_.push_back(std::make_pair(filename, label)); } if (this->layer_param_.image_data_param().shuffle()) { // randomly shuffle data LOG(INFO) << "Shuffling data"; const unsigned int prefetch_rng_seed = caffe_rng_rand(); prefetch_rng_.reset(new Engine::RNG(prefetch_rng_seed)); ShuffleImages(); } LOG(INFO) << "A total of " << lines_.size() << " images."; lines_id_ = 0; // Check if we would need to randomly skip a few data points if (this->layer_param_.image_data_param().rand_skip()) { unsigned int skip = caffe_rng_rand() % this->layer_param_.image_data_param().rand_skip(); LOG(INFO) << "Skipping first " << skip << " data points."; CHECK_GT(lines_.size(), skip) << "Not enough points to skip"; lines_id_ = skip; } // Read an image, and use it to initialize the top blob. cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first, new_height, new_width, is_color); const int channels = cv_img.channels(); const int height = cv_img.rows; const int width = cv_img.cols; // image const int crop_size = this->layer_param_.transform_param().crop_size(); const int batch_size = this->layer_param_.image_data_param().batch_size(); if (crop_size > 0) { top[0]->Reshape(batch_size, channels, crop_size, crop_size); this->prefetch_data_.Reshape(batch_size, channels, crop_size, crop_size); this->transformed_data_.Reshape(1, channels, crop_size, crop_size); } else { top[0]->Reshape(batch_size, channels, height, width); this->prefetch_data_.Reshape(batch_size, channels, height, width); this->transformed_data_.Reshape(1, channels, height, width); } LOG(INFO) << "output data size: " << top[0]->num() << "," << top[0]->channels() << "," << top[0]->height() << "," << top[0]->width(); // label vector<int> label_shape(1, batch_size); top[1]->Reshape(label_shape); this->prefetch_label_.Reshape(label_shape); }