TEST_F(IOTest, TestReadImageToCVMatResizedSquare) {
string filename = EXAMPLES_SOURCE_DIR "images/cat.jpg";
cv::Mat cv_img = ReadImageToCVMat(filename, 256, 256);
EXPECT_EQ(cv_img.channels(), 3);
EXPECT_EQ(cv_img.rows, 256);
EXPECT_EQ(cv_img.cols, 256);
}
void ImageDataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
const int new_height = this->layer_param_.image_data_param().new_height();
const int new_width = this->layer_param_.image_data_param().new_width();
const bool is_color = this->layer_param_.image_data_param().is_color();
string root_folder = this->layer_param_.image_data_param().root_folder();
CHECK((new_height == 0 && new_width == 0) ||
(new_height > 0 && new_width > 0)) << "Current implementation requires "
"new_height and new_width to be set at the same time.";
// Read the file with filenames and labels
const string& source = this->layer_param_.image_data_param().source();
LOG(INFO) << "Opening file " << source;
std::ifstream infile(source.c_str());
string filename;
int label;
while (infile >> filename >> label) {
lines_.push_back(std::make_pair(filename, label));
}
if (this->layer_param_.image_data_param().shuffle()) {
// randomly shuffle data
LOG(INFO) << "Shuffling data";
const unsigned int prefetch_rng_seed = caffe_rng_rand();
prefetch_rng_.reset(new Caffe::RNG(prefetch_rng_seed));
ShuffleImages();
}
LOG(INFO) << "A total of " << lines_.size() << " images.";
lines_id_ = 0;
// Check if we would need to randomly skip a few data points
if (this->layer_param_.image_data_param().rand_skip()) {
unsigned int skip = caffe_rng_rand() %
this->layer_param_.image_data_param().rand_skip();
LOG(INFO) << "Skipping first " << skip << " data points.";
CHECK_GT(lines_.size(), skip) << "Not enough points to skip";
lines_id_ = skip;
}
// Read an image, and use it to initialize the top blob.
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color);
CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
// Use data_transformer to infer the expected blob shape from a cv_image.
vector<int> top_shape = this->data_transformer_->InferBlobShape(cv_img);
this->transformed_data_.Reshape(top_shape);
// Reshape prefetch_data and top[0] according to the batch_size.
const int batch_size = this->layer_param_.image_data_param().batch_size();
CHECK_GT(batch_size, 0) << "Positive batch size required";
top_shape[0] = batch_size;
this->prefetch_data_.Reshape(top_shape);
top[0]->ReshapeLike(this->prefetch_data_);
LOG(INFO) << "output data size: " << top[0]->num() << ","
<< top[0]->channels() << "," << top[0]->height() << ","
<< top[0]->width();
// label
vector<int> label_shape(1, batch_size);
top[1]->Reshape(label_shape);
this->prefetch_label_.Reshape(label_shape);
}
TEST_F(IOTest, TestReadImageToCVMat) {
string filename = EXAMPLES_SOURCE_DIR "images/cat.jpg";
cv::Mat cv_img = ReadImageToCVMat(filename);
EXPECT_EQ(cv_img.channels(), 3);
EXPECT_EQ(cv_img.rows, 360);
EXPECT_EQ(cv_img.cols, 480);
}
bool ReadImageToDatum(const string& filename, const int label, const int height, const int width,
const bool is_color, const string& encoding, JDatum* datum)
{
cv::Mat cv_img = ReadImageToCVMat(filename, height, width, is_color);
if (cv_img.data)
{
if (encoding.size())
{
if ((cv_img.channels() == 3) && is_color == 1 && !height && !width && matchExt(filename, encoding))
{
return ReadFileToDatum(filename, label, datum);
}
vector<uchar> buf;
cv::imencode("." + encoding, cv_img, buf);
datum->SetData(string(reinterpret_cast<char*>(&buf[0]), buf.size()));
datum->SetLabel(label);
datum->SetEncoded(true);
return true;
}
CVMatToDatum(cv_img, datum);
datum->SetLabel(label);
return true;
}
else
{
return false;
}
}
TEST_F(IOTest, TestReadImageToCVMatResizedGray) {
string filename = EXAMPLES_SOURCE_DIR "images/cat.jpg";
const bool is_color = false;
cv::Mat cv_img = ReadImageToCVMat(filename, 256, 256, is_color);
EXPECT_EQ(cv_img.channels(), 1);
EXPECT_EQ(cv_img.rows, 256);
EXPECT_EQ(cv_img.cols, 256);
}
void DepthDataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
const int new_height = this->layer_param_.depth_data_param().new_height();
const int new_width = this->layer_param_.depth_data_param().new_width();
const bool is_color = this->layer_param_.depth_data_param().is_color();
string root_folder = this->layer_param_.depth_data_param().root_folder();
CHECK((new_height == 0 && new_width == 0) ||
(new_height > 0 && new_width > 0)) << "Current implementation requires "
"new_height and new_width to be set at the same time.";
// Read the file with image filenames and depth filenames
const string& source = this->layer_param_.depth_data_param().source();
LOG(INFO) << "Opening file " << source;
std::ifstream infile(source.c_str());
string image_filename;
string depth_filename;
while (infile >> image_filename >> depth_filename) {
lines_.push_back(std::make_pair(image_filename, depth_filename));
}
infile.close();
// randomly shuffle data
LOG(INFO) << "Shuffleing data";
const unsigned int prefetch_rng_seed = caffe_rng_rand();
prefetch_rng_.reset(new Caffe::RNG(prefetch_rng_seed));
ShuffleImages();
LOG(INFO) << "A total of " << lines_.size() << " images.";
lines_id_ = 0;
//image
// Read an image, and use it to initialize the top blob.
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color);
// Use data_transformer to infer the expected blob shape from a cv_image.
vector<int> top_shape = this->data_transformer_->InferBlobShape(cv_img);
this->transformed_data_.Reshape(top_shape);
// Reshape prefetch_data and top[0] according to the batch_size.
const int batch_size = this->layer_param_.depth_data_param().batch_size();
CHECK_GT(batch_size, 0) << "Positive batch size required";
top_shape[0] = batch_size;
this->prefetch_data_.Reshape(top_shape);
top[0]->ReshapeLike(this->prefetch_data_);
LOG(INFO) << "output data size: " << top[0]->num() << ","
<< top[0]->channels() << "," << top[0]->height() << ","
<< top[0]->width();
//depths
vector<int> label_shape;
label_shape.push_back(batch_size);
label_shape.push_back(74*74);
top[1]->Reshape(label_shape);
this->prefetch_label_.Reshape(label_shape);
LOG(INFO) << "output depth size: " << label_shape[0] << ","
<< label_shape[1];
}
TEST_F(IOTest, TestCVMatToDatum) {
string filename = EXAMPLES_SOURCE_DIR "images/cat.jpg";
cv::Mat cv_img = ReadImageToCVMat(filename);
Datum datum;
CVMatToDatum(cv_img, &datum);
EXPECT_EQ(datum.channels(), 3);
EXPECT_EQ(datum.height(), 360);
EXPECT_EQ(datum.width(), 480);
}
void ImageDataLayer<Dtype>::InternalThreadEntry() {
CPUTimer batch_timer;
batch_timer.Start();
double read_time = 0;
double trans_time = 0;
CPUTimer timer;
CHECK(this->prefetch_data_.count());
CHECK(this->transformed_data_.count());
Dtype* top_data = this->prefetch_data_.mutable_cpu_data();
Dtype* top_label = this->prefetch_label_.mutable_cpu_data();
ImageDataParameter image_data_param = this->layer_param_.image_data_param();
const int batch_size = image_data_param.batch_size();
const int new_height = image_data_param.new_height();
const int new_width = image_data_param.new_width();
const bool is_color = image_data_param.is_color();
string root_folder = image_data_param.root_folder();
// datum scales
const int lines_size = lines_.size();
for (int item_id = 0; item_id < batch_size; ++item_id) {
// get a blob
timer.Start();
CHECK_GT(lines_size, lines_id_);
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color);
if (!cv_img.data) {
continue;
}
read_time += timer.MicroSeconds();
timer.Start();
// Apply transformations (mirror, crop...) to the image
int offset = this->prefetch_data_.offset(item_id);
this->transformed_data_.set_cpu_data(top_data + offset);
this->data_transformer_.Transform(cv_img, &(this->transformed_data_));
trans_time += timer.MicroSeconds();
top_label[item_id] = lines_[lines_id_].second;
// go to the next iter
lines_id_++;
if (lines_id_ >= lines_size) {
// We have reached the end. Restart from the first.
DLOG(INFO) << "Restarting data prefetching from start.";
lines_id_ = 0;
if (this->layer_param_.image_data_param().shuffle()) {
ShuffleImages();
}
}
}
batch_timer.Stop();
DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
DLOG(INFO) << " Read time: " << read_time / 1000 << " ms.";
DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
}
void ImgReaderLayer<Dtype>::ReadData()
{
if(data_.count()) return;
cv::Mat cv_img = ReadImageToCVMat(this->layer_param_.reader_param().file());
CHECK(cv_img.data) << "Could not load " << this->layer_param_.reader_param().file();
DataTransformer<Dtype> data_transformer(this->layer_param_.transform_param(), this->phase_);
vector<int> top_shape = data_transformer.InferBlobShape(cv_img);
data_.Reshape(top_shape);
data_transformer.Transform(cv_img, &data_);
}
TEST_F(IOTest, TestReadImageToDatumContentGray) {
string filename = EXAMPLES_SOURCE_DIR "images/cat.jpg";
Datum datum;
const bool is_color = false;
ReadImageToDatum(filename, 0, is_color, &datum);
cv::Mat cv_img = ReadImageToCVMat(filename, is_color);
EXPECT_EQ(datum.channels(), cv_img.channels());
EXPECT_EQ(datum.height(), cv_img.rows);
EXPECT_EQ(datum.width(), cv_img.cols);
const string& data = datum.data();
int index = 0;
for (int h = 0; h < datum.height(); ++h) {
for (int w = 0; w < datum.width(); ++w) {
EXPECT_TRUE(data[index++] == static_cast<char>(cv_img.at<uchar>(h, w)));
}
}
}
TEST_F(IOTest, TestCVMatToDatumReference) {
string filename = EXAMPLES_SOURCE_DIR "images/cat.jpg";
cv::Mat cv_img = ReadImageToCVMat(filename);
Datum datum;
CVMatToDatum(cv_img, &datum);
Datum datum_ref;
ReadImageToDatumReference(filename, 0, 0, 0, true, &datum_ref);
EXPECT_EQ(datum.channels(), datum_ref.channels());
EXPECT_EQ(datum.height(), datum_ref.height());
EXPECT_EQ(datum.width(), datum_ref.width());
EXPECT_EQ(datum.data().size(), datum_ref.data().size());
const string& data = datum.data();
const string& data_ref = datum_ref.data();
for (int i = 0; i < datum.data().size(); ++i) {
EXPECT_TRUE(data[i] == data_ref[i]);
}
}
void FlowDataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
// Read the file with filenames and labels
const string& source = this->layer_param_.flow_data_param().source();
LOG(INFO) << "Opening file " << source;
std::ifstream infile(source.c_str());
string filename;
int label;
while (infile >> filename >> label) {
lines_.push_back(std::make_pair(filename, label));
}
LOG(INFO) << "A total of " << lines_.size() << " images.";
lines_id_ = 0;
// Read an image, and use it to initialize the top blob.
cv::Mat cv_img = ReadImageToCVMat(lines_[lines_id_].first, 0, 0 , true);
const int channels = this->layer_param_.flow_data_param().stack_size() * 2;
const int height = cv_img.rows;
const int width = cv_img.cols;
// image
const int crop_size = this->layer_param_.transform_param().crop_size();
const int batch_size = this->layer_param_.flow_data_param().batch_size();
// flow
flow_field_ = shared_ptr<Blob<Dtype> >(new Blob<Dtype>());
flow_stack_ = shared_ptr<Blob<Dtype> >(new Blob<Dtype>());
flow_field_->Reshape(1, 2, height, width);
flow_stack_->Reshape(1, channels, height, width);
if (crop_size > 0) {
top[0]->Reshape(batch_size, channels, crop_size, crop_size);
this->prefetch_data_.Reshape(batch_size, channels, crop_size, crop_size);
this->transformed_data_.Reshape(1, channels, crop_size, crop_size);
} else {
top[0]->Reshape(batch_size, channels, height, width);
this->prefetch_data_.Reshape(batch_size, channels, height, width);
this->transformed_data_.Reshape(1, channels, height, width);
}
LOG(INFO) << "output data size: " << top[0]->num() << ","
<< top[0]->channels() << "," << top[0]->height() << ","
<< top[0]->width();
// label
top[1]->Reshape(batch_size, 1, 1, 1);
this->prefetch_label_.Reshape(batch_size, 1, 1, 1);
}
TEST_F(IOTest, TestDecodeDatumToCVMatContentNative) {
string filename = EXAMPLES_SOURCE_DIR "images/cat.jpg";
Datum datum;
EXPECT_TRUE(ReadImageToDatum(filename, 0, std::string("jpg"), &datum));
cv::Mat cv_img = DecodeDatumToCVMatNative(datum);
cv::Mat cv_img_ref = ReadImageToCVMat(filename);
EXPECT_EQ(cv_img_ref.channels(), cv_img.channels());
EXPECT_EQ(cv_img_ref.rows, cv_img.rows);
EXPECT_EQ(cv_img_ref.cols, cv_img.cols);
for (int c = 0; c < datum.channels(); ++c) {
for (int h = 0; h < datum.height(); ++h) {
for (int w = 0; w < datum.width(); ++w) {
EXPECT_TRUE(cv_img.at<cv::Vec3b>(h, w)[c]==
cv_img_ref.at<cv::Vec3b>(h, w)[c]);
}
}
}
}
TEST_F(IOTest, TestReadImageToDatumContent) {
string filename = EXAMPLES_SOURCE_DIR "images/cat.jpg";
Datum datum;
ReadImageToDatum(filename, 0, &datum);
cv::Mat cv_img = ReadImageToCVMat(filename);
EXPECT_EQ(datum.channels(), cv_img.channels());
EXPECT_EQ(datum.height(), cv_img.rows);
EXPECT_EQ(datum.width(), cv_img.cols);
const string& data = datum.data();
int_tp index = 0;
for (int_tp c = 0; c < datum.channels(); ++c) {
for (int_tp h = 0; h < datum.height(); ++h) {
for (int_tp w = 0; w < datum.width(); ++w) {
EXPECT_TRUE(data[index++] ==
static_cast<char>(cv_img.at<cv::Vec3b>(h, w)[c]));
}
}
}
}
void ImageLabelmapDataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
const int new_height = this->layer_param_.image_data_param().new_height();
const int new_width = this->layer_param_.image_data_param().new_width();
const bool is_color = this->layer_param_.image_data_param().is_color();
string root_folder = this->layer_param_.image_data_param().root_folder();
CHECK((new_height == 0 && new_width == 0) ||
(new_height > 0 && new_width > 0)) << "Current implementation requires "
"new_height and new_width to be set at the same time.";
// Read the file with filenames and labels
const string& source = this->layer_param_.image_data_param().source();
LOG(INFO) << "Opening file " << source;
std::ifstream infile(source.c_str());
string img_filename;
string gt_filename;
while (infile >> img_filename >> gt_filename) {
lines_.push_back(std::make_pair(img_filename, gt_filename));
}
if (this->layer_param_.image_data_param().shuffle()) {
// randomly shuffle data
LOG(INFO) << "Shuffling data";
const unsigned int prefetch_rng_seed = caffe_rng_rand();
prefetch_rng_.reset(new Caffe::RNG(prefetch_rng_seed));
ShuffleImages();
}
LOG(INFO) << "A total of " << lines_.size() << " images.";
lines_id_ = 0;
// Check if we would need to randomly skip a few data points
if (this->layer_param_.image_data_param().rand_skip()) {
unsigned int skip = caffe_rng_rand() %
this->layer_param_.image_data_param().rand_skip();
LOG(INFO) << "Skipping first " << skip << " data points.";
CHECK_GT(lines_.size(), skip) << "Not enough points to skip";
lines_id_ = skip;
}
// Read an image, and use it to initialize the top blob.
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color);
cv::Mat cv_gt = ReadImageToCVMat(root_folder + lines_[lines_id_].second,
new_height, new_width, 0);
//const int channels = cv_img.channels();
const int height = cv_img.rows;
const int width = cv_img.cols;
const int gt_channels = cv_gt.channels();
const int gt_height = cv_gt.rows;
const int gt_width = cv_gt.cols;
CHECK((height == gt_height) && (width == gt_width)) << "groundtruth size != image size";
CHECK(gt_channels == 1) << "GT image channel number should be 1";
CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
if (new_height > 0 && new_width > 0) {
cv::resize(cv_img, cv_img, cv::Size(new_width, new_height));
cv::resize(cv_gt, cv_gt, cv::Size(new_width, new_height));
}
// Use data_transformer to infer the expected blob shape from a cv_image.
vector<int> top_shape = this->data_transformer_->InferBlobShape(cv_img);
vector<int> top_shape_labelmap = this->data_transformer_->InferBlobShape(cv_gt);
this->transformed_data_.Reshape(top_shape);
this->transformed_labelmap_.Reshape(top_shape_labelmap);
// Reshape prefetch_data and top[0] according to the batch_size.
const int batch_size = this->layer_param_.image_data_param().batch_size();
CHECK_GT(batch_size, 0) << "Positive batch size required";
top_shape[0] = batch_size;
top_shape_labelmap[0] = batch_size;
for (int i = 0; i < this->PREFETCH_COUNT; ++i) {
this->prefetch_[i].data_.Reshape(top_shape);
this->prefetch_[i].labelmap_.Reshape(top_shape_labelmap);
}
top[0]->Reshape(top_shape);
top[1]->Reshape(top_shape_labelmap);
LOG(INFO) << "output data size: " << top[0]->num() << ","
<< top[0]->channels() << "," << top[0]->height() << ","
<< top[0]->width();
LOG(INFO) << "output label size: " << top[1]->num() << ","
<< top[1]->channels() << "," << top[1]->height() << ","
<< top[1]->width();
}
void ImageLabelmapDataLayer<Dtype>::load_batch(LabelmapBatch<Dtype>* batch) {
CPUTimer batch_timer;
batch_timer.Start();
double read_time = 0;
double trans_time = 0;
CPUTimer timer;
CHECK(batch->data_.count());
CHECK(batch->labelmap_.count());
CHECK(this->transformed_data_.count());
CHECK(this->transformed_labelmap_.count());
ImageDataParameter image_data_param = this->layer_param_.image_data_param();
const int batch_size = image_data_param.batch_size();
const int new_height = image_data_param.new_height();
const int new_width = image_data_param.new_width();
const bool is_color = image_data_param.is_color();
string root_folder = image_data_param.root_folder();
// Reshape according to the first image of each batch
// on single input batches allows for inputs of varying dimension.
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color);
cv::Mat cv_gt = ReadImageToCVMat(root_folder + lines_[lines_id_].second,
new_height, new_width, 0);
CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
// Use data_transformer to infer the expected blob shape from a cv_img.
vector<int> top_shape = this->data_transformer_->InferBlobShape(cv_img);
vector<int> top_shape_labelmap = this->data_transformer_->InferBlobShape(cv_gt);
this->transformed_data_.Reshape(top_shape);
this->transformed_labelmap_.Reshape(top_shape_labelmap);
// Reshape prefetch_data and top[0] according to the batch_size.
top_shape[0] = batch_size;
top_shape_labelmap[0] = batch_size;
batch->data_.Reshape(top_shape);
batch->labelmap_.Reshape(top_shape_labelmap);
Dtype* prefetch_data = batch->data_.mutable_cpu_data();
Dtype* prefetch_labelmap = batch->labelmap_.mutable_cpu_data();
// datum scales
const int lines_size = lines_.size();
for (int item_id = 0; item_id < batch_size; ++item_id) {
// get a blob
timer.Start();
CHECK_GT(lines_size, lines_id_);
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
0, 0, is_color);
cv::Mat cv_gt = ReadImageToCVMat(root_folder + lines_[lines_id_].second,
0, 0, 0);
CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
const int height = cv_img.rows;
const int width = cv_img.cols;
const int gt_channels = cv_gt.channels();
const int gt_height = cv_gt.rows;
const int gt_width = cv_gt.cols;
CHECK((height == gt_height) && (width == gt_width)) << "GT image size should be equal to true image size";
CHECK(gt_channels == 1) << "GT image channel number should be 1";
if (new_height > 0 && new_width > 0) {
cv::resize(cv_img, cv_img, cv::Size(new_width, new_height));
cv::resize(cv_gt, cv_gt, cv::Size(new_width, new_height), 0, 0, cv::INTER_LINEAR);
}
if (!cv_img.data || !cv_gt.data) {
continue;
}
read_time += timer.MicroSeconds();
timer.Start();
// Apply transformations (mirror, crop...) to the image
int offset = batch->data_.offset(item_id);
int offset_gt = batch->labelmap_.offset(item_id);
//CHECK(offset == offset_gt) << "fetching should be synchronized";
this->transformed_data_.set_cpu_data(prefetch_data + offset);
this->transformed_labelmap_.set_cpu_data(prefetch_labelmap + offset_gt);
std::pair<int, int> hw_off = this->data_transformer_->LocTransform(cv_img, &(this->transformed_data_));
cv::Mat encoded_gt;
//regression
encoded_gt = cv_gt/255;
//[***Cautions***]
//One small trick leveraging opencv roundoff feature for **consensus sampling** in Holistically-Nested Edge Detection paper.
//For general binary edge maps this is okay
//For 5-subject aggregated edge maps (BSDS), this will abandon weak edge points labeled by only two or less labelers.
this->data_transformer_->LabelmapTransform(encoded_gt, &(this->transformed_labelmap_), hw_off);
trans_time += timer.MicroSeconds();
// go to the next iter
lines_id_++;
if (lines_id_ >= lines_size) {
// We have reached the end. Restart from the first.
DLOG(INFO) << "Restarting data prefetching from start.";
lines_id_ = 0;
if (this->layer_param_.image_data_param().shuffle()) {
ShuffleImages();
}
}
}
batch_timer.Stop();
DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
DLOG(INFO) << " Read time: " << read_time / 1000 << " ms.";
DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
}
void MultiImageDataLayer<Dtype>::load_batch(Batch<Dtype>* batch) {
CPUTimer batch_timer;
batch_timer.Start();
double read_time = 0;
double trans_time = 0;
CPUTimer timer;
CHECK(batch->data_.count());
CHECK(this->transformed_data_.count());
MultiImageDataParameter multi_image_data_param = this->layer_param_.multi_image_data_param();
const int batch_size = multi_image_data_param.batch_size();
const int new_height = multi_image_data_param.new_height();
const int new_width = multi_image_data_param.new_width();
const bool is_color = multi_image_data_param.is_color();
string root_folder = multi_image_data_param.root_folder();
const int num_images = this->layer_param_.multi_image_data_param().num_images();
// Reshape according to the first image of each batch
// on single input batches allows for inputs of varying dimension.
cv::Mat cv_img = ReadImageToCVMat(root_folder + *lines_[lines_id_].first.begin(),
new_height, new_width, is_color);
CHECK(cv_img.data) << "Could not load " << *lines_[lines_id_].first.begin();
// Use data_transformer to infer the expected blob shape from a cv_img.
vector<int> top_shape = this->data_transformer_->InferBlobShape(cv_img);
this->transformed_data_.Reshape(top_shape);
top_shape[1] *= num_images;
// Reshape batch according to the batch_size.
top_shape[0] = batch_size;
batch->data_.Reshape(top_shape);
Dtype* prefetch_data = batch->data_.mutable_cpu_data();
Dtype* prefetch_label = batch->label_.mutable_cpu_data();
// datum scales
const int lines_size = lines_.size();
for (int item_id = 0; item_id < batch_size; ++item_id) {
// get a blob
timer.Start();
CHECK_GT(lines_size, lines_id_);
if (this->layer_param_.multi_image_data_param().shuffle_images() == true) {
caffe::rng_t* prefetch_rng =
static_cast<caffe::rng_t*>(prefetch_rng_->generator());
shuffle(lines_[lines_id_].first.begin(), lines_[lines_id_].first.end(), prefetch_rng);
}
read_time += timer.MicroSeconds();
timer.Start();
for (int image_index = 0; image_index < num_images; image_index++) {
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first[image_index], new_height, new_width, is_color);
CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first[image_index];
// Apply transformations (mirror, crop...) to the image
int offset = batch->data_.offset(item_id, image_index * cv_img.channels());
this->transformed_data_.set_cpu_data(prefetch_data + offset);
this->data_transformer_->Transform(cv_img, &(this->transformed_data_));
}
trans_time += timer.MicroSeconds();
prefetch_label[item_id] = lines_[lines_id_].second;
// go to the next iter
lines_id_++;
if (lines_id_ >= lines_size) {
// We have reached the end. Restart from the first.
DLOG(INFO) << "Restarting data prefetching from start.";
lines_id_ = 0;
if (this->layer_param_.multi_image_data_param().shuffle()) {
ShuffleImages();
}
}
}
batch_timer.Stop();
DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
DLOG(INFO) << " Read time: " << read_time / 1000 << " ms.";
DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
}
void SiameseDataLayer<Dtype>::load_batch(Batch<Dtype>* batch) {
CPUTimer batch_timer;
batch_timer.Start();
double read_time = 0;
double trans_time = 0;
CPUTimer timer;
CHECK(batch->data_.count());
CHECK(this->transformed_data_.count());
ImageDataParameter image_data_param = this->layer_param_.image_data_param();
const int batch_size = image_data_param.batch_size();
const int new_height = image_data_param.new_height();
const int new_width = image_data_param.new_width();
const bool is_color = image_data_param.is_color();
string root_folder = image_data_param.root_folder();
const int interpolation = image_data_param.interpolation();
const int resize_mode = image_data_param.resize_mode();
// Reshape according to the first image of each batch
// on single input batches allows for inputs of varying dimension.
cv::Mat cv_img = ReadImageToCVMat(root_folder + pair_lines_[lines_id_].first,
new_height, new_width, is_color, interpolation, resize_mode);
CHECK(cv_img.data) << "Could not load " << pair_lines_[lines_id_].first;
// Use data_transformer to infer the expected blob shape from a cv_img.
vector<int> top_shape = this->data_transformer_->InferBlobShape(cv_img);
this->transformed_data_.Reshape(top_shape);
// Reshape batch according to the batch_size.
top_shape[0] = batch_size;
batch->data_.Reshape(top_shape);
Dtype* prefetch_data = batch->data_.mutable_cpu_data();
Dtype* prefetch_label = batch->label_.mutable_cpu_data();
// datum scales
const int lines_size = pair_lines_.size();
for (int item_id = 0; item_id < batch_size; ++item_id) {
// get a blob
timer.Start();
CHECK_GT(lines_size, lines_id_);
cv::Mat cv_img = ReadImageToCVMat(root_folder + pair_lines_[lines_id_].first,
new_height, new_width, is_color, interpolation, resize_mode);
CHECK(cv_img.data) << "Could not load " << pair_lines_[lines_id_].first;
read_time += timer.MicroSeconds();
timer.Start();
//cv::imwrite("aa.jpg", cv_img);
// Apply transformations (mirror, crop...) to the image
int offset = batch->data_.offset(item_id);
this->transformed_data_.set_cpu_data(prefetch_data + offset);
this->data_transformer_->Transform(cv_img, &(this->transformed_data_));
trans_time += timer.MicroSeconds();
prefetch_label[item_id] = pair_lines_[lines_id_].second;
// go to the next iter
lines_id_++;
if (lines_id_ >= lines_size) {
// We have reached the end. Restart from the first.
DLOG(INFO) << "Restarting data prefetching from start.";
lines_id_ = 0;
if (this->layer_param_.image_data_param().shuffle()) {
ShuffleImages();
}
}
}
batch_timer.Stop();
DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
DLOG(INFO) << " Read time: " << read_time / 1000 << " ms.";
DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
}
void DepthDataLayer<Dtype>::InternalThreadEntry() {
CPUTimer batch_timer;
batch_timer.Start();
double read_time = 0;
double trans_time = 0;
CPUTimer timer;
CHECK(this->prefetch_data_.count());
CHECK(this->transformed_data_.count());
//CHECK(this->transformed_data_.count());
DepthDataParameter depth_data_param = this->layer_param_.depth_data_param();
const int batch_size = depth_data_param.batch_size();
const int new_height = depth_data_param.new_height();
const int new_width = depth_data_param.new_width();
const bool is_color = depth_data_param.is_color();
string root_folder = depth_data_param.root_folder();
// Reshape according to the first image of each batch
// on single input batches allows for inputs of varying dimension.
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color);
CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
// Use data_transformer to infer the expected blob shape from a cv_img.
vector<int> top_shape = this->data_transformer_->InferBlobShape(cv_img);
this->transformed_data_.Reshape(top_shape);
// Reshape prefetch_data according to the batch_size.
top_shape[0] = batch_size;
this->prefetch_data_.Reshape(top_shape);
Dtype* prefetch_data = this->prefetch_data_.mutable_cpu_data();
Dtype* prefetch_label = this->prefetch_label_.mutable_cpu_data();
// datum scales
const int lines_size = lines_.size();
for (int item_id = 0; item_id < batch_size; ++item_id) {
// get a blob
timer.Start();
CHECK_GT(lines_size, lines_id_);
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color);
CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
read_time += timer.MicroSeconds();
timer.Start();
int offset = this->prefetch_data_.offset(item_id);
this->transformed_data_.set_cpu_data(prefetch_data + offset);
this->data_transformer_->Transform(cv_img, &(this->transformed_data_));
trans_time += timer.MicroSeconds();
//read Depths
//prefetch_label[item_id] = lines_[lines_id_].second;
float depths[74*74];
ReadDepthToArray(lines_[lines_id_].second, depths);
int depth_offset = this->prefetch_label_.offset(item_id);
memcpy(&prefetch_label[depth_offset], &depths[0], sizeof(depths));
// go to the next iter
lines_id_++;
if (lines_id_ >= lines_size) {
// We have reached the end. Restart from the first.
DLOG(INFO) << "Restarting data prefetching from start.";
lines_id_ = 0;
ShuffleImages();
}
}
batch_timer.Stop();
DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
DLOG(INFO) << " Read time: " << read_time / 1000 << " ms.";
DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
}
void ImageDataLayer<Dtype>::load_batch(Batch<Dtype>* batch) {
CPUTimer batch_timer;
batch_timer.Start();
double read_time = 0;
double trans_time = 0;
CPUTimer timer;
CHECK(batch->data_.count());
CHECK(this->transformed_data_.count());
// 获取层参数，具体参见层参数的定义的解释
ImageDataParameter image_data_param = this->layer_param_.image_data_param();
const int batch_size = image_data_param.batch_size();
const int new_height = image_data_param.new_height();
const int new_width = image_data_param.new_width();
const bool is_color = image_data_param.is_color();
string root_folder = image_data_param.root_folder();
// Reshape according to the first image of each batch
// on single input batches allows for inputs of varying dimension.
// 读取跳过之后的第一幅图像，然后根据该图像设置相撞
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color);
CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
// Use data_transformer to infer the expected blob shape from a cv_img.
// 推断图像形状
vector<int> top_shape = this->data_transformer_->InferBlobShape(cv_img);
// 设置transformed_data_形状
this->transformed_data_.Reshape(top_shape);
// Reshape batch according to the batch_size.
// 设置batch_size
top_shape[0] = batch_size;
batch->data_.Reshape(top_shape);
Dtype* prefetch_data = batch->data_.mutable_cpu_data();
Dtype* prefetch_label = batch->label_.mutable_cpu_data();
// datum scales
// 读取一批图像，并进行预处理
const int lines_size = lines_.size();
for (int item_id = 0; item_id < batch_size; ++item_id) {
// get a blob
timer.Start();
CHECK_GT(lines_size, lines_id_);
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color);
CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
read_time += timer.MicroSeconds();
timer.Start();
// Apply transformations (mirror, crop...) to the image
// 进行预处理
// 根据图像的批次获得图像数据的偏移量
int offset = batch->data_.offset(item_id);
// 设置图像数据的指针到transformed_data_
this->transformed_data_.set_cpu_data(prefetch_data + offset);
// 进行预处理
this->data_transformer_->Transform(cv_img, &(this->transformed_data_));
trans_time += timer.MicroSeconds();//统计预处理时间
// 复制类标到prefetch_label
prefetch_label[item_id] = lines_[lines_id_].second;
// go to the next iter
lines_id_++;
// 是否是图像目录中的最后一个图像
if (lines_id_ >= lines_size) {
// We have reached the end. Restart from the first.
DLOG(INFO) << "Restarting data prefetching from start.";
lines_id_ = 0;
// 打乱图像索引的顺序
if (this->layer_param_.image_data_param().shuffle()) {
ShuffleImages();
}
}
}
batch_timer.Stop();
DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
DLOG(INFO) << " Read time: " << read_time / 1000 << " ms.";
// 预处理时间
DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
}
void ImageDataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
// 根据参数文件设置参数
// 图像的高度、宽度、是否彩色图像、图像目录
const int new_height = this->layer_param_.image_data_param().new_height();
const int new_width = this->layer_param_.image_data_param().new_width();
const bool is_color = this->layer_param_.image_data_param().is_color();
string root_folder = this->layer_param_.image_data_param().root_folder();
// 当前只支持读取高度和宽度同样大小的图像
CHECK((new_height == 0 && new_width == 0) ||
(new_height > 0 && new_width > 0)) << "Current implementation requires "
"new_height and new_width to be set at the same time.";
// Read the file with filenames and labels
// 读取存放图像文件名和类标的列表文件
const string& source = this->layer_param_.image_data_param().source();
LOG(INFO) << "Opening file " << source;
std::ifstream infile(source.c_str());
string filename;
int label;
// lines_存放文件名和类标的pair
while (infile >> filename >> label) {
lines_.push_back(std::make_pair(filename, label));
}
// 是否需要打乱文件的顺序
if (this->layer_param_.image_data_param().shuffle()) {
// randomly shuffle data
LOG(INFO) << "Shuffling data";
const unsigned int prefetch_rng_seed = caffe_rng_rand();
prefetch_rng_.reset(new Caffe::RNG(prefetch_rng_seed));
ShuffleImages();
}
LOG(INFO) << "A total of " << lines_.size() << " images.";
// 随机跳过的图像，调过的图像个数在[0, rand_skip-1]之间
lines_id_ = 0;
// Check if we would need to randomly skip a few data points
// 如果参数中的rand_skip大于1，则随机跳过[0,rand_skip-1]个图片
//
if (this->layer_param_.image_data_param().rand_skip()) {
unsigned int skip = caffe_rng_rand() %
this->layer_param_.image_data_param().rand_skip();
LOG(INFO) << "Skipping first " << skip << " data points.";
CHECK_GT(lines_.size(), skip) << "Not enough points to skip";
lines_id_ = skip;
}
// Read an image, and use it to initialize the top blob.
// 读取文件名到Mat
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color);
CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
// Use data_transformer to infer the expected blob shape from a cv_image.
// 对数据的形状进行推断
vector<int> top_shape = this->data_transformer_->InferBlobShape(cv_img);
// 设置transformed_data_的形状
this->transformed_data_.Reshape(top_shape);
// Reshape prefetch_data and top[0] according to the batch_size.
// 设置batch_size
const int batch_size = this->layer_param_.image_data_param().batch_size();
CHECK_GT(batch_size, 0) << "Positive batch size required";
top_shape[0] = batch_size;
// 设置预取数组中数据的形状
for (int i = 0; i < this->PREFETCH_COUNT; ++i) {
this->prefetch_[i].data_.Reshape(top_shape);
}
// 设置输出的数据的形状
top[0]->Reshape(top_shape);
LOG(INFO) << "output data size: " << top[0]->num() << ","
<< top[0]->channels() << "," << top[0]->height() << ","
<< top[0]->width();
// label
// 设置输出的类标的形状
vector<int> label_shape(1, batch_size);
top[1]->Reshape(label_shape);
// 设置预取数组中类标的形状
for (int i = 0; i < this->PREFETCH_COUNT; ++i) {
this->prefetch_[i].label_.Reshape(label_shape);
}
}
void ImageDataLayer<Dtype>::load_batch(Batch<Dtype>* batch) {
CPUTimer batch_timer;
batch_timer.Start();
double read_time = 0;
double trans_time = 0;
CPUTimer timer;
CHECK(batch->data_.count());
CHECK(this->transformed_data_.count());
ImageDataParameter image_data_param = this->layer_param_.image_data_param();
const int batch_size = image_data_param.batch_size();
const int new_height = image_data_param.new_height();
const int new_width = image_data_param.new_width();
const int min_height = image_data_param.min_height();
const int min_width = image_data_param.min_width();
const bool is_color = image_data_param.is_color();
string root_folder = image_data_param.root_folder();
// Reshape according to the first image of each batch
// on single input batches allows for inputs of varying dimension.
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color, min_height, min_width);
CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
// Use data_transformer to infer the expected blob shape from a cv_img.
vector<int> top_shape = this->data_transformer_->InferBlobShape(cv_img);
this->transformed_data_.Reshape(top_shape);
// Reshape batch according to the batch_size.
top_shape[0] = batch_size;
batch->data_.Reshape(top_shape);
Dtype* prefetch_data = batch->data_.mutable_cpu_data();
Dtype* prefetch_label = batch->label_.mutable_cpu_data();
// datum scales
const int lines_size = lines_.size();
#ifdef _OPENMP
#pragma omp parallel if (batch_size > 1)
#pragma omp single nowait
#endif
for (int item_id = 0; item_id < batch_size; ++item_id) {
// get a blob
timer.Start();
CHECK_GT(lines_size, lines_id_);
#ifndef _OPENMP
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color, min_height, min_width);
CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
read_time += timer.MicroSeconds();
timer.Start();
// Apply transformations (mirror, crop...) to the image
int offset = batch->data_.offset(item_id);
this->transformed_data_.set_cpu_data(prefetch_data + offset);
this->data_transformer_->Transform(cv_img, &(this->transformed_data_));
trans_time += timer.MicroSeconds();
#else
read_time = 0;
trans_time = 0;
int offset = batch->data_.offset(item_id);
std::string img_file_name = lines_[lines_id_].first;
PreclcRandomNumbers precalculated_rand_numbers;
this->data_transformer_->GenerateRandNumbers(precalculated_rand_numbers);
#pragma omp task firstprivate(offset, img_file_name, \
precalculated_rand_numbers)
{
cv::Mat cv_img = ReadImageToCVMat(root_folder + img_file_name,
new_height, new_width, is_color);
CHECK(cv_img.data) << "Could not load " << img_file_name;
Blob<Dtype> tmp_data;
tmp_data.Reshape(top_shape);
tmp_data.set_cpu_data(prefetch_data + offset);
this->data_transformer_->Transform(cv_img, &tmp_data,
precalculated_rand_numbers);
}
#endif
prefetch_label[item_id] = lines_[lines_id_].second;
// go to the next iter
lines_id_++;
if (lines_id_ >= lines_size) {
// We have reached the end. Restart from the first.
DLOG(INFO) << "Restarting data prefetching from start.";
lines_id_ = 0;
if (this->layer_param_.image_data_param().shuffle()) {
ShuffleImages();
}
}
}
batch_timer.Stop();
DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
DLOG(INFO) << " Read time: " << read_time / 1000 << " ms.";
DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
}
void ImageDataLayer<Dtype>::InternalThreadEntry() {
CPUTimer batch_timer;
batch_timer.Start();
double read_time = 0;
double trans_time = 0;
CPUTimer timer;
CHECK(this->prefetch_data_.count());
CHECK(this->transformed_data_.count());
ImageDataParameter image_data_param = this->layer_param_.image_data_param();
const int batch_size = image_data_param.batch_size();
const int new_height = image_data_param.new_height();
const int new_width = image_data_param.new_width();
const int crop_size = this->layer_param_.transform_param().crop_size();
const bool is_color = image_data_param.is_color();
string root_folder = image_data_param.root_folder();
// Reshape on single input batches for inputs of varying dimension.
if (batch_size == 1 && crop_size == 0 && new_height == 0 && new_width == 0) {
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
0, 0, is_color);
this->prefetch_data_.Reshape(1, cv_img.channels(),
cv_img.rows, cv_img.cols);
this->transformed_data_.Reshape(1, cv_img.channels(),
cv_img.rows, cv_img.cols);
}
Dtype* prefetch_data = this->prefetch_data_.mutable_cpu_data();
Dtype* prefetch_label = this->prefetch_label_.mutable_cpu_data();
// datum scales
const int lines_size = lines_.size();
int label_dim = this->layer_param_.image_data_param().label_dim();
for (int item_id = 0; item_id < batch_size; ++item_id) {
// get a blob
timer.Start();
CHECK_GT(lines_size, lines_id_);
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color);
CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
read_time += timer.MicroSeconds();
timer.Start();
// Apply transformations (mirror, crop...) to the image
int offset = this->prefetch_data_.offset(item_id);
this->transformed_data_.set_cpu_data(prefetch_data + offset);
this->data_transformer_->Transform(cv_img, &(this->transformed_data_));
trans_time += timer.MicroSeconds();
for(int i = 0;i < label_dim;++i){
prefetch_label[item_id * label_dim + i] = lines_[lines_id_].second[i];
}
// go to the next iter
lines_id_++;
if (lines_id_ >= lines_size) {
// We have reached the end. Restart from the first.
DLOG(INFO) << "Restarting data prefetching from start.";
lines_id_ = 0;
if (this->layer_param_.image_data_param().shuffle()) {
ShuffleImages();
}
}
}
batch_timer.Stop();
DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
DLOG(INFO) << " Read time: " << read_time / 1000 << " ms.";
DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
}
void ImageDataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
const int new_height = this->layer_param_.image_data_param().new_height();
const int new_width = this->layer_param_.image_data_param().new_width();
const bool is_color = this->layer_param_.image_data_param().is_color();
string root_folder = this->layer_param_.image_data_param().root_folder();
CHECK((new_height == 0 && new_width == 0) ||
(new_height > 0 && new_width > 0)) << "Current implementation requires "
"new_height and new_width to be set at the same time.";
// Read the file with filenames and labels
const string& source = this->layer_param_.image_data_param().source();
LOG(INFO) << "Opening file " << source;
std::ifstream infile(source.c_str());
string filename;
int label;
while (infile >> filename >> label) {
lines_.push_back(std::make_pair(filename, label));
}
if (this->layer_param_.image_data_param().shuffle()) {
// randomly shuffle data
LOG(INFO) << "Shuffling data";
const unsigned int prefetch_rng_seed = caffe_rng_rand();
prefetch_rng_.reset(new Caffe::RNG(prefetch_rng_seed));
ShuffleImages();
}
LOG(INFO) << "A total of " << lines_.size() << " images.";
lines_id_ = 0;
// Check if we would need to randomly skip a few data points
if (this->layer_param_.image_data_param().rand_skip()) {
unsigned int skip = caffe_rng_rand() %
this->layer_param_.image_data_param().rand_skip();
LOG(INFO) << "Skipping first " << skip << " data points.";
CHECK_GT(lines_.size(), skip) << "Not enough points to skip";
lines_id_ = skip;
}
// Read an image, and use it to initialize the top blob.
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color);
const int channels = cv_img.channels();
const int height = cv_img.rows;
const int width = cv_img.cols;
// image
const int crop_size = this->layer_param_.transform_param().crop_size();
const int batch_size = this->layer_param_.image_data_param().batch_size();
if (crop_size > 0) {
top[0]->Reshape(batch_size, channels, crop_size, crop_size);
this->prefetch_data_.Reshape(batch_size, channels, crop_size, crop_size);
this->transformed_data_.Reshape(1, channels, crop_size, crop_size);
} else {
top[0]->Reshape(batch_size, channels, height, width);
this->prefetch_data_.Reshape(batch_size, channels, height, width);
this->transformed_data_.Reshape(1, channels, height, width);
}
LOG(INFO) << "output data size: " << top[0]->num() << ","
<< top[0]->channels() << "," << top[0]->height() << ","
<< top[0]->width();
// label
top[1]->Reshape(batch_size, 1, 1, 1);
this->prefetch_label_.Reshape(batch_size, 1, 1, 1);
}
void SiameseDataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
const int new_height = this->layer_param_.image_data_param().new_height();
const int new_width = this->layer_param_.image_data_param().new_width();
const bool is_color = this->layer_param_.image_data_param().is_color();
string root_folder = this->layer_param_.image_data_param().root_folder();
const int interpolation = this->layer_param_.image_data_param().interpolation();
const int resize_mode = this->layer_param_.image_data_param().resize_mode();
CHECK((new_width >= 0) && (new_height >= 0))
<< "Current implementation requires new_height and new_width to be set at the same time.";
// Read the file with filenames and labels
const string& source = this->layer_param_.image_data_param().source();
LOG(INFO) << "Opening file " << source;
std::ifstream infile(source.c_str());
// string filename;
// int label;
// while (infile >> filename >> label) {
// lines_.push_back(std::make_pair(filename, label));
// }
string line_buf, filename, label_str;
int tmp_i = 0;
while (!(getline(infile, line_buf).fail())) {
int label;
size_t pos;
stringstream label_stream;
//tmp_i++;
line_buf.erase(line_buf.find_last_not_of(" \r\n") + 1);
pos = line_buf.find_last_not_of(" 0123456789");
//LOG(INFO) << "Opening file " << tmp_i << ", path: " << line_buf;
//LOG(INFO) << "pos: " << pos;
//LOG(INFO) << "line_size: " << line_buf.size();
filename = line_buf.substr(0, pos + 1);
//LOG(INFO) << "filename: " << filename;
label_str = line_buf.substr(pos + 2);
label_stream.str(label_str);
label_stream >> label;
//LOG(INFO) << "label :" << label;
lines_.push_back(std::make_pair(filename, label));
}
if (this->layer_param_.image_data_param().shuffle()) {
// randomly shuffle data
LOG(INFO) << "Shuffling data";
const unsigned int prefetch_rng_seed = caffe_rng_rand();
prefetch_rng_.reset(new Caffe::RNG(prefetch_rng_seed));
ShuffleImages();
}
LOG(INFO) << "A total of " << lines_.size() << " images.";
generate_pair_lines(lines_, pair_lines_);
lines_id_ = 0;
// Check if we would need to randomly skip a few data points
if (this->layer_param_.image_data_param().rand_skip()) {
unsigned int skip = caffe_rng_rand() %
this->layer_param_.image_data_param().rand_skip();
LOG(INFO) << "Skipping first " << skip << " data points.";
CHECK_GT(lines_.size(), skip) << "Not enough points to skip";
lines_id_ = skip;
}
// Read an image, and use it to initialize the top blob.
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color, interpolation, resize_mode);
CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
// Use data_transformer to infer the expected blob shape from a cv_image.
vector<int> top_shape = this->data_transformer_->InferBlobShape(cv_img);
this->transformed_data_.Reshape(top_shape);
// Reshape prefetch_data and top[0] according to the batch_size.
const int batch_size = this->layer_param_.image_data_param().batch_size();
CHECK_GT(batch_size, 0) << "Positive batch size required";
top_shape[0] = batch_size;
for (int i = 0; i < this->PREFETCH_COUNT; ++i) {
this->prefetch_[i].data_.Reshape(top_shape);
}
top[0]->Reshape(top_shape);
LOG(INFO) << "output data size: " << top[0]->num() << ","
<< top[0]->channels() << "," << top[0]->height() << ","
<< top[0]->width();
// label
vector<int> label_shape(1, batch_size);
top[1]->Reshape(label_shape);
for (int i = 0; i < this->PREFETCH_COUNT; ++i) {
this->prefetch_[i].label_.Reshape(label_shape);
}
}
cv::Mat ReadImageToCVMat(const string& filename) {
return ReadImageToCVMat(filename, 0, 0, true);
}
cv::Mat ReadImageToCVMat(const string& filename,
const int height, const int width) {
return ReadImageToCVMat(filename, height, width, true);
}
cv::Mat ReadImageToCVMat(const string& filename,
const bool is_color) {
return ReadImageToCVMat(filename, 0, 0, is_color);
}
void DenseImageDataLayer<Dtype>::InternalThreadEntry() {
CPUTimer batch_timer;
batch_timer.Start();
double read_time = 0;
double trans_time = 0;
CPUTimer timer;
CHECK(this->prefetch_data_.count());
CHECK(this->transformed_data_.count());
DenseImageDataParameter dense_image_data_param = this->layer_param_.dense_image_data_param();
const int batch_size = dense_image_data_param.batch_size();
const int new_height = dense_image_data_param.new_height();
const int new_width = dense_image_data_param.new_width();
const int crop_height = dense_image_data_param.crop_height();
const int crop_width = dense_image_data_param.crop_width();
const int crop_size = this->layer_param_.transform_param().crop_size();
const bool is_color = dense_image_data_param.is_color();
string root_folder = dense_image_data_param.root_folder();
// Reshape on single input batches for inputs of varying dimension.
if (batch_size == 1 && crop_size == 0 && new_height == 0 && new_width == 0 && crop_height == 0 && crop_width == 0) {
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
0, 0, is_color);
this->prefetch_data_.Reshape(1, cv_img.channels(),
cv_img.rows, cv_img.cols);
this->transformed_data_.Reshape(1, cv_img.channels(),
cv_img.rows, cv_img.cols);
this->prefetch_label_.Reshape(1, 1, cv_img.rows, cv_img.cols);
this->transformed_label_.Reshape(1, 1, cv_img.rows, cv_img.cols);
}
Dtype* prefetch_data = this->prefetch_data_.mutable_cpu_data();
Dtype* prefetch_label = this->prefetch_label_.mutable_cpu_data();
// datum scales
const int lines_size = lines_.size();
for (int item_id = 0; item_id < batch_size; ++item_id) {
// get a blob
timer.Start();
CHECK_GT(lines_size, lines_id_);
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color);
CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
cv::Mat cv_lab = ReadImageToCVMat(root_folder + lines_[lines_id_].second,
new_height, new_width, false, true);
CHECK(cv_lab.data) << "Could not load " << lines_[lines_id_].second;
read_time += timer.MicroSeconds();
timer.Start();
// Apply random horizontal mirror of images
if (this->layer_param_.dense_image_data_param().mirror()) {
const bool do_mirror = caffe_rng_rand() % 2;
if (do_mirror) {
cv::flip(cv_img,cv_img,1);
cv::flip(cv_lab,cv_lab,1);
}
}
// Apply crop
int height = cv_img.rows;
int width = cv_img.cols;
int h_off = 0;
int w_off = 0;
if (crop_height>0 && crop_width>0) {
h_off = caffe_rng_rand() % (height - crop_height + 1);
w_off = caffe_rng_rand() % (width - crop_width + 1);
cv::Rect myROI(w_off, h_off, crop_width, crop_height);
cv_img = cv_img(myROI);
cv_lab = cv_lab(myROI);
}
// Apply transformations (mirror, crop...) to the image
int offset = this->prefetch_data_.offset(item_id);
this->transformed_data_.set_cpu_data(prefetch_data + offset);
this->data_transformer_->Transform(cv_img, &(this->transformed_data_));
// transform label the same way
int label_offset = this->prefetch_label_.offset(item_id);
this->transformed_label_.set_cpu_data(prefetch_label + label_offset);
this->data_transformer_->Transform(cv_lab, &this->transformed_label_, true);
CHECK(!this->layer_param_.transform_param().mirror() &&
this->layer_param_.transform_param().crop_size() == 0)
<< "FIXME: Any stochastic transformation will break layer due to "
<< "the need to transform input and label images in the same way";
trans_time += timer.MicroSeconds();
// go to the next iter
lines_id_++;
if (lines_id_ >= lines_size) {
// We have reached the end. Restart from the first.
DLOG(INFO) << "Restarting data prefetching from start.";
lines_id_ = 0;
if (this->layer_param_.dense_image_data_param().shuffle()) {
ShuffleImages();
}
}
}
batch_timer.Stop();
DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
DLOG(INFO) << " Read time: " << read_time / 1000 << " ms.";
DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
}
void ImageDataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
const int new_height = this->layer_param_.image_data_param().new_height();
const int new_width = this->layer_param_.image_data_param().new_width();
const int new_dim = this->layer_param_.image_data_param().new_dim();
const bool is_color = this->layer_param_.image_data_param().is_color();
string root_folder = this->layer_param_.image_data_param().root_folder();
CHECK((new_height == 0 && new_width == 0) ||
(new_height > 0 && new_width > 0)) << "Current implementation requires "
"new_height and new_width to be set at the same time.";
CHECK(!(new_dim > 0 && new_height > 0 && new_width > 0)) << "Both new_dim and "
"(new_height + new_width) cannot be non-zero at the same time.";
// Read the file with filenames and labels
const string& source = this->layer_param_.image_data_param().source();
LOG(INFO) << "Opening file " << source;
std::ifstream infile(source.c_str());
string line;
size_t pos;
int label;
while (std::getline(infile, line)) {
pos = line.find_last_of(' ');
label = atoi(line.substr(pos + 1).c_str());
lines_.push_back(std::make_pair(line.substr(0, pos), label));
}
CHECK(!lines_.empty()) << "File is empty";
if (this->layer_param_.image_data_param().shuffle()) {
// randomly shuffle data
LOG(INFO) << "Shuffling data";
const unsigned int prefetch_rng_seed = caffe_rng_rand();
prefetch_rng_.reset(new Caffe::RNG(prefetch_rng_seed));
ShuffleImages();
} else {
if (this->phase_ == TRAIN && Caffe::solver_rank() > 0 &&
this->layer_param_.image_data_param().rand_skip() == 0) {
LOG(WARNING) << "Shuffling or skipping recommended for multi-GPU";
}
}
LOG(INFO) << "A total of " << lines_.size() << " images.";
lines_id_ = 0;
// Check if we would need to randomly skip a few data points
if (this->layer_param_.image_data_param().rand_skip()) {
unsigned int skip = caffe_rng_rand() %
this->layer_param_.image_data_param().rand_skip();
LOG(INFO) << "Skipping first " << skip << " data points.";
CHECK_GT(lines_.size(), skip) << "Not enough points to skip";
lines_id_ = skip;
}
// Read an image, and use it to initialize the top blob.
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, new_dim, is_color);
CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
// Use data_transformer to infer the expected blob shape from a cv_image.
vector<int> top_shape = this->data_transformer_->InferBlobShape(cv_img);
this->transformed_data_.Reshape(top_shape);
// Reshape prefetch_data and top[0] according to the batch_size.
const int batch_size = this->layer_param_.image_data_param().batch_size();
CHECK_GT(batch_size, 0) << "Positive batch size required";
top_shape[0] = batch_size;
for (int i = 0; i < this->prefetch_.size(); ++i) {
this->prefetch_[i]->data_.Reshape(top_shape);
}
top[0]->Reshape(top_shape);
LOG(INFO) << "output data size: " << top[0]->num() << ","
<< top[0]->channels() << "," << top[0]->height() << ","
<< top[0]->width();
// label
vector<int> label_shape(1, batch_size);
top[1]->Reshape(label_shape);
for (int i = 0; i < this->prefetch_.size(); ++i) {
this->prefetch_[i]->label_.Reshape(label_shape);
}
}