int main(int argc, char** argv) {
::google::InitGoogleLogging(argv[0]);
if (argc < 5) {
printf(
"Convert a set of images to the leveldb format used\n"
"as input for Caffe.\n"
"Usage:\n"
" convert_imageset ROOTFOLDER/ ANNOTATION DB_NAME"
" MODE[0-train, 1-val, 2-test] RANDOM_SHUFFLE_DATA[0 or 1, default 1] RESIZE_WIDTH[default 256] RESIZE_HEIGHT[default 256](0 indicates no resize)\n"
"The ImageNet dataset for the training demo is at\n"
" http://www.image-net.org/download-images\n");
return 0;
}
std::ifstream infile(argv[2]);
string root_folder(argv[1]);
string coarse_folder(argv[8]);
std::vector<Seg_Anno> annos;
std::set<string> fNames;
string filename;
float prop;
while (infile >> filename)
{
LOG(INFO)<<filename;
Seg_Anno seg_Anno;
seg_Anno.filename_ = filename;
for (int i = 0; i < LABEL_LEN; i++)
{
infile >> prop;
if(!(prop < 1000000 && prop > -1000000))
{
printf("123");
}
seg_Anno.pos_.push_back(prop);
}
string labelFile = filename;
labelFile[labelFile.size() - 1] = 't';
labelFile[labelFile.size() - 2] = 'x';
labelFile[labelFile.size() - 3] = 't';
labelFile = coarse_folder + "/" + labelFile;
//std::ifstream fin(labelFile.c_str());
FILE * tf = fopen(labelFile.c_str(), "rb");
if(tf == NULL) continue;
fclose(tf);
if (fNames.find(filename)== fNames.end())
{
fNames.insert(filename);
annos.push_back(seg_Anno);
}
//debug
//if(annos.size() == 10)
// break;
}
if (argc < 6 || argv[5][0] != '0') {
// randomly shuffle data
LOG(INFO)<< "Shuffling data";
std::random_shuffle(annos.begin(), annos.end());
}
LOG(INFO)<< "A total of " << annos.size() << " images.";
leveldb::DB* db;
leveldb::Options options;
options.error_if_exists = true;
options.create_if_missing = true;
options.write_buffer_size = 268435456;
LOG(INFO)<< "Opening leveldb " << argv[3];
leveldb::Status status = leveldb::DB::Open(options, argv[3], &db);
CHECK(status.ok()) << "Failed to open leveldb " << argv[3];
Datum datum;
int count = 0;
const int maxKeyLength = 256;
char key_cstr[maxKeyLength];
leveldb::WriteBatch* batch = new leveldb::WriteBatch();
int data_size;
bool data_size_initialized = false;
// resize to height * width
int width = RESIZE_WIDTH;
int height = RESIZE_HEIGHT;
if (argc > 6) width = atoi(argv[6]);
if (argc > 7) height = atoi(argv[7]);
if (width == 0 || height == 0)
LOG(INFO) << "NO RESIZE SHOULD BE DONE";
else
LOG(INFO) << "RESIZE DIM: " << width << "*" << height;
for (int anno_id = 0; anno_id < annos.size(); ++anno_id)
{
string labelFile = annos[anno_id].filename_;
labelFile[labelFile.size() - 1] = 't';
labelFile[labelFile.size() - 2] = 'x';
labelFile[labelFile.size() - 3] = 't';
if (!MyReadImageToDatum(root_folder + "/" + annos[anno_id].filename_, coarse_folder + "/" + labelFile,
annos[anno_id].pos_, height, width, &datum))
{
continue;
}
if (!data_size_initialized)
{
data_size = datum.channels() * datum.height() * datum.width() ;
data_size_initialized = true;
}
else
{
int dataLen = datum.float_data_size();
CHECK_EQ(dataLen, data_size)<< "Incorrect data field size " << dataLen;
}
// sequential
snprintf(key_cstr, maxKeyLength, "%07d_%s", anno_id, annos[anno_id].filename_.c_str());
string value;
// get the value
datum.SerializeToString(&value);
batch->Put(string(key_cstr), value);
if (++count % 1000 == 0)
{
db->Write(leveldb::WriteOptions(), batch);
LOG(ERROR)<< "Processed " << count << " files.";
delete batch;
batch = new leveldb::WriteBatch();
}
}
// write the last batch
if (count % 1000 != 0) {
db->Write(leveldb::WriteOptions(), batch);
LOG(ERROR)<< "Processed " << count << " files.";
}
delete batch;
delete db;
return 0;
}
int main(int argc, char** argv) {
::google::InitGoogleLogging(argv[0]);
#ifndef GFLAGS_GFLAGS_H_
namespace gflags = google;
#endif
gflags::SetUsageMessage("Convert a set of images to the leveldb/lmdb\n"
"format used as input for Caffe.\n"
"Usage:\n"
" convert_imageset [FLAGS] ROOTFOLDER/ LISTFILE DB_NAME\n"
"The ImageNet dataset for the training demo is at\n"
" http://www.image-net.org/download-images\n");
gflags::ParseCommandLineFlags(&argc, &argv, true);
if (argc < 4) {
gflags::ShowUsageWithFlagsRestrict(argv[0], "tools/convert_imageset");
return 1;
}
const bool is_color = !FLAGS_gray;
const bool check_size = FLAGS_check_size;
const bool encoded = FLAGS_encoded;
const string encode_type = FLAGS_encode_type;
std::ifstream infile(argv[2]);
std::vector<std::pair<std::string, int> > lines;
std::string filename;
int label;
while (infile >> filename >> label) {
lines.push_back(std::make_pair(filename, label));
}
if (FLAGS_shuffle) {
// randomly shuffle data
LOG(INFO) << "Shuffling data";
if (FLAGS_seed >= 0) {
shared_ptr<Caffe::RNG> rng(new Caffe::RNG(FLAGS_seed));
shuffle(lines.begin(), lines.end(),
static_cast<caffe::rng_t *>(rng->generator()));
} else {
shuffle(lines.begin(), lines.end());
}
}
LOG(INFO) << "A total of " << lines.size() << " images.";
if (encode_type.size() && !encoded)
LOG(INFO) << "encode_type specified, assuming encoded=true.";
int resize_height = std::max<int>(0, FLAGS_resize_height);
int resize_width = std::max<int>(0, FLAGS_resize_width);
// Create new DB
scoped_ptr<db::DB> db(db::GetDB(FLAGS_backend));
db->Open(argv[3], db::NEW);
scoped_ptr<db::Transaction> txn(db->NewTransaction());
// Storing to db
std::string root_folder(argv[1]);
Datum datum;
int count = 0;
const int kMaxKeyLength = 256;
char key_cstr[kMaxKeyLength];
int data_size = 0;
bool data_size_initialized = false;
for (int line_id = 0; line_id < lines.size(); ++line_id) {
bool status;
std::string enc = encode_type;
if (encoded && !enc.size()) {
// Guess the encoding type from the file name
string fn = lines[line_id].first;
size_t p = fn.rfind('.');
if ( p == fn.npos )
LOG(WARNING) << "Failed to guess the encoding of '" << fn << "'";
enc = fn.substr(p);
std::transform(enc.begin(), enc.end(), enc.begin(), ::tolower);
}
status = ReadImageToDatum(root_folder + lines[line_id].first,
lines[line_id].second, resize_height, resize_width, is_color,
enc, &datum);
if (status == false) continue;
if (check_size) {
if (!data_size_initialized) {
data_size = datum.channels() * datum.height() * datum.width();
data_size_initialized = true;
} else {
const std::string& data = datum.data();
CHECK_EQ(data.size(), data_size) << "Incorrect data field size "
<< data.size();
}
}
// sequential
int length = snprintf(key_cstr, kMaxKeyLength, "%08d_%s", line_id,
lines[line_id].first.c_str());
// Put in db
string out;
CHECK(datum.SerializeToString(&out));
txn->Put(string(key_cstr, length), out);
if (++count % 1000 == 0) {
// Commit db
txn->Commit();
txn.reset(db->NewTransaction());
LOG(ERROR) << "Processed " << count << " files.";
}
}
// write the last batch
if (count % 1000 != 0) {
txn->Commit();
LOG(ERROR) << "Processed " << count << " files.";
}
return 0;
}
void DataLayer<Dtype>::InternalThreadEntry() {
CPUTimer batch_timer;
batch_timer.Start();
double read_time = 0;
double trans_time = 0;
CPUTimer timer;
CHECK(this->prefetch_data_.count());
CHECK(this->transformed_data_.count());
// Reshape on single input batches for inputs of varying dimension.
const int batch_size = this->layer_param_.data_param().batch_size();
const int crop_size = this->layer_param_.transform_param().crop_size();
bool force_color = this->layer_param_.data_param().force_encoded_color();
if (batch_size == 1 && crop_size == 0) {
Datum datum;
datum.ParseFromString(cursor_->value());
if (datum.encoded()) {
if (force_color) {
DecodeDatum(&datum, true);
} else {
DecodeDatumNative(&datum);
}
}
this->prefetch_data_.Reshape(1, datum.channels(),
datum.height(), datum.width());
this->transformed_data_.Reshape(1, datum.channels(),
datum.height(), datum.width());
}
Dtype* top_data = this->prefetch_data_.mutable_cpu_data();
Dtype* top_label = NULL; // suppress warnings about uninitialized variables
if (this->output_labels_) {
top_label = this->prefetch_label_.mutable_cpu_data();
}
for (int item_id = 0; item_id < batch_size; ++item_id) {
timer.Start();
// get a blob
Datum datum;
datum.ParseFromString(cursor_->value());
cv::Mat cv_img;
if (datum.encoded()) {
if (force_color) {
cv_img = DecodeDatumToCVMat(datum, true);
} else {
cv_img = DecodeDatumToCVMatNative(datum);
}
if (cv_img.channels() != this->transformed_data_.channels()) {
LOG(WARNING) << "Your dataset contains encoded images with mixed "
<< "channel sizes. Consider adding a 'force_color' flag to the "
<< "model definition, or rebuild your dataset using "
<< "convert_imageset.";
}
}
read_time += timer.MicroSeconds();
timer.Start();
// Apply data transformations (mirror, scale, crop...)
int offset = this->prefetch_data_.offset(item_id);
this->transformed_data_.set_cpu_data(top_data + offset);
if (datum.encoded()) {
this->data_transformer_->Transform(cv_img, &(this->transformed_data_));
} else {
this->data_transformer_->Transform(datum, &(this->transformed_data_));
}
if (this->output_labels_) {
top_label[item_id] = datum.label();
}
trans_time += timer.MicroSeconds();
// go to the next iter
cursor_->Next();
if (!cursor_->valid()) {
DLOG(INFO) << "Restarting data prefetching from start.";
cursor_->SeekToFirst();
}
}
batch_timer.Stop();
DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
DLOG(INFO) << " Read time: " << read_time / 1000 << " ms.";
DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
}
void DataTransformer<Dtype>::Transform(const Datum& datum,
Dtype* transformed_data) {
const string& data = datum.data();
const int datum_channels = datum.channels();
const int datum_height = datum.height();
const int datum_width = datum.width();
const int crop_size = param_.crop_size();
const Dtype scale = param_.scale();
const bool do_mirror = param_.mirror() && Rand(2);
const bool has_mean_file = param_.has_mean_file();
const bool has_uint8 = data.size() > 0;
const bool has_mean_values = mean_values_.size() > 0;
CHECK_GT(datum_channels, 0);
CHECK_GE(datum_height, crop_size);
CHECK_GE(datum_width, crop_size);
Dtype* mean = NULL;
if (has_mean_file) {
CHECK_EQ(datum_channels, data_mean_.channels());
CHECK_EQ(datum_height, data_mean_.height());
CHECK_EQ(datum_width, data_mean_.width());
mean = data_mean_.mutable_cpu_data();
}
if (has_mean_values) {
CHECK(mean_values_.size() == 1 || mean_values_.size() == datum_channels) <<
"Specify either 1 mean_value or as many as channels: " << datum_channels;
if (datum_channels > 1 && mean_values_.size() == 1) {
// Replicate the mean_value for simplicity
for (int c = 1; c < datum_channels; ++c) {
mean_values_.push_back(mean_values_[0]);
}
}
}
int height = datum_height;
int width = datum_width;
int h_off = 0;
int w_off = 0;
if (crop_size) {
height = crop_size;
width = crop_size;
// We only do random crop when we do training.
if (phase_ == Caffe::TRAIN) {
h_off = Rand(datum_height - crop_size + 1);
w_off = Rand(datum_width - crop_size + 1);
} else {
h_off = (datum_height - crop_size) / 2;
w_off = (datum_width - crop_size) / 2;
}
}
Dtype datum_element;
int top_index, data_index;
for (int c = 0; c < datum_channels; ++c) {
for (int h = 0; h < height; ++h) {
for (int w = 0; w < width; ++w) {
data_index = (c * datum_height + h_off + h) * datum_width + w_off + w;
if (do_mirror) {
top_index = (c * height + h) * width + (width - 1 - w);
} else {
top_index = (c * height + h) * width + w;
}
if (has_uint8) {
datum_element =
static_cast<Dtype>(static_cast<uint8_t>(data[data_index]));
} else {
datum_element = datum.float_data(data_index);
}
if (has_mean_file) {
transformed_data[top_index] =
(datum_element - mean[data_index]) * scale;
} else {
if (has_mean_values) {
transformed_data[top_index] =
(datum_element - mean_values_[c]) * scale;
} else {
transformed_data[top_index] = datum_element * scale;
}
}
}
}
}
}
int main(int argc, char** argv) {
::google::InitGoogleLogging(argv[0]);
if (argc < 4 || argc > 8) {
printf("Convert a set of images to the leveldb format used\n"
"as input for Caffe.\n"
"Usage:\n"
" convert_imageset [-g] ROOTFOLDER/ LISTFILE DB_NAME"
" RANDOM_SHUFFLE_DATA[0 or 1] [resize_height] [resize_width]\n"
"The ImageNet dataset for the training demo is at\n"
" http://www.image-net.org/download-images\n");
return 1;
}
// Test whether argv[1] == "-g"
bool iscolor= !(string("-g") == string(argv[1]));
int arg_offset = (iscolor ? 0 : 1);
std::ifstream infile(argv[arg_offset+2]);
std::vector<std::pair<string, int> > lines;
string filename;
int label;
while (infile >> filename >> label) {
lines.push_back(std::make_pair(filename, label));
}
if (argc >= (arg_offset+5) && argv[arg_offset+4][0] == '1') {
// randomly shuffle data
LOG(INFO) << "Shuffling data";
std::random_shuffle(lines.begin(), lines.end());
}
LOG(INFO) << "A total of " << lines.size() << " images.";
int resize_height = 0;
int resize_width = 0;
if (argc >= (arg_offset+6)) {
resize_height = atoi(argv[arg_offset+5]);
}
if (argc >= (arg_offset+7)) {
resize_width = atoi(argv[arg_offset+6]);
}
leveldb::DB* db;
leveldb::Options options;
options.error_if_exists = true;
options.create_if_missing = true;
options.write_buffer_size = 268435456;
LOG(INFO) << "Opening leveldb " << argv[arg_offset+3];
leveldb::Status status = leveldb::DB::Open(
options, argv[arg_offset+3], &db);
CHECK(status.ok()) << "Failed to open leveldb " << argv[arg_offset+3];
string root_folder(argv[arg_offset+1]);
Datum datum;
int count = 0;
const int kMaxKeyLength = 256;
char key_cstr[kMaxKeyLength];
leveldb::WriteBatch* batch = new leveldb::WriteBatch();
int data_size;
bool data_size_initialized = false;
for (int line_id = 0; line_id < lines.size(); ++line_id) {
if (!ReadImageToDatum(root_folder + lines[line_id].first,
lines[line_id].second, resize_height, resize_width, iscolor, &datum)) {
continue;
}
if (!data_size_initialized) {
data_size = datum.channels() * datum.height() * datum.width();
data_size_initialized = true;
} else {
const string& data = datum.data();
CHECK_EQ(data.size(), data_size) << "Incorrect data field size "
<< data.size();
}
// sequential
snprintf(key_cstr, kMaxKeyLength, "%08d_%s", line_id,
lines[line_id].first.c_str());
string value;
// get the value
datum.SerializeToString(&value);
batch->Put(string(key_cstr), value);
if (++count % 1000 == 0) {
db->Write(leveldb::WriteOptions(), batch);
LOG(ERROR) << "Processed " << count << " files.";
delete batch;
batch = new leveldb::WriteBatch();
}
}
// write the last batch
if (count % 1000 != 0) {
db->Write(leveldb::WriteOptions(), batch);
LOG(ERROR) << "Processed " << count << " files.";
}
delete batch;
delete db;
return 0;
}
int main(int argc, char** argv) {
::google::InitGoogleLogging(argv[0]);
if (argc < 5) {
LOG(ERROR) << "Usage: compute_image_mean input_list new_height new_width output_file [dropping_rate]";
return 1;
}
char* fn_list = argv[1];
const int height = atoi(argv[2]);
const int width = atoi(argv[3]);
char* fn_output = argv[4];
int sampling_rate = 1;
if (argc >= 6){
sampling_rate = atoi(argv[5]);
LOG(INFO) << "using sampling rate " << sampling_rate;
}
Datum datum;
BlobProto sum_blob;
int count = 0;
std::ifstream infile(fn_list);
string fn_frm;
int label;
infile >> fn_frm >> label;
ReadImageToDatum(fn_frm, label, height, width, &datum);
sum_blob.set_num(1);
sum_blob.set_channels(datum.channels());
sum_blob.set_length(1);
sum_blob.set_height(datum.height());
sum_blob.set_width(datum.width());
const int data_size = datum.channels() * datum.height() * datum.width();
int size_in_datum = std::max<int>(datum.data().size(),
datum.float_data_size());
for (int i = 0; i < size_in_datum; ++i) {
sum_blob.add_data(0.);
}
LOG(INFO) << "Starting Iteration";
int i = 0;
while (infile >> fn_frm >> label) {
i++;
if (i % sampling_rate!=0){
continue;
}
ReadImageToDatum(fn_frm, label, height, width, &datum);
const string& data = datum.data();
size_in_datum = std::max<int>(datum.data().size(), datum.float_data_size());
CHECK_EQ(size_in_datum, data_size) << "Incorrect data field size " <<
size_in_datum;
if (data.size() != 0) {
for (int i = 0; i < size_in_datum; ++i) {
sum_blob.set_data(i, sum_blob.data(i) + (uint8_t)data[i]);
}
} else {
for (int i = 0; i < size_in_datum; ++i) {
sum_blob.set_data(i, sum_blob.data(i) +
static_cast<float>(datum.float_data(i)));
}
}
++count;
if (count % 10000 == 0) {
LOG(ERROR) << "Processed " << count << " files.";
}
}
infile.close();
if (count % 10000 != 0) {
LOG(ERROR) << "Processed " << count << " files.";
}
for (int i = 0; i < sum_blob.data_size(); ++i) {
sum_blob.set_data(i, sum_blob.data(i) / count);
}
// Write to disk
LOG(INFO) << "Write to " << fn_output;
WriteProtoToBinaryFile(sum_blob, fn_output);
return 0;
}
void ImageDataLayer<Dtype>::SetUp(const vector<Blob<Dtype>*>& bottom,
vector<Blob<Dtype>*>* top) {
CHECK_EQ(bottom.size(), 0) << "Input Layer takes no input blobs.";
CHECK_EQ(top->size(), 2) << "Input Layer takes two blobs as output.";
const int new_height = this->layer_param_.image_data_param().new_height();
const int new_width = this->layer_param_.image_data_param().new_width();
CHECK((new_height == 0 && new_width == 0) ||
(new_height > 0 && new_width > 0)) << "Current implementation requires "
"new_height and new_width to be set at the same time.";
// Read the file with filenames and labels
const string& source = this->layer_param_.image_data_param().source();
LOG(INFO) << "Opening file " << source;
std::ifstream infile(source.c_str());
string filename;
int label;
while (infile >> filename >> label) {
lines_.push_back(std::make_pair(filename, label));
}
if (this->layer_param_.image_data_param().shuffle()) {
// randomly shuffle data
LOG(INFO) << "Shuffling data";
const unsigned int prefetch_rng_seed = caffe_rng_rand();
prefetch_rng_.reset(new Caffe::RNG(prefetch_rng_seed));
ShuffleImages();
}
LOG(INFO) << "A total of " << lines_.size() << " images.";
lines_id_ = 0;
// Check if we would need to randomly skip a few data points
if (this->layer_param_.image_data_param().rand_skip()) {
unsigned int skip = caffe_rng_rand() %
this->layer_param_.image_data_param().rand_skip();
LOG(INFO) << "Skipping first " << skip << " data points.";
CHECK_GT(lines_.size(), skip) << "Not enough points to skip";
lines_id_ = skip;
}
// Read a data point, and use it to initialize the top blob.
Datum datum;
CHECK(ReadImageToDatum(lines_[lines_id_].first, lines_[lines_id_].second,
new_height, new_width, &datum));
// image
const int crop_size = this->layer_param_.image_data_param().crop_size();
const int batch_size = this->layer_param_.image_data_param().batch_size();
const string& mean_file = this->layer_param_.image_data_param().mean_file();
if (crop_size > 0) {
(*top)[0]->Reshape(batch_size, datum.channels(), crop_size, crop_size);
prefetch_data_.reset(new Blob<Dtype>(batch_size, datum.channels(),
crop_size, crop_size));
} else {
(*top)[0]->Reshape(batch_size, datum.channels(), datum.height(),
datum.width());
prefetch_data_.reset(new Blob<Dtype>(batch_size, datum.channels(),
datum.height(), datum.width()));
}
LOG(INFO) << "output data size: " << (*top)[0]->num() << ","
<< (*top)[0]->channels() << "," << (*top)[0]->height() << ","
<< (*top)[0]->width();
// label
(*top)[1]->Reshape(batch_size, 1, 1, 1);
prefetch_label_.reset(new Blob<Dtype>(batch_size, 1, 1, 1));
// datum size
datum_channels_ = datum.channels();
datum_height_ = datum.height();
datum_width_ = datum.width();
datum_size_ = datum.channels() * datum.height() * datum.width();
//CHECK_GT(datum_height_, crop_size);
//CHECK_GT(datum_width_, crop_size);
// check if we want to have mean
if (this->layer_param_.image_data_param().has_mean_file()) {
BlobProto blob_proto;
LOG(INFO) << "Loading mean file from" << mean_file;
ReadProtoFromBinaryFile(mean_file.c_str(), &blob_proto);
data_mean_.FromProto(blob_proto);
CHECK_EQ(data_mean_.num(), 1);
CHECK_EQ(data_mean_.channels(), datum_channels_);
CHECK_EQ(data_mean_.height(), datum_height_);
CHECK_EQ(data_mean_.width(), datum_width_);
} else {
// Simply initialize an all-empty mean.
data_mean_.Reshape(1, datum_channels_, datum_height_, datum_width_);
}
// Now, start the prefetch thread. Before calling prefetch, we make two
// cpu_data calls so that the prefetch thread does not accidentally make
// simultaneous cudaMalloc calls when the main thread is running. In some
// GPUs this seems to cause failures if we do not so.
prefetch_data_->mutable_cpu_data();
prefetch_label_->mutable_cpu_data();
data_mean_.cpu_data();
DLOG(INFO) << "Initializing prefetch";
CreatePrefetchThread();
DLOG(INFO) << "Prefetch initialized.";
}
void DataLayer<Dtype>::SetUp(const vector<Blob<Dtype>*>& bottom,
vector<Blob<Dtype>*>* top) {
CHECK_EQ(bottom.size(), 0) << "Data Layer takes no input blobs.";
CHECK_GE(top->size(), 1) << "Data Layer takes at least one blob as output.";
CHECK_LE(top->size(), 2) << "Data Layer takes at most two blobs as output.";
if (top->size() == 1) {
output_labels_ = false;
} else {
output_labels_ = true;
}
// Initialize the leveldb
leveldb::DB* db_temp;
leveldb::Options options;
options.create_if_missing = false;
options.max_open_files = 100;
LOG(INFO) << "Opening leveldb " << this->layer_param_.data_param().source();
leveldb::Status status = leveldb::DB::Open(
options, this->layer_param_.data_param().source(), &db_temp);
CHECK(status.ok()) << "Failed to open leveldb "
<< this->layer_param_.data_param().source() << std::endl
<< status.ToString();
db_.reset(db_temp);
iter_.reset(db_->NewIterator(leveldb::ReadOptions()));//通过迭代器来操纵leveldb
iter_->SeekToFirst();
// Check if we would need to randomly skip a few data points
//是否要随机跳过一些数据
if (this->layer_param_.data_param().rand_skip()) {
unsigned int skip = caffe_rng_rand() %
this->layer_param_.data_param().rand_skip();
LOG(INFO) << "Skipping first " << skip << " data points.";
while (skip-- > 0) {
iter_->Next();
if (!iter_->Valid()) {
iter_->SeekToFirst();
}
}
}
// Read a data point, and use it to initialize the top blob.
//读取一个数据点，用来初始化topblob。所谓初始化，只要是指reshape。
//可以观察到下面iter_调用调用next。所以这次读取只是用来读取出来channels等参数的，不作处理。
Datum datum;
datum.ParseFromString(iter_->value().ToString());//利用迭代器读取第一个数据点
// image图像数据
int crop_size = this->layer_param_.data_param().crop_size();//裁剪大小
if (crop_size > 0) {//需要裁剪
(*top)[0]->Reshape(this->layer_param_.data_param().batch_size(),
datum.channels(), crop_size, crop_size);
prefetch_data_.reset(new Blob<Dtype>(
this->layer_param_.data_param().batch_size(), datum.channels(),
crop_size, crop_size));
} else {//不需要裁剪
(*top)[0]->Reshape(
this->layer_param_.data_param().batch_size(), datum.channels(),
datum.height(), datum.width());
prefetch_data_.reset(new Blob<Dtype>(
this->layer_param_.data_param().batch_size(), datum.channels(),
datum.height(), datum.width()));
}
LOG(INFO) << "output data size: " << (*top)[0]->num() << ","
<< (*top)[0]->channels() << "," << (*top)[0]->height() << ","
<< (*top)[0]->width();
/*
// label标签数据
if (output_labels_) {
(*top)[1]->Reshape(this->layer_param_.data_param().batch_size(), 1, 1, 1);
prefetch_label_.reset(
new Blob<Dtype>(this->layer_param_.data_param().batch_size(), 1, 1, 1));
}
*/
// label标签数据
if (output_labels_) {
(*top)[1]->Reshape(this->layer_param_.data_param().batch_size(), datum.label_size(), 1, 1);
prefetch_label_.reset(
new Blob<Dtype>(this->layer_param_.data_param().batch_size(), datum.label_size(), 1, 1));
}
// datum size
datum_channels_ = datum.channels();
datum_height_ = datum.height();
datum_width_ = datum.width();
datum_size_ = datum.channels() * datum.height() * datum.width();
CHECK_GT(datum_height_, crop_size);
CHECK_GT(datum_width_, crop_size);
// check if we want to have mean 是否要减去均值
if (this->layer_param_.data_param().has_mean_file()) {
const string& mean_file = this->layer_param_.data_param().mean_file();
LOG(INFO) << "Loading mean file from" << mean_file;
BlobProto blob_proto;
ReadProtoFromBinaryFileOrDie(mean_file.c_str(), &blob_proto);
data_mean_.FromProto(blob_proto);
CHECK_EQ(data_mean_.num(), 1);
CHECK_EQ(data_mean_.channels(), datum_channels_);
CHECK_EQ(data_mean_.height(), datum_height_);
CHECK_EQ(data_mean_.width(), datum_width_);
} else {
// Simply initialize an all-empty mean.
data_mean_.Reshape(1, datum_channels_, datum_height_, datum_width_);
}
// Now, start the prefetch thread. Before calling prefetch, we make two
// cpu_data calls so that the prefetch thread does not accidentally make
// simultaneous cudaMalloc calls when the main thread is running. In some
// GPUs this seems to cause failures if we do not so.
prefetch_data_->mutable_cpu_data();
if (output_labels_) {
prefetch_label_->mutable_cpu_data();
}
data_mean_.cpu_data();
DLOG(INFO) << "Initializing prefetch";
CreatePrefetchThread();
DLOG(INFO) << "Prefetch initialized.";
}
int main(int argc, char** argv)
{
::google::InitGoogleLogging(argv[0]);
#ifndef GFLAGS_GFLAGS_H_
namespace gflags = google;
#endif
gflags::SetUsageMessage("Convert a set of images to the leveldb/lmdb\n"
"format used as input for Caffe.\n"
"Usage:\n"
" convert_imageset [FLAGS] ROOTFOLDER/ LISTFILE DB_NAME\n"
"The ImageNet dataset for the training demo is at\n"
" http://www.image-net.org/download-images\n");
gflags::ParseCommandLineFlags(&argc, &argv, true);
if (argc != 4) {
gflags::ShowUsageWithFlagsRestrict(argv[0], "tools/convert_imageset");
return 1;
}
bool is_color = !FLAGS_gray;
bool is_gradient = FLAGS_gradient;
printf("is_color = %d\n", is_color);
printf("is_gradient = %d\n", is_gradient);
std::ifstream infile(argv[2]);
std::vector<std::pair<string, int> > lines;
string filename;
int label;
while (infile >> filename >> label) {
lines.push_back(std::make_pair(filename, label));
}
if (FLAGS_shuffle) {
// randomly shuffle data
LOG(INFO) << "Shuffling data";
shuffle(lines.begin(), lines.end());
}
LOG(INFO) << "A total of " << lines.size() << " images.";
const string& db_backend = FLAGS_backend;
const char* db_path = argv[3];
int resize_height = std::max<int>(0, FLAGS_resize_height);
int resize_width = std::max<int>(0, FLAGS_resize_width);
// Open new db
// lmdb
MDB_env *mdb_env;
MDB_dbi mdb_dbi;
MDB_val mdb_key, mdb_data;
MDB_txn *mdb_txn;
// leveldb
leveldb::DB* db;
leveldb::Options options;
options.error_if_exists = true;
options.create_if_missing = true;
options.write_buffer_size = 268435456;
leveldb::WriteBatch* batch = NULL;
// Open db
if (db_backend == "leveldb") { // leveldb
LOG(INFO) << "Opening leveldb " << db_path;
leveldb::Status status = leveldb::DB::Open(
options, db_path, &db);
CHECK(status.ok()) << "Failed to open leveldb " << db_path
<< ". Is it already existing?";
batch = new leveldb::WriteBatch();
} else if (db_backend == "lmdb") { // lmdb
LOG(INFO) << "Opening lmdb " << db_path;
CHECK_EQ(mkdir(db_path, 0744), 0)
<< "mkdir " << db_path << "failed";
CHECK_EQ(mdb_env_create(&mdb_env), MDB_SUCCESS) << "mdb_env_create failed";
CHECK_EQ(mdb_env_set_mapsize(mdb_env, 1099511627776), MDB_SUCCESS) // 1TB
<< "mdb_env_set_mapsize failed";
CHECK_EQ(mdb_env_open(mdb_env, db_path, 0, 0664), MDB_SUCCESS)
<< "mdb_env_open failed";
CHECK_EQ(mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn), MDB_SUCCESS)
<< "mdb_txn_begin failed";
CHECK_EQ(mdb_open(mdb_txn, NULL, 0, &mdb_dbi), MDB_SUCCESS)
<< "mdb_open failed. Does the lmdb already exist? ";
} else {
LOG(FATAL) << "Unknown db backend " << db_backend;
}
// Storing to db
string root_folder(argv[1]);
Datum datum;
int count = 0;
const int kMaxKeyLength = 256;
char key_cstr[kMaxKeyLength];
int data_size;
bool data_size_initialized = false;
for (int line_id = 0; line_id < lines.size(); ++line_id) {
if (!MyReadImageToDatum(root_folder + lines[line_id].first,
lines[line_id].second, resize_height, resize_width, is_color, is_gradient, &datum)) {
continue;
}
if (!data_size_initialized) {
data_size = datum.channels() * datum.height() * datum.width();
data_size_initialized = true;
} else {
const string& data = datum.data();
CHECK_EQ(data.size(), data_size) << "Incorrect data field size "
<< data.size();
}
// sequential
snprintf(key_cstr, kMaxKeyLength, "%08d_%s", line_id,
lines[line_id].first.c_str());
string value;
datum.SerializeToString(&value);
string keystr(key_cstr);
// Put in db
if (db_backend == "leveldb") { // leveldb
batch->Put(keystr, value);
} else if (db_backend == "lmdb") { // lmdb
mdb_data.mv_size = value.size();
mdb_data.mv_data = reinterpret_cast<void*>(&value[0]);
mdb_key.mv_size = keystr.size();
mdb_key.mv_data = reinterpret_cast<void*>(&keystr[0]);
CHECK_EQ(mdb_put(mdb_txn, mdb_dbi, &mdb_key, &mdb_data, 0), MDB_SUCCESS)
<< "mdb_put failed";
} else {
LOG(FATAL) << "Unknown db backend " << db_backend;
}
if (++count % 1000 == 0) {
// Commit txn
if (db_backend == "leveldb") { // leveldb
db->Write(leveldb::WriteOptions(), batch);
delete batch;
batch = new leveldb::WriteBatch();
} else if (db_backend == "lmdb") { // lmdb
CHECK_EQ(mdb_txn_commit(mdb_txn), MDB_SUCCESS)
<< "mdb_txn_commit failed";
CHECK_EQ(mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn), MDB_SUCCESS)
<< "mdb_txn_begin failed";
} else {
LOG(FATAL) << "Unknown db backend " << db_backend;
}
LOG(ERROR) << "Processed " << count << " files.";
}
}
// write the last batch
if (count % 1000 != 0) {
if (db_backend == "leveldb") { // leveldb
db->Write(leveldb::WriteOptions(), batch);
delete batch;
delete db;
} else if (db_backend == "lmdb") { // lmdb
CHECK_EQ(mdb_txn_commit(mdb_txn), MDB_SUCCESS) << "mdb_txn_commit failed";
mdb_close(mdb_env, mdb_dbi);
mdb_env_close(mdb_env);
} else {
LOG(FATAL) << "Unknown db backend " << db_backend;
}
LOG(ERROR) << "Processed " << count << " files.";
}
return 0;
}
