int main(int argc, char** argv) { ::google::InitGoogleLogging(argv[0]); if (argc < 5) { printf( "Convert a set of images to the leveldb format used\n" "as input for Caffe.\n" "Usage:\n" " convert_imageset ROOTFOLDER/ ANNOTATION DB_NAME" " MODE[0-train, 1-val, 2-test] RANDOM_SHUFFLE_DATA[0 or 1, default 1] RESIZE_WIDTH[default 256] RESIZE_HEIGHT[default 256](0 indicates no resize)\n" "The ImageNet dataset for the training demo is at\n" " http://www.image-net.org/download-images\n"); return 0; } std::ifstream infile(argv[2]); string root_folder(argv[1]); string coarse_folder(argv[8]); std::vector<Seg_Anno> annos; std::set<string> fNames; string filename; float prop; while (infile >> filename) { LOG(INFO)<<filename; Seg_Anno seg_Anno; seg_Anno.filename_ = filename; for (int i = 0; i < LABEL_LEN; i++) { infile >> prop; if(!(prop < 1000000 && prop > -1000000)) { printf("123"); } seg_Anno.pos_.push_back(prop); } string labelFile = filename; labelFile[labelFile.size() - 1] = 't'; labelFile[labelFile.size() - 2] = 'x'; labelFile[labelFile.size() - 3] = 't'; labelFile = coarse_folder + "/" + labelFile; //std::ifstream fin(labelFile.c_str()); FILE * tf = fopen(labelFile.c_str(), "rb"); if(tf == NULL) continue; fclose(tf); if (fNames.find(filename)== fNames.end()) { fNames.insert(filename); annos.push_back(seg_Anno); } //debug //if(annos.size() == 10) // break; } if (argc < 6 || argv[5][0] != '0') { // randomly shuffle data LOG(INFO)<< "Shuffling data"; std::random_shuffle(annos.begin(), annos.end()); } LOG(INFO)<< "A total of " << annos.size() << " images."; leveldb::DB* db; leveldb::Options options; options.error_if_exists = true; options.create_if_missing = true; options.write_buffer_size = 268435456; LOG(INFO)<< "Opening leveldb " << argv[3]; leveldb::Status status = leveldb::DB::Open(options, argv[3], &db); CHECK(status.ok()) << "Failed to open leveldb " << argv[3]; Datum datum; int count = 0; const int maxKeyLength = 256; char key_cstr[maxKeyLength]; leveldb::WriteBatch* batch = new leveldb::WriteBatch(); int data_size; bool data_size_initialized = false; // resize to height * width int width = RESIZE_WIDTH; int height = RESIZE_HEIGHT; if (argc > 6) width = atoi(argv[6]); if (argc > 7) height = atoi(argv[7]); if (width == 0 || height == 0) LOG(INFO) << "NO RESIZE SHOULD BE DONE"; else LOG(INFO) << "RESIZE DIM: " << width << "*" << height; for (int anno_id = 0; anno_id < annos.size(); ++anno_id) { string labelFile = annos[anno_id].filename_; labelFile[labelFile.size() - 1] = 't'; labelFile[labelFile.size() - 2] = 'x'; labelFile[labelFile.size() - 3] = 't'; if (!MyReadImageToDatum(root_folder + "/" + annos[anno_id].filename_, coarse_folder + "/" + labelFile, annos[anno_id].pos_, height, width, &datum)) { continue; } if (!data_size_initialized) { data_size = datum.channels() * datum.height() * datum.width() ; data_size_initialized = true; } else { int dataLen = datum.float_data_size(); CHECK_EQ(dataLen, data_size)<< "Incorrect data field size " << dataLen; } // sequential snprintf(key_cstr, maxKeyLength, "%07d_%s", anno_id, annos[anno_id].filename_.c_str()); string value; // get the value datum.SerializeToString(&value); batch->Put(string(key_cstr), value); if (++count % 1000 == 0) { db->Write(leveldb::WriteOptions(), batch); LOG(ERROR)<< "Processed " << count << " files."; delete batch; batch = new leveldb::WriteBatch(); } } // write the last batch if (count % 1000 != 0) { db->Write(leveldb::WriteOptions(), batch); LOG(ERROR)<< "Processed " << count << " files."; } delete batch; delete db; return 0; }
int main(int argc, char** argv) { ::google::InitGoogleLogging(argv[0]); #ifndef GFLAGS_GFLAGS_H_ namespace gflags = google; #endif gflags::SetUsageMessage("Convert a set of images to the leveldb/lmdb\n" "format used as input for Caffe.\n" "Usage:\n" " convert_imageset [FLAGS] ROOTFOLDER/ LISTFILE DB_NAME\n" "The ImageNet dataset for the training demo is at\n" " http://www.image-net.org/download-images\n"); gflags::ParseCommandLineFlags(&argc, &argv, true); if (argc < 4) { gflags::ShowUsageWithFlagsRestrict(argv[0], "tools/convert_imageset"); return 1; } const bool is_color = !FLAGS_gray; const bool check_size = FLAGS_check_size; const bool encoded = FLAGS_encoded; const string encode_type = FLAGS_encode_type; std::ifstream infile(argv[2]); std::vector<std::pair<std::string, int> > lines; std::string filename; int label; while (infile >> filename >> label) { lines.push_back(std::make_pair(filename, label)); } if (FLAGS_shuffle) { // randomly shuffle data LOG(INFO) << "Shuffling data"; if (FLAGS_seed >= 0) { shared_ptr<Caffe::RNG> rng(new Caffe::RNG(FLAGS_seed)); shuffle(lines.begin(), lines.end(), static_cast<caffe::rng_t *>(rng->generator())); } else { shuffle(lines.begin(), lines.end()); } } LOG(INFO) << "A total of " << lines.size() << " images."; if (encode_type.size() && !encoded) LOG(INFO) << "encode_type specified, assuming encoded=true."; int resize_height = std::max<int>(0, FLAGS_resize_height); int resize_width = std::max<int>(0, FLAGS_resize_width); // Create new DB scoped_ptr<db::DB> db(db::GetDB(FLAGS_backend)); db->Open(argv[3], db::NEW); scoped_ptr<db::Transaction> txn(db->NewTransaction()); // Storing to db std::string root_folder(argv[1]); Datum datum; int count = 0; const int kMaxKeyLength = 256; char key_cstr[kMaxKeyLength]; int data_size = 0; bool data_size_initialized = false; for (int line_id = 0; line_id < lines.size(); ++line_id) { bool status; std::string enc = encode_type; if (encoded && !enc.size()) { // Guess the encoding type from the file name string fn = lines[line_id].first; size_t p = fn.rfind('.'); if ( p == fn.npos ) LOG(WARNING) << "Failed to guess the encoding of '" << fn << "'"; enc = fn.substr(p); std::transform(enc.begin(), enc.end(), enc.begin(), ::tolower); } status = ReadImageToDatum(root_folder + lines[line_id].first, lines[line_id].second, resize_height, resize_width, is_color, enc, &datum); if (status == false) continue; if (check_size) { if (!data_size_initialized) { data_size = datum.channels() * datum.height() * datum.width(); data_size_initialized = true; } else { const std::string& data = datum.data(); CHECK_EQ(data.size(), data_size) << "Incorrect data field size " << data.size(); } } // sequential int length = snprintf(key_cstr, kMaxKeyLength, "%08d_%s", line_id, lines[line_id].first.c_str()); // Put in db string out; CHECK(datum.SerializeToString(&out)); txn->Put(string(key_cstr, length), out); if (++count % 1000 == 0) { // Commit db txn->Commit(); txn.reset(db->NewTransaction()); LOG(ERROR) << "Processed " << count << " files."; } } // write the last batch if (count % 1000 != 0) { txn->Commit(); LOG(ERROR) << "Processed " << count << " files."; } return 0; }
void DataLayer<Dtype>::InternalThreadEntry() { CPUTimer batch_timer; batch_timer.Start(); double read_time = 0; double trans_time = 0; CPUTimer timer; CHECK(this->prefetch_data_.count()); CHECK(this->transformed_data_.count()); // Reshape on single input batches for inputs of varying dimension. const int batch_size = this->layer_param_.data_param().batch_size(); const int crop_size = this->layer_param_.transform_param().crop_size(); bool force_color = this->layer_param_.data_param().force_encoded_color(); if (batch_size == 1 && crop_size == 0) { Datum datum; datum.ParseFromString(cursor_->value()); if (datum.encoded()) { if (force_color) { DecodeDatum(&datum, true); } else { DecodeDatumNative(&datum); } } this->prefetch_data_.Reshape(1, datum.channels(), datum.height(), datum.width()); this->transformed_data_.Reshape(1, datum.channels(), datum.height(), datum.width()); } Dtype* top_data = this->prefetch_data_.mutable_cpu_data(); Dtype* top_label = NULL; // suppress warnings about uninitialized variables if (this->output_labels_) { top_label = this->prefetch_label_.mutable_cpu_data(); } for (int item_id = 0; item_id < batch_size; ++item_id) { timer.Start(); // get a blob Datum datum; datum.ParseFromString(cursor_->value()); cv::Mat cv_img; if (datum.encoded()) { if (force_color) { cv_img = DecodeDatumToCVMat(datum, true); } else { cv_img = DecodeDatumToCVMatNative(datum); } if (cv_img.channels() != this->transformed_data_.channels()) { LOG(WARNING) << "Your dataset contains encoded images with mixed " << "channel sizes. Consider adding a 'force_color' flag to the " << "model definition, or rebuild your dataset using " << "convert_imageset."; } } read_time += timer.MicroSeconds(); timer.Start(); // Apply data transformations (mirror, scale, crop...) int offset = this->prefetch_data_.offset(item_id); this->transformed_data_.set_cpu_data(top_data + offset); if (datum.encoded()) { this->data_transformer_->Transform(cv_img, &(this->transformed_data_)); } else { this->data_transformer_->Transform(datum, &(this->transformed_data_)); } if (this->output_labels_) { top_label[item_id] = datum.label(); } trans_time += timer.MicroSeconds(); // go to the next iter cursor_->Next(); if (!cursor_->valid()) { DLOG(INFO) << "Restarting data prefetching from start."; cursor_->SeekToFirst(); } } batch_timer.Stop(); DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms."; DLOG(INFO) << " Read time: " << read_time / 1000 << " ms."; DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms."; }
void DataTransformer<Dtype>::Transform(const Datum& datum, Dtype* transformed_data) { const string& data = datum.data(); const int datum_channels = datum.channels(); const int datum_height = datum.height(); const int datum_width = datum.width(); const int crop_size = param_.crop_size(); const Dtype scale = param_.scale(); const bool do_mirror = param_.mirror() && Rand(2); const bool has_mean_file = param_.has_mean_file(); const bool has_uint8 = data.size() > 0; const bool has_mean_values = mean_values_.size() > 0; CHECK_GT(datum_channels, 0); CHECK_GE(datum_height, crop_size); CHECK_GE(datum_width, crop_size); Dtype* mean = NULL; if (has_mean_file) { CHECK_EQ(datum_channels, data_mean_.channels()); CHECK_EQ(datum_height, data_mean_.height()); CHECK_EQ(datum_width, data_mean_.width()); mean = data_mean_.mutable_cpu_data(); } if (has_mean_values) { CHECK(mean_values_.size() == 1 || mean_values_.size() == datum_channels) << "Specify either 1 mean_value or as many as channels: " << datum_channels; if (datum_channels > 1 && mean_values_.size() == 1) { // Replicate the mean_value for simplicity for (int c = 1; c < datum_channels; ++c) { mean_values_.push_back(mean_values_[0]); } } } int height = datum_height; int width = datum_width; int h_off = 0; int w_off = 0; if (crop_size) { height = crop_size; width = crop_size; // We only do random crop when we do training. if (phase_ == Caffe::TRAIN) { h_off = Rand(datum_height - crop_size + 1); w_off = Rand(datum_width - crop_size + 1); } else { h_off = (datum_height - crop_size) / 2; w_off = (datum_width - crop_size) / 2; } } Dtype datum_element; int top_index, data_index; for (int c = 0; c < datum_channels; ++c) { for (int h = 0; h < height; ++h) { for (int w = 0; w < width; ++w) { data_index = (c * datum_height + h_off + h) * datum_width + w_off + w; if (do_mirror) { top_index = (c * height + h) * width + (width - 1 - w); } else { top_index = (c * height + h) * width + w; } if (has_uint8) { datum_element = static_cast<Dtype>(static_cast<uint8_t>(data[data_index])); } else { datum_element = datum.float_data(data_index); } if (has_mean_file) { transformed_data[top_index] = (datum_element - mean[data_index]) * scale; } else { if (has_mean_values) { transformed_data[top_index] = (datum_element - mean_values_[c]) * scale; } else { transformed_data[top_index] = datum_element * scale; } } } } } }
int main(int argc, char** argv) { ::google::InitGoogleLogging(argv[0]); if (argc < 4 || argc > 8) { printf("Convert a set of images to the leveldb format used\n" "as input for Caffe.\n" "Usage:\n" " convert_imageset [-g] ROOTFOLDER/ LISTFILE DB_NAME" " RANDOM_SHUFFLE_DATA[0 or 1] [resize_height] [resize_width]\n" "The ImageNet dataset for the training demo is at\n" " http://www.image-net.org/download-images\n"); return 1; } // Test whether argv[1] == "-g" bool iscolor= !(string("-g") == string(argv[1])); int arg_offset = (iscolor ? 0 : 1); std::ifstream infile(argv[arg_offset+2]); std::vector<std::pair<string, int> > lines; string filename; int label; while (infile >> filename >> label) { lines.push_back(std::make_pair(filename, label)); } if (argc >= (arg_offset+5) && argv[arg_offset+4][0] == '1') { // randomly shuffle data LOG(INFO) << "Shuffling data"; std::random_shuffle(lines.begin(), lines.end()); } LOG(INFO) << "A total of " << lines.size() << " images."; int resize_height = 0; int resize_width = 0; if (argc >= (arg_offset+6)) { resize_height = atoi(argv[arg_offset+5]); } if (argc >= (arg_offset+7)) { resize_width = atoi(argv[arg_offset+6]); } leveldb::DB* db; leveldb::Options options; options.error_if_exists = true; options.create_if_missing = true; options.write_buffer_size = 268435456; LOG(INFO) << "Opening leveldb " << argv[arg_offset+3]; leveldb::Status status = leveldb::DB::Open( options, argv[arg_offset+3], &db); CHECK(status.ok()) << "Failed to open leveldb " << argv[arg_offset+3]; string root_folder(argv[arg_offset+1]); Datum datum; int count = 0; const int kMaxKeyLength = 256; char key_cstr[kMaxKeyLength]; leveldb::WriteBatch* batch = new leveldb::WriteBatch(); int data_size; bool data_size_initialized = false; for (int line_id = 0; line_id < lines.size(); ++line_id) { if (!ReadImageToDatum(root_folder + lines[line_id].first, lines[line_id].second, resize_height, resize_width, iscolor, &datum)) { continue; } if (!data_size_initialized) { data_size = datum.channels() * datum.height() * datum.width(); data_size_initialized = true; } else { const string& data = datum.data(); CHECK_EQ(data.size(), data_size) << "Incorrect data field size " << data.size(); } // sequential snprintf(key_cstr, kMaxKeyLength, "%08d_%s", line_id, lines[line_id].first.c_str()); string value; // get the value datum.SerializeToString(&value); batch->Put(string(key_cstr), value); if (++count % 1000 == 0) { db->Write(leveldb::WriteOptions(), batch); LOG(ERROR) << "Processed " << count << " files."; delete batch; batch = new leveldb::WriteBatch(); } } // write the last batch if (count % 1000 != 0) { db->Write(leveldb::WriteOptions(), batch); LOG(ERROR) << "Processed " << count << " files."; } delete batch; delete db; return 0; }
int main(int argc, char** argv) { ::google::InitGoogleLogging(argv[0]); if (argc < 5) { LOG(ERROR) << "Usage: compute_image_mean input_list new_height new_width output_file [dropping_rate]"; return 1; } char* fn_list = argv[1]; const int height = atoi(argv[2]); const int width = atoi(argv[3]); char* fn_output = argv[4]; int sampling_rate = 1; if (argc >= 6){ sampling_rate = atoi(argv[5]); LOG(INFO) << "using sampling rate " << sampling_rate; } Datum datum; BlobProto sum_blob; int count = 0; std::ifstream infile(fn_list); string fn_frm; int label; infile >> fn_frm >> label; ReadImageToDatum(fn_frm, label, height, width, &datum); sum_blob.set_num(1); sum_blob.set_channels(datum.channels()); sum_blob.set_length(1); sum_blob.set_height(datum.height()); sum_blob.set_width(datum.width()); const int data_size = datum.channels() * datum.height() * datum.width(); int size_in_datum = std::max<int>(datum.data().size(), datum.float_data_size()); for (int i = 0; i < size_in_datum; ++i) { sum_blob.add_data(0.); } LOG(INFO) << "Starting Iteration"; int i = 0; while (infile >> fn_frm >> label) { i++; if (i % sampling_rate!=0){ continue; } ReadImageToDatum(fn_frm, label, height, width, &datum); const string& data = datum.data(); size_in_datum = std::max<int>(datum.data().size(), datum.float_data_size()); CHECK_EQ(size_in_datum, data_size) << "Incorrect data field size " << size_in_datum; if (data.size() != 0) { for (int i = 0; i < size_in_datum; ++i) { sum_blob.set_data(i, sum_blob.data(i) + (uint8_t)data[i]); } } else { for (int i = 0; i < size_in_datum; ++i) { sum_blob.set_data(i, sum_blob.data(i) + static_cast<float>(datum.float_data(i))); } } ++count; if (count % 10000 == 0) { LOG(ERROR) << "Processed " << count << " files."; } } infile.close(); if (count % 10000 != 0) { LOG(ERROR) << "Processed " << count << " files."; } for (int i = 0; i < sum_blob.data_size(); ++i) { sum_blob.set_data(i, sum_blob.data(i) / count); } // Write to disk LOG(INFO) << "Write to " << fn_output; WriteProtoToBinaryFile(sum_blob, fn_output); return 0; }
void ImageDataLayer<Dtype>::SetUp(const vector<Blob<Dtype>*>& bottom, vector<Blob<Dtype>*>* top) { CHECK_EQ(bottom.size(), 0) << "Input Layer takes no input blobs."; CHECK_EQ(top->size(), 2) << "Input Layer takes two blobs as output."; const int new_height = this->layer_param_.image_data_param().new_height(); const int new_width = this->layer_param_.image_data_param().new_width(); CHECK((new_height == 0 && new_width == 0) || (new_height > 0 && new_width > 0)) << "Current implementation requires " "new_height and new_width to be set at the same time."; // Read the file with filenames and labels const string& source = this->layer_param_.image_data_param().source(); LOG(INFO) << "Opening file " << source; std::ifstream infile(source.c_str()); string filename; int label; while (infile >> filename >> label) { lines_.push_back(std::make_pair(filename, label)); } if (this->layer_param_.image_data_param().shuffle()) { // randomly shuffle data LOG(INFO) << "Shuffling data"; const unsigned int prefetch_rng_seed = caffe_rng_rand(); prefetch_rng_.reset(new Caffe::RNG(prefetch_rng_seed)); ShuffleImages(); } LOG(INFO) << "A total of " << lines_.size() << " images."; lines_id_ = 0; // Check if we would need to randomly skip a few data points if (this->layer_param_.image_data_param().rand_skip()) { unsigned int skip = caffe_rng_rand() % this->layer_param_.image_data_param().rand_skip(); LOG(INFO) << "Skipping first " << skip << " data points."; CHECK_GT(lines_.size(), skip) << "Not enough points to skip"; lines_id_ = skip; } // Read a data point, and use it to initialize the top blob. Datum datum; CHECK(ReadImageToDatum(lines_[lines_id_].first, lines_[lines_id_].second, new_height, new_width, &datum)); // image const int crop_size = this->layer_param_.image_data_param().crop_size(); const int batch_size = this->layer_param_.image_data_param().batch_size(); const string& mean_file = this->layer_param_.image_data_param().mean_file(); if (crop_size > 0) { (*top)[0]->Reshape(batch_size, datum.channels(), crop_size, crop_size); prefetch_data_.reset(new Blob<Dtype>(batch_size, datum.channels(), crop_size, crop_size)); } else { (*top)[0]->Reshape(batch_size, datum.channels(), datum.height(), datum.width()); prefetch_data_.reset(new Blob<Dtype>(batch_size, datum.channels(), datum.height(), datum.width())); } LOG(INFO) << "output data size: " << (*top)[0]->num() << "," << (*top)[0]->channels() << "," << (*top)[0]->height() << "," << (*top)[0]->width(); // label (*top)[1]->Reshape(batch_size, 1, 1, 1); prefetch_label_.reset(new Blob<Dtype>(batch_size, 1, 1, 1)); // datum size datum_channels_ = datum.channels(); datum_height_ = datum.height(); datum_width_ = datum.width(); datum_size_ = datum.channels() * datum.height() * datum.width(); //CHECK_GT(datum_height_, crop_size); //CHECK_GT(datum_width_, crop_size); // check if we want to have mean if (this->layer_param_.image_data_param().has_mean_file()) { BlobProto blob_proto; LOG(INFO) << "Loading mean file from" << mean_file; ReadProtoFromBinaryFile(mean_file.c_str(), &blob_proto); data_mean_.FromProto(blob_proto); CHECK_EQ(data_mean_.num(), 1); CHECK_EQ(data_mean_.channels(), datum_channels_); CHECK_EQ(data_mean_.height(), datum_height_); CHECK_EQ(data_mean_.width(), datum_width_); } else { // Simply initialize an all-empty mean. data_mean_.Reshape(1, datum_channels_, datum_height_, datum_width_); } // Now, start the prefetch thread. Before calling prefetch, we make two // cpu_data calls so that the prefetch thread does not accidentally make // simultaneous cudaMalloc calls when the main thread is running. In some // GPUs this seems to cause failures if we do not so. prefetch_data_->mutable_cpu_data(); prefetch_label_->mutable_cpu_data(); data_mean_.cpu_data(); DLOG(INFO) << "Initializing prefetch"; CreatePrefetchThread(); DLOG(INFO) << "Prefetch initialized."; }
void DataLayer<Dtype>::SetUp(const vector<Blob<Dtype>*>& bottom, vector<Blob<Dtype>*>* top) { CHECK_EQ(bottom.size(), 0) << "Data Layer takes no input blobs."; CHECK_GE(top->size(), 1) << "Data Layer takes at least one blob as output."; CHECK_LE(top->size(), 2) << "Data Layer takes at most two blobs as output."; if (top->size() == 1) { output_labels_ = false; } else { output_labels_ = true; } // Initialize the leveldb leveldb::DB* db_temp; leveldb::Options options; options.create_if_missing = false; options.max_open_files = 100; LOG(INFO) << "Opening leveldb " << this->layer_param_.data_param().source(); leveldb::Status status = leveldb::DB::Open( options, this->layer_param_.data_param().source(), &db_temp); CHECK(status.ok()) << "Failed to open leveldb " << this->layer_param_.data_param().source() << std::endl << status.ToString(); db_.reset(db_temp); iter_.reset(db_->NewIterator(leveldb::ReadOptions()));//通过迭代器来操纵leveldb iter_->SeekToFirst(); // Check if we would need to randomly skip a few data points //是否要随机跳过一些数据 if (this->layer_param_.data_param().rand_skip()) { unsigned int skip = caffe_rng_rand() % this->layer_param_.data_param().rand_skip(); LOG(INFO) << "Skipping first " << skip << " data points."; while (skip-- > 0) { iter_->Next(); if (!iter_->Valid()) { iter_->SeekToFirst(); } } } // Read a data point, and use it to initialize the top blob. //读取一个数据点,用来初始化topblob。所谓初始化,只要是指reshape。 //可以观察到下面iter_调用调用next。所以这次读取只是用来读取出来channels等参数的,不作处理。 Datum datum; datum.ParseFromString(iter_->value().ToString());//利用迭代器读取第一个数据点 // image图像数据 int crop_size = this->layer_param_.data_param().crop_size();//裁剪大小 if (crop_size > 0) {//需要裁剪 (*top)[0]->Reshape(this->layer_param_.data_param().batch_size(), datum.channels(), crop_size, crop_size); prefetch_data_.reset(new Blob<Dtype>( this->layer_param_.data_param().batch_size(), datum.channels(), crop_size, crop_size)); } else {//不需要裁剪 (*top)[0]->Reshape( this->layer_param_.data_param().batch_size(), datum.channels(), datum.height(), datum.width()); prefetch_data_.reset(new Blob<Dtype>( this->layer_param_.data_param().batch_size(), datum.channels(), datum.height(), datum.width())); } LOG(INFO) << "output data size: " << (*top)[0]->num() << "," << (*top)[0]->channels() << "," << (*top)[0]->height() << "," << (*top)[0]->width(); /* // label标签数据 if (output_labels_) { (*top)[1]->Reshape(this->layer_param_.data_param().batch_size(), 1, 1, 1); prefetch_label_.reset( new Blob<Dtype>(this->layer_param_.data_param().batch_size(), 1, 1, 1)); } */ // label标签数据 if (output_labels_) { (*top)[1]->Reshape(this->layer_param_.data_param().batch_size(), datum.label_size(), 1, 1); prefetch_label_.reset( new Blob<Dtype>(this->layer_param_.data_param().batch_size(), datum.label_size(), 1, 1)); } // datum size datum_channels_ = datum.channels(); datum_height_ = datum.height(); datum_width_ = datum.width(); datum_size_ = datum.channels() * datum.height() * datum.width(); CHECK_GT(datum_height_, crop_size); CHECK_GT(datum_width_, crop_size); // check if we want to have mean 是否要减去均值 if (this->layer_param_.data_param().has_mean_file()) { const string& mean_file = this->layer_param_.data_param().mean_file(); LOG(INFO) << "Loading mean file from" << mean_file; BlobProto blob_proto; ReadProtoFromBinaryFileOrDie(mean_file.c_str(), &blob_proto); data_mean_.FromProto(blob_proto); CHECK_EQ(data_mean_.num(), 1); CHECK_EQ(data_mean_.channels(), datum_channels_); CHECK_EQ(data_mean_.height(), datum_height_); CHECK_EQ(data_mean_.width(), datum_width_); } else { // Simply initialize an all-empty mean. data_mean_.Reshape(1, datum_channels_, datum_height_, datum_width_); } // Now, start the prefetch thread. Before calling prefetch, we make two // cpu_data calls so that the prefetch thread does not accidentally make // simultaneous cudaMalloc calls when the main thread is running. In some // GPUs this seems to cause failures if we do not so. prefetch_data_->mutable_cpu_data(); if (output_labels_) { prefetch_label_->mutable_cpu_data(); } data_mean_.cpu_data(); DLOG(INFO) << "Initializing prefetch"; CreatePrefetchThread(); DLOG(INFO) << "Prefetch initialized."; }
int main(int argc, char** argv) { ::google::InitGoogleLogging(argv[0]); #ifndef GFLAGS_GFLAGS_H_ namespace gflags = google; #endif gflags::SetUsageMessage("Convert a set of images to the leveldb/lmdb\n" "format used as input for Caffe.\n" "Usage:\n" " convert_imageset [FLAGS] ROOTFOLDER/ LISTFILE DB_NAME\n" "The ImageNet dataset for the training demo is at\n" " http://www.image-net.org/download-images\n"); gflags::ParseCommandLineFlags(&argc, &argv, true); if (argc != 4) { gflags::ShowUsageWithFlagsRestrict(argv[0], "tools/convert_imageset"); return 1; } bool is_color = !FLAGS_gray; bool is_gradient = FLAGS_gradient; printf("is_color = %d\n", is_color); printf("is_gradient = %d\n", is_gradient); std::ifstream infile(argv[2]); std::vector<std::pair<string, int> > lines; string filename; int label; while (infile >> filename >> label) { lines.push_back(std::make_pair(filename, label)); } if (FLAGS_shuffle) { // randomly shuffle data LOG(INFO) << "Shuffling data"; shuffle(lines.begin(), lines.end()); } LOG(INFO) << "A total of " << lines.size() << " images."; const string& db_backend = FLAGS_backend; const char* db_path = argv[3]; int resize_height = std::max<int>(0, FLAGS_resize_height); int resize_width = std::max<int>(0, FLAGS_resize_width); // Open new db // lmdb MDB_env *mdb_env; MDB_dbi mdb_dbi; MDB_val mdb_key, mdb_data; MDB_txn *mdb_txn; // leveldb leveldb::DB* db; leveldb::Options options; options.error_if_exists = true; options.create_if_missing = true; options.write_buffer_size = 268435456; leveldb::WriteBatch* batch = NULL; // Open db if (db_backend == "leveldb") { // leveldb LOG(INFO) << "Opening leveldb " << db_path; leveldb::Status status = leveldb::DB::Open( options, db_path, &db); CHECK(status.ok()) << "Failed to open leveldb " << db_path << ". Is it already existing?"; batch = new leveldb::WriteBatch(); } else if (db_backend == "lmdb") { // lmdb LOG(INFO) << "Opening lmdb " << db_path; CHECK_EQ(mkdir(db_path, 0744), 0) << "mkdir " << db_path << "failed"; CHECK_EQ(mdb_env_create(&mdb_env), MDB_SUCCESS) << "mdb_env_create failed"; CHECK_EQ(mdb_env_set_mapsize(mdb_env, 1099511627776), MDB_SUCCESS) // 1TB << "mdb_env_set_mapsize failed"; CHECK_EQ(mdb_env_open(mdb_env, db_path, 0, 0664), MDB_SUCCESS) << "mdb_env_open failed"; CHECK_EQ(mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn), MDB_SUCCESS) << "mdb_txn_begin failed"; CHECK_EQ(mdb_open(mdb_txn, NULL, 0, &mdb_dbi), MDB_SUCCESS) << "mdb_open failed. Does the lmdb already exist? "; } else { LOG(FATAL) << "Unknown db backend " << db_backend; } // Storing to db string root_folder(argv[1]); Datum datum; int count = 0; const int kMaxKeyLength = 256; char key_cstr[kMaxKeyLength]; int data_size; bool data_size_initialized = false; for (int line_id = 0; line_id < lines.size(); ++line_id) { if (!MyReadImageToDatum(root_folder + lines[line_id].first, lines[line_id].second, resize_height, resize_width, is_color, is_gradient, &datum)) { continue; } if (!data_size_initialized) { data_size = datum.channels() * datum.height() * datum.width(); data_size_initialized = true; } else { const string& data = datum.data(); CHECK_EQ(data.size(), data_size) << "Incorrect data field size " << data.size(); } // sequential snprintf(key_cstr, kMaxKeyLength, "%08d_%s", line_id, lines[line_id].first.c_str()); string value; datum.SerializeToString(&value); string keystr(key_cstr); // Put in db if (db_backend == "leveldb") { // leveldb batch->Put(keystr, value); } else if (db_backend == "lmdb") { // lmdb mdb_data.mv_size = value.size(); mdb_data.mv_data = reinterpret_cast<void*>(&value[0]); mdb_key.mv_size = keystr.size(); mdb_key.mv_data = reinterpret_cast<void*>(&keystr[0]); CHECK_EQ(mdb_put(mdb_txn, mdb_dbi, &mdb_key, &mdb_data, 0), MDB_SUCCESS) << "mdb_put failed"; } else { LOG(FATAL) << "Unknown db backend " << db_backend; } if (++count % 1000 == 0) { // Commit txn if (db_backend == "leveldb") { // leveldb db->Write(leveldb::WriteOptions(), batch); delete batch; batch = new leveldb::WriteBatch(); } else if (db_backend == "lmdb") { // lmdb CHECK_EQ(mdb_txn_commit(mdb_txn), MDB_SUCCESS) << "mdb_txn_commit failed"; CHECK_EQ(mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn), MDB_SUCCESS) << "mdb_txn_begin failed"; } else { LOG(FATAL) << "Unknown db backend " << db_backend; } LOG(ERROR) << "Processed " << count << " files."; } } // write the last batch if (count % 1000 != 0) { if (db_backend == "leveldb") { // leveldb db->Write(leveldb::WriteOptions(), batch); delete batch; delete db; } else if (db_backend == "lmdb") { // lmdb CHECK_EQ(mdb_txn_commit(mdb_txn), MDB_SUCCESS) << "mdb_txn_commit failed"; mdb_close(mdb_env, mdb_dbi); mdb_env_close(mdb_env); } else { LOG(FATAL) << "Unknown db backend " << db_backend; } LOG(ERROR) << "Processed " << count << " files."; } return 0; }