void BaseDataLayer<Dtype>::LayerSetUp(const vector<Blob<Dtype>*>& bottom,
vector<Blob<Dtype>*>* top) {
MPI_Comm_rank (MPI_COMM_WORLD, &rank);
if (top->size() == 1) {
output_labels_ = false;
} else {
output_labels_ = true;
}
DataLayerSetUp(bottom, top);
// The subclasses should setup the datum channels, height and width
CHECK_GT(datum_channels_, 0);
CHECK_GT(datum_height_, 0);
CHECK_GT(datum_width_, 0);
if (transform_param_.crop_size() > 0) {
CHECK_GE(datum_height_, transform_param_.crop_size());
CHECK_GE(datum_width_, transform_param_.crop_size());
}
// check if we want to have mean
if (transform_param_.has_mean_file()) {
const string& mean_file = transform_param_.mean_file();
LOG(INFO) << "Loading mean file from" << mean_file;
BlobProto blob_proto;
#if 0
int fsize=0;
FILE * fin=NULL;
if(rank==0){
fin=fopen(mean_file.c_str(),"rb");
if(fin==NULL)LOG(FATAL)<<"NO this mean file "<< mean_file;//TODO client
fseek(fin,0,SEEK_END);
fsize=ftell(fin);
rewind(fin);
}
MPI_Bcast(&fsize,1,MPI_INT,0,MPI_COMM_WORLD);
uint8_t *mean_buffer=(uint8_t*)malloc(fsize);
if(rank==0){
fread(mean_buffer,fsize,1,fin);
fclose(fin);
}
MPI_Bcast(mean_buffer,fsize,MPI_CHAR,0,MPI_COMM_WORLD);
CodedInputStream* coded_input = new CodedInputStream(mean_buffer,fsize);
coded_input->SetTotalBytesLimit(1073741824, 536870912);
blob_proto.ParseFromCodedStream(coded_input);
delete coded_input;
free(mean_buffer);
#else
ReadProtoFromBinaryFileOrDie(mean_file.c_str(), &blob_proto);
#endif
data_mean_.FromProto(blob_proto);
CHECK_GE(data_mean_.num(), 1);
CHECK_GE(data_mean_.channels(), datum_channels_);
CHECK_GE(data_mean_.height(), datum_height_);
CHECK_GE(data_mean_.width(), datum_width_);
} else {
// Simply initialize an all-empty mean.
data_mean_.Reshape(1, datum_channels_, datum_height_, datum_width_);
}
mean_ = data_mean_.cpu_data();
data_transformer_.InitRand();
}
void DataLayer<Dtype>::SetUp(const vector<Blob<Dtype>*>& bottom,
vector<Blob<Dtype>*>* top) {
CHECK_EQ(bottom.size(), 0) << "Data Layer takes no input blobs.";
CHECK_GE(top->size(), 1) << "Data Layer takes at least one blob as output.";
CHECK_LE(top->size(), 2) << "Data Layer takes at most two blobs as output.";
if (top->size() == 1) {
output_labels_ = false;
} else {
output_labels_ = true;
}
// Initialize the leveldb
leveldb::DB* db_temp;
leveldb::Options options;
options.create_if_missing = false;
options.max_open_files = 100;
LOG(INFO) << "Opening leveldb " << this->layer_param_.data_param().source();
leveldb::Status status = leveldb::DB::Open(
options, this->layer_param_.data_param().source(), &db_temp);
CHECK(status.ok()) << "Failed to open leveldb "
<< this->layer_param_.data_param().source() << std::endl
<< status.ToString();
db_.reset(db_temp);
iter_.reset(db_->NewIterator(leveldb::ReadOptions()));
iter_->SeekToFirst();
// Check if we would need to randomly skip a few data points
if (this->layer_param_.data_param().rand_skip()) {
unsigned int skip = caffe_rng_rand() %
this->layer_param_.data_param().rand_skip();
LOG(INFO) << "Skipping first " << skip << " data points.";
while (skip-- > 0) {
iter_->Next();
if (!iter_->Valid()) {
iter_->SeekToFirst();
}
}
}
// Read a data point, and use it to initialize the top blob.
Datum datum;
datum.ParseFromString(iter_->value().ToString());
// image
int crop_size = this->layer_param_.data_param().crop_size();
if (crop_size > 0) {
(*top)[0]->Reshape(this->layer_param_.data_param().batch_size(),
datum.channels(), 1, crop_size, crop_size);
prefetch_data_.reset(new Blob<Dtype>(
this->layer_param_.data_param().batch_size(), datum.channels(),
1, crop_size, crop_size));
} else {
(*top)[0]->Reshape(
this->layer_param_.data_param().batch_size(), datum.channels(),
1, datum.height(), datum.width());
prefetch_data_.reset(new Blob<Dtype>(
this->layer_param_.data_param().batch_size(), datum.channels(),
1, datum.height(), datum.width()));
}
LOG(INFO) << "output data size: " << (*top)[0]->num() << ","
<< (*top)[0]->channels() << "," << (*top)[0]->length() << "," << (*top)[0]->height() << ","
<< (*top)[0]->width();
// label
if (output_labels_) {
(*top)[1]->Reshape(this->layer_param_.data_param().batch_size(), 1, 1, 1, 1);
prefetch_label_.reset(
new Blob<Dtype>(this->layer_param_.data_param().batch_size(), 1, 1, 1, 1));
}
// datum size
datum_channels_ = datum.channels();
datum_height_ = datum.height();
datum_width_ = datum.width();
datum_size_ = datum.channels() * datum.height() * datum.width();
CHECK_GT(datum_height_, crop_size);
CHECK_GT(datum_width_, crop_size);
// check if we want to have mean
if (this->layer_param_.data_param().has_mean_file()) {
const string& mean_file = this->layer_param_.data_param().mean_file();
LOG(INFO) << "Loading mean file from" << mean_file;
BlobProto blob_proto;
ReadProtoFromBinaryFileOrDie(mean_file.c_str(), &blob_proto);
data_mean_.FromProto(blob_proto);
CHECK_EQ(data_mean_.num(), 1);
CHECK_EQ(data_mean_.channels(), datum_channels_);
CHECK_EQ(data_mean_.length(), 1);
CHECK_EQ(data_mean_.height(), datum_height_);
CHECK_EQ(data_mean_.width(), datum_width_);
} else {
// Simply initialize an all-empty mean.
data_mean_.Reshape(1, datum_channels_, 1, datum_height_, datum_width_);
}
// Now, start the prefetch thread. Before calling prefetch, we make two
// cpu_data calls so that the prefetch thread does not accidentally make
// simultaneous cudaMalloc calls when the main thread is running. In some
// GPUs this seems to cause failures if we do not so.
prefetch_data_->mutable_cpu_data();
if (output_labels_) {
prefetch_label_->mutable_cpu_data();
}
data_mean_.cpu_data();
DLOG(INFO) << "Initializing prefetch";
CreatePrefetchThread();
DLOG(INFO) << "Prefetch initialized.";
}
inline void ReadProtoFromBinaryFileOrDie(const string& filename,
Message* proto) {
ReadProtoFromBinaryFileOrDie(filename.c_str(), proto);
}
