void BaseDataLayer<Dtype>::LayerSetUp(const vector<Blob<Dtype>*>& bottom, vector<Blob<Dtype>*>* top) { MPI_Comm_rank (MPI_COMM_WORLD, &rank); if (top->size() == 1) { output_labels_ = false; } else { output_labels_ = true; } DataLayerSetUp(bottom, top); // The subclasses should setup the datum channels, height and width CHECK_GT(datum_channels_, 0); CHECK_GT(datum_height_, 0); CHECK_GT(datum_width_, 0); if (transform_param_.crop_size() > 0) { CHECK_GE(datum_height_, transform_param_.crop_size()); CHECK_GE(datum_width_, transform_param_.crop_size()); } // check if we want to have mean if (transform_param_.has_mean_file()) { const string& mean_file = transform_param_.mean_file(); LOG(INFO) << "Loading mean file from" << mean_file; BlobProto blob_proto; #if 0 int fsize=0; FILE * fin=NULL; if(rank==0){ fin=fopen(mean_file.c_str(),"rb"); if(fin==NULL)LOG(FATAL)<<"NO this mean file "<< mean_file;//TODO client fseek(fin,0,SEEK_END); fsize=ftell(fin); rewind(fin); } MPI_Bcast(&fsize,1,MPI_INT,0,MPI_COMM_WORLD); uint8_t *mean_buffer=(uint8_t*)malloc(fsize); if(rank==0){ fread(mean_buffer,fsize,1,fin); fclose(fin); } MPI_Bcast(mean_buffer,fsize,MPI_CHAR,0,MPI_COMM_WORLD); CodedInputStream* coded_input = new CodedInputStream(mean_buffer,fsize); coded_input->SetTotalBytesLimit(1073741824, 536870912); blob_proto.ParseFromCodedStream(coded_input); delete coded_input; free(mean_buffer); #else ReadProtoFromBinaryFileOrDie(mean_file.c_str(), &blob_proto); #endif data_mean_.FromProto(blob_proto); CHECK_GE(data_mean_.num(), 1); CHECK_GE(data_mean_.channels(), datum_channels_); CHECK_GE(data_mean_.height(), datum_height_); CHECK_GE(data_mean_.width(), datum_width_); } else { // Simply initialize an all-empty mean. data_mean_.Reshape(1, datum_channels_, datum_height_, datum_width_); } mean_ = data_mean_.cpu_data(); data_transformer_.InitRand(); }
void DataLayer<Dtype>::SetUp(const vector<Blob<Dtype>*>& bottom, vector<Blob<Dtype>*>* top) { CHECK_EQ(bottom.size(), 0) << "Data Layer takes no input blobs."; CHECK_GE(top->size(), 1) << "Data Layer takes at least one blob as output."; CHECK_LE(top->size(), 2) << "Data Layer takes at most two blobs as output."; if (top->size() == 1) { output_labels_ = false; } else { output_labels_ = true; } // Initialize the leveldb leveldb::DB* db_temp; leveldb::Options options; options.create_if_missing = false; options.max_open_files = 100; LOG(INFO) << "Opening leveldb " << this->layer_param_.data_param().source(); leveldb::Status status = leveldb::DB::Open( options, this->layer_param_.data_param().source(), &db_temp); CHECK(status.ok()) << "Failed to open leveldb " << this->layer_param_.data_param().source() << std::endl << status.ToString(); db_.reset(db_temp); iter_.reset(db_->NewIterator(leveldb::ReadOptions())); iter_->SeekToFirst(); // Check if we would need to randomly skip a few data points if (this->layer_param_.data_param().rand_skip()) { unsigned int skip = caffe_rng_rand() % this->layer_param_.data_param().rand_skip(); LOG(INFO) << "Skipping first " << skip << " data points."; while (skip-- > 0) { iter_->Next(); if (!iter_->Valid()) { iter_->SeekToFirst(); } } } // Read a data point, and use it to initialize the top blob. Datum datum; datum.ParseFromString(iter_->value().ToString()); // image int crop_size = this->layer_param_.data_param().crop_size(); if (crop_size > 0) { (*top)[0]->Reshape(this->layer_param_.data_param().batch_size(), datum.channels(), 1, crop_size, crop_size); prefetch_data_.reset(new Blob<Dtype>( this->layer_param_.data_param().batch_size(), datum.channels(), 1, crop_size, crop_size)); } else { (*top)[0]->Reshape( this->layer_param_.data_param().batch_size(), datum.channels(), 1, datum.height(), datum.width()); prefetch_data_.reset(new Blob<Dtype>( this->layer_param_.data_param().batch_size(), datum.channels(), 1, datum.height(), datum.width())); } LOG(INFO) << "output data size: " << (*top)[0]->num() << "," << (*top)[0]->channels() << "," << (*top)[0]->length() << "," << (*top)[0]->height() << "," << (*top)[0]->width(); // label if (output_labels_) { (*top)[1]->Reshape(this->layer_param_.data_param().batch_size(), 1, 1, 1, 1); prefetch_label_.reset( new Blob<Dtype>(this->layer_param_.data_param().batch_size(), 1, 1, 1, 1)); } // datum size datum_channels_ = datum.channels(); datum_height_ = datum.height(); datum_width_ = datum.width(); datum_size_ = datum.channels() * datum.height() * datum.width(); CHECK_GT(datum_height_, crop_size); CHECK_GT(datum_width_, crop_size); // check if we want to have mean if (this->layer_param_.data_param().has_mean_file()) { const string& mean_file = this->layer_param_.data_param().mean_file(); LOG(INFO) << "Loading mean file from" << mean_file; BlobProto blob_proto; ReadProtoFromBinaryFileOrDie(mean_file.c_str(), &blob_proto); data_mean_.FromProto(blob_proto); CHECK_EQ(data_mean_.num(), 1); CHECK_EQ(data_mean_.channels(), datum_channels_); CHECK_EQ(data_mean_.length(), 1); CHECK_EQ(data_mean_.height(), datum_height_); CHECK_EQ(data_mean_.width(), datum_width_); } else { // Simply initialize an all-empty mean. data_mean_.Reshape(1, datum_channels_, 1, datum_height_, datum_width_); } // Now, start the prefetch thread. Before calling prefetch, we make two // cpu_data calls so that the prefetch thread does not accidentally make // simultaneous cudaMalloc calls when the main thread is running. In some // GPUs this seems to cause failures if we do not so. prefetch_data_->mutable_cpu_data(); if (output_labels_) { prefetch_label_->mutable_cpu_data(); } data_mean_.cpu_data(); DLOG(INFO) << "Initializing prefetch"; CreatePrefetchThread(); DLOG(INFO) << "Prefetch initialized."; }
inline void ReadProtoFromBinaryFileOrDie(const string& filename, Message* proto) { ReadProtoFromBinaryFileOrDie(filename.c_str(), proto); }