void BaseShufflingDataLayer<Dtype>::Forward_cpu( const vector<Blob<Dtype>*>& bottom, vector<Blob<Dtype>*>* top) { int batch_size = this->layer_param_.shuffling_data_param().batch_size(); int offset = 0; int N = prefetch_data_[current_buffer_].num(); int dim = prefetch_data_[current_buffer_].count() / N; while (batch_size - offset) { if (current_row_ == N) { JoinPrefetchThread(); current_row_ = 0; current_buffer_ = 1 - current_buffer_; CreatePrefetchThread(); } int avail = std::min(batch_size-offset, N - current_row_); caffe_copy(avail * dim, prefetch_data_[current_buffer_].cpu_data() + prefetch_data_[current_buffer_].offset(current_row_), (*top)[0]->mutable_cpu_data() + (*top)[0]->offset(offset)); if (this->output_labels_) { dim = prefetch_label_[current_buffer_].count() / N; caffe_copy(avail * dim, prefetch_label_[current_buffer_].cpu_data() + prefetch_label_[current_buffer_].offset(current_row_), (*top)[1]->mutable_cpu_data() + (*top)[1]->offset(offset)); } current_row_ += avail; offset += avail; } }
void BasePrefetchingDataLayer<Dtype>::Forward_gpu( const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) { // First, join the thread JoinPrefetchThread(); // Reshape to loaded data. top[0]->Reshape( this->prefetch_data_.num(), this->prefetch_data_.channels(), this->prefetch_data_.height(), this->prefetch_data_.width()); // Copy the data caffe_copy( prefetch_data_.count(), prefetch_data_.cpu_data(), top[0]->mutable_gpu_data()); if (this->output_labels_) { caffe_copy( prefetch_label_.count(), prefetch_label_.cpu_data(), top[1]->mutable_gpu_data()); } // Start a new prefetch thread CreatePrefetchThread(); }
void BasePrefetchingDataLayer<Dtype>::Forward_cpu( const vector<Blob<Dtype>*>& bottom, vector<Blob<Dtype>*>* top) { // First, join the thread JoinPrefetchThread(); // Copy the data caffe_copy(prefetch_data_.count(), prefetch_data_.cpu_data(), (*top)[0]->mutable_cpu_data()); if (this->output_labels_) { caffe_copy(prefetch_label_.count(), prefetch_label_.cpu_data(), (*top)[1]->mutable_cpu_data()); } // Start a new prefetch thread CreatePrefetchThread(); }
Dtype VolumeDataLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom, vector<Blob<Dtype>*>* top) { // First, join the thread JoinPrefetchThread(); // Copy the data caffe_copy(prefetch_data_->count(), prefetch_data_->cpu_data(), (*top)[0]->mutable_cpu_data()); if (output_labels_) { caffe_copy(prefetch_label_->count(), prefetch_label_->cpu_data(), (*top)[1]->mutable_cpu_data()); } // Start a new prefetch thread CreatePrefetchThread(); return Dtype(0.); }
void BasePrefetchingDataLayer<Dtype>::Forward_cpu( const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) { // First, join the thread JoinPrefetchThread(); DLOG(INFO) << "Thread joined"; // Copy the data caffe_copy(prefetch_data_.count(), prefetch_data_.cpu_data(), top[0]->mutable_cpu_data()); DLOG(INFO) << "Prefetch copied"; if (this->output_labels_) { caffe_copy(prefetch_label_.count(), prefetch_label_.cpu_data(), top[1]->mutable_cpu_data()); } // Start a new prefetch thread DLOG(INFO) << "CreatePrefetchThread"; CreatePrefetchThread(); }
void VolumeDataLayer<Dtype>::SetUp(const vector<Blob<Dtype>*>& bottom, vector<Blob<Dtype>*>* top) { CHECK_EQ(bottom.size(), 0) << "Data Layer takes no input blobs."; CHECK_GE(top->size(), 1) << "Data Layer takes at least one blob as output."; CHECK_LE(top->size(), 2) << "Data Layer takes at most two blobs as output."; if (top->size() == 1) { output_labels_ = false; } else { output_labels_ = true; } // Initialize the leveldb leveldb::DB* db_temp; leveldb::Options options; options.create_if_missing = false; options.max_open_files = 100; LOG(INFO) << "Opening leveldb " << this->layer_param_.data_param().source(); leveldb::Status status = leveldb::DB::Open( options, this->layer_param_.data_param().source(), &db_temp); CHECK(status.ok()) << "Failed to open leveldb " << this->layer_param_.data_param().source() << std::endl << status.ToString(); db_.reset(db_temp); iter_.reset(db_->NewIterator(leveldb::ReadOptions())); iter_->SeekToFirst(); // Check if we would need to randomly skip a few data points if (this->layer_param_.data_param().rand_skip()) { unsigned int skip = caffe_rng_rand() % this->layer_param_.data_param().rand_skip(); LOG(INFO) << "Skipping first " << skip << " data points."; while (skip-- > 0) { iter_->Next(); if (!iter_->Valid()) { iter_->SeekToFirst(); } } } // Read a data point, and use it to initialize the top blob. VolumeDatum datum; datum.ParseFromString(iter_->value().ToString()); // image int crop_size = this->layer_param_.data_param().crop_size(); if (crop_size > 0) { (*top)[0]->Reshape(this->layer_param_.data_param().batch_size(), datum.channels(), datum.length(), crop_size, crop_size); prefetch_data_.reset(new Blob<Dtype>( this->layer_param_.data_param().batch_size(), datum.channels(), datum.length(), crop_size, crop_size)); } else { (*top)[0]->Reshape( this->layer_param_.data_param().batch_size(), datum.channels(), datum.length(), datum.height(), datum.width()); prefetch_data_.reset(new Blob<Dtype>( this->layer_param_.data_param().batch_size(), datum.channels(), datum.length(), datum.height(), datum.width())); } LOG(INFO) << "output data size: " << (*top)[0]->num() << "," << (*top)[0]->channels() << "," << (*top)[0]->length() << "," << (*top)[0]->height() << "," << (*top)[0]->width(); // label if (output_labels_) { (*top)[1]->Reshape(this->layer_param_.data_param().batch_size(), 1, 1, 1, 1); prefetch_label_.reset( new Blob<Dtype>(this->layer_param_.data_param().batch_size(), 1, 1, 1, 1)); } // datum size datum_channels_ = datum.channels(); datum_length_ = datum.length(); datum_height_ = datum.height(); datum_width_ = datum.width(); datum_size_ = datum.channels() * datum.length() * datum.height() * datum.width(); CHECK_GT(datum_height_, crop_size); CHECK_GT(datum_width_, crop_size); // check if we want to have mean if (this->layer_param_.data_param().has_mean_file()) { const string& mean_file = this->layer_param_.data_param().mean_file(); LOG(INFO) << "Loading mean file from" << mean_file; BlobProto blob_proto; ReadProtoFromBinaryFileOrDie(mean_file.c_str(), &blob_proto); data_mean_.FromProto(blob_proto); CHECK_EQ(data_mean_.num(), 1); CHECK_EQ(data_mean_.channels(), datum_channels_); CHECK_EQ(data_mean_.length(), datum_length_); CHECK_EQ(data_mean_.height(), datum_height_); CHECK_EQ(data_mean_.width(), datum_width_); } else { // Simply initialize an all-empty mean. data_mean_.Reshape(1, datum_channels_, datum_length_, datum_height_, datum_width_); } // Now, start the prefetch thread. Before calling prefetch, we make two // cpu_data calls so that the prefetch thread does not accidentally make // simultaneous cudaMalloc calls when the main thread is running. In some // GPUs this seems to cause failures if we do not so. prefetch_data_->mutable_cpu_data(); if (output_labels_) { prefetch_label_->mutable_cpu_data(); } data_mean_.cpu_data(); DLOG(INFO) << "Initializing prefetch"; CreatePrefetchThread(); DLOG(INFO) << "Prefetch initialized."; }