bool SnapshotControl::InitBloomFilters(uint64_t snapshot_size) { if (!FileSystemHelper::GetInstance()->IsFileExists(vm_meta_pathname_)) { if (!FileSystemHelper::GetInstance()->IsDirectoryExists(vm_path_)) FileSystemHelper::GetInstance()->CreateDirectory(vm_path_); // init bloom filter params and store into vm meta LOG4CXX_INFO(logger_, "VM meta not found, will create " << vm_meta_pathname_); vm_meta_.filter_num_items_ = snapshot_size / AVG_BLOCK_SIZE; vm_meta_.filter_num_funcs_ = BLOOM_FILTER_NUM_FUNCS; vm_meta_.filter_fp_rate_ = BLOOM_FILTER_FP_RATE; FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(vm_meta_pathname_, O_WRONLY); fh->Create(); stringstream buffer; vm_meta_.Serialize(buffer); LOG4CXX_DEBUG(logger_, "VM meta size " << buffer.str().size()); fh->WriteData((char *)buffer.str().c_str(), buffer.str().size()); fh->Close(); FileSystemHelper::GetInstance()->DestroyFileHelper(fh); } else { // read bloom filter params FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(vm_meta_pathname_, O_RDONLY); fh->Open(); long read_length = FileSystemHelper::GetInstance()->GetSize(vm_meta_pathname_); char *data = new char[read_length]; fh->Read(data, read_length); LOG4CXX_DEBUG(logger_, "Read " << read_length << " from file"); stringstream buffer; buffer.write(data, read_length); vm_meta_.Deserialize(buffer); LOG4CXX_DEBUG(logger_, "VM meta loaded: " << vm_meta_.filter_num_items_ << " " << vm_meta_.filter_num_funcs_ << "" << vm_meta_.filter_fp_rate_); fh->Close(); FileSystemHelper::GetInstance()->DestroyFileHelper(fh); delete[] data; } // params ready, now init bloom filters primary_filter_ptr_ = new BloomFilter<Checksum>(vm_meta_.filter_num_items_, vm_meta_.filter_fp_rate_, kBloomFilterFunctions, vm_meta_.filter_num_funcs_); // for fine-grained deletion we need a bigger filter, using different group of hash functions secondary_filter_ptr_ = new BloomFilter<Checksum>(vm_meta_.filter_num_items_ * 2, vm_meta_.filter_fp_rate_, &kBloomFilterFunctions[8], vm_meta_.filter_num_funcs_); return true; }
bool SnapshotControl::SaveBloomFilter(BloomFilter<Checksum>* pbf, const string& bf_name) { if (!FileSystemHelper::GetInstance()->IsDirectoryExists(vm_path_)) FileSystemHelper::GetInstance()->CreateDirectory(vm_path_); if (FileSystemHelper::GetInstance()->IsFileExists(bf_name)) FileSystemHelper::GetInstance()->RemoveFile(bf_name); FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(bf_name, O_WRONLY); fh->Create(); stringstream buffer; pbf->Serialize(buffer); LOG4CXX_DEBUG(logger_, "Bloom filter primary size " << buffer.str().size()); fh->Write((char *)buffer.str().c_str(), buffer.str().size()); fh->Close(); FileSystemHelper::GetInstance()->DestroyFileHelper(fh); return true; }
int main(int argc, char** argv) { if (argc != 4) { usage(argv[0]); return -1; } DOMConfigurator::configure("Log4cxxConfig.xml"); string cds_name(argv[1]); string cds_pathname(argv[2]); string sample_pathname(argv[3]); string qfs_cds_dir = "/cds"; string qfs_cds_file = qfs_cds_dir + "/" + cds_name; DataSource source(cds_pathname, sample_pathname); CdsIndex cds; BlockMeta bm; uint64_t offset = 0; uint32_t bytes_written = 0; QFSHelper::Connect(); if (FileSystemHelper::GetInstance()->IsDirectoryExists(qfs_cds_dir)) { FileSystemHelper::GetInstance()->RemoveDirectory(qfs_cds_dir); } FileSystemHelper::GetInstance()->CreateDirectory(qfs_cds_dir); FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(qfs_cds_file, O_WRONLY); fh->Create(); while (source.GetBlock(bm)) { MEASURE(cds.Set(bm.cksum_, offset)); // add to cds index cache MEASURE(bytes_written = fh->WriteData(bm.data_, bm.size_)); // write to qfs if (bytes_written != bm.size_) cout << "Error: write " << bytes_written << ", expect " << bm.size_ << endl; offset += bm.size_; } fh->Close(); FileSystemHelper::GetInstance()->DestroyFileHelper(fh); TIMER_PRINT_ALL(); exit(0); }
bool SnapshotControl::SaveSnapshotMeta() { if (!FileSystemHelper::GetInstance()->IsDirectoryExists(vm_path_)) FileSystemHelper::GetInstance()->CreateDirectory(vm_path_); if (FileSystemHelper::GetInstance()->IsFileExists(ss_meta_pathname_)) { LOG4CXX_WARN(logger_, "Sanpshot metadata exists, will re-create " << ss_meta_pathname_); FileSystemHelper::GetInstance()->RemoveFile(ss_meta_pathname_); } FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(ss_meta_pathname_, O_WRONLY); fh->Create(); stringstream buffer; ss_meta_.Serialize(buffer); ss_meta_.SerializeRecipe(buffer); LOG4CXX_DEBUG(logger_, "save snapshot meta, size is" << buffer.str().size()); fh->Write((char *)buffer.str().c_str(), buffer.str().size()); fh->Close(); FileSystemHelper::GetInstance()->DestroyFileHelper(fh); return true; }