bool SnapshotControl::InitBloomFilters(uint64_t snapshot_size) { if (!FileSystemHelper::GetInstance()->IsFileExists(vm_meta_pathname_)) { if (!FileSystemHelper::GetInstance()->IsDirectoryExists(vm_path_)) FileSystemHelper::GetInstance()->CreateDirectory(vm_path_); // init bloom filter params and store into vm meta LOG4CXX_INFO(logger_, "VM meta not found, will create " << vm_meta_pathname_); vm_meta_.filter_num_items_ = snapshot_size / AVG_BLOCK_SIZE; vm_meta_.filter_num_funcs_ = BLOOM_FILTER_NUM_FUNCS; vm_meta_.filter_fp_rate_ = BLOOM_FILTER_FP_RATE; FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(vm_meta_pathname_, O_WRONLY); fh->Create(); stringstream buffer; vm_meta_.Serialize(buffer); LOG4CXX_DEBUG(logger_, "VM meta size " << buffer.str().size()); fh->WriteData((char *)buffer.str().c_str(), buffer.str().size()); fh->Close(); FileSystemHelper::GetInstance()->DestroyFileHelper(fh); } else { // read bloom filter params FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(vm_meta_pathname_, O_RDONLY); fh->Open(); long read_length = FileSystemHelper::GetInstance()->GetSize(vm_meta_pathname_); char *data = new char[read_length]; fh->Read(data, read_length); LOG4CXX_DEBUG(logger_, "Read " << read_length << " from file"); stringstream buffer; buffer.write(data, read_length); vm_meta_.Deserialize(buffer); LOG4CXX_DEBUG(logger_, "VM meta loaded: " << vm_meta_.filter_num_items_ << " " << vm_meta_.filter_num_funcs_ << "" << vm_meta_.filter_fp_rate_); fh->Close(); FileSystemHelper::GetInstance()->DestroyFileHelper(fh); delete[] data; } // params ready, now init bloom filters primary_filter_ptr_ = new BloomFilter<Checksum>(vm_meta_.filter_num_items_, vm_meta_.filter_fp_rate_, kBloomFilterFunctions, vm_meta_.filter_num_funcs_); // for fine-grained deletion we need a bigger filter, using different group of hash functions secondary_filter_ptr_ = new BloomFilter<Checksum>(vm_meta_.filter_num_items_ * 2, vm_meta_.filter_fp_rate_, &kBloomFilterFunctions[8], vm_meta_.filter_num_funcs_); return true; }
void PanguAppendStore::WriteMetaInfo(const std::string& root, const StoreMetaData& meta) { std::string metaFileName = root + MetaFileName; try { FileHelper* metaOutputFH = mFileSystemHelper->CreateFileHelper(metaFileName, O_WRONLY); metaOutputFH->Open(); char *write_buffer = new char[sizeof(StoreMetaData)]; /* Copying into buffer from StoreMetaData */ LOG4CXX_DEBUG(logger_, "before reading mMeta values : " << mMeta.storeminor << "," << mMeta.storemajor << "," << mMeta.maxChunkSize << "," << mMeta.blockIndexInterval << "," << mMeta.compressionFlag); meta.toBuffer(write_buffer); metaOutputFH->WriteData(write_buffer, sizeof(StoreMetaData)); metaOutputFH->Close(); mFileSystemHelper->DestroyFileHelper(metaOutputFH); } catch (ExceptionBase& e) { THROW_EXCEPTION(AppendStoreWriteException, e.ToString()+" Cannot generate .meta_ file"); } LOG4CXX_TRACE(logger_, "Store::WroteMetaDataInfo()" ); }
bool SnapshotControl::LoadSnapshotMeta() { // open the snapshot meta file in qfs and read it // TODO: currently the read/write apis provide a log style file if (!FileSystemHelper::GetInstance()->IsFileExists(ss_meta_pathname_)) { LOG4CXX_ERROR(logger_, "Couldn't find snapshot meta: " << ss_meta_.vm_id_ << " " << ss_meta_.snapshot_id_); return false; } FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(ss_meta_pathname_, O_RDONLY); fh->Open(); int read_length = fh->GetNextLogSize(); char *data = new char[read_length]; fh->Read(data, read_length); LOG4CXX_DEBUG(logger_, "Read " << read_length << " from " << ss_meta_pathname_); stringstream buffer; buffer.write(data, read_length); ss_meta_.Deserialize(buffer); ss_meta_.DeserializeRecipe(buffer); LOG4CXX_INFO(logger_, "Snapshot meta loaded: " << ss_meta_.vm_id_ << " " << ss_meta_.snapshot_id_); fh->Close(); FileSystemHelper::GetInstance()->DestroyFileHelper(fh); delete[] data; return true; }
bool SnapshotControl::SaveBloomFilter(BloomFilter<Checksum>* pbf, const string& bf_name) { if (!FileSystemHelper::GetInstance()->IsDirectoryExists(vm_path_)) FileSystemHelper::GetInstance()->CreateDirectory(vm_path_); if (FileSystemHelper::GetInstance()->IsFileExists(bf_name)) FileSystemHelper::GetInstance()->RemoveFile(bf_name); FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(bf_name, O_WRONLY); fh->Create(); stringstream buffer; pbf->Serialize(buffer); LOG4CXX_DEBUG(logger_, "Bloom filter primary size " << buffer.str().size()); fh->Write((char *)buffer.str().c_str(), buffer.str().size()); fh->Close(); FileSystemHelper::GetInstance()->DestroyFileHelper(fh); return true; }
bool PanguAppendStore::ReadMetaInfo() { std::string metaFileName = mRoot + MetaFileName; bool fexist = false; try { fexist = mFileSystemHelper->IsFileExists(metaFileName); LOG4CXX_DEBUG(logger_, "meta file name " << metaFileName << " : " << fexist); } catch (ExceptionBase & e) { LOG4CXX_ERROR(logger_, "IsFileExist : " << metaFileName); throw; } if (fexist) { try { FileHelper* metaInputFH = mFileSystemHelper->CreateFileHelper(metaFileName, O_RDONLY); char *read_buffer = new char[sizeof(StoreMetaData)]; metaInputFH->Open(); metaInputFH->Read(read_buffer, sizeof(StoreMetaData)); metaInputFH->Close(); mFileSystemHelper->DestroyFileHelper(metaInputFH); mMeta.fromBuffer(read_buffer); LOG4CXX_DEBUG(logger_, "after reading mMeta values : " << mMeta.storeminor << "," << mMeta.storemajor << "," << mMeta.maxChunkSize << "," << mMeta.blockIndexInterval << "," << mMeta.compressionFlag); return true; } catch (ExceptionBase& e) { THROW_EXCEPTION(AppendStoreWriteException, "Cannot open meta file for append " + e.ToString()); } } LOG4CXX_TRACE(logger_, "Store::ReadMetaDataInfo()" ); return false; }
int main(int argc, char** argv) { if (argc != 4) { usage(argv[0]); return -1; } DOMConfigurator::configure("Log4cxxConfig.xml"); string cds_name(argv[1]); string cds_pathname(argv[2]); string sample_pathname(argv[3]); string qfs_cds_dir = "/cds"; string qfs_cds_file = qfs_cds_dir + "/" + cds_name; DataSource source(cds_pathname, sample_pathname); CdsIndex cds; BlockMeta bm; uint64_t offset = 0; uint32_t bytes_written = 0; QFSHelper::Connect(); if (FileSystemHelper::GetInstance()->IsDirectoryExists(qfs_cds_dir)) { FileSystemHelper::GetInstance()->RemoveDirectory(qfs_cds_dir); } FileSystemHelper::GetInstance()->CreateDirectory(qfs_cds_dir); FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(qfs_cds_file, O_WRONLY); fh->Create(); while (source.GetBlock(bm)) { MEASURE(cds.Set(bm.cksum_, offset)); // add to cds index cache MEASURE(bytes_written = fh->WriteData(bm.data_, bm.size_)); // write to qfs if (bytes_written != bm.size_) cout << "Error: write " << bytes_written << ", expect " << bm.size_ << endl; offset += bm.size_; } fh->Close(); FileSystemHelper::GetInstance()->DestroyFileHelper(fh); TIMER_PRINT_ALL(); exit(0); }
bool SnapshotControl::SaveSnapshotMeta() { if (!FileSystemHelper::GetInstance()->IsDirectoryExists(vm_path_)) FileSystemHelper::GetInstance()->CreateDirectory(vm_path_); if (FileSystemHelper::GetInstance()->IsFileExists(ss_meta_pathname_)) { LOG4CXX_WARN(logger_, "Sanpshot metadata exists, will re-create " << ss_meta_pathname_); FileSystemHelper::GetInstance()->RemoveFile(ss_meta_pathname_); } FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(ss_meta_pathname_, O_WRONLY); fh->Create(); stringstream buffer; ss_meta_.Serialize(buffer); ss_meta_.SerializeRecipe(buffer); LOG4CXX_DEBUG(logger_, "save snapshot meta, size is" << buffer.str().size()); fh->Write((char *)buffer.str().c_str(), buffer.str().size()); fh->Close(); FileSystemHelper::GetInstance()->DestroyFileHelper(fh); return true; }
bool SnapshotControl::LoadBloomFilter(BloomFilter<Checksum>* pbf, const string& bf_name) { if (!FileSystemHelper::GetInstance()->IsFileExists(bf_name)) { LOG4CXX_ERROR(logger_, "Couldn't find bloom filter: " << bf_name); return false; } FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(bf_name, O_RDONLY); fh->Open(); int read_length = fh->GetNextLogSize(); char *data = new char[read_length]; fh->Read(data, read_length); LOG4CXX_DEBUG(logger_, "Read " << read_length << " from file " << bf_name); stringstream buffer; buffer.write(data, read_length); pbf->Deserialize(buffer); LOG4CXX_DEBUG(logger_, "Bloom filter loaded: " << bf_name); fh->Close(); FileSystemHelper::GetInstance()->DestroyFileHelper(fh); delete[] data; return true; }