void PanguAppendStore::WriteMetaInfo(const std::string& root, const StoreMetaData& meta) { std::string metaFileName = root + MetaFileName; try { FileHelper* metaOutputFH = mFileSystemHelper->CreateFileHelper(metaFileName, O_WRONLY); metaOutputFH->Open(); char *write_buffer = new char[sizeof(StoreMetaData)]; /* Copying into buffer from StoreMetaData */ LOG4CXX_DEBUG(logger_, "before reading mMeta values : " << mMeta.storeminor << "," << mMeta.storemajor << "," << mMeta.maxChunkSize << "," << mMeta.blockIndexInterval << "," << mMeta.compressionFlag); meta.toBuffer(write_buffer); metaOutputFH->WriteData(write_buffer, sizeof(StoreMetaData)); metaOutputFH->Close(); mFileSystemHelper->DestroyFileHelper(metaOutputFH); } catch (ExceptionBase& e) { THROW_EXCEPTION(AppendStoreWriteException, e.ToString()+" Cannot generate .meta_ file"); } LOG4CXX_TRACE(logger_, "Store::WroteMetaDataInfo()" ); }
bool SnapshotControl::InitBloomFilters(uint64_t snapshot_size) { if (!FileSystemHelper::GetInstance()->IsFileExists(vm_meta_pathname_)) { if (!FileSystemHelper::GetInstance()->IsDirectoryExists(vm_path_)) FileSystemHelper::GetInstance()->CreateDirectory(vm_path_); // init bloom filter params and store into vm meta LOG4CXX_INFO(logger_, "VM meta not found, will create " << vm_meta_pathname_); vm_meta_.filter_num_items_ = snapshot_size / AVG_BLOCK_SIZE; vm_meta_.filter_num_funcs_ = BLOOM_FILTER_NUM_FUNCS; vm_meta_.filter_fp_rate_ = BLOOM_FILTER_FP_RATE; FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(vm_meta_pathname_, O_WRONLY); fh->Create(); stringstream buffer; vm_meta_.Serialize(buffer); LOG4CXX_DEBUG(logger_, "VM meta size " << buffer.str().size()); fh->WriteData((char *)buffer.str().c_str(), buffer.str().size()); fh->Close(); FileSystemHelper::GetInstance()->DestroyFileHelper(fh); } else { // read bloom filter params FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(vm_meta_pathname_, O_RDONLY); fh->Open(); long read_length = FileSystemHelper::GetInstance()->GetSize(vm_meta_pathname_); char *data = new char[read_length]; fh->Read(data, read_length); LOG4CXX_DEBUG(logger_, "Read " << read_length << " from file"); stringstream buffer; buffer.write(data, read_length); vm_meta_.Deserialize(buffer); LOG4CXX_DEBUG(logger_, "VM meta loaded: " << vm_meta_.filter_num_items_ << " " << vm_meta_.filter_num_funcs_ << "" << vm_meta_.filter_fp_rate_); fh->Close(); FileSystemHelper::GetInstance()->DestroyFileHelper(fh); delete[] data; } // params ready, now init bloom filters primary_filter_ptr_ = new BloomFilter<Checksum>(vm_meta_.filter_num_items_, vm_meta_.filter_fp_rate_, kBloomFilterFunctions, vm_meta_.filter_num_funcs_); // for fine-grained deletion we need a bigger filter, using different group of hash functions secondary_filter_ptr_ = new BloomFilter<Checksum>(vm_meta_.filter_num_items_ * 2, vm_meta_.filter_fp_rate_, &kBloomFilterFunctions[8], vm_meta_.filter_num_funcs_); return true; }
int main(int argc, char** argv) { if (argc != 4) { usage(argv[0]); return -1; } DOMConfigurator::configure("Log4cxxConfig.xml"); string cds_name(argv[1]); string cds_pathname(argv[2]); string sample_pathname(argv[3]); string qfs_cds_dir = "/cds"; string qfs_cds_file = qfs_cds_dir + "/" + cds_name; DataSource source(cds_pathname, sample_pathname); CdsIndex cds; BlockMeta bm; uint64_t offset = 0; uint32_t bytes_written = 0; QFSHelper::Connect(); if (FileSystemHelper::GetInstance()->IsDirectoryExists(qfs_cds_dir)) { FileSystemHelper::GetInstance()->RemoveDirectory(qfs_cds_dir); } FileSystemHelper::GetInstance()->CreateDirectory(qfs_cds_dir); FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(qfs_cds_file, O_WRONLY); fh->Create(); while (source.GetBlock(bm)) { MEASURE(cds.Set(bm.cksum_, offset)); // add to cds index cache MEASURE(bytes_written = fh->WriteData(bm.data_, bm.size_)); // write to qfs if (bytes_written != bm.size_) cout << "Error: write " << bytes_written << ", expect " << bm.size_ << endl; offset += bm.size_; } fh->Close(); FileSystemHelper::GetInstance()->DestroyFileHelper(fh); TIMER_PRINT_ALL(); exit(0); }