void PanguAppendStore::WriteMetaInfo(const std::string& root, const StoreMetaData& meta)
{
std::string metaFileName = root + MetaFileName;
try
{
FileHelper* metaOutputFH = mFileSystemHelper->CreateFileHelper(metaFileName, O_WRONLY);
metaOutputFH->Open();
char *write_buffer = new char[sizeof(StoreMetaData)];
/* Copying into buffer from StoreMetaData */
LOG4CXX_DEBUG(logger_, "before reading mMeta values : " << mMeta.storeminor <<
"," << mMeta.storemajor <<
"," << mMeta.maxChunkSize <<
"," << mMeta.blockIndexInterval <<
"," << mMeta.compressionFlag);
meta.toBuffer(write_buffer);
metaOutputFH->WriteData(write_buffer, sizeof(StoreMetaData));
metaOutputFH->Close();
mFileSystemHelper->DestroyFileHelper(metaOutputFH);
}
catch (ExceptionBase& e)
{
THROW_EXCEPTION(AppendStoreWriteException, e.ToString()+" Cannot generate .meta_ file");
}
LOG4CXX_TRACE(logger_, "Store::WroteMetaDataInfo()" );
}
bool SnapshotControl::InitBloomFilters(uint64_t snapshot_size)
{
if (!FileSystemHelper::GetInstance()->IsFileExists(vm_meta_pathname_)) {
if (!FileSystemHelper::GetInstance()->IsDirectoryExists(vm_path_))
FileSystemHelper::GetInstance()->CreateDirectory(vm_path_);
// init bloom filter params and store into vm meta
LOG4CXX_INFO(logger_, "VM meta not found, will create " << vm_meta_pathname_);
vm_meta_.filter_num_items_ = snapshot_size / AVG_BLOCK_SIZE;
vm_meta_.filter_num_funcs_ = BLOOM_FILTER_NUM_FUNCS;
vm_meta_.filter_fp_rate_ = BLOOM_FILTER_FP_RATE;
FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(vm_meta_pathname_, O_WRONLY);
fh->Create();
stringstream buffer;
vm_meta_.Serialize(buffer);
LOG4CXX_DEBUG(logger_, "VM meta size " << buffer.str().size());
fh->WriteData((char *)buffer.str().c_str(), buffer.str().size());
fh->Close();
FileSystemHelper::GetInstance()->DestroyFileHelper(fh);
}
else {
// read bloom filter params
FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(vm_meta_pathname_, O_RDONLY);
fh->Open();
long read_length = FileSystemHelper::GetInstance()->GetSize(vm_meta_pathname_);
char *data = new char[read_length];
fh->Read(data, read_length);
LOG4CXX_DEBUG(logger_, "Read " << read_length << " from file");
stringstream buffer;
buffer.write(data, read_length);
vm_meta_.Deserialize(buffer);
LOG4CXX_DEBUG(logger_, "VM meta loaded: "
<< vm_meta_.filter_num_items_ << " "
<< vm_meta_.filter_num_funcs_ << ""
<< vm_meta_.filter_fp_rate_);
fh->Close();
FileSystemHelper::GetInstance()->DestroyFileHelper(fh);
delete[] data;
}
// params ready, now init bloom filters
primary_filter_ptr_ = new BloomFilter<Checksum>(vm_meta_.filter_num_items_,
vm_meta_.filter_fp_rate_,
kBloomFilterFunctions,
vm_meta_.filter_num_funcs_);
// for fine-grained deletion we need a bigger filter, using different group of hash functions
secondary_filter_ptr_ = new BloomFilter<Checksum>(vm_meta_.filter_num_items_ * 2,
vm_meta_.filter_fp_rate_,
&kBloomFilterFunctions[8],
vm_meta_.filter_num_funcs_);
return true;
}
int main(int argc, char** argv)
{
if (argc != 4) {
usage(argv[0]);
return -1;
}
DOMConfigurator::configure("Log4cxxConfig.xml");
string cds_name(argv[1]);
string cds_pathname(argv[2]);
string sample_pathname(argv[3]);
string qfs_cds_dir = "/cds";
string qfs_cds_file = qfs_cds_dir + "/" + cds_name;
DataSource source(cds_pathname, sample_pathname);
CdsIndex cds;
BlockMeta bm;
uint64_t offset = 0;
uint32_t bytes_written = 0;
QFSHelper::Connect();
if (FileSystemHelper::GetInstance()->IsDirectoryExists(qfs_cds_dir)) {
FileSystemHelper::GetInstance()->RemoveDirectory(qfs_cds_dir);
}
FileSystemHelper::GetInstance()->CreateDirectory(qfs_cds_dir);
FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(qfs_cds_file, O_WRONLY);
fh->Create();
while (source.GetBlock(bm)) {
MEASURE(cds.Set(bm.cksum_, offset)); // add to cds index cache
MEASURE(bytes_written = fh->WriteData(bm.data_, bm.size_)); // write to qfs
if (bytes_written != bm.size_)
cout << "Error: write " << bytes_written << ", expect " << bm.size_ << endl;
offset += bm.size_;
}
fh->Close();
FileSystemHelper::GetInstance()->DestroyFileHelper(fh);
TIMER_PRINT_ALL();
exit(0);
}
