void PanguAppendStore::WriteMetaInfo(const std::string& root, const StoreMetaData& meta)
{
std::string metaFileName = root + MetaFileName;
try
{
FileHelper* metaOutputFH = mFileSystemHelper->CreateFileHelper(metaFileName, O_WRONLY);
metaOutputFH->Open();
char *write_buffer = new char[sizeof(StoreMetaData)];
/* Copying into buffer from StoreMetaData */
LOG4CXX_DEBUG(logger_, "before reading mMeta values : " << mMeta.storeminor <<
"," << mMeta.storemajor <<
"," << mMeta.maxChunkSize <<
"," << mMeta.blockIndexInterval <<
"," << mMeta.compressionFlag);
meta.toBuffer(write_buffer);
metaOutputFH->WriteData(write_buffer, sizeof(StoreMetaData));
metaOutputFH->Close();
mFileSystemHelper->DestroyFileHelper(metaOutputFH);
}
catch (ExceptionBase& e)
{
THROW_EXCEPTION(AppendStoreWriteException, e.ToString()+" Cannot generate .meta_ file");
}
LOG4CXX_TRACE(logger_, "Store::WroteMetaDataInfo()" );
}
bool SnapshotControl::LoadSnapshotMeta()
{
// open the snapshot meta file in qfs and read it
// TODO: currently the read/write apis provide a log style file
if (!FileSystemHelper::GetInstance()->IsFileExists(ss_meta_pathname_)) {
LOG4CXX_ERROR(logger_, "Couldn't find snapshot meta: " << ss_meta_.vm_id_ << " " << ss_meta_.snapshot_id_);
return false;
}
FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(ss_meta_pathname_, O_RDONLY);
fh->Open();
int read_length = fh->GetNextLogSize();
char *data = new char[read_length];
fh->Read(data, read_length);
LOG4CXX_DEBUG(logger_, "Read " << read_length << " from " << ss_meta_pathname_);
stringstream buffer;
buffer.write(data, read_length);
ss_meta_.Deserialize(buffer);
ss_meta_.DeserializeRecipe(buffer);
LOG4CXX_INFO(logger_, "Snapshot meta loaded: " << ss_meta_.vm_id_ << " " << ss_meta_.snapshot_id_);
fh->Close();
FileSystemHelper::GetInstance()->DestroyFileHelper(fh);
delete[] data;
return true;
}
bool SnapshotControl::InitBloomFilters(uint64_t snapshot_size)
{
if (!FileSystemHelper::GetInstance()->IsFileExists(vm_meta_pathname_)) {
if (!FileSystemHelper::GetInstance()->IsDirectoryExists(vm_path_))
FileSystemHelper::GetInstance()->CreateDirectory(vm_path_);
// init bloom filter params and store into vm meta
LOG4CXX_INFO(logger_, "VM meta not found, will create " << vm_meta_pathname_);
vm_meta_.filter_num_items_ = snapshot_size / AVG_BLOCK_SIZE;
vm_meta_.filter_num_funcs_ = BLOOM_FILTER_NUM_FUNCS;
vm_meta_.filter_fp_rate_ = BLOOM_FILTER_FP_RATE;
FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(vm_meta_pathname_, O_WRONLY);
fh->Create();
stringstream buffer;
vm_meta_.Serialize(buffer);
LOG4CXX_DEBUG(logger_, "VM meta size " << buffer.str().size());
fh->WriteData((char *)buffer.str().c_str(), buffer.str().size());
fh->Close();
FileSystemHelper::GetInstance()->DestroyFileHelper(fh);
}
else {
// read bloom filter params
FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(vm_meta_pathname_, O_RDONLY);
fh->Open();
long read_length = FileSystemHelper::GetInstance()->GetSize(vm_meta_pathname_);
char *data = new char[read_length];
fh->Read(data, read_length);
LOG4CXX_DEBUG(logger_, "Read " << read_length << " from file");
stringstream buffer;
buffer.write(data, read_length);
vm_meta_.Deserialize(buffer);
LOG4CXX_DEBUG(logger_, "VM meta loaded: "
<< vm_meta_.filter_num_items_ << " "
<< vm_meta_.filter_num_funcs_ << ""
<< vm_meta_.filter_fp_rate_);
fh->Close();
FileSystemHelper::GetInstance()->DestroyFileHelper(fh);
delete[] data;
}
// params ready, now init bloom filters
primary_filter_ptr_ = new BloomFilter<Checksum>(vm_meta_.filter_num_items_,
vm_meta_.filter_fp_rate_,
kBloomFilterFunctions,
vm_meta_.filter_num_funcs_);
// for fine-grained deletion we need a bigger filter, using different group of hash functions
secondary_filter_ptr_ = new BloomFilter<Checksum>(vm_meta_.filter_num_items_ * 2,
vm_meta_.filter_fp_rate_,
&kBloomFilterFunctions[8],
vm_meta_.filter_num_funcs_);
return true;
}
bool PanguAppendStore::ReadMetaInfo()
{
std::string metaFileName = mRoot + MetaFileName;
bool fexist = false;
try
{
fexist = mFileSystemHelper->IsFileExists(metaFileName);
LOG4CXX_DEBUG(logger_, "meta file name " << metaFileName << " : " << fexist);
}
catch (ExceptionBase & e)
{
LOG4CXX_ERROR(logger_, "IsFileExist : " << metaFileName);
throw;
}
if (fexist)
{
try
{
FileHelper* metaInputFH = mFileSystemHelper->CreateFileHelper(metaFileName, O_RDONLY);
char *read_buffer = new char[sizeof(StoreMetaData)];
metaInputFH->Open();
metaInputFH->Read(read_buffer, sizeof(StoreMetaData));
metaInputFH->Close();
mFileSystemHelper->DestroyFileHelper(metaInputFH);
mMeta.fromBuffer(read_buffer);
LOG4CXX_DEBUG(logger_, "after reading mMeta values : " << mMeta.storeminor <<
"," << mMeta.storemajor <<
"," << mMeta.maxChunkSize <<
"," << mMeta.blockIndexInterval <<
"," << mMeta.compressionFlag);
return true;
}
catch (ExceptionBase& e)
{
THROW_EXCEPTION(AppendStoreWriteException, "Cannot open meta file for append " + e.ToString());
}
}
LOG4CXX_TRACE(logger_, "Store::ReadMetaDataInfo()" );
return false;
}
bool SnapshotControl::LoadBloomFilter(BloomFilter<Checksum>* pbf, const string& bf_name)
{
if (!FileSystemHelper::GetInstance()->IsFileExists(bf_name)) {
LOG4CXX_ERROR(logger_, "Couldn't find bloom filter: " << bf_name);
return false;
}
FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(bf_name, O_RDONLY);
fh->Open();
int read_length = fh->GetNextLogSize();
char *data = new char[read_length];
fh->Read(data, read_length);
LOG4CXX_DEBUG(logger_, "Read " << read_length << " from file " << bf_name);
stringstream buffer;
buffer.write(data, read_length);
pbf->Deserialize(buffer);
LOG4CXX_DEBUG(logger_, "Bloom filter loaded: " << bf_name);
fh->Close();
FileSystemHelper::GetInstance()->DestroyFileHelper(fh);
delete[] data;
return true;
}
