TEST(FileTest, GetParent) {
static const string kFileName = this->test_info_->name();
FileHelper helper;
string path = helper.CreateTempFile(kFileName, "Hello World");
File file(path);
ASSERT_EQ(helper.TempDir(), file.GetParent());
}
TEST(FileTest, DoesNotExist_Static) {
FileHelper file;
string tmp = file.TempDir();
GTEST_ASSERT_NE("", tmp);
File dne(tmp, "doesnotexist");
ASSERT_FALSE(File::Exists(dne.full_pathname()));
}
TEST(FileTest, DoesNotExist) {
FileHelper file;
string tmp = file.TempDir();
GTEST_ASSERT_NE("", tmp);
File dne(tmp, "doesnotexist");
ASSERT_FALSE(dne.Exists());
}
TEST(FileTest, Stream) {
FileHelper file;
File f(file.TempDir(), "newdir");
std::stringstream s;
s << f;
ASSERT_EQ(f.full_pathname(), s.str());
}
void PanguAppendStore::WriteMetaInfo(const std::string& root, const StoreMetaData& meta)
{
std::string metaFileName = root + MetaFileName;
try
{
FileHelper* metaOutputFH = mFileSystemHelper->CreateFileHelper(metaFileName, O_WRONLY);
metaOutputFH->Open();
char *write_buffer = new char[sizeof(StoreMetaData)];
/* Copying into buffer from StoreMetaData */
LOG4CXX_DEBUG(logger_, "before reading mMeta values : " << mMeta.storeminor <<
"," << mMeta.storemajor <<
"," << mMeta.maxChunkSize <<
"," << mMeta.blockIndexInterval <<
"," << mMeta.compressionFlag);
meta.toBuffer(write_buffer);
metaOutputFH->WriteData(write_buffer, sizeof(StoreMetaData));
metaOutputFH->Close();
mFileSystemHelper->DestroyFileHelper(metaOutputFH);
}
catch (ExceptionBase& e)
{
THROW_EXCEPTION(AppendStoreWriteException, e.ToString()+" Cannot generate .meta_ file");
}
LOG4CXX_TRACE(logger_, "Store::WroteMetaDataInfo()" );
}
bool SnapshotControl::LoadSnapshotMeta()
{
// open the snapshot meta file in qfs and read it
// TODO: currently the read/write apis provide a log style file
if (!FileSystemHelper::GetInstance()->IsFileExists(ss_meta_pathname_)) {
LOG4CXX_ERROR(logger_, "Couldn't find snapshot meta: " << ss_meta_.vm_id_ << " " << ss_meta_.snapshot_id_);
return false;
}
FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(ss_meta_pathname_, O_RDONLY);
fh->Open();
int read_length = fh->GetNextLogSize();
char *data = new char[read_length];
fh->Read(data, read_length);
LOG4CXX_DEBUG(logger_, "Read " << read_length << " from " << ss_meta_pathname_);
stringstream buffer;
buffer.write(data, read_length);
ss_meta_.Deserialize(buffer);
ss_meta_.DeserializeRecipe(buffer);
LOG4CXX_INFO(logger_, "Snapshot meta loaded: " << ss_meta_.vm_id_ << " " << ss_meta_.snapshot_id_);
fh->Close();
FileSystemHelper::GetInstance()->DestroyFileHelper(fh);
delete[] data;
return true;
}
TEST(FileTest, LastWriteTime_NotOpen) {
static const string kHelloWorld = "Hello World";
FileHelper helper;
time_t now = time(nullptr);
string path = helper.CreateTempFile(this->test_info_->name(), kHelloWorld);
File file(path);
ASSERT_LE(now, file.last_write_time());
}
TEST(FileTest, IsDirectory_NotOpen) {
static const string kHelloWorld = "Hello World";
FileHelper helper;
string path = helper.CreateTempFile(this->test_info_->name(), kHelloWorld);
File file(path);
ASSERT_FALSE(file.IsDirectory());
ASSERT_TRUE(file.IsFile());
}
TEST(FileTest, Length_NotOpen) {
static const string kHelloWorld = "Hello World";
FileHelper helper;
string path = helper.CreateTempFile(this->test_info_->name(), kHelloWorld);
File file(path);
ASSERT_EQ(static_cast<long>(kHelloWorld.size()),
file.GetLength());
}
TEST(FileTest, IsOpen_NotOpen) {
static const string kHelloWorld = "Hello World";
FileHelper helper;
string path = helper.CreateTempFile(this->test_info_->name(), kHelloWorld);
File file(path + "DNE");
EXPECT_FALSE(file.IsOpen());
EXPECT_FALSE(file);
}
TEST(FileTest, Exists) {
FileHelper file;
string tmp = file.TempDir();
GTEST_ASSERT_NE("", tmp);
ASSERT_TRUE(file.Mkdir("newdir"));
File f(tmp, "newdir");
ASSERT_TRUE(f.Exists()) << f.full_pathname();
}
TEST(FileTest, Exists_Static) {
FileHelper file;
string tmp = file.TempDir();
GTEST_ASSERT_NE("", tmp);
ASSERT_TRUE(file.Mkdir("newdir"));
File dne(tmp, "newdir");
ASSERT_TRUE(File::Exists(dne.full_pathname())) << dne.full_pathname();
}
TEST(FileTest, IsRelative) {
static const string kFileName = this->test_info_->name();
FileHelper helper;
const string path = helper.CreateTempFile(kFileName, "Hello World");
EXPECT_TRUE(File::IsRelativePath(kFileName));
EXPECT_FALSE(File::IsRelativePath(path));
}
TEST(FileTest, MakeAbsolutePath_AlreadyAbsolute) {
static const string kFileName = this->test_info_->name();
FileHelper helper;
const string path = helper.CreateTempFile(kFileName, "Hello World");
string relative(path); // Note: relative == absolute path (path)
File::MakeAbsolutePath(helper.TempDir(), &relative);
EXPECT_EQ(path, relative);
}
TEST(FileTest, Exists_TrailingSlash) {
FileHelper file;
string tmp = file.TempDir();
GTEST_ASSERT_NE("", tmp);
ASSERT_TRUE(file.Mkdir("newdir"));
File dne(tmp, "newdir");
string path = StringPrintf("%s%c", dne.full_pathname().c_str(), File::pathSeparatorChar);
ASSERT_TRUE(File::Exists(path)) << path;
}
TEST(FileTest, RealPath_Same) {
static const string kFileName = this->test_info_->name();
FileHelper helper;
const string path = helper.CreateTempFile(kFileName, "Hello World");
string realpath;
ASSERT_TRUE(File::RealPath(path, &realpath));
EXPECT_EQ(path, realpath);
}
TEST(FileTest, IsOpen_Open) {
static const string kHelloWorld = "Hello World";
FileHelper helper;
string path = helper.CreateTempFile(this->test_info_->name(), kHelloWorld);
File file(path);
ASSERT_TRUE(file.Open(File::modeBinary | File::modeReadOnly));
EXPECT_TRUE(file.IsOpen());
EXPECT_TRUE((bool) file);
}
TEST(FileTest, RealPath_Different) {
static const string kFileName = this->test_info_->name();
FileHelper helper;
const string path = helper.CreateTempFile(kFileName, "Hello World");
string realpath;
// Add an extra ./ into the path.
ASSERT_TRUE(File::RealPath(StrCat(helper.TempDir(), File::pathSeparatorString, ".", File::pathSeparatorString, kFileName), &realpath));
EXPECT_EQ(path, realpath);
}
int main (int argc, char *argv[])
{
CommandLine cmd;
cmd.Parse (argc, argv);
//
// This Emitter has a trace source object that will emit values at
// random times.
//
Ptr<Emitter> emitter = CreateObject<Emitter> ();
Names::Add ("/Names/Emitter", emitter);
//
// This Probe will be hooked to the Emitter's trace source object by
// accessing it by path name in the Config database.
//
Ptr<DoubleProbe> probe = CreateObject<DoubleProbe> ();
probe->SetName ("PathProbe");
Names::Add ("/Names/Probe", probe);
// Note, no return value is checked here.
probe->ConnectByPath ("/Names/Emitter/Counter");
//
// This file helper will be used to put data values into a file.
//
// Create the file helper.
FileHelper fileHelper;
// Configure the file to be written.
fileHelper.ConfigureFile ("file-helper-example",
FileAggregator::FORMATTED);
// Set the labels for this formatted output file.
fileHelper.Set2dFormat ("Time (Seconds) = %.3e\tCount = %.0f");
// Write the values generated by the probe. The path that we
// provide helps to disambiguate the source of the trace.
fileHelper.WriteProbe ("ns3::DoubleProbe",
"/Names/Probe/Output",
"Output");
// The Emitter object is not associated with an ns-3 node, so
// it won't get started automatically, so we need to do this ourselves
Simulator::Schedule (Seconds (0.0), &Emitter::Initialize, emitter);
Simulator::Stop (Seconds (100.0));
Simulator::Run ();
Simulator::Destroy ();
return 0;
}
TEST(PPPConfigTest, NodeConfig) {
FileHelper files;
files.Mkdir("network");
const string line("@2 [email protected]");
files.CreateTempFile("network/address.net", line);
const string network_dir = files.DirName("network");
PPPConfig config(1, "mybbs", network_dir);
const PPPNodeConfig* node_config = config.node_config_for(2);
ASSERT_TRUE(node_config != nullptr);
EXPECT_EQ("*****@*****.**", node_config->email_address);
}
TEST(FileTest, ExistsWildCard_Extension) {
FileHelper helper;
const string path = helper.CreateTempFile("msg00000.001", "msg00000.001");
ASSERT_TRUE(File::Exists(path));
string wildcard_path = StrCat(helper.TempDir(), File::pathSeparatorString, "msg*.001");
ASSERT_TRUE(File::ExistsWildcard(wildcard_path)) << path << "; w: " << wildcard_path;
wildcard_path = StrCat(helper.TempDir(), File::pathSeparatorString, "msg*.??1");
ASSERT_TRUE(File::ExistsWildcard(wildcard_path)) << path << "; w: " << wildcard_path;
}
TEST(FileTest, mkdirs) {
FileHelper helper;
const string path = StrCat(helper.TempDir(), File::pathSeparatorString, "a",
File::pathSeparatorString, "b", File::pathSeparatorString, "c");
ASSERT_FALSE(File::Exists(path));
ASSERT_TRUE(File::mkdirs(path));
ASSERT_TRUE(File::mkdirs(path));
ASSERT_TRUE(File::Exists(path));
}
TEST_F(CalloutTest, NodeConfig) {
FileHelper files;
files.Mkdir("network");
const string line("@1 & \"foo\"");
files.CreateTempFile("network/callout.net", line);
const string network_dir = files.DirName("network");
Callout callout(network_dir);
const net_call_out_rec* con = callout.node_config_for(1);
ASSERT_TRUE(con != nullptr);
EXPECT_EQ(options_sendback, con->options);
EXPECT_STREQ("foo", con->password);
}
TEST(FileTest, Read) {
static const string kHelloWorld = "Hello World";
FileHelper helper;
string path = helper.CreateTempFile(this->test_info_->name(), kHelloWorld);
File file(path);
ASSERT_TRUE(file.Open(File::modeBinary | File::modeReadOnly));
char buf[255];
ASSERT_EQ(static_cast<int>(kHelloWorld.length()),
file.Read(buf, kHelloWorld.length()));
buf[11] = 0;
ASSERT_STREQ(kHelloWorld.c_str(), buf);
}
TEST(FileTest, SetCurrentDirectory) {
char expected[MAX_PATH];
getcwd(expected, MAX_PATH);
string original_dir = File::current_directory();
ASSERT_STREQ(expected, original_dir.c_str());
FileHelper helper;
File::set_current_directory(helper.TempDir());
EXPECT_EQ(helper.TempDir(), File::current_directory());
File::set_current_directory(original_dir);
}
TEST(FileTest, mkdir) {
FileHelper helper;
const string path = StrCat(helper.TempDir(), File::pathSeparatorString, "a");
const string path_missing_middle = StrCat(helper.TempDir(), File::pathSeparatorString, "a",
File::pathSeparatorString, "b", File::pathSeparatorString, "c");
ASSERT_FALSE(File::Exists(path));
ASSERT_TRUE(File::mkdir(path));
ASSERT_TRUE(File::mkdir(path)); // 2nd time should still return true.
EXPECT_FALSE(File::mkdir(path_missing_middle)); // Can't create missing path elements.
ASSERT_TRUE(File::Exists(path));
}
bool SnapshotControl::InitBloomFilters(uint64_t snapshot_size)
{
if (!FileSystemHelper::GetInstance()->IsFileExists(vm_meta_pathname_)) {
if (!FileSystemHelper::GetInstance()->IsDirectoryExists(vm_path_))
FileSystemHelper::GetInstance()->CreateDirectory(vm_path_);
// init bloom filter params and store into vm meta
LOG4CXX_INFO(logger_, "VM meta not found, will create " << vm_meta_pathname_);
vm_meta_.filter_num_items_ = snapshot_size / AVG_BLOCK_SIZE;
vm_meta_.filter_num_funcs_ = BLOOM_FILTER_NUM_FUNCS;
vm_meta_.filter_fp_rate_ = BLOOM_FILTER_FP_RATE;
FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(vm_meta_pathname_, O_WRONLY);
fh->Create();
stringstream buffer;
vm_meta_.Serialize(buffer);
LOG4CXX_DEBUG(logger_, "VM meta size " << buffer.str().size());
fh->WriteData((char *)buffer.str().c_str(), buffer.str().size());
fh->Close();
FileSystemHelper::GetInstance()->DestroyFileHelper(fh);
}
else {
// read bloom filter params
FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(vm_meta_pathname_, O_RDONLY);
fh->Open();
long read_length = FileSystemHelper::GetInstance()->GetSize(vm_meta_pathname_);
char *data = new char[read_length];
fh->Read(data, read_length);
LOG4CXX_DEBUG(logger_, "Read " << read_length << " from file");
stringstream buffer;
buffer.write(data, read_length);
vm_meta_.Deserialize(buffer);
LOG4CXX_DEBUG(logger_, "VM meta loaded: "
<< vm_meta_.filter_num_items_ << " "
<< vm_meta_.filter_num_funcs_ << ""
<< vm_meta_.filter_fp_rate_);
fh->Close();
FileSystemHelper::GetInstance()->DestroyFileHelper(fh);
delete[] data;
}
// params ready, now init bloom filters
primary_filter_ptr_ = new BloomFilter<Checksum>(vm_meta_.filter_num_items_,
vm_meta_.filter_fp_rate_,
kBloomFilterFunctions,
vm_meta_.filter_num_funcs_);
// for fine-grained deletion we need a bigger filter, using different group of hash functions
secondary_filter_ptr_ = new BloomFilter<Checksum>(vm_meta_.filter_num_items_ * 2,
vm_meta_.filter_fp_rate_,
&kBloomFilterFunctions[8],
vm_meta_.filter_num_funcs_);
return true;
}
bool SnapshotControl::SaveBloomFilter(BloomFilter<Checksum>* pbf, const string& bf_name)
{
if (!FileSystemHelper::GetInstance()->IsDirectoryExists(vm_path_))
FileSystemHelper::GetInstance()->CreateDirectory(vm_path_);
if (FileSystemHelper::GetInstance()->IsFileExists(bf_name))
FileSystemHelper::GetInstance()->RemoveFile(bf_name);
FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(bf_name, O_WRONLY);
fh->Create();
stringstream buffer;
pbf->Serialize(buffer);
LOG4CXX_DEBUG(logger_, "Bloom filter primary size " << buffer.str().size());
fh->Write((char *)buffer.str().c_str(), buffer.str().size());
fh->Close();
FileSystemHelper::GetInstance()->DestroyFileHelper(fh);
return true;
}
int main(int argc, char** argv)
{
if (argc != 4) {
usage(argv[0]);
return -1;
}
DOMConfigurator::configure("Log4cxxConfig.xml");
string cds_name(argv[1]);
string cds_pathname(argv[2]);
string sample_pathname(argv[3]);
string qfs_cds_dir = "/cds";
string qfs_cds_file = qfs_cds_dir + "/" + cds_name;
DataSource source(cds_pathname, sample_pathname);
CdsIndex cds;
BlockMeta bm;
uint64_t offset = 0;
uint32_t bytes_written = 0;
QFSHelper::Connect();
if (FileSystemHelper::GetInstance()->IsDirectoryExists(qfs_cds_dir)) {
FileSystemHelper::GetInstance()->RemoveDirectory(qfs_cds_dir);
}
FileSystemHelper::GetInstance()->CreateDirectory(qfs_cds_dir);
FileHelper* fh = FileSystemHelper::GetInstance()->CreateFileHelper(qfs_cds_file, O_WRONLY);
fh->Create();
while (source.GetBlock(bm)) {
MEASURE(cds.Set(bm.cksum_, offset)); // add to cds index cache
MEASURE(bytes_written = fh->WriteData(bm.data_, bm.size_)); // write to qfs
if (bytes_written != bm.size_)
cout << "Error: write " << bytes_written << ", expect " << bm.size_ << endl;
offset += bm.size_;
}
fh->Close();
FileSystemHelper::GetInstance()->DestroyFileHelper(fh);
TIMER_PRINT_ALL();
exit(0);
}
bool PanguAppendStore::ReadMetaInfo()
{
std::string metaFileName = mRoot + MetaFileName;
bool fexist = false;
try
{
fexist = mFileSystemHelper->IsFileExists(metaFileName);
LOG4CXX_DEBUG(logger_, "meta file name " << metaFileName << " : " << fexist);
}
catch (ExceptionBase & e)
{
LOG4CXX_ERROR(logger_, "IsFileExist : " << metaFileName);
throw;
}
if (fexist)
{
try
{
FileHelper* metaInputFH = mFileSystemHelper->CreateFileHelper(metaFileName, O_RDONLY);
char *read_buffer = new char[sizeof(StoreMetaData)];
metaInputFH->Open();
metaInputFH->Read(read_buffer, sizeof(StoreMetaData));
metaInputFH->Close();
mFileSystemHelper->DestroyFileHelper(metaInputFH);
mMeta.fromBuffer(read_buffer);
LOG4CXX_DEBUG(logger_, "after reading mMeta values : " << mMeta.storeminor <<
"," << mMeta.storemajor <<
"," << mMeta.maxChunkSize <<
"," << mMeta.blockIndexInterval <<
"," << mMeta.compressionFlag);
return true;
}
catch (ExceptionBase& e)
{
THROW_EXCEPTION(AppendStoreWriteException, "Cannot open meta file for append " + e.ToString());
}
}
LOG4CXX_TRACE(logger_, "Store::ReadMetaDataInfo()" );
return false;
}
