TEST(RecordIOTest, Simple) {
TemporaryFile file;
{
RecordIOWriter writer(File(file.fd()));
writer.write(iobufs({"hello ", "world"}));
writer.write(iobufs({"goodbye"}));
}
{
RecordIOReader reader(File(file.fd()));
auto it = reader.begin();
ASSERT_FALSE(it == reader.end());
EXPECT_EQ("hello world", sp((it++)->first));
ASSERT_FALSE(it == reader.end());
EXPECT_EQ("goodbye", sp((it++)->first));
EXPECT_TRUE(it == reader.end());
}
{
RecordIOWriter writer(File(file.fd()));
writer.write(iobufs({"meow"}));
writer.write(iobufs({"woof"}));
}
{
RecordIOReader reader(File(file.fd()));
auto it = reader.begin();
ASSERT_FALSE(it == reader.end());
EXPECT_EQ("hello world", sp((it++)->first));
ASSERT_FALSE(it == reader.end());
EXPECT_EQ("goodbye", sp((it++)->first));
ASSERT_FALSE(it == reader.end());
EXPECT_EQ("meow", sp((it++)->first));
ASSERT_FALSE(it == reader.end());
EXPECT_EQ("woof", sp((it++)->first));
EXPECT_TRUE(it == reader.end());
}
}
TEST(RecordIOTest, SmallRecords) {
constexpr size_t kSize = 10;
char tmp[kSize];
memset(tmp, 'x', kSize);
TemporaryFile file;
{
RecordIOWriter writer(File(file.fd()));
for (size_t i = 0; i < kSize; ++i) { // record of size 0 should be ignored
writer.write(IOBuf::wrapBuffer(tmp, i));
}
}
{
RecordIOReader reader(File(file.fd()));
auto it = reader.begin();
for (size_t i = 1; i < kSize; ++i) {
ASSERT_FALSE(it == reader.end());
EXPECT_EQ(StringPiece(tmp, i), sp((it++)->first));
}
EXPECT_TRUE(it == reader.end());
}
}
TEST(RecordIOTest, MultipleFileIds) {
TemporaryFile file;
{
RecordIOWriter writer(File(file.fd()), 1);
writer.write(iobufs({"hello"}));
}
{
RecordIOWriter writer(File(file.fd()), 2);
writer.write(iobufs({"world"}));
}
{
RecordIOWriter writer(File(file.fd()), 1);
writer.write(iobufs({"goodbye"}));
}
{
RecordIOReader reader(File(file.fd()), 0); // return all
auto it = reader.begin();
ASSERT_FALSE(it == reader.end());
EXPECT_EQ("hello", sp((it++)->first));
ASSERT_FALSE(it == reader.end());
EXPECT_EQ("world", sp((it++)->first));
ASSERT_FALSE(it == reader.end());
EXPECT_EQ("goodbye", sp((it++)->first));
EXPECT_TRUE(it == reader.end());
}
{
RecordIOReader reader(File(file.fd()), 1);
auto it = reader.begin();
ASSERT_FALSE(it == reader.end());
EXPECT_EQ("hello", sp((it++)->first));
ASSERT_FALSE(it == reader.end());
EXPECT_EQ("goodbye", sp((it++)->first));
EXPECT_TRUE(it == reader.end());
}
{
RecordIOReader reader(File(file.fd()), 2);
auto it = reader.begin();
ASSERT_FALSE(it == reader.end());
EXPECT_EQ("world", sp((it++)->first));
EXPECT_TRUE(it == reader.end());
}
{
RecordIOReader reader(File(file.fd()), 3);
auto it = reader.begin();
EXPECT_TRUE(it == reader.end());
}
}
TEST(LineReader, Simple) {
TemporaryFile file;
int fd = file.fd();
writeAll(fd,
"Meow\n"
"Hello world\n"
"This is a long line. It is longer than the other lines.\n"
"\n"
"Incomplete last line");
{
CHECK_ERR(lseek(fd, 0, SEEK_SET));
char buf[10];
LineReader lr(fd, buf, sizeof(buf));
expect(lr, "Meow\n");
expect(lr, "Hello worl");
expect(lr, "d\n");
expect(lr, "This is a ");
expect(lr, "long line.");
expect(lr, " It is lon");
expect(lr, "ger than t");
expect(lr, "he other l");
expect(lr, "ines.\n");
expect(lr, "\n");
expect(lr, "Incomplete");
expect(lr, " last line");
expect(lr, "");
}
{
CHECK_ERR(lseek(fd, 0, SEEK_SET));
char buf[80];
LineReader lr(fd, buf, sizeof(buf));
expect(lr, "Meow\n");
expect(lr, "Hello world\n");
expect(lr, "This is a long line. It is longer than the other lines.\n");
expect(lr, "\n");
expect(lr, "Incomplete last line");
expect(lr, "");
}
}
TEST(TemporaryFile, Simple) {
int fd = -1;
char c = 'x';
{
TemporaryFile f;
EXPECT_FALSE(f.path().empty());
EXPECT_TRUE(f.path().is_absolute());
fd = f.fd();
EXPECT_LE(0, fd);
ssize_t r = write(fd, &c, 1);
EXPECT_EQ(1, r);
}
// The file must have been closed. This assumes that no other thread
// has opened another file in the meanwhile, which is a sane assumption
// to make in this test.
ssize_t r = write(fd, &c, 1);
int savedErrno = errno;
EXPECT_EQ(-1, r);
EXPECT_EQ(EBADF, savedErrno);
}
TEST(RecordIOTest, ExtraMagic) {
TemporaryFile file;
{
RecordIOWriter writer(File(file.fd()));
writer.write(iobufs({"hello"}));
}
uint8_t buf[recordio_helpers::headerSize() + 5];
EXPECT_EQ(0, lseek(file.fd(), 0, SEEK_SET));
EXPECT_EQ(sizeof(buf), read(file.fd(), buf, sizeof(buf)));
// Append an extra magic
const uint32_t magic = recordio_helpers::detail::Header::kMagic;
EXPECT_EQ(sizeof(magic), write(file.fd(), &magic, sizeof(magic)));
// and an extra record
EXPECT_EQ(sizeof(buf), write(file.fd(), buf, sizeof(buf)));
{
RecordIOReader reader(File(file.fd()));
auto it = reader.begin();
ASSERT_FALSE(it == reader.end());
EXPECT_EQ("hello", sp((it++)->first));
ASSERT_FALSE(it == reader.end());
EXPECT_EQ("hello", sp((it++)->first));
EXPECT_TRUE(it == reader.end());
}
}
TEST(File, Locks) {
typedef std::unique_lock<File> Lock;
typedef boost::shared_lock<File> SharedLock;
// Find out where we are.
static constexpr size_t pathLength = 2048;
char buf[pathLength + 1];
int r = readlink("/proc/self/exe", buf, pathLength);
CHECK_ERR(r);
buf[r] = '\0';
fs::path helper(buf);
helper.remove_filename();
helper /= "file_test_lock_helper";
TemporaryFile tempFile;
File f(tempFile.fd());
enum LockMode { EXCLUSIVE, SHARED };
auto testLock = [&] (LockMode mode, bool expectedSuccess) {
auto ret =
Subprocess({helper.native(),
mode == SHARED ? "-s" : "-x",
tempFile.path().native()}).wait();
EXPECT_TRUE(ret.exited());
if (ret.exited()) {
EXPECT_EQ(expectedSuccess ? 0 : 42, ret.exitStatus());
}
};
// Make sure nothing breaks and things compile.
{
Lock lock(f);
}
{
SharedLock lock(f);
}
{
Lock lock(f, std::defer_lock);
EXPECT_TRUE(lock.try_lock());
}
{
SharedLock lock(f, boost::defer_lock);
EXPECT_TRUE(lock.try_lock());
}
// X blocks X
{
Lock lock(f);
testLock(EXCLUSIVE, false);
}
// X blocks S
{
Lock lock(f);
testLock(SHARED, false);
}
// S blocks X
{
SharedLock lock(f);
testLock(EXCLUSIVE, false);
}
// S does not block S
{
SharedLock lock(f);
testLock(SHARED, true);
}
}
TEST(RecordIOTest, Randomized) {
SCOPED_TRACE(to<std::string>("Random seed is ", FLAGS_random_seed));
std::mt19937 rnd(FLAGS_random_seed);
size_t recordCount =
std::uniform_int_distribution<uint32_t>(30, 300)(rnd);
std::uniform_int_distribution<uint32_t> recordSizeDist(1, 3 << 16);
std::uniform_int_distribution<uint32_t> charDist(0, 255);
std::uniform_int_distribution<uint32_t> junkDist(0, 1 << 20);
// corrupt 1/5 of all records
std::uniform_int_distribution<uint32_t> corruptDist(0, 4);
std::vector<std::pair<fbstring, off_t>> records;
std::vector<off_t> corruptPositions;
records.reserve(recordCount);
TemporaryFile file;
fbstring record;
// Recreate the writer multiple times so we test that we create a
// continuous stream
for (size_t i = 0; i < 3; ++i) {
RecordIOWriter writer(File(file.fd()));
for (size_t j = 0; j < recordCount; ++j) {
off_t beginPos = writer.filePos();
record.clear();
size_t recordSize = recordSizeDist(rnd);
record.reserve(recordSize);
for (size_t k = 0; k < recordSize; ++k) {
record.push_back(charDist(rnd));
}
writer.write(iobufs({record}));
bool corrupt = (corruptDist(rnd) == 0);
if (corrupt) {
// Corrupt one random byte in the record (including header)
std::uniform_int_distribution<uint32_t> corruptByteDist(
0, recordSize + recordio_helpers::headerSize() - 1);
off_t corruptRel = corruptByteDist(rnd);
VLOG(1) << "n=" << records.size() << " bpos=" << beginPos
<< " rsize=" << record.size()
<< " corrupt rel=" << corruptRel
<< " abs=" << beginPos + corruptRel;
corruptPositions.push_back(beginPos + corruptRel);
} else {
VLOG(2) << "n=" << records.size() << " bpos=" << beginPos
<< " rsize=" << record.size()
<< " good";
records.emplace_back(std::move(record), beginPos);
}
}
VLOG(1) << "n=" << records.size() << " close abs=" << writer.filePos();
}
for (auto& pos : corruptPositions) {
corrupt(file.fd(), pos);
}
{
size_t i = 0;
RecordIOReader reader(File(file.fd()));
for (auto& r : reader) {
SCOPED_TRACE(i);
ASSERT_LT(i, records.size());
EXPECT_EQ(records[i].first, sp(r.first));
EXPECT_EQ(records[i].second, r.second);
++i;
}
EXPECT_EQ(records.size(), i);
}
}
TEST(File, Locks) {
typedef std::unique_lock<File> Lock;
typedef boost::shared_lock<File> SharedLock;
// Find out where we are.
static constexpr size_t pathLength = 2048;
char buf[pathLength + 1];
int r = readlink("/proc/self/exe", buf, pathLength);
CHECK(r != -1);
buf[r] = '\0';
fs::path me(buf);
auto helper_basename = "file_test_lock_helper";
fs::path helper;
if (fs::exists(me.parent_path() / helper_basename)) {
helper = me.parent_path() / helper_basename;
} else {
throw std::runtime_error(
folly::to<std::string>("cannot find helper ", helper_basename));
}
TemporaryFile tempFile;
File f(tempFile.fd());
enum LockMode { EXCLUSIVE, SHARED };
auto testLock = [&](LockMode mode, bool expectedSuccess) {
auto ret = Subprocess({helper.native(),
mode == SHARED ? "-s" : "-x",
tempFile.path().native()}).wait();
EXPECT_TRUE(ret.exited());
if (ret.exited()) {
EXPECT_EQ(expectedSuccess ? 0 : 42, ret.exitStatus());
}
};
// Make sure nothing breaks and things compile.
{ Lock lock(f); }
{ SharedLock lock(f); }
{
Lock lock(f, std::defer_lock);
EXPECT_TRUE(lock.try_lock());
}
{
SharedLock lock(f, boost::defer_lock);
EXPECT_TRUE(lock.try_lock());
}
// X blocks X
{
Lock lock(f);
testLock(EXCLUSIVE, false);
}
// X blocks S
{
Lock lock(f);
testLock(SHARED, false);
}
// S blocks X
{
SharedLock lock(f);
testLock(EXCLUSIVE, false);
}
// S does not block S
{
SharedLock lock(f);
testLock(SHARED, true);
}
}
