TEST(RecordIOTest, Simple) { TemporaryFile file; { RecordIOWriter writer(File(file.fd())); writer.write(iobufs({"hello ", "world"})); writer.write(iobufs({"goodbye"})); } { RecordIOReader reader(File(file.fd())); auto it = reader.begin(); ASSERT_FALSE(it == reader.end()); EXPECT_EQ("hello world", sp((it++)->first)); ASSERT_FALSE(it == reader.end()); EXPECT_EQ("goodbye", sp((it++)->first)); EXPECT_TRUE(it == reader.end()); } { RecordIOWriter writer(File(file.fd())); writer.write(iobufs({"meow"})); writer.write(iobufs({"woof"})); } { RecordIOReader reader(File(file.fd())); auto it = reader.begin(); ASSERT_FALSE(it == reader.end()); EXPECT_EQ("hello world", sp((it++)->first)); ASSERT_FALSE(it == reader.end()); EXPECT_EQ("goodbye", sp((it++)->first)); ASSERT_FALSE(it == reader.end()); EXPECT_EQ("meow", sp((it++)->first)); ASSERT_FALSE(it == reader.end()); EXPECT_EQ("woof", sp((it++)->first)); EXPECT_TRUE(it == reader.end()); } }
TEST(RecordIOTest, SmallRecords) { constexpr size_t kSize = 10; char tmp[kSize]; memset(tmp, 'x', kSize); TemporaryFile file; { RecordIOWriter writer(File(file.fd())); for (size_t i = 0; i < kSize; ++i) { // record of size 0 should be ignored writer.write(IOBuf::wrapBuffer(tmp, i)); } } { RecordIOReader reader(File(file.fd())); auto it = reader.begin(); for (size_t i = 1; i < kSize; ++i) { ASSERT_FALSE(it == reader.end()); EXPECT_EQ(StringPiece(tmp, i), sp((it++)->first)); } EXPECT_TRUE(it == reader.end()); } }
TEST(RecordIOTest, MultipleFileIds) { TemporaryFile file; { RecordIOWriter writer(File(file.fd()), 1); writer.write(iobufs({"hello"})); } { RecordIOWriter writer(File(file.fd()), 2); writer.write(iobufs({"world"})); } { RecordIOWriter writer(File(file.fd()), 1); writer.write(iobufs({"goodbye"})); } { RecordIOReader reader(File(file.fd()), 0); // return all auto it = reader.begin(); ASSERT_FALSE(it == reader.end()); EXPECT_EQ("hello", sp((it++)->first)); ASSERT_FALSE(it == reader.end()); EXPECT_EQ("world", sp((it++)->first)); ASSERT_FALSE(it == reader.end()); EXPECT_EQ("goodbye", sp((it++)->first)); EXPECT_TRUE(it == reader.end()); } { RecordIOReader reader(File(file.fd()), 1); auto it = reader.begin(); ASSERT_FALSE(it == reader.end()); EXPECT_EQ("hello", sp((it++)->first)); ASSERT_FALSE(it == reader.end()); EXPECT_EQ("goodbye", sp((it++)->first)); EXPECT_TRUE(it == reader.end()); } { RecordIOReader reader(File(file.fd()), 2); auto it = reader.begin(); ASSERT_FALSE(it == reader.end()); EXPECT_EQ("world", sp((it++)->first)); EXPECT_TRUE(it == reader.end()); } { RecordIOReader reader(File(file.fd()), 3); auto it = reader.begin(); EXPECT_TRUE(it == reader.end()); } }
TEST(LineReader, Simple) { TemporaryFile file; int fd = file.fd(); writeAll(fd, "Meow\n" "Hello world\n" "This is a long line. It is longer than the other lines.\n" "\n" "Incomplete last line"); { CHECK_ERR(lseek(fd, 0, SEEK_SET)); char buf[10]; LineReader lr(fd, buf, sizeof(buf)); expect(lr, "Meow\n"); expect(lr, "Hello worl"); expect(lr, "d\n"); expect(lr, "This is a "); expect(lr, "long line."); expect(lr, " It is lon"); expect(lr, "ger than t"); expect(lr, "he other l"); expect(lr, "ines.\n"); expect(lr, "\n"); expect(lr, "Incomplete"); expect(lr, " last line"); expect(lr, ""); } { CHECK_ERR(lseek(fd, 0, SEEK_SET)); char buf[80]; LineReader lr(fd, buf, sizeof(buf)); expect(lr, "Meow\n"); expect(lr, "Hello world\n"); expect(lr, "This is a long line. It is longer than the other lines.\n"); expect(lr, "\n"); expect(lr, "Incomplete last line"); expect(lr, ""); } }
TEST(TemporaryFile, Simple) { int fd = -1; char c = 'x'; { TemporaryFile f; EXPECT_FALSE(f.path().empty()); EXPECT_TRUE(f.path().is_absolute()); fd = f.fd(); EXPECT_LE(0, fd); ssize_t r = write(fd, &c, 1); EXPECT_EQ(1, r); } // The file must have been closed. This assumes that no other thread // has opened another file in the meanwhile, which is a sane assumption // to make in this test. ssize_t r = write(fd, &c, 1); int savedErrno = errno; EXPECT_EQ(-1, r); EXPECT_EQ(EBADF, savedErrno); }
TEST(RecordIOTest, ExtraMagic) { TemporaryFile file; { RecordIOWriter writer(File(file.fd())); writer.write(iobufs({"hello"})); } uint8_t buf[recordio_helpers::headerSize() + 5]; EXPECT_EQ(0, lseek(file.fd(), 0, SEEK_SET)); EXPECT_EQ(sizeof(buf), read(file.fd(), buf, sizeof(buf))); // Append an extra magic const uint32_t magic = recordio_helpers::detail::Header::kMagic; EXPECT_EQ(sizeof(magic), write(file.fd(), &magic, sizeof(magic))); // and an extra record EXPECT_EQ(sizeof(buf), write(file.fd(), buf, sizeof(buf))); { RecordIOReader reader(File(file.fd())); auto it = reader.begin(); ASSERT_FALSE(it == reader.end()); EXPECT_EQ("hello", sp((it++)->first)); ASSERT_FALSE(it == reader.end()); EXPECT_EQ("hello", sp((it++)->first)); EXPECT_TRUE(it == reader.end()); } }
TEST(File, Locks) { typedef std::unique_lock<File> Lock; typedef boost::shared_lock<File> SharedLock; // Find out where we are. static constexpr size_t pathLength = 2048; char buf[pathLength + 1]; int r = readlink("/proc/self/exe", buf, pathLength); CHECK_ERR(r); buf[r] = '\0'; fs::path helper(buf); helper.remove_filename(); helper /= "file_test_lock_helper"; TemporaryFile tempFile; File f(tempFile.fd()); enum LockMode { EXCLUSIVE, SHARED }; auto testLock = [&] (LockMode mode, bool expectedSuccess) { auto ret = Subprocess({helper.native(), mode == SHARED ? "-s" : "-x", tempFile.path().native()}).wait(); EXPECT_TRUE(ret.exited()); if (ret.exited()) { EXPECT_EQ(expectedSuccess ? 0 : 42, ret.exitStatus()); } }; // Make sure nothing breaks and things compile. { Lock lock(f); } { SharedLock lock(f); } { Lock lock(f, std::defer_lock); EXPECT_TRUE(lock.try_lock()); } { SharedLock lock(f, boost::defer_lock); EXPECT_TRUE(lock.try_lock()); } // X blocks X { Lock lock(f); testLock(EXCLUSIVE, false); } // X blocks S { Lock lock(f); testLock(SHARED, false); } // S blocks X { SharedLock lock(f); testLock(EXCLUSIVE, false); } // S does not block S { SharedLock lock(f); testLock(SHARED, true); } }
TEST(RecordIOTest, Randomized) { SCOPED_TRACE(to<std::string>("Random seed is ", FLAGS_random_seed)); std::mt19937 rnd(FLAGS_random_seed); size_t recordCount = std::uniform_int_distribution<uint32_t>(30, 300)(rnd); std::uniform_int_distribution<uint32_t> recordSizeDist(1, 3 << 16); std::uniform_int_distribution<uint32_t> charDist(0, 255); std::uniform_int_distribution<uint32_t> junkDist(0, 1 << 20); // corrupt 1/5 of all records std::uniform_int_distribution<uint32_t> corruptDist(0, 4); std::vector<std::pair<fbstring, off_t>> records; std::vector<off_t> corruptPositions; records.reserve(recordCount); TemporaryFile file; fbstring record; // Recreate the writer multiple times so we test that we create a // continuous stream for (size_t i = 0; i < 3; ++i) { RecordIOWriter writer(File(file.fd())); for (size_t j = 0; j < recordCount; ++j) { off_t beginPos = writer.filePos(); record.clear(); size_t recordSize = recordSizeDist(rnd); record.reserve(recordSize); for (size_t k = 0; k < recordSize; ++k) { record.push_back(charDist(rnd)); } writer.write(iobufs({record})); bool corrupt = (corruptDist(rnd) == 0); if (corrupt) { // Corrupt one random byte in the record (including header) std::uniform_int_distribution<uint32_t> corruptByteDist( 0, recordSize + recordio_helpers::headerSize() - 1); off_t corruptRel = corruptByteDist(rnd); VLOG(1) << "n=" << records.size() << " bpos=" << beginPos << " rsize=" << record.size() << " corrupt rel=" << corruptRel << " abs=" << beginPos + corruptRel; corruptPositions.push_back(beginPos + corruptRel); } else { VLOG(2) << "n=" << records.size() << " bpos=" << beginPos << " rsize=" << record.size() << " good"; records.emplace_back(std::move(record), beginPos); } } VLOG(1) << "n=" << records.size() << " close abs=" << writer.filePos(); } for (auto& pos : corruptPositions) { corrupt(file.fd(), pos); } { size_t i = 0; RecordIOReader reader(File(file.fd())); for (auto& r : reader) { SCOPED_TRACE(i); ASSERT_LT(i, records.size()); EXPECT_EQ(records[i].first, sp(r.first)); EXPECT_EQ(records[i].second, r.second); ++i; } EXPECT_EQ(records.size(), i); } }
TEST(File, Locks) { typedef std::unique_lock<File> Lock; typedef boost::shared_lock<File> SharedLock; // Find out where we are. static constexpr size_t pathLength = 2048; char buf[pathLength + 1]; int r = readlink("/proc/self/exe", buf, pathLength); CHECK(r != -1); buf[r] = '\0'; fs::path me(buf); auto helper_basename = "file_test_lock_helper"; fs::path helper; if (fs::exists(me.parent_path() / helper_basename)) { helper = me.parent_path() / helper_basename; } else { throw std::runtime_error( folly::to<std::string>("cannot find helper ", helper_basename)); } TemporaryFile tempFile; File f(tempFile.fd()); enum LockMode { EXCLUSIVE, SHARED }; auto testLock = [&](LockMode mode, bool expectedSuccess) { auto ret = Subprocess({helper.native(), mode == SHARED ? "-s" : "-x", tempFile.path().native()}).wait(); EXPECT_TRUE(ret.exited()); if (ret.exited()) { EXPECT_EQ(expectedSuccess ? 0 : 42, ret.exitStatus()); } }; // Make sure nothing breaks and things compile. { Lock lock(f); } { SharedLock lock(f); } { Lock lock(f, std::defer_lock); EXPECT_TRUE(lock.try_lock()); } { SharedLock lock(f, boost::defer_lock); EXPECT_TRUE(lock.try_lock()); } // X blocks X { Lock lock(f); testLock(EXCLUSIVE, false); } // X blocks S { Lock lock(f); testLock(SHARED, false); } // S blocks X { SharedLock lock(f); testLock(EXCLUSIVE, false); } // S does not block S { SharedLock lock(f); testLock(SHARED, true); } }