void readPackAndWrite(
uint64_t& writeSize, packet::Packet& pkt,
cybozu::util::File& fileW, bool doWriteDiff, DiscardType discardType,
uint64_t fsyncIntervalSize,
std::vector<char>& zero, AlignedArray& buf)
{
const char *const FUNC = __func__;
size_t size;
pkt.read(size);
verifyDiffPackSize(size, FUNC);
buf.resize(size);
pkt.read(buf.data(), buf.size());
verifyDiffPack(buf.data(), buf.size(), true);
if (doWriteDiff) {
fileW.write(buf.data(), buf.size());
} else {
MemoryDiffPack pack(buf.data(), buf.size());
issueDiffPack(fileW, discardType, pack, zero);
}
writeSize += buf.size();
if (writeSize >= fsyncIntervalSize) {
fileW.fdatasync();
writeSize = 0;
}
}
void readAndSendHash(
uint64_t& hashLb,
packet::Packet& pkt, packet::StreamControl2& ctrl, Reader &reader,
uint64_t sizeLb, size_t bulkLb,
cybozu::murmurhash3::Hasher& hasher, AlignedArray& buf)
{
const uint64_t lb = std::min<uint64_t>(sizeLb - hashLb, bulkLb);
buf.resize(lb * LOGICAL_BLOCK_SIZE);
reader.read(buf.data(), buf.size());
const cybozu::murmurhash3::Hash hash = hasher(buf.data(), buf.size());
doRetrySockIo(2, "ctrl.send.next", [&]() { ctrl.sendNext(); });
pkt.write(hash);
hashLb += lb;
}
uint64_t readAndValidateLogPackIo(cybozu::util::File& file, const device::SuperBlock& super, const LogPackHeader& logh)
{
const uint32_t pbs = super.getPhysicalBlockSize();
assert(logh.isValid());
AlignedArray buf;
for (size_t i = 0; i < logh.nRecords(); i++) {
const WlogRecord &rec = logh.record(i);
if (!rec.hasDataForChecksum()) continue;
const uint64_t pb = super.getOffsetFromLsid(rec.lsid);
const size_t ioSize = rec.ioSizeLb() * LBS;
buf.resize(ioSize);
file.pread(buf.data(), ioSize, pb * pbs);
const uint32_t csum = cybozu::util::calcChecksum(buf.data(), buf.size(), logh.salt());
if (csum == rec.checksum) continue;
LOGs.info() << "INVALID_LOGPACK_IO" << rec << csum;
}
return logh.nextLogpackLsid();
}
void setRandomData(AlignedArray& buf, Rand& rand)
{
size_t off = 0;
while (off < buf.size()) {
const size_t size = std::min<size_t>(buf.size() - off, 16);
rand.fill(buf.data() + off, size);
off += 512;
}
}
/*
* @zero is used as zero-filled buffer. It may be resized.
*/
inline void issueDiffPack(cybozu::util::File& file, DiscardType discardType, MemoryDiffPack& pack, std::vector<char>& zero)
{
const DiffPackHeader& head = pack.header();
DiffRecord rec;
AlignedArray array; // buffer for uncompressed data.
for (size_t i = 0; i < head.n_records; i++) {
const DiffRecord& inRec = head[i];
const char *iodata = nullptr;
if (inRec.isNormal()) {
if (inRec.isCompressed()) {
uncompressDiffIo(inRec, pack.data(i), rec, array, false);
iodata = array.data();
} else {
rec = inRec;
iodata = pack.data(i);
}
} else {
rec = inRec;
}
issueIo(file, discardType, rec, iodata, zero);
}
}
inline bool dirtyHashSyncClient(
packet::Packet &pkt, Reader &reader,
uint64_t sizeLb, uint64_t bulkLb, uint32_t hashSeed,
const std::atomic<int> &stopState, const ProcessStatus &ps,
const std::atomic<uint64_t>& maxLbPerSec)
{
const char *const FUNC = __func__;
packet::StreamControl2 recvCtl(pkt.sock());
packet::StreamControl2 sendCtl(pkt.sock());
DiffPacker packer;
walb::PackCompressor compr(::WALB_DIFF_CMPR_SNAPPY);
cybozu::murmurhash3::Hasher hasher(hashSeed);
ThroughputStabilizer thStab;
uint64_t addr = 0;
uint64_t remainingLb = sizeLb;
AlignedArray buf;
size_t cHash = 0, cSend = 0, cDummy = 0;
try {
for (;;) {
if (stopState == ForceStopping || ps.isForceShutdown()) {
return false;
}
dirty_hash_sync_local::doRetrySockIo(4, "ctrl.recv", [&]() { recvCtl.recv(); });
if (recvCtl.isNext()) {
if (remainingLb == 0) throw cybozu::Exception(FUNC) << "has next but remainingLb is zero";
} else {
if (remainingLb == 0) break;
throw cybozu::Exception(FUNC) << "no next but remainingLb is not zero" << remainingLb;
}
cybozu::murmurhash3::Hash recvHash;
pkt.read(recvHash);
cHash++;
const uint32_t lb = std::min<uint64_t>(remainingLb, bulkLb);
buf.resize(lb * LOGICAL_BLOCK_SIZE);
reader.read(buf.data(), buf.size());
// to avoid socket timeout.
dirty_hash_sync_local::doRetrySockIo(4, "ctrl.send.dummy", [&]() { sendCtl.sendDummy(); });
cDummy++; cSend++;
const cybozu::murmurhash3::Hash bdHash = hasher(buf.data(), buf.size());
const uint64_t bgnAddr = packer.empty() ? addr : packer.header()[0].io_address;
if (addr - bgnAddr >= DIRTY_HASH_SYNC_MAX_PACK_AREA_LB && !packer.empty()) {
dirty_hash_sync_local::doRetrySockIo(4, "ctrl.send.next0", [&]() { sendCtl.sendNext(); });
cSend++;
dirty_hash_sync_local::compressAndSend(pkt, packer, compr);
}
if (recvHash != bdHash && !packer.add(addr, lb, buf.data())) {
dirty_hash_sync_local::doRetrySockIo(4, "ctrl.send.next1", [&]() { sendCtl.sendNext(); });
cSend++;
dirty_hash_sync_local::compressAndSend(pkt, packer, compr);
packer.add(addr, lb, buf.data());
}
pkt.flush();
remainingLb -= lb;
addr += lb;
thStab.setMaxLbPerSec(maxLbPerSec.load());
thStab.addAndSleepIfNecessary(lb, 10, 100);
}
} catch (...) {
LOGs.warn() << "SEND_CTL" << cHash << cSend << cDummy;
throw;
}
if (!packer.empty()) {
dirty_hash_sync_local::doRetrySockIo(4, "ctrl.send.next2", [&]() { sendCtl.sendNext(); });
cSend++;
dirty_hash_sync_local::compressAndSend(pkt, packer, compr);
}
if (recvCtl.isError()) {
throw cybozu::Exception(FUNC) << "recvCtl";
}
dirty_hash_sync_local::doRetrySockIo(4, "ctrl.send.next2", [&]() { sendCtl.sendEnd(); });
pkt.flush();
LOGs.debug() << "SEND_CTL" << cHash << cSend << cDummy;
return true;
}
