bool ThreadTransferHistory::addSource(std::unique_ptr<ByteSource> &source) {
std::lock_guard<std::mutex> lock(mutex_);
if (globalCheckpoint_) {
// already received an error for this thread
VLOG(1) << "adding source after global checkpoint is received. returning "
"the source to the queue";
markSourceAsFailed(source, lastCheckpoint_.get());
lastCheckpoint_.reset();
queue_.returnToQueue(source);
return false;
}
history_.emplace_back(std::move(source));
return true;
}
ErrorCode ThreadTransferHistory::setCheckpointAndReturnToQueue(
const Checkpoint &checkpoint, bool globalCheckpoint) {
const int64_t historySize = history_.size();
int64_t numReceivedSources = checkpoint.numBlocks;
int64_t lastBlockReceivedBytes = checkpoint.lastBlockReceivedBytes;
if (numReceivedSources > historySize) {
WLOG(ERROR)
<< "checkpoint is greater than total number of sources transferred "
<< history_.size() << " " << numReceivedSources;
return INVALID_CHECKPOINT;
}
ErrorCode errCode = validateCheckpoint(checkpoint, globalCheckpoint);
if (errCode == INVALID_CHECKPOINT) {
return INVALID_CHECKPOINT;
}
globalCheckpoint_ |= globalCheckpoint;
lastCheckpoint_ = std::make_unique<Checkpoint>(checkpoint);
int64_t numFailedSources = historySize - numReceivedSources;
if (numFailedSources == 0 && lastBlockReceivedBytes > 0) {
if (!globalCheckpoint) {
// no block to apply checkpoint offset. This can happen if we receive same
// local checkpoint without adding anything to the history
WLOG(WARNING)
<< "Local checkpoint has received bytes for last block, but "
"there are no unacked blocks in the history. Ignoring.";
}
}
numAcknowledged_ = numReceivedSources;
std::vector<std::unique_ptr<ByteSource>> sourcesToReturn;
for (int64_t i = 0; i < numFailedSources; i++) {
std::unique_ptr<ByteSource> source = std::move(history_.back());
history_.pop_back();
const Checkpoint *checkpointPtr =
(i == numFailedSources - 1 ? &checkpoint : nullptr);
markSourceAsFailed(source, checkpointPtr);
sourcesToReturn.emplace_back(std::move(source));
}
queue_.returnToQueue(sourcesToReturn);
WLOG(INFO) << numFailedSources
<< " number of sources returned to queue, checkpoint: "
<< checkpoint;
return errCode;
}