void Receiver::progressTracker() {
// Progress tracker will check for progress after the time specified
// in milliseconds.
int progressReportIntervalMillis = options_.progress_report_interval_millis;
int throughputUpdateIntervalMillis =
options_.throughput_update_interval_millis;
if (progressReportIntervalMillis <= 0 || throughputUpdateIntervalMillis < 0 ||
!isJoinable_) {
return;
}
int throughputUpdateInterval =
throughputUpdateIntervalMillis / progressReportIntervalMillis;
int64_t lastEffectiveBytes = 0;
std::chrono::time_point<Clock> lastUpdateTime = Clock::now();
int intervalsSinceLastUpdate = 0;
double currentThroughput = 0;
WLOG(INFO) << "Progress reporter updating every "
<< progressReportIntervalMillis << " ms";
auto waitingTime = std::chrono::milliseconds(progressReportIntervalMillis);
int64_t totalSenderBytes = -1;
while (true) {
{
std::unique_lock<std::mutex> lock(mutex_);
conditionFinished_.wait_for(lock, waitingTime);
if (transferStatus_ == THREADS_JOINED) {
break;
}
}
double totalTime = durationSeconds(Clock::now() - startTime_);
TransferStats globalStats;
for (const auto &receiverThread : receiverThreads_) {
globalStats += receiverThread->getTransferStats();
}
totalSenderBytes = globalStats.getTotalSenderBytes();
if (totalSenderBytes == -1) {
continue;
}
auto transferReport = folly::make_unique<TransferReport>(
std::move(globalStats), totalTime, totalSenderBytes);
intervalsSinceLastUpdate++;
if (intervalsSinceLastUpdate >= throughputUpdateInterval) {
auto curTime = Clock::now();
int64_t curEffectiveBytes =
transferReport->getSummary().getEffectiveDataBytes();
double time = durationSeconds(curTime - lastUpdateTime);
currentThroughput = (curEffectiveBytes - lastEffectiveBytes) / time;
lastEffectiveBytes = curEffectiveBytes;
lastUpdateTime = curTime;
intervalsSinceLastUpdate = 0;
}
transferReport->setCurrentThroughput(currentThroughput);
progressReporter_->progress(transferReport);
if (reportPerfSignal_.notified()) {
logPerfStats();
}
}
}
void Sender::endCurTransfer() {
endTime_ = Clock::now();
LOG(INFO) << "Last thread finished " << durationSeconds(endTime_ - startTime_)
<< " for transfer id " << getTransferId();
setTransferStatus(FINISHED);
if (throttler_) {
throttler_->deRegisterTransfer();
}
}
std::unique_ptr<ClientSocket> SenderThread::connectToReceiver(
const int port, IAbortChecker const *abortChecker, ErrorCode &errCode) {
auto startTime = Clock::now();
int connectAttempts = 0;
std::unique_ptr<ClientSocket> socket;
const EncryptionParams &encryptionData =
wdtParent_->transferRequest_.encryptionData;
if (!wdtParent_->socketCreator_) {
// socket creator not set, creating ClientSocket
socket = folly::make_unique<ClientSocket>(
*threadCtx_, wdtParent_->destHost_, port, encryptionData);
} else {
socket = wdtParent_->socketCreator_->makeSocket(
*threadCtx_, wdtParent_->destHost_, port, encryptionData);
}
double retryInterval = options_.sleep_millis;
int maxRetries = options_.max_retries;
if (maxRetries < 1) {
WTLOG(ERROR) << "Invalid max_retries " << maxRetries << " using 1 instead";
maxRetries = 1;
}
for (int i = 1; i <= maxRetries; ++i) {
++connectAttempts;
errCode = socket->connect();
if (errCode == OK) {
break;
} else if (errCode == CONN_ERROR) {
return nullptr;
}
if (getThreadAbortCode() != OK) {
errCode = ABORT;
return nullptr;
}
if (i != maxRetries) {
// sleep between attempts but not after the last
WVTLOG(1) << "Sleeping after failed attempt " << i;
/* sleep override */ usleep(retryInterval * 1000);
}
}
double elapsedSecsConn = durationSeconds(Clock::now() - startTime);
if (errCode != OK) {
WTLOG(ERROR) << "Unable to connect to " << wdtParent_->destHost_ << " "
<< port << " despite " << connectAttempts << " retries in "
<< elapsedSecsConn << " seconds.";
errCode = CONN_ERROR;
return nullptr;
}
((connectAttempts > 1) ? WTLOG(WARNING) : WTLOG(INFO))
<< "Connection took " << connectAttempts << " attempt(s) and "
<< elapsedSecsConn << " seconds. port " << port;
return socket;
}
std::unique_ptr<TransferReport> Sender::getTransferReport() {
int64_t totalFileSize = dirQueue_->getTotalSize();
double totalTime = durationSeconds(Clock::now() - startTime_);
auto globalStats = getGlobalTransferStats();
std::unique_ptr<TransferReport> transferReport =
folly::make_unique<TransferReport>(std::move(globalStats), totalTime,
totalFileSize);
TransferStatus status = getTransferStatus();
ErrorCode errCode = transferReport->getSummary().getErrorCode();
if (status == NOT_STARTED && errCode == OK) {
LOG(INFO) << "Transfer not started, setting the error code to ERROR";
transferReport->setErrorCode(ERROR);
}
return transferReport;
}
void SenderThread::start() {
Clock::time_point startTime = Clock::now();
if (buf_ == nullptr) {
WTLOG(ERROR) << "Unable to allocate buffer";
threadStats_.setLocalErrorCode(MEMORY_ALLOCATION_ERROR);
return;
}
setFooterType();
controller_->executeAtStart([&]() { wdtParent_->startNewTransfer(); });
SenderState state = CONNECT;
while (state != END) {
ErrorCode abortCode = getThreadAbortCode();
if (abortCode != OK) {
WTLOG(ERROR) << "Transfer aborted " << errorCodeToStr(abortCode);
threadStats_.setLocalErrorCode(ABORT);
if (abortCode == VERSION_MISMATCH) {
state = PROCESS_VERSION_MISMATCH;
} else {
break;
}
}
state = (this->*stateMap_[state])();
}
EncryptionType encryptionType =
(socket_ ? socket_->getEncryptionType() : ENC_NONE);
threadStats_.setEncryptionType(encryptionType);
double totalTime = durationSeconds(Clock::now() - startTime);
WTLOG(INFO) << "Port " << port_ << " done. " << threadStats_
<< " Total throughput = "
<< threadStats_.getEffectiveTotalBytes() / totalTime / kMbToB
<< " Mbytes/sec";
ThreadTransferHistory &transferHistory = getTransferHistory();
transferHistory.markNotInUse();
controller_->deRegisterThread(threadIndex_);
controller_->executeAtEnd([&]() { wdtParent_->endCurTransfer(); });
// Important to delete the socket before the thread dies for sub class
// of clientsocket which have thread local data
socket_ = nullptr;
return;
}
void Sender::reportProgress() {
WDT_CHECK(progressReportIntervalMillis_ > 0);
int throughputUpdateIntervalMillis =
options_.throughput_update_interval_millis;
WDT_CHECK(throughputUpdateIntervalMillis >= 0);
int throughputUpdateInterval =
throughputUpdateIntervalMillis / progressReportIntervalMillis_;
int64_t lastEffectiveBytes = 0;
std::chrono::time_point<Clock> lastUpdateTime = Clock::now();
int intervalsSinceLastUpdate = 0;
double currentThroughput = 0;
auto waitingTime = std::chrono::milliseconds(progressReportIntervalMillis_);
LOG(INFO) << "Progress reporter tracking every "
<< progressReportIntervalMillis_ << " ms";
while (true) {
{
std::unique_lock<std::mutex> lock(mutex_);
conditionFinished_.wait_for(lock, waitingTime);
if (transferStatus_ == THREADS_JOINED) {
break;
}
}
if (!dirQueue_->fileDiscoveryFinished()) {
continue;
}
std::unique_ptr<TransferReport> transferReport = getTransferReport();
intervalsSinceLastUpdate++;
if (intervalsSinceLastUpdate >= throughputUpdateInterval) {
auto curTime = Clock::now();
int64_t curEffectiveBytes =
transferReport->getSummary().getEffectiveDataBytes();
double time = durationSeconds(curTime - lastUpdateTime);
currentThroughput = (curEffectiveBytes - lastEffectiveBytes) / time;
lastEffectiveBytes = curEffectiveBytes;
lastUpdateTime = curTime;
intervalsSinceLastUpdate = 0;
}
transferReport->setCurrentThroughput(currentThroughput);
progressReporter_->progress(transferReport);
}
}
std::unique_ptr<TransferReport> Receiver::finish() {
std::unique_lock<std::mutex> instanceLock(instanceManagementMutex_);
TransferStatus status = getTransferStatus();
if (status == NOT_STARTED) {
WLOG(WARNING) << "Even though transfer has not started, finish is called";
// getTransferReport will set the error code to ERROR
return getTransferReport();
}
if (status == THREADS_JOINED) {
WLOG(WARNING) << "Threads have already been joined. Returning the "
<< "transfer report";
return getTransferReport();
}
if (!isJoinable_) {
// TODO: don't complain about this when coming from runForever()
WLOG(WARNING) << "The receiver is not joinable. The threads will never"
<< " finish and this method will never return";
}
for (auto &receiverThread : receiverThreads_) {
receiverThread->finish();
}
setTransferStatus(THREADS_JOINED);
if (isJoinable_) {
// Make sure to join the progress thread.
progressTrackerThread_.join();
}
std::unique_ptr<TransferReport> report = getTransferReport();
auto &summary = report->getSummary();
bool transferSuccess = (report->getSummary().getErrorCode() == OK);
fixAndCloseTransferLog(transferSuccess);
auto totalSenderBytes = summary.getTotalSenderBytes();
if (progressReporter_ && totalSenderBytes >= 0) {
report->setTotalFileSize(totalSenderBytes);
report->setTotalTime(durationSeconds(Clock::now() - startTime_));
progressReporter_->end(report);
}
logPerfStats();
WLOG(WARNING) << "WDT receiver's transfer has been finished";
WLOG(INFO) << *report;
return report;
}
std::unique_ptr<TransferReport> Sender::finish() {
std::unique_lock<std::mutex> instanceLock(instanceManagementMutex_);
VLOG(1) << "Sender::finish()";
TransferStatus status = getTransferStatus();
if (status == NOT_STARTED) {
LOG(WARNING) << "Even though transfer has not started, finish is called";
// getTransferReport will set the error code to ERROR
return getTransferReport();
}
if (status == THREADS_JOINED) {
VLOG(1) << "Threads have already been joined. Returning the"
<< " existing transfer report";
return getTransferReport();
}
const bool twoPhases = options_.two_phases;
bool progressReportEnabled =
progressReporter_ && progressReportIntervalMillis_ > 0;
for (auto &senderThread : senderThreads_) {
senderThread->finish();
}
if (!twoPhases) {
dirThread_.join();
}
WDT_CHECK(numActiveThreads_ == 0);
setTransferStatus(THREADS_JOINED);
if (progressReportEnabled) {
progressReporterThread_.join();
}
std::vector<TransferStats> threadStats;
for (auto &senderThread : senderThreads_) {
threadStats.push_back(senderThread->moveStats());
}
bool allSourcesAcked = false;
for (auto &senderThread : senderThreads_) {
auto &stats = senderThread->getTransferStats();
if (stats.getErrorCode() == OK) {
// at least one thread finished correctly
// that means all transferred sources are acked
allSourcesAcked = true;
break;
}
}
std::vector<TransferStats> transferredSourceStats;
for (auto port : transferRequest_.ports) {
auto &transferHistory =
transferHistoryController_->getTransferHistory(port);
if (allSourcesAcked) {
transferHistory.markAllAcknowledged();
} else {
transferHistory.returnUnackedSourcesToQueue();
}
if (options_.full_reporting) {
std::vector<TransferStats> stats = transferHistory.popAckedSourceStats();
transferredSourceStats.insert(transferredSourceStats.end(),
std::make_move_iterator(stats.begin()),
std::make_move_iterator(stats.end()));
}
}
if (options_.full_reporting) {
validateTransferStats(transferredSourceStats,
dirQueue_->getFailedSourceStats());
}
int64_t totalFileSize = dirQueue_->getTotalSize();
double totalTime = durationSeconds(endTime_ - startTime_);
std::unique_ptr<TransferReport> transferReport =
folly::make_unique<TransferReport>(
transferredSourceStats, dirQueue_->getFailedSourceStats(),
threadStats, dirQueue_->getFailedDirectories(), totalTime,
totalFileSize, dirQueue_->getCount());
if (progressReportEnabled) {
progressReporter_->end(transferReport);
}
if (options_.enable_perf_stat_collection) {
PerfStatReport report(options_);
for (auto &senderThread : senderThreads_) {
report += senderThread->getPerfReport();
}
report += dirQueue_->getPerfReport();
LOG(INFO) << report;
}
double directoryTime;
directoryTime = dirQueue_->getDirectoryTime();
LOG(INFO) << "Total sender time = " << totalTime << " seconds ("
<< directoryTime << " dirTime)"
<< ". Transfer summary : " << *transferReport
<< "\nTotal sender throughput = "
<< transferReport->getThroughputMBps() << " Mbytes/sec ("
<< transferReport->getSummary().getEffectiveTotalBytes() /
(totalTime - directoryTime) / kMbToB
<< " Mbytes/sec pure transfer rate)";
return transferReport;
}
