Sender::~Sender() {
TransferStatus status = getTransferStatus();
if (status == ONGOING) {
LOG(WARNING) << "Sender being deleted. Forcefully aborting the transfer";
abort(ABORTED_BY_APPLICATION);
}
finish();
}
Receiver::~Receiver() {
TransferStatus status = getTransferStatus();
if (status == ONGOING) {
WLOG(WARNING) << "There is an ongoing transfer and the destructor"
<< " is being called. Trying to finish the transfer";
abort(ABORTED_BY_APPLICATION);
}
finish();
}
std::unique_ptr<TransferReport> Sender::getTransferReport() {
int64_t totalFileSize = dirQueue_->getTotalSize();
double totalTime = durationSeconds(Clock::now() - startTime_);
auto globalStats = getGlobalTransferStats();
std::unique_ptr<TransferReport> transferReport =
folly::make_unique<TransferReport>(std::move(globalStats), totalTime,
totalFileSize);
TransferStatus status = getTransferStatus();
ErrorCode errCode = transferReport->getSummary().getErrorCode();
if (status == NOT_STARTED && errCode == OK) {
LOG(INFO) << "Transfer not started, setting the error code to ERROR";
transferReport->setErrorCode(ERROR);
}
return transferReport;
}
std::unique_ptr<TransferReport> Receiver::getTransferReport() {
TransferStats globalStats;
for (const auto &receiverThread : receiverThreads_) {
globalStats += receiverThread->getTransferStats();
}
std::unique_ptr<TransferReport> transferReport =
folly::make_unique<TransferReport>(std::move(globalStats));
TransferStatus status = getTransferStatus();
ErrorCode errCode = transferReport->getSummary().getErrorCode();
if (status == NOT_STARTED && errCode == OK) {
WLOG(INFO) << "Transfer not started, setting the error code to ERROR";
transferReport->setErrorCode(ERROR);
}
WVLOG(1) << "Summary code " << errCode;
return transferReport;
}
std::unique_ptr<TransferReport> Receiver::finish() {
std::unique_lock<std::mutex> instanceLock(instanceManagementMutex_);
TransferStatus status = getTransferStatus();
if (status == NOT_STARTED) {
WLOG(WARNING) << "Even though transfer has not started, finish is called";
// getTransferReport will set the error code to ERROR
return getTransferReport();
}
if (status == THREADS_JOINED) {
WLOG(WARNING) << "Threads have already been joined. Returning the "
<< "transfer report";
return getTransferReport();
}
if (!isJoinable_) {
// TODO: don't complain about this when coming from runForever()
WLOG(WARNING) << "The receiver is not joinable. The threads will never"
<< " finish and this method will never return";
}
for (auto &receiverThread : receiverThreads_) {
receiverThread->finish();
}
setTransferStatus(THREADS_JOINED);
if (isJoinable_) {
// Make sure to join the progress thread.
progressTrackerThread_.join();
}
std::unique_ptr<TransferReport> report = getTransferReport();
auto &summary = report->getSummary();
bool transferSuccess = (report->getSummary().getErrorCode() == OK);
fixAndCloseTransferLog(transferSuccess);
auto totalSenderBytes = summary.getTotalSenderBytes();
if (progressReporter_ && totalSenderBytes >= 0) {
report->setTotalFileSize(totalSenderBytes);
report->setTotalTime(durationSeconds(Clock::now() - startTime_));
progressReporter_->end(report);
}
logPerfStats();
WLOG(WARNING) << "WDT receiver's transfer has been finished";
WLOG(INFO) << *report;
return report;
}
ErrorCode Receiver::start() {
WDT_CHECK_EQ(getTransferStatus(), NOT_STARTED)
<< "There is already a transfer running on this instance of receiver";
startTime_ = Clock::now();
WLOG(INFO) << "Starting (receiving) server on ports [ "
<< transferRequest_.ports << "] Target dir : " << destDir_;
// TODO do the init stuff here
if (!throttler_) {
configureThrottler();
} else {
WLOG(INFO) << "Throttler set externally. Throttler : " << *throttler_;
}
setTransferStatus(ONGOING);
while (true) {
for (auto &receiverThread : receiverThreads_) {
receiverThread->startThread();
}
if (isJoinable_) {
break;
}
// If it is long running mode, finish the threads
// processing the current transfer and re spawn them again
// with the same sockets
for (auto &receiverThread : receiverThreads_) {
receiverThread->finish();
receiverThread->reset();
}
threadsController_->reset();
// reset transfer status
setTransferStatus(NOT_STARTED);
continue;
}
if (isJoinable_) {
if (progressReporter_) {
progressReporter_->start();
}
std::thread trackerThread(&Receiver::progressTracker, this);
progressTrackerThread_ = std::move(trackerThread);
}
return OK;
}
std::unique_ptr<TransferReport> Sender::finish() {
std::unique_lock<std::mutex> instanceLock(instanceManagementMutex_);
VLOG(1) << "Sender::finish()";
TransferStatus status = getTransferStatus();
if (status == NOT_STARTED) {
LOG(WARNING) << "Even though transfer has not started, finish is called";
// getTransferReport will set the error code to ERROR
return getTransferReport();
}
if (status == THREADS_JOINED) {
VLOG(1) << "Threads have already been joined. Returning the"
<< " existing transfer report";
return getTransferReport();
}
const bool twoPhases = options_.two_phases;
bool progressReportEnabled =
progressReporter_ && progressReportIntervalMillis_ > 0;
for (auto &senderThread : senderThreads_) {
senderThread->finish();
}
if (!twoPhases) {
dirThread_.join();
}
WDT_CHECK(numActiveThreads_ == 0);
setTransferStatus(THREADS_JOINED);
if (progressReportEnabled) {
progressReporterThread_.join();
}
std::vector<TransferStats> threadStats;
for (auto &senderThread : senderThreads_) {
threadStats.push_back(senderThread->moveStats());
}
bool allSourcesAcked = false;
for (auto &senderThread : senderThreads_) {
auto &stats = senderThread->getTransferStats();
if (stats.getErrorCode() == OK) {
// at least one thread finished correctly
// that means all transferred sources are acked
allSourcesAcked = true;
break;
}
}
std::vector<TransferStats> transferredSourceStats;
for (auto port : transferRequest_.ports) {
auto &transferHistory =
transferHistoryController_->getTransferHistory(port);
if (allSourcesAcked) {
transferHistory.markAllAcknowledged();
} else {
transferHistory.returnUnackedSourcesToQueue();
}
if (options_.full_reporting) {
std::vector<TransferStats> stats = transferHistory.popAckedSourceStats();
transferredSourceStats.insert(transferredSourceStats.end(),
std::make_move_iterator(stats.begin()),
std::make_move_iterator(stats.end()));
}
}
if (options_.full_reporting) {
validateTransferStats(transferredSourceStats,
dirQueue_->getFailedSourceStats());
}
int64_t totalFileSize = dirQueue_->getTotalSize();
double totalTime = durationSeconds(endTime_ - startTime_);
std::unique_ptr<TransferReport> transferReport =
folly::make_unique<TransferReport>(
transferredSourceStats, dirQueue_->getFailedSourceStats(),
threadStats, dirQueue_->getFailedDirectories(), totalTime,
totalFileSize, dirQueue_->getCount());
if (progressReportEnabled) {
progressReporter_->end(transferReport);
}
if (options_.enable_perf_stat_collection) {
PerfStatReport report(options_);
for (auto &senderThread : senderThreads_) {
report += senderThread->getPerfReport();
}
report += dirQueue_->getPerfReport();
LOG(INFO) << report;
}
double directoryTime;
directoryTime = dirQueue_->getDirectoryTime();
LOG(INFO) << "Total sender time = " << totalTime << " seconds ("
<< directoryTime << " dirTime)"
<< ". Transfer summary : " << *transferReport
<< "\nTotal sender throughput = "
<< transferReport->getThroughputMBps() << " Mbytes/sec ("
<< transferReport->getSummary().getEffectiveTotalBytes() /
(totalTime - directoryTime) / kMbToB
<< " Mbytes/sec pure transfer rate)";
return transferReport;
}
bool WdtBase::isStale() {
TransferStatus status = getTransferStatus();
return (status == FINISHED || status == THREADS_JOINED);
}
bool WdtBase::hasStarted() {
TransferStatus status = getTransferStatus();
return (status != NOT_STARTED);
}
