Sender::~Sender() { TransferStatus status = getTransferStatus(); if (status == ONGOING) { LOG(WARNING) << "Sender being deleted. Forcefully aborting the transfer"; abort(ABORTED_BY_APPLICATION); } finish(); }
Receiver::~Receiver() { TransferStatus status = getTransferStatus(); if (status == ONGOING) { WLOG(WARNING) << "There is an ongoing transfer and the destructor" << " is being called. Trying to finish the transfer"; abort(ABORTED_BY_APPLICATION); } finish(); }
std::unique_ptr<TransferReport> Sender::getTransferReport() { int64_t totalFileSize = dirQueue_->getTotalSize(); double totalTime = durationSeconds(Clock::now() - startTime_); auto globalStats = getGlobalTransferStats(); std::unique_ptr<TransferReport> transferReport = folly::make_unique<TransferReport>(std::move(globalStats), totalTime, totalFileSize); TransferStatus status = getTransferStatus(); ErrorCode errCode = transferReport->getSummary().getErrorCode(); if (status == NOT_STARTED && errCode == OK) { LOG(INFO) << "Transfer not started, setting the error code to ERROR"; transferReport->setErrorCode(ERROR); } return transferReport; }
std::unique_ptr<TransferReport> Receiver::getTransferReport() { TransferStats globalStats; for (const auto &receiverThread : receiverThreads_) { globalStats += receiverThread->getTransferStats(); } std::unique_ptr<TransferReport> transferReport = folly::make_unique<TransferReport>(std::move(globalStats)); TransferStatus status = getTransferStatus(); ErrorCode errCode = transferReport->getSummary().getErrorCode(); if (status == NOT_STARTED && errCode == OK) { WLOG(INFO) << "Transfer not started, setting the error code to ERROR"; transferReport->setErrorCode(ERROR); } WVLOG(1) << "Summary code " << errCode; return transferReport; }
std::unique_ptr<TransferReport> Receiver::finish() { std::unique_lock<std::mutex> instanceLock(instanceManagementMutex_); TransferStatus status = getTransferStatus(); if (status == NOT_STARTED) { WLOG(WARNING) << "Even though transfer has not started, finish is called"; // getTransferReport will set the error code to ERROR return getTransferReport(); } if (status == THREADS_JOINED) { WLOG(WARNING) << "Threads have already been joined. Returning the " << "transfer report"; return getTransferReport(); } if (!isJoinable_) { // TODO: don't complain about this when coming from runForever() WLOG(WARNING) << "The receiver is not joinable. The threads will never" << " finish and this method will never return"; } for (auto &receiverThread : receiverThreads_) { receiverThread->finish(); } setTransferStatus(THREADS_JOINED); if (isJoinable_) { // Make sure to join the progress thread. progressTrackerThread_.join(); } std::unique_ptr<TransferReport> report = getTransferReport(); auto &summary = report->getSummary(); bool transferSuccess = (report->getSummary().getErrorCode() == OK); fixAndCloseTransferLog(transferSuccess); auto totalSenderBytes = summary.getTotalSenderBytes(); if (progressReporter_ && totalSenderBytes >= 0) { report->setTotalFileSize(totalSenderBytes); report->setTotalTime(durationSeconds(Clock::now() - startTime_)); progressReporter_->end(report); } logPerfStats(); WLOG(WARNING) << "WDT receiver's transfer has been finished"; WLOG(INFO) << *report; return report; }
ErrorCode Receiver::start() { WDT_CHECK_EQ(getTransferStatus(), NOT_STARTED) << "There is already a transfer running on this instance of receiver"; startTime_ = Clock::now(); WLOG(INFO) << "Starting (receiving) server on ports [ " << transferRequest_.ports << "] Target dir : " << destDir_; // TODO do the init stuff here if (!throttler_) { configureThrottler(); } else { WLOG(INFO) << "Throttler set externally. Throttler : " << *throttler_; } setTransferStatus(ONGOING); while (true) { for (auto &receiverThread : receiverThreads_) { receiverThread->startThread(); } if (isJoinable_) { break; } // If it is long running mode, finish the threads // processing the current transfer and re spawn them again // with the same sockets for (auto &receiverThread : receiverThreads_) { receiverThread->finish(); receiverThread->reset(); } threadsController_->reset(); // reset transfer status setTransferStatus(NOT_STARTED); continue; } if (isJoinable_) { if (progressReporter_) { progressReporter_->start(); } std::thread trackerThread(&Receiver::progressTracker, this); progressTrackerThread_ = std::move(trackerThread); } return OK; }
std::unique_ptr<TransferReport> Sender::finish() { std::unique_lock<std::mutex> instanceLock(instanceManagementMutex_); VLOG(1) << "Sender::finish()"; TransferStatus status = getTransferStatus(); if (status == NOT_STARTED) { LOG(WARNING) << "Even though transfer has not started, finish is called"; // getTransferReport will set the error code to ERROR return getTransferReport(); } if (status == THREADS_JOINED) { VLOG(1) << "Threads have already been joined. Returning the" << " existing transfer report"; return getTransferReport(); } const bool twoPhases = options_.two_phases; bool progressReportEnabled = progressReporter_ && progressReportIntervalMillis_ > 0; for (auto &senderThread : senderThreads_) { senderThread->finish(); } if (!twoPhases) { dirThread_.join(); } WDT_CHECK(numActiveThreads_ == 0); setTransferStatus(THREADS_JOINED); if (progressReportEnabled) { progressReporterThread_.join(); } std::vector<TransferStats> threadStats; for (auto &senderThread : senderThreads_) { threadStats.push_back(senderThread->moveStats()); } bool allSourcesAcked = false; for (auto &senderThread : senderThreads_) { auto &stats = senderThread->getTransferStats(); if (stats.getErrorCode() == OK) { // at least one thread finished correctly // that means all transferred sources are acked allSourcesAcked = true; break; } } std::vector<TransferStats> transferredSourceStats; for (auto port : transferRequest_.ports) { auto &transferHistory = transferHistoryController_->getTransferHistory(port); if (allSourcesAcked) { transferHistory.markAllAcknowledged(); } else { transferHistory.returnUnackedSourcesToQueue(); } if (options_.full_reporting) { std::vector<TransferStats> stats = transferHistory.popAckedSourceStats(); transferredSourceStats.insert(transferredSourceStats.end(), std::make_move_iterator(stats.begin()), std::make_move_iterator(stats.end())); } } if (options_.full_reporting) { validateTransferStats(transferredSourceStats, dirQueue_->getFailedSourceStats()); } int64_t totalFileSize = dirQueue_->getTotalSize(); double totalTime = durationSeconds(endTime_ - startTime_); std::unique_ptr<TransferReport> transferReport = folly::make_unique<TransferReport>( transferredSourceStats, dirQueue_->getFailedSourceStats(), threadStats, dirQueue_->getFailedDirectories(), totalTime, totalFileSize, dirQueue_->getCount()); if (progressReportEnabled) { progressReporter_->end(transferReport); } if (options_.enable_perf_stat_collection) { PerfStatReport report(options_); for (auto &senderThread : senderThreads_) { report += senderThread->getPerfReport(); } report += dirQueue_->getPerfReport(); LOG(INFO) << report; } double directoryTime; directoryTime = dirQueue_->getDirectoryTime(); LOG(INFO) << "Total sender time = " << totalTime << " seconds (" << directoryTime << " dirTime)" << ". Transfer summary : " << *transferReport << "\nTotal sender throughput = " << transferReport->getThroughputMBps() << " Mbytes/sec (" << transferReport->getSummary().getEffectiveTotalBytes() / (totalTime - directoryTime) / kMbToB << " Mbytes/sec pure transfer rate)"; return transferReport; }
bool WdtBase::isStale() { TransferStatus status = getTransferStatus(); return (status == FINISHED || status == THREADS_JOINED); }
bool WdtBase::hasStarted() { TransferStatus status = getTransferStatus(); return (status != NOT_STARTED); }