ErrorCode Sender::start() { { std::lock_guard<std::mutex> lock(mutex_); if (transferStatus_ != NOT_STARTED) { LOG(ERROR) << "duplicate start() call detected " << transferStatus_; return ALREADY_EXISTS; } transferStatus_ = ONGOING; } checkAndUpdateBufferSize(); const bool twoPhases = options_.two_phases; LOG(INFO) << "Client (sending) to " << destHost_ << ", Using ports [ " << transferRequest_.ports << "]"; startTime_ = Clock::now(); downloadResumptionEnabled_ = options_.enable_download_resumption; if (!progressReporter_) { VLOG(1) << "No progress reporter provided, making a default one"; progressReporter_ = folly::make_unique<ProgressReporter>(transferRequest_); } bool progressReportEnabled = progressReporter_ && progressReportIntervalMillis_ > 0; if (throttler_) { LOG(INFO) << "Skipping throttler setup. External throttler set." << "Throttler details : " << *throttler_; } else { configureThrottler(); } threadsController_ = new ThreadsController(transferRequest_.ports.size()); threadsController_->setNumBarriers(SenderThread::NUM_BARRIERS); threadsController_->setNumFunnels(SenderThread::NUM_FUNNELS); threadsController_->setNumConditions(SenderThread::NUM_CONDITIONS); // TODO: fix this ! use transferRequest! (and dup from Receiver) senderThreads_ = threadsController_->makeThreads<Sender, SenderThread>( this, transferRequest_.ports.size(), transferRequest_.ports); if (downloadResumptionEnabled_ && options_.delete_extra_files) { if (protocolVersion_ >= Protocol::DELETE_CMD_VERSION) { dirQueue_->enableFileDeletion(); } else { LOG(WARNING) << "Turning off extra file deletion on the receiver side " "because of protocol version " << protocolVersion_; } } dirThread_ = dirQueue_->buildQueueAsynchronously(); if (twoPhases) { dirThread_.join(); } for (auto &senderThread : senderThreads_) { senderThread->startThread(); } if (progressReportEnabled) { progressReporter_->start(); std::thread reporterThread(&Sender::reportProgress, this); progressReporterThread_ = std::move(reporterThread); } return OK; }
const WdtTransferRequest &Receiver::init() { if (validateTransferRequest() != OK) { WLOG(ERROR) << "Couldn't validate the transfer request " << transferRequest_.getLogSafeString(); return transferRequest_; } checkAndUpdateBufferSize(); backlog_ = options_.backlog; if (getTransferId().empty()) { setTransferId(WdtBase::generateTransferId()); } setProtocolVersion(transferRequest_.protocolVersion); setDir(transferRequest_.directory); auto numThreads = transferRequest_.ports.size(); // This creates the destination directory (which is needed for transferLogMgr) fileCreator_.reset(new FileCreator(destDir_, numThreads, transferLogManager_, options_.skip_writes)); // Make sure we can get the lock on the transfer log manager early // so if we can't we don't generate a valid but useless url and end up // starting a sender doomed to fail if (options_.enable_download_resumption) { WDT_CHECK(!options_.skip_writes) << "Can not skip transfers with download resumption turned on"; if (options_.resume_using_dir_tree) { WDT_CHECK(!options_.shouldPreallocateFiles()) << "Can not resume using directory tree if preallocation is enabled"; } ErrorCode errCode = transferLogManager_.openLog(); if (errCode != OK) { WLOG(ERROR) << "Failed to open transfer log " << errorCodeToStr(errCode); transferRequest_.errorCode = errCode; return transferRequest_; } ErrorCode code = transferLogManager_.parseAndMatch( recoveryId_, getTransferConfig(), fileChunksInfo_); if (code == OK && options_.resume_using_dir_tree) { WDT_CHECK(fileChunksInfo_.empty()); traverseDestinationDir(fileChunksInfo_); } } EncryptionType encryptionType = parseEncryptionType(options_.encryption_type); // is encryption enabled? bool encrypt = (encryptionType != ENC_NONE && protocolVersion_ >= Protocol::ENCRYPTION_V1_VERSION); if (encrypt) { WLOG(INFO) << encryptionTypeToStr(encryptionType) << " encryption is enabled for this transfer "; if (!transferRequest_.encryptionData.isSet()) { WLOG(INFO) << "Receiver generating encryption key for type " << encryptionTypeToStr(encryptionType); transferRequest_.encryptionData = EncryptionParams::generateEncryptionParams(encryptionType); } if (!transferRequest_.encryptionData.isSet()) { WLOG(ERROR) << "Unable to generate encryption key for type " << encryptionTypeToStr(encryptionType); transferRequest_.errorCode = ENCRYPTION_ERROR; return transferRequest_; } } else { if (encryptionType != ENC_NONE) { WLOG(WARNING) << "Encryption is enabled, but protocol version is " << protocolVersion_ << ", minimum version required for encryption is " << Protocol::ENCRYPTION_V1_VERSION; } transferRequest_.encryptionData.erase(); } threadsController_ = new ThreadsController(numThreads); threadsController_->setNumFunnels(ReceiverThread::NUM_FUNNELS); threadsController_->setNumBarriers(ReceiverThread::NUM_BARRIERS); threadsController_->setNumConditions(ReceiverThread::NUM_CONDITIONS); // TODO: take transferRequest directly ! receiverThreads_ = threadsController_->makeThreads<Receiver, ReceiverThread>( this, transferRequest_.ports.size(), transferRequest_.ports); size_t numSuccessfulInitThreads = 0; for (auto &receiverThread : receiverThreads_) { ErrorCode code = receiverThread->init(); if (code == OK) { ++numSuccessfulInitThreads; } } WLOG(INFO) << "Registered " << numSuccessfulInitThreads << " successful sockets"; ErrorCode code = OK; const size_t targetSize = transferRequest_.ports.size(); // TODO: replace with getNumPorts/thread if (numSuccessfulInitThreads != targetSize) { code = FEWER_PORTS; if (numSuccessfulInitThreads == 0) { code = ERROR; } } transferRequest_.protocolVersion = protocolVersion_; transferRequest_.ports.clear(); for (const auto &receiverThread : receiverThreads_) { transferRequest_.ports.push_back(receiverThread->getPort()); } if (transferRequest_.hostName.empty()) { char hostName[1024]; int ret = gethostname(hostName, sizeof(hostName)); if (ret == 0) { transferRequest_.hostName.assign(hostName); } else { PLOG(ERROR) << "Couldn't find the host name"; code = ERROR; } } transferRequest_.directory = getDir(); transferRequest_.errorCode = code; return transferRequest_; }