void WdtResourceControllerTest::MultipleNamespacesTest() { int namespaceNum = 0; int maxNamespaces = 5; int numSenders = 0; int numReceivers = 0; int numObjectsPerNamespace = 3; string transferPrefix = "add-objects-transfer"; for (; namespaceNum < maxNamespaces; namespaceNum++) { string wdtNamespace = "test-namespace-" + to_string(namespaceNum); registerWdtNamespace(wdtNamespace); for (int index = 0; index < numObjectsPerNamespace; index++) { auto transferRequest = makeTransferRequest(getTransferId(transferPrefix, index)); ReceiverPtr receiverPtr; ErrorCode code = createReceiver(wdtNamespace, transferRequest.transferId, transferRequest, receiverPtr); ASSERT_TRUE(code == OK); ASSERT_TRUE(receiverPtr != nullptr); SenderPtr senderPtr; code = createSender(wdtNamespace, transferRequest.transferId, transferRequest, senderPtr); ASSERT_TRUE(senderPtr != nullptr); ASSERT_TRUE(code == OK); ++numSenders; ++numReceivers; } } ASSERT_TRUE(numSenders_ == numSenders); ASSERT_TRUE(numReceivers_ == numReceivers); deRegisterWdtNamespace("test-namespace-1"); ASSERT_TRUE(numSenders_ == numSenders - numObjectsPerNamespace); ASSERT_TRUE(numReceivers_ == numReceivers - numObjectsPerNamespace); }
void Sender::startNewTransfer() { if (throttler_) { throttler_->registerTransfer(); } LOG(INFO) << "Starting a new transfer " << getTransferId() << " to " << destHost_; }
void WdtResourceControllerTest::AddObjectsWithNoLimitsTest() { string wdtNamespace = "test-namespace-1"; registerWdtNamespace(wdtNamespace); int index = 0; string transferPrefix = "add-objects-transfer"; for (; index < 3; index++) { auto transferRequest = makeTransferRequest(getTransferId(transferPrefix, index)); ReceiverPtr receiverPtr; ErrorCode code = createReceiver(wdtNamespace, transferRequest.transferId, transferRequest, receiverPtr); ASSERT_TRUE(code == OK); ASSERT_TRUE(receiverPtr != nullptr); SenderPtr senderPtr; code = createSender(wdtNamespace, transferRequest.transferId, transferRequest, senderPtr); ASSERT_TRUE(code == OK); ASSERT_TRUE(senderPtr != nullptr); if (index == 1) { SenderPtr oldSender; code = createSender(wdtNamespace, transferRequest.transferId, transferRequest, oldSender); EXPECT_EQ(code, ALREADY_EXISTS); EXPECT_EQ(senderPtr, oldSender); ReceiverPtr oldReceiver; code = createReceiver(wdtNamespace, transferRequest.transferId, transferRequest, oldReceiver); EXPECT_EQ(code, ALREADY_EXISTS); EXPECT_EQ(receiverPtr, oldReceiver); } } EXPECT_EQ(getAllReceivers(wdtNamespace).size(), index); EXPECT_EQ(getAllSenders(wdtNamespace).size(), index); int numSenders = index; int numReceivers = index; ErrorCode code = releaseSender(wdtNamespace, getTransferId(transferPrefix, 0)); ASSERT_TRUE(code == OK); code = releaseReceiver(wdtNamespace, getTransferId(transferPrefix, 0)); ASSERT_TRUE(code == OK); EXPECT_EQ(numSenders_, numSenders - 1); EXPECT_EQ(numReceivers_, numReceivers - 1); }
void Sender::endCurTransfer() { endTime_ = Clock::now(); LOG(INFO) << "Last thread finished " << durationSeconds(endTime_ - startTime_) << " for transfer id " << getTransferId(); setTransferStatus(FINISHED); if (throttler_) { throttler_->deRegisterTransfer(); } }
// Set the file transfer process identifier into database void TransferExec::updatePid(const int& pid) const { setProcessId(pid); std::string query = (boost::format("UPDATE filetransfer SET processid=%1%" " WHERE transferId='%2%';") %vishnu::convertToString(pid) %getTransferId()).str(); FileTransferServer::getDatabaseInstance()->process(query); }
// TODO: argghhhh Sender::Sender(const WdtTransferRequest &transferRequest) : Sender(transferRequest.hostName, transferRequest.directory, transferRequest.ports, transferRequest.fileInfo, transferRequest.disableDirectoryTraversal) { transferRequest_ = transferRequest; if (getTransferId().empty()) { LOG(WARNING) << "Sender without transferId... will likely fail to connect"; } // TODO: use transferRequest_ setProtocolVersion(transferRequest.protocolVersion); }
void WdtResourceControllerTest::ReleaseStaleTest() { int maxNamespaces = 2; string transferPrefix = "add-objects-limit-transfer"; for (int namespaceNum = 1; namespaceNum <= maxNamespaces; namespaceNum++) { string wdtNamespace = "test-namespace-" + to_string(namespaceNum); registerWdtNamespace(wdtNamespace); } int index = 0; { string wdtNamespace = "test-namespace-1"; auto transferRequest = makeTransferRequest(getTransferId(transferPrefix, index)); ReceiverPtr receiverPtr; ErrorCode code = createReceiver(wdtNamespace, transferRequest.transferId, transferRequest, receiverPtr); ASSERT_TRUE(code == OK); ASSERT_TRUE(receiverPtr != nullptr); SenderPtr senderPtr; code = createSender(wdtNamespace, transferRequest.transferId, transferRequest, senderPtr); ASSERT_TRUE(code == OK); ASSERT_TRUE(senderPtr != nullptr); index++; } ASSERT_TRUE(numSenders_ == 1); ASSERT_TRUE(numReceivers_ == 1); releaseAllSenders("test-namespace-1"); releaseAllReceivers("test-namespace-1"); ASSERT_TRUE(numSenders_ == 0); ASSERT_TRUE(numReceivers_ == 0); { string wdtNamespace = "test-namespace-1"; auto transferRequest = makeTransferRequest(getTransferId(transferPrefix, index)); ReceiverPtr receiverPtr; ErrorCode code = createReceiver(wdtNamespace, transferRequest.transferId, transferRequest, receiverPtr); ASSERT_TRUE(code == OK); ASSERT_TRUE(receiverPtr != nullptr); SenderPtr senderPtr; code = createSender(wdtNamespace, transferRequest.transferId, transferRequest, senderPtr); ASSERT_TRUE(code == OK); ASSERT_TRUE(senderPtr != nullptr); senderPtr = getSender(wdtNamespace, getTransferId(transferPrefix, index)); receiverPtr = getReceiver(wdtNamespace, getTransferId(transferPrefix, index)); ASSERT_TRUE(senderPtr != nullptr); ASSERT_TRUE(receiverPtr != nullptr); code = releaseReceiver(wdtNamespace, getTransferId(transferPrefix, index)); EXPECT_EQ(code, OK); code = releaseReceiver(wdtNamespace, getTransferId(transferPrefix, index)); ASSERT_TRUE(code != OK); index++; } }
void WdtResourceControllerTest::InvalidNamespaceTest() { int index = 0; string wdtNamespace = "test-namespace-1"; string transferPrefix = "invalid-namespace"; requireRegistration(true); auto transferRequest = makeTransferRequest(getTransferId(transferPrefix, index)); ReceiverPtr receiverPtr; ErrorCode code = createReceiver(wdtNamespace, transferRequest.transferId, transferRequest, receiverPtr); EXPECT_EQ(NOT_FOUND, code); /// Receiver should not be added ASSERT_TRUE(receiverPtr == nullptr); EXPECT_EQ(deRegisterWdtNamespace(wdtNamespace), ERROR); }
void Receiver::endCurGlobalSession() { setTransferStatus(FINISHED); if (!hasNewTransferStarted_) { WLOG(WARNING) << "WDT transfer did not start, no need to end session"; return; } WLOG(INFO) << "Ending the transfer " << getTransferId(); if (throttler_) { throttler_->deRegisterTransfer(); } checkpoints_.clear(); if (fileCreator_) { fileCreator_->clearAllocationMap(); } // TODO might consider moving closing the transfer log here hasNewTransferStarted_.store(false); }
void Receiver::startNewGlobalSession(const std::string &peerIp) { if (throttler_) { // If throttler is configured/set then register this session // in the throttler. This is guranteed to work in either of the // modes long running or not. We will de register from the throttler // when the current session ends throttler_->registerTransfer(); } startTime_ = Clock::now(); if (options_.enable_download_resumption) { transferLogManager_.startThread(); bool verifySuccessful = transferLogManager_.verifySenderIp(peerIp); if (!verifySuccessful) { fileChunksInfo_.clear(); } } hasNewTransferStarted_.store(true); WLOG(INFO) << "Starting new transfer, peerIp " << peerIp << " , transfer id " << getTransferId(); }
void WdtResourceControllerTest::AddObjectsWithLimitsTest() { int maxNamespaces = 2; string transferPrefix = "add-objects-limit-transfer"; for (int namespaceNum = 1; namespaceNum <= maxNamespaces; namespaceNum++) { string wdtNamespace = "test-namespace-" + to_string(namespaceNum); registerWdtNamespace(wdtNamespace); } int index = 0; { string wdtNamespace = "test-namespace-1"; auto transferRequest = makeTransferRequest(getTransferId(transferPrefix, index)); ReceiverPtr receiverPtr; ErrorCode code = createReceiver(wdtNamespace, transferRequest.transferId, transferRequest, receiverPtr); ASSERT_TRUE(code == OK); ASSERT_TRUE(receiverPtr != nullptr); SenderPtr senderPtr; code = createSender(wdtNamespace, transferRequest.transferId, transferRequest, senderPtr); ASSERT_TRUE(code == OK); ASSERT_TRUE(senderPtr != nullptr); index++; } { string wdtNamespace = "test-namespace-1"; auto transferRequest = makeTransferRequest(getTransferId(transferPrefix, index)); ReceiverPtr receiverPtr; ErrorCode code = createReceiver(wdtNamespace, transferRequest.transferId, transferRequest, receiverPtr); ASSERT_TRUE(code == QUOTA_EXCEEDED); ASSERT_TRUE(receiverPtr == nullptr); SenderPtr senderPtr; code = createSender(wdtNamespace, transferRequest.transferId, transferRequest, senderPtr); ASSERT_TRUE(code == OK); ASSERT_TRUE(senderPtr != nullptr); index++; } { string wdtNamespace = "test-namespace-2"; auto transferRequest = makeTransferRequest(getTransferId(transferPrefix, index)); ReceiverPtr receiverPtr; ErrorCode code = createReceiver(wdtNamespace, transferRequest.transferId, transferRequest, receiverPtr); ASSERT_TRUE(code == OK); ASSERT_TRUE(receiverPtr != nullptr); SenderPtr senderPtr; code = createSender(wdtNamespace, transferRequest.transferId, transferRequest, senderPtr); ASSERT_TRUE(code == QUOTA_EXCEEDED); ASSERT_TRUE(senderPtr == nullptr); index++; } deRegisterWdtNamespace("test-namespace-1"); { string wdtNamespace = "test-namespace-2"; auto transferRequest = makeTransferRequest(getTransferId(transferPrefix, index)); ReceiverPtr receiverPtr; ErrorCode code = createReceiver(wdtNamespace, transferRequest.transferId, transferRequest, receiverPtr); ASSERT_TRUE(code == QUOTA_EXCEEDED); ASSERT_TRUE(receiverPtr == nullptr); SenderPtr senderPtr; code = createSender(wdtNamespace, transferRequest.transferId, transferRequest, senderPtr); ASSERT_TRUE(code == OK); ASSERT_TRUE(senderPtr != nullptr); index++; } }
const WdtTransferRequest &Receiver::init() { if (validateTransferRequest() != OK) { WLOG(ERROR) << "Couldn't validate the transfer request " << transferRequest_.getLogSafeString(); return transferRequest_; } checkAndUpdateBufferSize(); backlog_ = options_.backlog; if (getTransferId().empty()) { setTransferId(WdtBase::generateTransferId()); } setProtocolVersion(transferRequest_.protocolVersion); setDir(transferRequest_.directory); auto numThreads = transferRequest_.ports.size(); // This creates the destination directory (which is needed for transferLogMgr) fileCreator_.reset(new FileCreator(destDir_, numThreads, transferLogManager_, options_.skip_writes)); // Make sure we can get the lock on the transfer log manager early // so if we can't we don't generate a valid but useless url and end up // starting a sender doomed to fail if (options_.enable_download_resumption) { WDT_CHECK(!options_.skip_writes) << "Can not skip transfers with download resumption turned on"; if (options_.resume_using_dir_tree) { WDT_CHECK(!options_.shouldPreallocateFiles()) << "Can not resume using directory tree if preallocation is enabled"; } ErrorCode errCode = transferLogManager_.openLog(); if (errCode != OK) { WLOG(ERROR) << "Failed to open transfer log " << errorCodeToStr(errCode); transferRequest_.errorCode = errCode; return transferRequest_; } ErrorCode code = transferLogManager_.parseAndMatch( recoveryId_, getTransferConfig(), fileChunksInfo_); if (code == OK && options_.resume_using_dir_tree) { WDT_CHECK(fileChunksInfo_.empty()); traverseDestinationDir(fileChunksInfo_); } } EncryptionType encryptionType = parseEncryptionType(options_.encryption_type); // is encryption enabled? bool encrypt = (encryptionType != ENC_NONE && protocolVersion_ >= Protocol::ENCRYPTION_V1_VERSION); if (encrypt) { WLOG(INFO) << encryptionTypeToStr(encryptionType) << " encryption is enabled for this transfer "; if (!transferRequest_.encryptionData.isSet()) { WLOG(INFO) << "Receiver generating encryption key for type " << encryptionTypeToStr(encryptionType); transferRequest_.encryptionData = EncryptionParams::generateEncryptionParams(encryptionType); } if (!transferRequest_.encryptionData.isSet()) { WLOG(ERROR) << "Unable to generate encryption key for type " << encryptionTypeToStr(encryptionType); transferRequest_.errorCode = ENCRYPTION_ERROR; return transferRequest_; } } else { if (encryptionType != ENC_NONE) { WLOG(WARNING) << "Encryption is enabled, but protocol version is " << protocolVersion_ << ", minimum version required for encryption is " << Protocol::ENCRYPTION_V1_VERSION; } transferRequest_.encryptionData.erase(); } threadsController_ = new ThreadsController(numThreads); threadsController_->setNumFunnels(ReceiverThread::NUM_FUNNELS); threadsController_->setNumBarriers(ReceiverThread::NUM_BARRIERS); threadsController_->setNumConditions(ReceiverThread::NUM_CONDITIONS); // TODO: take transferRequest directly ! receiverThreads_ = threadsController_->makeThreads<Receiver, ReceiverThread>( this, transferRequest_.ports.size(), transferRequest_.ports); size_t numSuccessfulInitThreads = 0; for (auto &receiverThread : receiverThreads_) { ErrorCode code = receiverThread->init(); if (code == OK) { ++numSuccessfulInitThreads; } } WLOG(INFO) << "Registered " << numSuccessfulInitThreads << " successful sockets"; ErrorCode code = OK; const size_t targetSize = transferRequest_.ports.size(); // TODO: replace with getNumPorts/thread if (numSuccessfulInitThreads != targetSize) { code = FEWER_PORTS; if (numSuccessfulInitThreads == 0) { code = ERROR; } } transferRequest_.protocolVersion = protocolVersion_; transferRequest_.ports.clear(); for (const auto &receiverThread : receiverThreads_) { transferRequest_.ports.push_back(receiverThread->getPort()); } if (transferRequest_.hostName.empty()) { char hostName[1024]; int ret = gethostname(hostName, sizeof(hostName)); if (ret == 0) { transferRequest_.hostName.assign(hostName); } else { PLOG(ERROR) << "Couldn't find the host name"; code = ERROR; } } transferRequest_.directory = getDir(); transferRequest_.errorCode = code; return transferRequest_; }