void WdtResourceControllerTest::MultipleNamespacesTest() {
int namespaceNum = 0;
int maxNamespaces = 5;
int numSenders = 0;
int numReceivers = 0;
int numObjectsPerNamespace = 3;
string transferPrefix = "add-objects-transfer";
for (; namespaceNum < maxNamespaces; namespaceNum++) {
string wdtNamespace = "test-namespace-" + to_string(namespaceNum);
registerWdtNamespace(wdtNamespace);
for (int index = 0; index < numObjectsPerNamespace; index++) {
auto transferRequest =
makeTransferRequest(getTransferId(transferPrefix, index));
ReceiverPtr receiverPtr;
ErrorCode code = createReceiver(wdtNamespace, transferRequest.transferId,
transferRequest, receiverPtr);
ASSERT_TRUE(code == OK);
ASSERT_TRUE(receiverPtr != nullptr);
SenderPtr senderPtr;
code = createSender(wdtNamespace, transferRequest.transferId,
transferRequest, senderPtr);
ASSERT_TRUE(senderPtr != nullptr);
ASSERT_TRUE(code == OK);
++numSenders;
++numReceivers;
}
}
ASSERT_TRUE(numSenders_ == numSenders);
ASSERT_TRUE(numReceivers_ == numReceivers);
deRegisterWdtNamespace("test-namespace-1");
ASSERT_TRUE(numSenders_ == numSenders - numObjectsPerNamespace);
ASSERT_TRUE(numReceivers_ == numReceivers - numObjectsPerNamespace);
}
void Sender::startNewTransfer() {
if (throttler_) {
throttler_->registerTransfer();
}
LOG(INFO) << "Starting a new transfer " << getTransferId() << " to "
<< destHost_;
}
void WdtResourceControllerTest::AddObjectsWithNoLimitsTest() {
string wdtNamespace = "test-namespace-1";
registerWdtNamespace(wdtNamespace);
int index = 0;
string transferPrefix = "add-objects-transfer";
for (; index < 3; index++) {
auto transferRequest =
makeTransferRequest(getTransferId(transferPrefix, index));
ReceiverPtr receiverPtr;
ErrorCode code = createReceiver(wdtNamespace, transferRequest.transferId,
transferRequest, receiverPtr);
ASSERT_TRUE(code == OK);
ASSERT_TRUE(receiverPtr != nullptr);
SenderPtr senderPtr;
code = createSender(wdtNamespace, transferRequest.transferId,
transferRequest, senderPtr);
ASSERT_TRUE(code == OK);
ASSERT_TRUE(senderPtr != nullptr);
if (index == 1) {
SenderPtr oldSender;
code = createSender(wdtNamespace, transferRequest.transferId,
transferRequest, oldSender);
EXPECT_EQ(code, ALREADY_EXISTS);
EXPECT_EQ(senderPtr, oldSender);
ReceiverPtr oldReceiver;
code = createReceiver(wdtNamespace, transferRequest.transferId,
transferRequest, oldReceiver);
EXPECT_EQ(code, ALREADY_EXISTS);
EXPECT_EQ(receiverPtr, oldReceiver);
}
}
EXPECT_EQ(getAllReceivers(wdtNamespace).size(), index);
EXPECT_EQ(getAllSenders(wdtNamespace).size(), index);
int numSenders = index;
int numReceivers = index;
ErrorCode code =
releaseSender(wdtNamespace, getTransferId(transferPrefix, 0));
ASSERT_TRUE(code == OK);
code = releaseReceiver(wdtNamespace, getTransferId(transferPrefix, 0));
ASSERT_TRUE(code == OK);
EXPECT_EQ(numSenders_, numSenders - 1);
EXPECT_EQ(numReceivers_, numReceivers - 1);
}
void Sender::endCurTransfer() {
endTime_ = Clock::now();
LOG(INFO) << "Last thread finished " << durationSeconds(endTime_ - startTime_)
<< " for transfer id " << getTransferId();
setTransferStatus(FINISHED);
if (throttler_) {
throttler_->deRegisterTransfer();
}
}
// Set the file transfer process identifier into database
void
TransferExec::updatePid(const int& pid) const {
setProcessId(pid);
std::string query = (boost::format("UPDATE filetransfer SET processid=%1%"
" WHERE transferId='%2%';")
%vishnu::convertToString(pid)
%getTransferId()).str();
FileTransferServer::getDatabaseInstance()->process(query);
}
// TODO: argghhhh
Sender::Sender(const WdtTransferRequest &transferRequest)
: Sender(transferRequest.hostName, transferRequest.directory,
transferRequest.ports, transferRequest.fileInfo,
transferRequest.disableDirectoryTraversal) {
transferRequest_ = transferRequest;
if (getTransferId().empty()) {
LOG(WARNING) << "Sender without transferId... will likely fail to connect";
}
// TODO: use transferRequest_
setProtocolVersion(transferRequest.protocolVersion);
}
void WdtResourceControllerTest::ReleaseStaleTest() {
int maxNamespaces = 2;
string transferPrefix = "add-objects-limit-transfer";
for (int namespaceNum = 1; namespaceNum <= maxNamespaces; namespaceNum++) {
string wdtNamespace = "test-namespace-" + to_string(namespaceNum);
registerWdtNamespace(wdtNamespace);
}
int index = 0;
{
string wdtNamespace = "test-namespace-1";
auto transferRequest =
makeTransferRequest(getTransferId(transferPrefix, index));
ReceiverPtr receiverPtr;
ErrorCode code = createReceiver(wdtNamespace, transferRequest.transferId,
transferRequest, receiverPtr);
ASSERT_TRUE(code == OK);
ASSERT_TRUE(receiverPtr != nullptr);
SenderPtr senderPtr;
code = createSender(wdtNamespace, transferRequest.transferId,
transferRequest, senderPtr);
ASSERT_TRUE(code == OK);
ASSERT_TRUE(senderPtr != nullptr);
index++;
}
ASSERT_TRUE(numSenders_ == 1);
ASSERT_TRUE(numReceivers_ == 1);
releaseAllSenders("test-namespace-1");
releaseAllReceivers("test-namespace-1");
ASSERT_TRUE(numSenders_ == 0);
ASSERT_TRUE(numReceivers_ == 0);
{
string wdtNamespace = "test-namespace-1";
auto transferRequest =
makeTransferRequest(getTransferId(transferPrefix, index));
ReceiverPtr receiverPtr;
ErrorCode code = createReceiver(wdtNamespace, transferRequest.transferId,
transferRequest, receiverPtr);
ASSERT_TRUE(code == OK);
ASSERT_TRUE(receiverPtr != nullptr);
SenderPtr senderPtr;
code = createSender(wdtNamespace, transferRequest.transferId,
transferRequest, senderPtr);
ASSERT_TRUE(code == OK);
ASSERT_TRUE(senderPtr != nullptr);
senderPtr = getSender(wdtNamespace, getTransferId(transferPrefix, index));
receiverPtr =
getReceiver(wdtNamespace, getTransferId(transferPrefix, index));
ASSERT_TRUE(senderPtr != nullptr);
ASSERT_TRUE(receiverPtr != nullptr);
code = releaseReceiver(wdtNamespace, getTransferId(transferPrefix, index));
EXPECT_EQ(code, OK);
code = releaseReceiver(wdtNamespace, getTransferId(transferPrefix, index));
ASSERT_TRUE(code != OK);
index++;
}
}
void WdtResourceControllerTest::InvalidNamespaceTest() {
int index = 0;
string wdtNamespace = "test-namespace-1";
string transferPrefix = "invalid-namespace";
requireRegistration(true);
auto transferRequest =
makeTransferRequest(getTransferId(transferPrefix, index));
ReceiverPtr receiverPtr;
ErrorCode code = createReceiver(wdtNamespace, transferRequest.transferId,
transferRequest, receiverPtr);
EXPECT_EQ(NOT_FOUND, code);
/// Receiver should not be added
ASSERT_TRUE(receiverPtr == nullptr);
EXPECT_EQ(deRegisterWdtNamespace(wdtNamespace), ERROR);
}
void Receiver::endCurGlobalSession() {
setTransferStatus(FINISHED);
if (!hasNewTransferStarted_) {
WLOG(WARNING) << "WDT transfer did not start, no need to end session";
return;
}
WLOG(INFO) << "Ending the transfer " << getTransferId();
if (throttler_) {
throttler_->deRegisterTransfer();
}
checkpoints_.clear();
if (fileCreator_) {
fileCreator_->clearAllocationMap();
}
// TODO might consider moving closing the transfer log here
hasNewTransferStarted_.store(false);
}
void Receiver::startNewGlobalSession(const std::string &peerIp) {
if (throttler_) {
// If throttler is configured/set then register this session
// in the throttler. This is guranteed to work in either of the
// modes long running or not. We will de register from the throttler
// when the current session ends
throttler_->registerTransfer();
}
startTime_ = Clock::now();
if (options_.enable_download_resumption) {
transferLogManager_.startThread();
bool verifySuccessful = transferLogManager_.verifySenderIp(peerIp);
if (!verifySuccessful) {
fileChunksInfo_.clear();
}
}
hasNewTransferStarted_.store(true);
WLOG(INFO) << "Starting new transfer, peerIp " << peerIp << " , transfer id "
<< getTransferId();
}
void WdtResourceControllerTest::AddObjectsWithLimitsTest() {
int maxNamespaces = 2;
string transferPrefix = "add-objects-limit-transfer";
for (int namespaceNum = 1; namespaceNum <= maxNamespaces; namespaceNum++) {
string wdtNamespace = "test-namespace-" + to_string(namespaceNum);
registerWdtNamespace(wdtNamespace);
}
int index = 0;
{
string wdtNamespace = "test-namespace-1";
auto transferRequest =
makeTransferRequest(getTransferId(transferPrefix, index));
ReceiverPtr receiverPtr;
ErrorCode code = createReceiver(wdtNamespace, transferRequest.transferId,
transferRequest, receiverPtr);
ASSERT_TRUE(code == OK);
ASSERT_TRUE(receiverPtr != nullptr);
SenderPtr senderPtr;
code = createSender(wdtNamespace, transferRequest.transferId,
transferRequest, senderPtr);
ASSERT_TRUE(code == OK);
ASSERT_TRUE(senderPtr != nullptr);
index++;
}
{
string wdtNamespace = "test-namespace-1";
auto transferRequest =
makeTransferRequest(getTransferId(transferPrefix, index));
ReceiverPtr receiverPtr;
ErrorCode code = createReceiver(wdtNamespace, transferRequest.transferId,
transferRequest, receiverPtr);
ASSERT_TRUE(code == QUOTA_EXCEEDED);
ASSERT_TRUE(receiverPtr == nullptr);
SenderPtr senderPtr;
code = createSender(wdtNamespace, transferRequest.transferId,
transferRequest, senderPtr);
ASSERT_TRUE(code == OK);
ASSERT_TRUE(senderPtr != nullptr);
index++;
}
{
string wdtNamespace = "test-namespace-2";
auto transferRequest =
makeTransferRequest(getTransferId(transferPrefix, index));
ReceiverPtr receiverPtr;
ErrorCode code = createReceiver(wdtNamespace, transferRequest.transferId,
transferRequest, receiverPtr);
ASSERT_TRUE(code == OK);
ASSERT_TRUE(receiverPtr != nullptr);
SenderPtr senderPtr;
code = createSender(wdtNamespace, transferRequest.transferId,
transferRequest, senderPtr);
ASSERT_TRUE(code == QUOTA_EXCEEDED);
ASSERT_TRUE(senderPtr == nullptr);
index++;
}
deRegisterWdtNamespace("test-namespace-1");
{
string wdtNamespace = "test-namespace-2";
auto transferRequest =
makeTransferRequest(getTransferId(transferPrefix, index));
ReceiverPtr receiverPtr;
ErrorCode code = createReceiver(wdtNamespace, transferRequest.transferId,
transferRequest, receiverPtr);
ASSERT_TRUE(code == QUOTA_EXCEEDED);
ASSERT_TRUE(receiverPtr == nullptr);
SenderPtr senderPtr;
code = createSender(wdtNamespace, transferRequest.transferId,
transferRequest, senderPtr);
ASSERT_TRUE(code == OK);
ASSERT_TRUE(senderPtr != nullptr);
index++;
}
}
const WdtTransferRequest &Receiver::init() {
if (validateTransferRequest() != OK) {
WLOG(ERROR) << "Couldn't validate the transfer request "
<< transferRequest_.getLogSafeString();
return transferRequest_;
}
checkAndUpdateBufferSize();
backlog_ = options_.backlog;
if (getTransferId().empty()) {
setTransferId(WdtBase::generateTransferId());
}
setProtocolVersion(transferRequest_.protocolVersion);
setDir(transferRequest_.directory);
auto numThreads = transferRequest_.ports.size();
// This creates the destination directory (which is needed for transferLogMgr)
fileCreator_.reset(new FileCreator(destDir_, numThreads, transferLogManager_,
options_.skip_writes));
// Make sure we can get the lock on the transfer log manager early
// so if we can't we don't generate a valid but useless url and end up
// starting a sender doomed to fail
if (options_.enable_download_resumption) {
WDT_CHECK(!options_.skip_writes)
<< "Can not skip transfers with download resumption turned on";
if (options_.resume_using_dir_tree) {
WDT_CHECK(!options_.shouldPreallocateFiles())
<< "Can not resume using directory tree if preallocation is enabled";
}
ErrorCode errCode = transferLogManager_.openLog();
if (errCode != OK) {
WLOG(ERROR) << "Failed to open transfer log " << errorCodeToStr(errCode);
transferRequest_.errorCode = errCode;
return transferRequest_;
}
ErrorCode code = transferLogManager_.parseAndMatch(
recoveryId_, getTransferConfig(), fileChunksInfo_);
if (code == OK && options_.resume_using_dir_tree) {
WDT_CHECK(fileChunksInfo_.empty());
traverseDestinationDir(fileChunksInfo_);
}
}
EncryptionType encryptionType = parseEncryptionType(options_.encryption_type);
// is encryption enabled?
bool encrypt = (encryptionType != ENC_NONE &&
protocolVersion_ >= Protocol::ENCRYPTION_V1_VERSION);
if (encrypt) {
WLOG(INFO) << encryptionTypeToStr(encryptionType)
<< " encryption is enabled for this transfer ";
if (!transferRequest_.encryptionData.isSet()) {
WLOG(INFO) << "Receiver generating encryption key for type "
<< encryptionTypeToStr(encryptionType);
transferRequest_.encryptionData =
EncryptionParams::generateEncryptionParams(encryptionType);
}
if (!transferRequest_.encryptionData.isSet()) {
WLOG(ERROR) << "Unable to generate encryption key for type "
<< encryptionTypeToStr(encryptionType);
transferRequest_.errorCode = ENCRYPTION_ERROR;
return transferRequest_;
}
} else {
if (encryptionType != ENC_NONE) {
WLOG(WARNING) << "Encryption is enabled, but protocol version is "
<< protocolVersion_
<< ", minimum version required for encryption is "
<< Protocol::ENCRYPTION_V1_VERSION;
}
transferRequest_.encryptionData.erase();
}
threadsController_ = new ThreadsController(numThreads);
threadsController_->setNumFunnels(ReceiverThread::NUM_FUNNELS);
threadsController_->setNumBarriers(ReceiverThread::NUM_BARRIERS);
threadsController_->setNumConditions(ReceiverThread::NUM_CONDITIONS);
// TODO: take transferRequest directly !
receiverThreads_ = threadsController_->makeThreads<Receiver, ReceiverThread>(
this, transferRequest_.ports.size(), transferRequest_.ports);
size_t numSuccessfulInitThreads = 0;
for (auto &receiverThread : receiverThreads_) {
ErrorCode code = receiverThread->init();
if (code == OK) {
++numSuccessfulInitThreads;
}
}
WLOG(INFO) << "Registered " << numSuccessfulInitThreads
<< " successful sockets";
ErrorCode code = OK;
const size_t targetSize = transferRequest_.ports.size();
// TODO: replace with getNumPorts/thread
if (numSuccessfulInitThreads != targetSize) {
code = FEWER_PORTS;
if (numSuccessfulInitThreads == 0) {
code = ERROR;
}
}
transferRequest_.protocolVersion = protocolVersion_;
transferRequest_.ports.clear();
for (const auto &receiverThread : receiverThreads_) {
transferRequest_.ports.push_back(receiverThread->getPort());
}
if (transferRequest_.hostName.empty()) {
char hostName[1024];
int ret = gethostname(hostName, sizeof(hostName));
if (ret == 0) {
transferRequest_.hostName.assign(hostName);
} else {
PLOG(ERROR) << "Couldn't find the host name";
code = ERROR;
}
}
transferRequest_.directory = getDir();
transferRequest_.errorCode = code;
return transferRequest_;
}
