int main(int argc, char *argv[]) {
FLAGS_logtostderr = true;
// Ugliness in gflags' api; to be able to use program name
google::SetArgv(argc, const_cast<const char **>(argv));
google::SetVersionString(Protocol::getFullVersion());
std::string usage("WDT Warp-speed Data Transfer. v ");
usage.append(google::VersionString());
usage.append(". Sample usage:\n\t");
usage.append(google::ProgramInvocationShortName());
usage.append(" # for a server/receiver\n\t");
usage.append(google::ProgramInvocationShortName());
usage.append(" -connection_url url_produced_by_receiver # for a sender");
google::SetUsageMessage(usage);
google::gflags_exitfunc = [](int code) {
if (FLAGS_exit_on_bad_flags) {
exit(code);
}
badGflagFound = true;
};
google::ParseCommandLineFlags(&argc, &argv, true);
google::InitGoogleLogging(argv[0]);
if (badGflagFound) {
LOG(ERROR) << "Continuing despite bad flags";
}
signal(SIGPIPE, SIG_IGN);
FLAGS::initializeFromFlags();
if (FLAGS_print_options) {
FLAGS::printOptions(std::cout);
return 0;
}
ErrorCode retCode = OK;
// Odd ball case of log parsing
if (FLAGS_parse_transfer_log) {
// Log parsing mode
WdtOptions::getMutable().enable_download_resumption = true;
TransferLogManager transferLogManager;
transferLogManager.setRootDir(FLAGS_directory);
transferLogManager.openLog();
bool success = transferLogManager.parseAndPrint();
LOG_IF(ERROR, success) << "Transfer log parsing failed";
transferLogManager.closeLog();
return success ? OK : ERROR;
}
// General case : Sender or Receiver
const auto &options = WdtOptions::get();
std::unique_ptr<WdtTransferRequest> reqPtr;
if (FLAGS_connection_url.empty()) {
reqPtr = folly::make_unique<WdtTransferRequest>(
options.start_port, options.num_ports, FLAGS_directory);
reqPtr->hostName = FLAGS_destination;
reqPtr->transferId = FLAGS_transfer_id;
} else {
LOG(INFO) << "Input url: " << FLAGS_connection_url;
reqPtr = folly::make_unique<WdtTransferRequest>(FLAGS_connection_url);
if (reqPtr->errorCode != OK) {
LOG(ERROR) << "Invalid url " << errorCodeToStr(reqPtr->errorCode);
return ERROR;
}
reqPtr->directory = FLAGS_directory;
}
WdtTransferRequest &req = *reqPtr;
if (FLAGS_protocol_version > 0) {
req.protocolVersion = FLAGS_protocol_version;
}
if (FLAGS_destination.empty() && FLAGS_connection_url.empty()) {
Receiver receiver(req);
WdtTransferRequest augmentedReq = receiver.init();
if (FLAGS_treat_fewer_port_as_error &&
augmentedReq.errorCode == FEWER_PORTS) {
LOG(ERROR) << "Receiver could not bind to all the ports";
return FEWER_PORTS;
}
if (augmentedReq.errorCode == ERROR) {
LOG(ERROR) << "Error setting up receiver";
return ERROR;
}
LOG(INFO) << "Starting receiver with connection url ";
std::cout << augmentedReq.generateUrl() << std::endl;
std::cout.flush();
setUpAbort(receiver);
if (!FLAGS_recovery_id.empty()) {
WdtOptions::getMutable().enable_download_resumption = true;
receiver.setRecoveryId(FLAGS_recovery_id);
}
if (!FLAGS_run_as_daemon) {
retCode = receiver.transferAsync();
if (retCode == OK) {
std::unique_ptr<TransferReport> report = receiver.finish();
retCode = report->getSummary().getErrorCode();
}
} else {
retCode = receiver.runForever();
// not reached
}
} else {
// Sender mode
if (!FLAGS_manifest.empty()) {
// Each line should have the filename and optionally
// the filesize separated by a single space
if (FLAGS_manifest == "-") {
readManifest(std::cin, req);
} else {
std::ifstream fin(FLAGS_manifest);
readManifest(fin, req);
fin.close();
}
LOG(INFO) << "Using files lists, number of files " << req.fileInfo.size();
}
Sender sender(req);
WdtTransferRequest processedRequest = sender.init();
LOG(INFO) << "Starting sender with details "
<< processedRequest.generateUrl(true);
ADDITIONAL_SENDER_SETUP
setUpAbort(sender);
std::unique_ptr<TransferReport> report = sender.transfer();
retCode = report->getSummary().getErrorCode();
}
cancelAbort();
LOG(INFO) << "Returning with code " << retCode << " "
<< errorCodeToStr(retCode);
return retCode;
}
int main(int argc, char *argv[]) {
FLAGS_logtostderr = true;
// Ugliness in gflags' api; to be able to use program name
google::SetArgv(argc, const_cast<const char **>(argv));
google::SetVersionString(Protocol::getFullVersion());
std::string usage("WDT Warp-speed Data Transfer. v ");
usage.append(google::VersionString());
usage.append(". Sample usage:\n\t");
usage.append(google::ProgramInvocationShortName());
usage.append(" # for a server/receiver\n\t");
usage.append(google::ProgramInvocationShortName());
usage.append(" -destination host # for a sender");
google::SetUsageMessage(usage);
google::ParseCommandLineFlags(&argc, &argv, true);
google::InitGoogleLogging(argv[0]);
signal(SIGPIPE, SIG_IGN);
#define STANDALONE_APP
#define ASSIGN_OPT
#include FLAGS_INCLUDE_FILE //nolint
LOG(INFO) << "Running WDT " << Protocol::getFullVersion();
ErrorCode retCode = OK;
// Odd ball case of log parsing
if (FLAGS_parse_transfer_log) {
// Log parsing mode
TransferLogManager transferLogManager;
transferLogManager.setRootDir(FLAGS_directory);
if (!transferLogManager.parseAndPrint()) {
LOG(ERROR) << "Transfer log parsing failed";
return ERROR;
}
return OK;
}
// General case : Sender or Receiver
const auto &options = WdtOptions::get();
WdtTransferRequest req(options.start_port, options.num_ports,
FLAGS_directory);
req.transferId = FLAGS_transfer_id;
if (FLAGS_protocol_version > 0) {
req.protocolVersion = FLAGS_protocol_version;
}
if (FLAGS_destination.empty() && FLAGS_connection_url.empty()) {
Receiver receiver(req);
WdtTransferRequest augmentedReq = receiver.init();
if (augmentedReq.errorCode == ERROR) {
LOG(ERROR) << "Error setting up receiver";
return augmentedReq.errorCode;
}
LOG(INFO) << "Starting receiver with connection url ";
std::cout << augmentedReq.generateUrl() << std::endl;
std::cout.flush();
setUpAbort(receiver);
if (!FLAGS_recovery_id.empty()) {
WdtOptions::getMutable().enable_download_resumption = true;
receiver.setRecoveryId(FLAGS_recovery_id);
}
if (!FLAGS_run_as_daemon) {
receiver.transferAsync();
std::unique_ptr<TransferReport> report = receiver.finish();
retCode = report->getSummary().getErrorCode();
} else {
retCode = receiver.runForever();
// not reached
}
} else {
// Sender mode
std::vector<FileInfo> fileInfo;
if (FLAGS_files) {
// Each line should have the filename and optionally
// the filesize separated by a single space
std::string line;
while (std::getline(std::cin, line)) {
std::vector<std::string> fields;
folly::split('\t', line, fields, true);
if (fields.empty() || fields.size() > 2) {
LOG(FATAL) << "Invalid input in stdin: " << line;
}
int64_t filesize =
fields.size() > 1 ? folly::to<int64_t>(fields[1]) : -1;
fileInfo.emplace_back(fields[0], filesize);
}
}
req.hostName = FLAGS_destination;
if (!FLAGS_connection_url.empty()) {
LOG(INFO) << "Input url: " << FLAGS_connection_url;
// TODO: merge instead
req = WdtTransferRequest(FLAGS_connection_url);
req.directory = FLAGS_directory; // re-set it for now
}
Sender sender(req);
WdtTransferRequest processedRequest = sender.init();
LOG(INFO) << "Starting sender with details "
<< processedRequest.generateUrl(true);
ADDITIONAL_SENDER_SETUP
setUpAbort(sender);
sender.setIncludeRegex(FLAGS_include_regex);
sender.setExcludeRegex(FLAGS_exclude_regex);
sender.setPruneDirRegex(FLAGS_prune_dir_regex);
std::unique_ptr<TransferReport> report = sender.transfer();
retCode = report->getSummary().getErrorCode();
}
cancelAbort();
return retCode;
}
int main(int argc, char *argv[]) {
FLAGS_logtostderr = true;
// Ugliness in gflags' api; to be able to use program name
google::SetArgv(argc, const_cast<const char **>(argv));
google::SetVersionString(Protocol::getFullVersion());
usage.assign("WDT Warp-speed Data Transfer. v ");
usage.append(google::VersionString());
usage.append(". Sample usage:\nTo transfer from srchost to desthost:\n\t");
usage.append("ssh dsthost ");
usage.append(google::ProgramInvocationShortName());
usage.append(" -directory destdir | ssh srchost ");
usage.append(google::ProgramInvocationShortName());
usage.append(" -directory srcdir -");
usage.append(
"\nPassing - as the argument to wdt means start the sender and"
" read the");
usage.append(
"\nconnection URL produced by the receiver, including encryption"
" key, from stdin.");
usage.append("\nUse --help to see all the options.");
google::SetUsageMessage(usage);
google::gflags_exitfunc = [](int code) {
if (code == 0 || FLAGS_help) {
// By default gflags exit 1 with --help and 0 for --version (good)
// let's also exit(0) for --help to be like most gnu command line
exit(0);
}
// error cases:
if (FLAGS_exit_on_bad_flags) {
printUsage();
exit(code);
}
badGflagFound = true;
};
google::ParseCommandLineFlags(&argc, &argv, true);
google::InitGoogleLogging(argv[0]);
if (badGflagFound) {
LOG(ERROR) << "Continuing despite bad flags";
}
// Only non -flag argument allowed so far is "-" meaning
// Read url from stdin and start a sender
if (argc > 2 || (argc == 2 && (argv[1][0] != '-' || argv[1][1] != '\0'))) {
printUsage();
std::cerr << "Error: argument should be - (to read url from stdin) "
<< "or no arguments" << std::endl;
exit(1);
}
signal(SIGPIPE, SIG_IGN);
std::string connectUrl;
if (argc == 2) {
std::getline(std::cin, connectUrl);
if (connectUrl.empty()) {
LOG(ERROR) << "Sender unable to read connection url from stdin - exiting";
return URI_PARSE_ERROR;
}
} else {
connectUrl = FLAGS_connection_url;
}
// Might be a sub class (fbonly wdtCmdLine.cpp)
Wdt &wdt = WDTCLASS::initializeWdt(FLAGS_app_name);
if (FLAGS_print_options) {
wdt.printWdtOptions(std::cout);
return 0;
}
WdtOptions &options = wdt.getWdtOptions();
ErrorCode retCode = OK;
// Odd ball case of log parsing
if (FLAGS_parse_transfer_log) {
// Log parsing mode
options.enable_download_resumption = true;
TransferLogManager transferLogManager(options);
transferLogManager.setRootDir(FLAGS_directory);
transferLogManager.openLog();
bool success = transferLogManager.parseAndPrint();
LOG_IF(ERROR, !success) << "Transfer log parsing failed";
transferLogManager.closeLog();
return success ? OK : ERROR;
}
// General case : Sender or Receiver
std::unique_ptr<WdtTransferRequest> reqPtr;
if (connectUrl.empty()) {
reqPtr = folly::make_unique<WdtTransferRequest>(
options.start_port, options.num_ports, FLAGS_directory);
reqPtr->hostName = FLAGS_destination;
reqPtr->transferId = FLAGS_transfer_id;
if (!FLAGS_test_only_encryption_secret.empty()) {
reqPtr->encryptionData =
EncryptionParams(parseEncryptionType(options.encryption_type),
FLAGS_test_only_encryption_secret);
}
} else {
reqPtr = folly::make_unique<WdtTransferRequest>(connectUrl);
if (reqPtr->errorCode != OK) {
LOG(ERROR) << "Invalid url \"" << connectUrl
<< "\" : " << errorCodeToStr(reqPtr->errorCode);
return ERROR;
}
reqPtr->directory = FLAGS_directory;
LOG(INFO) << "Parsed url as " << reqPtr->getLogSafeString();
}
WdtTransferRequest &req = *reqPtr;
if (FLAGS_protocol_version > 0) {
req.protocolVersion = FLAGS_protocol_version;
}
if (FLAGS_destination.empty() && connectUrl.empty()) {
Receiver receiver(req);
if (!FLAGS_recovery_id.empty()) {
options.enable_download_resumption = true;
receiver.setRecoveryId(FLAGS_recovery_id);
}
WdtTransferRequest augmentedReq = receiver.init();
retCode = augmentedReq.errorCode;
if (retCode == FEWER_PORTS) {
if (FLAGS_treat_fewer_port_as_error) {
LOG(ERROR) << "Receiver could not bind to all the ports";
return FEWER_PORTS;
}
retCode = OK;
} else if (augmentedReq.errorCode != OK) {
LOG(ERROR) << "Error setting up receiver " << errorCodeToStr(retCode);
return retCode;
}
// In the log:
LOG(INFO) << "Starting receiver with connection url "
<< augmentedReq.getLogSafeString(); // The url without secret
// on stdout: the one with secret:
std::cout << augmentedReq.genWdtUrlWithSecret() << std::endl;
std::cout.flush();
if (FLAGS_fork) {
pid_t cpid = fork();
if (cpid == -1) {
perror("Failed to fork()");
exit(1);
}
if (cpid > 0) {
LOG(INFO) << "Detaching receiver";
exit(0);
}
close(0);
close(1);
}
setAbortChecker(receiver);
if (!FLAGS_run_as_daemon) {
retCode = receiver.transferAsync();
if (retCode == OK) {
std::unique_ptr<TransferReport> report = receiver.finish();
retCode = report->getSummary().getErrorCode();
}
} else {
retCode = receiver.runForever();
// not reached
}
} else {
// Sender mode
if (!FLAGS_manifest.empty()) {
// Each line should have the filename and optionally
// the filesize separated by a single space
if (FLAGS_manifest == "-") {
readManifest(std::cin, req, options.odirect_reads);
} else {
std::ifstream fin(FLAGS_manifest);
readManifest(fin, req, options.odirect_reads);
fin.close();
}
LOG(INFO) << "Using files lists, number of files " << req.fileInfo.size();
}
LOG(INFO) << "Making Sender with encryption set = "
<< req.encryptionData.isSet();
// TODO: find something more useful for namespace (userid ? directory?)
// (shardid at fb)
retCode = wdt.wdtSend(WdtResourceController::kGlobalNamespace, req,
setupAbortChecker());
}
cancelAbort();
if (retCode == OK) {
LOG(INFO) << "Returning with OK exit code";
} else {
LOG(ERROR) << "Returning with code " << retCode << " "
<< errorCodeToStr(retCode);
}
return retCode;
}