int main(int argc, char *argv[]) { FLAGS_logtostderr = true; // Ugliness in gflags' api; to be able to use program name google::SetArgv(argc, const_cast<const char **>(argv)); google::SetVersionString(Protocol::getFullVersion()); std::string usage("WDT Warp-speed Data Transfer. v "); usage.append(google::VersionString()); usage.append(". Sample usage:\n\t"); usage.append(google::ProgramInvocationShortName()); usage.append(" # for a server/receiver\n\t"); usage.append(google::ProgramInvocationShortName()); usage.append(" -connection_url url_produced_by_receiver # for a sender"); google::SetUsageMessage(usage); google::gflags_exitfunc = [](int code) { if (FLAGS_exit_on_bad_flags) { exit(code); } badGflagFound = true; }; google::ParseCommandLineFlags(&argc, &argv, true); google::InitGoogleLogging(argv[0]); if (badGflagFound) { LOG(ERROR) << "Continuing despite bad flags"; } signal(SIGPIPE, SIG_IGN); FLAGS::initializeFromFlags(); if (FLAGS_print_options) { FLAGS::printOptions(std::cout); return 0; } ErrorCode retCode = OK; // Odd ball case of log parsing if (FLAGS_parse_transfer_log) { // Log parsing mode WdtOptions::getMutable().enable_download_resumption = true; TransferLogManager transferLogManager; transferLogManager.setRootDir(FLAGS_directory); transferLogManager.openLog(); bool success = transferLogManager.parseAndPrint(); LOG_IF(ERROR, success) << "Transfer log parsing failed"; transferLogManager.closeLog(); return success ? OK : ERROR; } // General case : Sender or Receiver const auto &options = WdtOptions::get(); std::unique_ptr<WdtTransferRequest> reqPtr; if (FLAGS_connection_url.empty()) { reqPtr = folly::make_unique<WdtTransferRequest>( options.start_port, options.num_ports, FLAGS_directory); reqPtr->hostName = FLAGS_destination; reqPtr->transferId = FLAGS_transfer_id; } else { LOG(INFO) << "Input url: " << FLAGS_connection_url; reqPtr = folly::make_unique<WdtTransferRequest>(FLAGS_connection_url); if (reqPtr->errorCode != OK) { LOG(ERROR) << "Invalid url " << errorCodeToStr(reqPtr->errorCode); return ERROR; } reqPtr->directory = FLAGS_directory; } WdtTransferRequest &req = *reqPtr; if (FLAGS_protocol_version > 0) { req.protocolVersion = FLAGS_protocol_version; } if (FLAGS_destination.empty() && FLAGS_connection_url.empty()) { Receiver receiver(req); WdtTransferRequest augmentedReq = receiver.init(); if (FLAGS_treat_fewer_port_as_error && augmentedReq.errorCode == FEWER_PORTS) { LOG(ERROR) << "Receiver could not bind to all the ports"; return FEWER_PORTS; } if (augmentedReq.errorCode == ERROR) { LOG(ERROR) << "Error setting up receiver"; return ERROR; } LOG(INFO) << "Starting receiver with connection url "; std::cout << augmentedReq.generateUrl() << std::endl; std::cout.flush(); setUpAbort(receiver); if (!FLAGS_recovery_id.empty()) { WdtOptions::getMutable().enable_download_resumption = true; receiver.setRecoveryId(FLAGS_recovery_id); } if (!FLAGS_run_as_daemon) { retCode = receiver.transferAsync(); if (retCode == OK) { std::unique_ptr<TransferReport> report = receiver.finish(); retCode = report->getSummary().getErrorCode(); } } else { retCode = receiver.runForever(); // not reached } } else { // Sender mode if (!FLAGS_manifest.empty()) { // Each line should have the filename and optionally // the filesize separated by a single space if (FLAGS_manifest == "-") { readManifest(std::cin, req); } else { std::ifstream fin(FLAGS_manifest); readManifest(fin, req); fin.close(); } LOG(INFO) << "Using files lists, number of files " << req.fileInfo.size(); } Sender sender(req); WdtTransferRequest processedRequest = sender.init(); LOG(INFO) << "Starting sender with details " << processedRequest.generateUrl(true); ADDITIONAL_SENDER_SETUP setUpAbort(sender); std::unique_ptr<TransferReport> report = sender.transfer(); retCode = report->getSummary().getErrorCode(); } cancelAbort(); LOG(INFO) << "Returning with code " << retCode << " " << errorCodeToStr(retCode); return retCode; }
int main(int argc, char *argv[]) { FLAGS_logtostderr = true; // Ugliness in gflags' api; to be able to use program name google::SetArgv(argc, const_cast<const char **>(argv)); google::SetVersionString(Protocol::getFullVersion()); std::string usage("WDT Warp-speed Data Transfer. v "); usage.append(google::VersionString()); usage.append(". Sample usage:\n\t"); usage.append(google::ProgramInvocationShortName()); usage.append(" # for a server/receiver\n\t"); usage.append(google::ProgramInvocationShortName()); usage.append(" -destination host # for a sender"); google::SetUsageMessage(usage); google::ParseCommandLineFlags(&argc, &argv, true); google::InitGoogleLogging(argv[0]); signal(SIGPIPE, SIG_IGN); #define STANDALONE_APP #define ASSIGN_OPT #include FLAGS_INCLUDE_FILE //nolint LOG(INFO) << "Running WDT " << Protocol::getFullVersion(); ErrorCode retCode = OK; // Odd ball case of log parsing if (FLAGS_parse_transfer_log) { // Log parsing mode TransferLogManager transferLogManager; transferLogManager.setRootDir(FLAGS_directory); if (!transferLogManager.parseAndPrint()) { LOG(ERROR) << "Transfer log parsing failed"; return ERROR; } return OK; } // General case : Sender or Receiver const auto &options = WdtOptions::get(); WdtTransferRequest req(options.start_port, options.num_ports, FLAGS_directory); req.transferId = FLAGS_transfer_id; if (FLAGS_protocol_version > 0) { req.protocolVersion = FLAGS_protocol_version; } if (FLAGS_destination.empty() && FLAGS_connection_url.empty()) { Receiver receiver(req); WdtTransferRequest augmentedReq = receiver.init(); if (augmentedReq.errorCode == ERROR) { LOG(ERROR) << "Error setting up receiver"; return augmentedReq.errorCode; } LOG(INFO) << "Starting receiver with connection url "; std::cout << augmentedReq.generateUrl() << std::endl; std::cout.flush(); setUpAbort(receiver); if (!FLAGS_recovery_id.empty()) { WdtOptions::getMutable().enable_download_resumption = true; receiver.setRecoveryId(FLAGS_recovery_id); } if (!FLAGS_run_as_daemon) { receiver.transferAsync(); std::unique_ptr<TransferReport> report = receiver.finish(); retCode = report->getSummary().getErrorCode(); } else { retCode = receiver.runForever(); // not reached } } else { // Sender mode std::vector<FileInfo> fileInfo; if (FLAGS_files) { // Each line should have the filename and optionally // the filesize separated by a single space std::string line; while (std::getline(std::cin, line)) { std::vector<std::string> fields; folly::split('\t', line, fields, true); if (fields.empty() || fields.size() > 2) { LOG(FATAL) << "Invalid input in stdin: " << line; } int64_t filesize = fields.size() > 1 ? folly::to<int64_t>(fields[1]) : -1; fileInfo.emplace_back(fields[0], filesize); } } req.hostName = FLAGS_destination; if (!FLAGS_connection_url.empty()) { LOG(INFO) << "Input url: " << FLAGS_connection_url; // TODO: merge instead req = WdtTransferRequest(FLAGS_connection_url); req.directory = FLAGS_directory; // re-set it for now } Sender sender(req); WdtTransferRequest processedRequest = sender.init(); LOG(INFO) << "Starting sender with details " << processedRequest.generateUrl(true); ADDITIONAL_SENDER_SETUP setUpAbort(sender); sender.setIncludeRegex(FLAGS_include_regex); sender.setExcludeRegex(FLAGS_exclude_regex); sender.setPruneDirRegex(FLAGS_prune_dir_regex); std::unique_ptr<TransferReport> report = sender.transfer(); retCode = report->getSummary().getErrorCode(); } cancelAbort(); return retCode; }
int main(int argc, char *argv[]) { FLAGS_logtostderr = true; // Ugliness in gflags' api; to be able to use program name google::SetArgv(argc, const_cast<const char **>(argv)); google::SetVersionString(Protocol::getFullVersion()); usage.assign("WDT Warp-speed Data Transfer. v "); usage.append(google::VersionString()); usage.append(". Sample usage:\nTo transfer from srchost to desthost:\n\t"); usage.append("ssh dsthost "); usage.append(google::ProgramInvocationShortName()); usage.append(" -directory destdir | ssh srchost "); usage.append(google::ProgramInvocationShortName()); usage.append(" -directory srcdir -"); usage.append( "\nPassing - as the argument to wdt means start the sender and" " read the"); usage.append( "\nconnection URL produced by the receiver, including encryption" " key, from stdin."); usage.append("\nUse --help to see all the options."); google::SetUsageMessage(usage); google::gflags_exitfunc = [](int code) { if (code == 0 || FLAGS_help) { // By default gflags exit 1 with --help and 0 for --version (good) // let's also exit(0) for --help to be like most gnu command line exit(0); } // error cases: if (FLAGS_exit_on_bad_flags) { printUsage(); exit(code); } badGflagFound = true; }; google::ParseCommandLineFlags(&argc, &argv, true); google::InitGoogleLogging(argv[0]); if (badGflagFound) { LOG(ERROR) << "Continuing despite bad flags"; } // Only non -flag argument allowed so far is "-" meaning // Read url from stdin and start a sender if (argc > 2 || (argc == 2 && (argv[1][0] != '-' || argv[1][1] != '\0'))) { printUsage(); std::cerr << "Error: argument should be - (to read url from stdin) " << "or no arguments" << std::endl; exit(1); } signal(SIGPIPE, SIG_IGN); std::string connectUrl; if (argc == 2) { std::getline(std::cin, connectUrl); if (connectUrl.empty()) { LOG(ERROR) << "Sender unable to read connection url from stdin - exiting"; return URI_PARSE_ERROR; } } else { connectUrl = FLAGS_connection_url; } // Might be a sub class (fbonly wdtCmdLine.cpp) Wdt &wdt = WDTCLASS::initializeWdt(FLAGS_app_name); if (FLAGS_print_options) { wdt.printWdtOptions(std::cout); return 0; } WdtOptions &options = wdt.getWdtOptions(); ErrorCode retCode = OK; // Odd ball case of log parsing if (FLAGS_parse_transfer_log) { // Log parsing mode options.enable_download_resumption = true; TransferLogManager transferLogManager(options); transferLogManager.setRootDir(FLAGS_directory); transferLogManager.openLog(); bool success = transferLogManager.parseAndPrint(); LOG_IF(ERROR, !success) << "Transfer log parsing failed"; transferLogManager.closeLog(); return success ? OK : ERROR; } // General case : Sender or Receiver std::unique_ptr<WdtTransferRequest> reqPtr; if (connectUrl.empty()) { reqPtr = folly::make_unique<WdtTransferRequest>( options.start_port, options.num_ports, FLAGS_directory); reqPtr->hostName = FLAGS_destination; reqPtr->transferId = FLAGS_transfer_id; if (!FLAGS_test_only_encryption_secret.empty()) { reqPtr->encryptionData = EncryptionParams(parseEncryptionType(options.encryption_type), FLAGS_test_only_encryption_secret); } } else { reqPtr = folly::make_unique<WdtTransferRequest>(connectUrl); if (reqPtr->errorCode != OK) { LOG(ERROR) << "Invalid url \"" << connectUrl << "\" : " << errorCodeToStr(reqPtr->errorCode); return ERROR; } reqPtr->directory = FLAGS_directory; LOG(INFO) << "Parsed url as " << reqPtr->getLogSafeString(); } WdtTransferRequest &req = *reqPtr; if (FLAGS_protocol_version > 0) { req.protocolVersion = FLAGS_protocol_version; } if (FLAGS_destination.empty() && connectUrl.empty()) { Receiver receiver(req); if (!FLAGS_recovery_id.empty()) { options.enable_download_resumption = true; receiver.setRecoveryId(FLAGS_recovery_id); } WdtTransferRequest augmentedReq = receiver.init(); retCode = augmentedReq.errorCode; if (retCode == FEWER_PORTS) { if (FLAGS_treat_fewer_port_as_error) { LOG(ERROR) << "Receiver could not bind to all the ports"; return FEWER_PORTS; } retCode = OK; } else if (augmentedReq.errorCode != OK) { LOG(ERROR) << "Error setting up receiver " << errorCodeToStr(retCode); return retCode; } // In the log: LOG(INFO) << "Starting receiver with connection url " << augmentedReq.getLogSafeString(); // The url without secret // on stdout: the one with secret: std::cout << augmentedReq.genWdtUrlWithSecret() << std::endl; std::cout.flush(); if (FLAGS_fork) { pid_t cpid = fork(); if (cpid == -1) { perror("Failed to fork()"); exit(1); } if (cpid > 0) { LOG(INFO) << "Detaching receiver"; exit(0); } close(0); close(1); } setAbortChecker(receiver); if (!FLAGS_run_as_daemon) { retCode = receiver.transferAsync(); if (retCode == OK) { std::unique_ptr<TransferReport> report = receiver.finish(); retCode = report->getSummary().getErrorCode(); } } else { retCode = receiver.runForever(); // not reached } } else { // Sender mode if (!FLAGS_manifest.empty()) { // Each line should have the filename and optionally // the filesize separated by a single space if (FLAGS_manifest == "-") { readManifest(std::cin, req, options.odirect_reads); } else { std::ifstream fin(FLAGS_manifest); readManifest(fin, req, options.odirect_reads); fin.close(); } LOG(INFO) << "Using files lists, number of files " << req.fileInfo.size(); } LOG(INFO) << "Making Sender with encryption set = " << req.encryptionData.isSet(); // TODO: find something more useful for namespace (userid ? directory?) // (shardid at fb) retCode = wdt.wdtSend(WdtResourceController::kGlobalNamespace, req, setupAbortChecker()); } cancelAbort(); if (retCode == OK) { LOG(INFO) << "Returning with OK exit code"; } else { LOG(ERROR) << "Returning with code " << retCode << " " << errorCodeToStr(retCode); } return retCode; }