Beispiel #1
0
int main(int argc, char *argv[]) {
  FLAGS_logtostderr = true;
  // Ugliness in gflags' api; to be able to use program name
  google::SetArgv(argc, const_cast<const char **>(argv));
  google::SetVersionString(Protocol::getFullVersion());
  std::string usage("WDT Warp-speed Data Transfer. v ");
  usage.append(google::VersionString());
  usage.append(". Sample usage:\n\t");
  usage.append(google::ProgramInvocationShortName());
  usage.append(" # for a server/receiver\n\t");
  usage.append(google::ProgramInvocationShortName());
  usage.append(" -connection_url url_produced_by_receiver # for a sender");
  google::SetUsageMessage(usage);
  google::gflags_exitfunc = [](int code) {
    if (FLAGS_exit_on_bad_flags) {
      exit(code);
    }
    badGflagFound = true;
  };
  google::ParseCommandLineFlags(&argc, &argv, true);
  google::InitGoogleLogging(argv[0]);
  if (badGflagFound) {
    LOG(ERROR) << "Continuing despite bad flags";
  }
  signal(SIGPIPE, SIG_IGN);

  FLAGS::initializeFromFlags();
  if (FLAGS_print_options) {
    FLAGS::printOptions(std::cout);
    return 0;
  }

  ErrorCode retCode = OK;

  // Odd ball case of log parsing
  if (FLAGS_parse_transfer_log) {
    // Log parsing mode
    WdtOptions::getMutable().enable_download_resumption = true;
    TransferLogManager transferLogManager;
    transferLogManager.setRootDir(FLAGS_directory);
    transferLogManager.openLog();
    bool success = transferLogManager.parseAndPrint();
    LOG_IF(ERROR, success) << "Transfer log parsing failed";
    transferLogManager.closeLog();
    return success ? OK : ERROR;
  }

  // General case : Sender or Receiver
  const auto &options = WdtOptions::get();
  std::unique_ptr<WdtTransferRequest> reqPtr;
  if (FLAGS_connection_url.empty()) {
    reqPtr = folly::make_unique<WdtTransferRequest>(
        options.start_port, options.num_ports, FLAGS_directory);
    reqPtr->hostName = FLAGS_destination;
    reqPtr->transferId = FLAGS_transfer_id;
  } else {
    LOG(INFO) << "Input url: " << FLAGS_connection_url;
    reqPtr = folly::make_unique<WdtTransferRequest>(FLAGS_connection_url);
    if (reqPtr->errorCode != OK) {
      LOG(ERROR) << "Invalid url " << errorCodeToStr(reqPtr->errorCode);
      return ERROR;
    }
    reqPtr->directory = FLAGS_directory;
  }
  WdtTransferRequest &req = *reqPtr;
  if (FLAGS_protocol_version > 0) {
    req.protocolVersion = FLAGS_protocol_version;
  }

  if (FLAGS_destination.empty() && FLAGS_connection_url.empty()) {
    Receiver receiver(req);
    WdtTransferRequest augmentedReq = receiver.init();
    if (FLAGS_treat_fewer_port_as_error &&
        augmentedReq.errorCode == FEWER_PORTS) {
      LOG(ERROR) << "Receiver could not bind to all the ports";
      return FEWER_PORTS;
    }
    if (augmentedReq.errorCode == ERROR) {
      LOG(ERROR) << "Error setting up receiver";
      return ERROR;
    }
    LOG(INFO) << "Starting receiver with connection url ";
    std::cout << augmentedReq.generateUrl() << std::endl;
    std::cout.flush();
    setUpAbort(receiver);
    if (!FLAGS_recovery_id.empty()) {
      WdtOptions::getMutable().enable_download_resumption = true;
      receiver.setRecoveryId(FLAGS_recovery_id);
    }
    if (!FLAGS_run_as_daemon) {
      retCode = receiver.transferAsync();
      if (retCode == OK) {
        std::unique_ptr<TransferReport> report = receiver.finish();
        retCode = report->getSummary().getErrorCode();
      }
    } else {
      retCode = receiver.runForever();
      // not reached
    }
  } else {
    // Sender mode
    if (!FLAGS_manifest.empty()) {
      // Each line should have the filename and optionally
      // the filesize separated by a single space
      if (FLAGS_manifest == "-") {
        readManifest(std::cin, req);
      } else {
        std::ifstream fin(FLAGS_manifest);
        readManifest(fin, req);
        fin.close();
      }
      LOG(INFO) << "Using files lists, number of files " << req.fileInfo.size();
    }
    Sender sender(req);
    WdtTransferRequest processedRequest = sender.init();
    LOG(INFO) << "Starting sender with details "
              << processedRequest.generateUrl(true);
    ADDITIONAL_SENDER_SETUP
    setUpAbort(sender);
    std::unique_ptr<TransferReport> report = sender.transfer();
    retCode = report->getSummary().getErrorCode();
  }
  cancelAbort();
  LOG(INFO) << "Returning with code " << retCode << " "
            << errorCodeToStr(retCode);
  return retCode;
}
Beispiel #2
0
int main(int argc, char *argv[]) {
  FLAGS_logtostderr = true;
  // Ugliness in gflags' api; to be able to use program name
  google::SetArgv(argc, const_cast<const char **>(argv));
  google::SetVersionString(Protocol::getFullVersion());
  std::string usage("WDT Warp-speed Data Transfer. v ");
  usage.append(google::VersionString());
  usage.append(". Sample usage:\n\t");
  usage.append(google::ProgramInvocationShortName());
  usage.append(" # for a server/receiver\n\t");
  usage.append(google::ProgramInvocationShortName());
  usage.append(" -destination host # for a sender");
  google::SetUsageMessage(usage);
  google::ParseCommandLineFlags(&argc, &argv, true);
  google::InitGoogleLogging(argv[0]);
  signal(SIGPIPE, SIG_IGN);

#define STANDALONE_APP
#define ASSIGN_OPT
#include FLAGS_INCLUDE_FILE  //nolint

  LOG(INFO) << "Running WDT " << Protocol::getFullVersion();
  ErrorCode retCode = OK;

  // Odd ball case of log parsing
  if (FLAGS_parse_transfer_log) {
    // Log parsing mode
    TransferLogManager transferLogManager;
    transferLogManager.setRootDir(FLAGS_directory);
    if (!transferLogManager.parseAndPrint()) {
      LOG(ERROR) << "Transfer log parsing failed";
      return ERROR;
    }
    return OK;
  }

  // General case : Sender or Receiver
  const auto &options = WdtOptions::get();
  WdtTransferRequest req(options.start_port, options.num_ports,
                         FLAGS_directory);
  req.transferId = FLAGS_transfer_id;
  if (FLAGS_protocol_version > 0) {
    req.protocolVersion = FLAGS_protocol_version;
  }

  if (FLAGS_destination.empty() && FLAGS_connection_url.empty()) {
    Receiver receiver(req);
    WdtTransferRequest augmentedReq = receiver.init();
    if (augmentedReq.errorCode == ERROR) {
      LOG(ERROR) << "Error setting up receiver";
      return augmentedReq.errorCode;
    }
    LOG(INFO) << "Starting receiver with connection url ";
    std::cout << augmentedReq.generateUrl() << std::endl;
    std::cout.flush();
    setUpAbort(receiver);
    if (!FLAGS_recovery_id.empty()) {
      WdtOptions::getMutable().enable_download_resumption = true;
      receiver.setRecoveryId(FLAGS_recovery_id);
    }
    if (!FLAGS_run_as_daemon) {
      receiver.transferAsync();
      std::unique_ptr<TransferReport> report = receiver.finish();
      retCode = report->getSummary().getErrorCode();
    } else {
      retCode = receiver.runForever();
      // not reached
    }
  } else {
    // Sender mode
    std::vector<FileInfo> fileInfo;
    if (FLAGS_files) {
      // Each line should have the filename and optionally
      // the filesize separated by a single space
      std::string line;
      while (std::getline(std::cin, line)) {
        std::vector<std::string> fields;
        folly::split('\t', line, fields, true);
        if (fields.empty() || fields.size() > 2) {
          LOG(FATAL) << "Invalid input in stdin: " << line;
        }
        int64_t filesize =
            fields.size() > 1 ? folly::to<int64_t>(fields[1]) : -1;
        fileInfo.emplace_back(fields[0], filesize);
      }
    }
    req.hostName = FLAGS_destination;
    if (!FLAGS_connection_url.empty()) {
      LOG(INFO) << "Input url: " << FLAGS_connection_url;
      // TODO: merge instead
      req = WdtTransferRequest(FLAGS_connection_url);
      req.directory = FLAGS_directory;  // re-set it for now
    }
    Sender sender(req);
    WdtTransferRequest processedRequest = sender.init();
    LOG(INFO) << "Starting sender with details "
              << processedRequest.generateUrl(true);
    ADDITIONAL_SENDER_SETUP
    setUpAbort(sender);
    sender.setIncludeRegex(FLAGS_include_regex);
    sender.setExcludeRegex(FLAGS_exclude_regex);
    sender.setPruneDirRegex(FLAGS_prune_dir_regex);
    std::unique_ptr<TransferReport> report = sender.transfer();
    retCode = report->getSummary().getErrorCode();
  }
  cancelAbort();
  return retCode;
}
Beispiel #3
0
int main(int argc, char *argv[]) {
  FLAGS_logtostderr = true;
  // Ugliness in gflags' api; to be able to use program name
  google::SetArgv(argc, const_cast<const char **>(argv));
  google::SetVersionString(Protocol::getFullVersion());
  usage.assign("WDT Warp-speed Data Transfer. v ");
  usage.append(google::VersionString());
  usage.append(". Sample usage:\nTo transfer from srchost to desthost:\n\t");
  usage.append("ssh dsthost ");
  usage.append(google::ProgramInvocationShortName());
  usage.append(" -directory destdir | ssh srchost ");
  usage.append(google::ProgramInvocationShortName());
  usage.append(" -directory srcdir -");
  usage.append(
      "\nPassing - as the argument to wdt means start the sender and"
      " read the");
  usage.append(
      "\nconnection URL produced by the receiver, including encryption"
      " key, from stdin.");
  usage.append("\nUse --help to see all the options.");
  google::SetUsageMessage(usage);
  google::gflags_exitfunc = [](int code) {
    if (code == 0 || FLAGS_help) {
      // By default gflags exit 1 with --help and 0 for --version (good)
      // let's also exit(0) for --help to be like most gnu command line
      exit(0);
    }
    // error cases:
    if (FLAGS_exit_on_bad_flags) {
      printUsage();
      exit(code);
    }
    badGflagFound = true;
  };
  google::ParseCommandLineFlags(&argc, &argv, true);
  google::InitGoogleLogging(argv[0]);
  if (badGflagFound) {
    LOG(ERROR) << "Continuing despite bad flags";
  }
  // Only non -flag argument allowed so far is "-" meaning
  // Read url from stdin and start a sender
  if (argc > 2 || (argc == 2 && (argv[1][0] != '-' || argv[1][1] != '\0'))) {
    printUsage();
    std::cerr << "Error: argument should be - (to read url from stdin) "
              << "or no arguments" << std::endl;
    exit(1);
  }
  signal(SIGPIPE, SIG_IGN);

  std::string connectUrl;
  if (argc == 2) {
    std::getline(std::cin, connectUrl);
    if (connectUrl.empty()) {
      LOG(ERROR) << "Sender unable to read connection url from stdin - exiting";
      return URI_PARSE_ERROR;
    }
  } else {
    connectUrl = FLAGS_connection_url;
  }

  // Might be a sub class (fbonly wdtCmdLine.cpp)
  Wdt &wdt = WDTCLASS::initializeWdt(FLAGS_app_name);
  if (FLAGS_print_options) {
    wdt.printWdtOptions(std::cout);
    return 0;
  }
  WdtOptions &options = wdt.getWdtOptions();

  ErrorCode retCode = OK;

  // Odd ball case of log parsing
  if (FLAGS_parse_transfer_log) {
    // Log parsing mode
    options.enable_download_resumption = true;
    TransferLogManager transferLogManager(options);
    transferLogManager.setRootDir(FLAGS_directory);
    transferLogManager.openLog();
    bool success = transferLogManager.parseAndPrint();
    LOG_IF(ERROR, !success) << "Transfer log parsing failed";
    transferLogManager.closeLog();
    return success ? OK : ERROR;
  }

  // General case : Sender or Receiver
  std::unique_ptr<WdtTransferRequest> reqPtr;
  if (connectUrl.empty()) {
    reqPtr = folly::make_unique<WdtTransferRequest>(
        options.start_port, options.num_ports, FLAGS_directory);
    reqPtr->hostName = FLAGS_destination;
    reqPtr->transferId = FLAGS_transfer_id;
    if (!FLAGS_test_only_encryption_secret.empty()) {
      reqPtr->encryptionData =
          EncryptionParams(parseEncryptionType(options.encryption_type),
                           FLAGS_test_only_encryption_secret);
    }
  } else {
    reqPtr = folly::make_unique<WdtTransferRequest>(connectUrl);
    if (reqPtr->errorCode != OK) {
      LOG(ERROR) << "Invalid url \"" << connectUrl
                 << "\" : " << errorCodeToStr(reqPtr->errorCode);
      return ERROR;
    }
    reqPtr->directory = FLAGS_directory;
    LOG(INFO) << "Parsed url as " << reqPtr->getLogSafeString();
  }
  WdtTransferRequest &req = *reqPtr;
  if (FLAGS_protocol_version > 0) {
    req.protocolVersion = FLAGS_protocol_version;
  }

  if (FLAGS_destination.empty() && connectUrl.empty()) {
    Receiver receiver(req);
    if (!FLAGS_recovery_id.empty()) {
      options.enable_download_resumption = true;
      receiver.setRecoveryId(FLAGS_recovery_id);
    }
    WdtTransferRequest augmentedReq = receiver.init();
    retCode = augmentedReq.errorCode;
    if (retCode == FEWER_PORTS) {
      if (FLAGS_treat_fewer_port_as_error) {
        LOG(ERROR) << "Receiver could not bind to all the ports";
        return FEWER_PORTS;
      }
      retCode = OK;
    } else if (augmentedReq.errorCode != OK) {
      LOG(ERROR) << "Error setting up receiver " << errorCodeToStr(retCode);
      return retCode;
    }
    // In the log:
    LOG(INFO) << "Starting receiver with connection url "
              << augmentedReq.getLogSafeString();  // The url without secret
    // on stdout: the one with secret:
    std::cout << augmentedReq.genWdtUrlWithSecret() << std::endl;
    std::cout.flush();
    if (FLAGS_fork) {
      pid_t cpid = fork();
      if (cpid == -1) {
        perror("Failed to fork()");
        exit(1);
      }
      if (cpid > 0) {
        LOG(INFO) << "Detaching receiver";
        exit(0);
      }
      close(0);
      close(1);
    }
    setAbortChecker(receiver);
    if (!FLAGS_run_as_daemon) {
      retCode = receiver.transferAsync();
      if (retCode == OK) {
        std::unique_ptr<TransferReport> report = receiver.finish();
        retCode = report->getSummary().getErrorCode();
      }
    } else {
      retCode = receiver.runForever();
      // not reached
    }
  } else {
    // Sender mode
    if (!FLAGS_manifest.empty()) {
      // Each line should have the filename and optionally
      // the filesize separated by a single space
      if (FLAGS_manifest == "-") {
        readManifest(std::cin, req, options.odirect_reads);
      } else {
        std::ifstream fin(FLAGS_manifest);
        readManifest(fin, req, options.odirect_reads);
        fin.close();
      }
      LOG(INFO) << "Using files lists, number of files " << req.fileInfo.size();
    }
    LOG(INFO) << "Making Sender with encryption set = "
              << req.encryptionData.isSet();

    // TODO: find something more useful for namespace (userid ? directory?)
    // (shardid at fb)
    retCode = wdt.wdtSend(WdtResourceController::kGlobalNamespace, req,
                          setupAbortChecker());
  }
  cancelAbort();
  if (retCode == OK) {
    LOG(INFO) << "Returning with OK exit code";
  } else {
    LOG(ERROR) << "Returning with code " << retCode << " "
               << errorCodeToStr(retCode);
  }
  return retCode;
}