Example #1
0
int RUNMAIN(arccat)(int argc, char **argv) {

  setlocale(LC_ALL, "");

  Arc::Logger logger(Arc::Logger::getRootLogger(), "arccat");
  Arc::LogStream logcerr(std::cerr);
  logcerr.setFormat(Arc::ShortFormat);
  Arc::Logger::getRootLogger().addDestination(logcerr);
  Arc::Logger::getRootLogger().setThreshold(Arc::WARNING);

  Arc::ArcLocation::Init(argv[0]);

  ClientOptions opt(ClientOptions::CO_CAT,
                    istring("[job ...]"),
                    istring("The arccat command performs the cat "
                            "command on the stdout, stderr or grid\n"
                            "manager's error log of the job."));

  std::list<std::string> jobidentifiers = opt.Parse(argc, argv);

  if (opt.showversion) {
    std::cout << Arc::IString("%s version %s", "arccat", VERSION)
              << std::endl;
    return 0;
  }

  // If debug is specified as argument, it should be set before loading the configuration.
  if (!opt.debug.empty())
    Arc::Logger::getRootLogger().setThreshold(Arc::string_to_level(opt.debug));

  if (opt.show_plugins) {
    std::list<std::string> types;
    types.push_back("HED:JobControllerPlugin");
    showplugins("arccat", types, logger);
    return 0;
  }

  Arc::UserConfig usercfg(opt.conffile, opt.joblist);
  if (!usercfg) {
    logger.msg(Arc::ERROR, "Failed configuration initialization");
    return 1;
  }

  if (!checkproxy(usercfg)) {
    return 1;
  }

  if (opt.debug.empty() && !usercfg.Verbosity().empty())
    Arc::Logger::getRootLogger().setThreshold(Arc::string_to_level(usercfg.Verbosity()));

  for (std::list<std::string>::const_iterator it = opt.jobidinfiles.begin(); it != opt.jobidinfiles.end(); it++) {
    if (!Arc::Job::ReadJobIDsFromFile(*it, jobidentifiers)) {
      logger.msg(Arc::WARNING, "Cannot read specified jobid file: %s", *it);
    }
  }

  if (opt.timeout > 0)
    usercfg.Timeout(opt.timeout);

  if ((!opt.joblist.empty() || !opt.status.empty()) && jobidentifiers.empty() && opt.clusters.empty())
    opt.all = true;

  if (jobidentifiers.empty() && opt.clusters.empty() && !opt.all) {
    logger.msg(Arc::ERROR, "No jobs given");
    return 1;
  }

  std::list<std::string> selectedURLs;
  if (!opt.clusters.empty()) {
    selectedURLs = getSelectedURLsFromUserConfigAndCommandLine(usercfg, opt.clusters);
  }
  std::list<std::string> rejectManagementURLs = getRejectManagementURLsFromUserConfigAndCommandLine(usercfg, opt.rejectmanagement);

  std::list<Arc::Job> jobs;
  Arc::JobInformationStorageXML jobList(usercfg.JobListFile());
  if (( opt.all && !jobList.ReadAll(jobs, rejectManagementURLs)) ||
      (!opt.all && !jobList.Read(jobs, jobidentifiers, selectedURLs, rejectManagementURLs))) {
    logger.msg(Arc::ERROR, "Unable to read job information from file (%s)", usercfg.JobListFile());
    return 1;
  }

  if (!opt.all) {
    for (std::list<std::string>::const_iterator itJIDAndName = jobidentifiers.begin();
         itJIDAndName != jobidentifiers.end(); ++itJIDAndName) {
      std::cout << Arc::IString("Warning: Job not found in job list: %s", *itJIDAndName) << std::endl;
    }
  }

  Arc::JobSupervisor jobmaster(usercfg, jobs);
  jobmaster.Update();
  jobmaster.SelectValid();
  if (!opt.status.empty()) {
    jobmaster.SelectByStatus(opt.status);
  }

  jobs = jobmaster.GetSelectedJobs();
  if (jobs.empty()) {
    std::cout << Arc::IString("No jobs") << std::endl;
    return 1;
  }

  std::string resourceName;
  Arc::Job::ResourceType resource;
  if (opt.show_joblog)      { resource = Arc::Job::JOBLOG; resourceName = "joblog"; }
  else if (opt.show_stderr) { resource = Arc::Job::STDERR; resourceName = "stderr"; }
  else                      { resource = Arc::Job::STDOUT; resourceName = "stdout"; }

  // saving to a temp file is necessary because chunks from server
  // may arrive out of order
  std::string filename = Glib::build_filename(Glib::get_tmp_dir(), "arccat.XXXXXX");
  int tmp_h = Glib::mkstemp(filename);
  if (tmp_h == -1) {
    logger.msg(Arc::INFO, "Could not create temporary file \"%s\"", filename);
    logger.msg(Arc::ERROR, "Cannot create output of %s for any jobs", resourceName);
    return 1;
  }

  Arc::URL dst("stdio:///"+Arc::tostring(tmp_h));
  if (!dst) {
    logger.msg(Arc::ERROR, "Cannot create output of %s for any jobs", resourceName);
    logger.msg(Arc::INFO, "Invalid destination URL %s", dst.str());
    close(tmp_h);
    unlink(filename.c_str());
    return 1;
  }

  int retval = 0;
  for (std::list<Arc::Job>::const_iterator it = jobs.begin();
       it != jobs.end(); it++) {
    if (!it->State || (!opt.status.empty() &&
        std::find(opt.status.begin(), opt.status.end(), it->State()) == opt.status.end() &&
        std::find(opt.status.begin(), opt.status.end(), it->State.GetGeneralState()) == opt.status.end())) {
      continue;
    }

    if (it->State == Arc::JobState::DELETED) {
      logger.msg(Arc::WARNING, "Job deleted: %s", it->JobID);
      retval = 1;
      continue;
    }

    // The job-log might be available before the job has started (middleware dependent).
    if (!opt.show_joblog &&
        !it->State.IsFinished() &&
        it->State != Arc::JobState::RUNNING &&
        it->State != Arc::JobState::FINISHING) {
      logger.msg(Arc::WARNING, "Job has not started yet: %s", it->JobID);
      retval = 1;
      continue;
    }

    if ((opt.show_joblog && it->LogDir.empty()) ||
        (!opt.show_joblog && opt.show_stderr && it->StdErr.empty()) ||
        (!opt.show_joblog && !opt.show_stderr && it->StdOut.empty())) {
      logger.msg(Arc::ERROR, "Cannot determine the %s location: %s", resourceName, it->JobID);
      retval = 1;
      continue;
    }

    Arc::URL src;
    it->GetURLToResource(resource, src);
    if (!src) {
      logger.msg(Arc::ERROR, "Cannot create output of %s for job (%s): Invalid source %s", resourceName, it->JobID, src.str());
      retval = 1;
      continue;
    }

    if (!it->CopyJobFile(usercfg, src, dst)) {
      retval = 1;
      continue;
    }

    logger.msg(Arc::VERBOSE, "Catting %s for job %s", resourceName, it->JobID);

    std::ifstream is(filename.c_str());
    char c;
    while (is.get(c)) {
      std::cout.put(c);
    }
    is.close();
  }

  close(tmp_h);
  unlink(filename.c_str());

  return retval;
}
Example #2
0
static bool arcls(const Arc::URL& dir_url,
           Arc::UserConfig& usercfg,
           bool show_details, // longlist
           bool show_urls,    // locations
           bool show_meta,    // metadata
           bool no_list,      // don't list dirs
           bool force_list,   // force dir list
           bool check_access, // checkaccess
           int recursion,     // recursion 
           int timeout) {     // timeout

  if (!dir_url) {
    logger.msg(Arc::ERROR, "Invalid URL: %s", dir_url.fullstr());
    return false;
  }
  if (dir_url.Protocol() == "urllist") {
    std::list<Arc::URL> dirs = Arc::ReadURLList(dir_url);
    if (dirs.empty()) {
      logger.msg(Arc::ERROR, "Can't read list of locations from file %s",
                 dir_url.Path());
      return false;
    }
    bool r = true;
    for (std::list<Arc::URL>::iterator dir = dirs.begin();
         dir != dirs.end(); dir++) {
      if(!arcls(*dir, usercfg, show_details, show_urls, show_meta,
               no_list, force_list, check_access, recursion, timeout)) r = false;
    }
    return r;
  }

  Arc::DataHandle url(dir_url, usercfg);
  if (!url) {
    logger.msg(Arc::ERROR, "Unsupported URL given");
    return false;
  }
  if (url->RequiresCredentials()) {
    if (usercfg.ProxyPath().empty() ) {
      logger.msg(Arc::ERROR, "Unable to list content of %s: No valid credentials found", dir_url.str());
      return false;
    }
    Arc::Credential holder(usercfg.ProxyPath(), "", "", "");
    if (holder.GetEndTime() < Arc::Time()){
      logger.msg(Arc::ERROR, "Proxy expired");
      logger.msg(Arc::ERROR, "Unable to list content of %s: No valid credentials found", dir_url.str());
      return false;
    }
  }
  url->SetSecure(false);

  if(check_access) {
    std::cout << dir_url << " - ";
    if(url->Check(false)) {
      std::cout << "passed" << std::endl;
      return true;
    } else {
      std::cout << "failed" << std::endl;
      return false;
    }
  }

  Arc::DataPoint::DataPointInfoType verb = (Arc::DataPoint::DataPointInfoType)
                                           (Arc::DataPoint::INFO_TYPE_MINIMAL |
                                            Arc::DataPoint::INFO_TYPE_NAME);
  if(show_urls) verb = (Arc::DataPoint::DataPointInfoType)
                       (verb | Arc::DataPoint::INFO_TYPE_STRUCT);
  if(show_meta) verb = (Arc::DataPoint::DataPointInfoType)
                       (verb | Arc::DataPoint::INFO_TYPE_ALL);
  if(show_details) verb = (Arc::DataPoint::DataPointInfoType)
                          (verb |
                           Arc::DataPoint::INFO_TYPE_TYPE |
                           Arc::DataPoint::INFO_TYPE_TIMES |
                           Arc::DataPoint::INFO_TYPE_CONTENT |
                           Arc::DataPoint::INFO_TYPE_ACCESS); 
  if(recursion > 0) verb = (Arc::DataPoint::DataPointInfoType)
                           (verb | Arc::DataPoint::INFO_TYPE_TYPE);

  Arc::DataStatus res;
  Arc::FileInfo file;
  std::list<Arc::FileInfo> files;

  if(no_list) { // only requested object is queried
    res = url->Stat(file, verb);
    if(res) files.push_back(file);
  } else if(force_list) { // assume it is directory, fail otherwise
    res = url->List(files, verb);
  } else { // try to guess what to do
    res = url->Stat(file, (Arc::DataPoint::DataPointInfoType)(verb | Arc::DataPoint::INFO_TYPE_TYPE));
    if(res && (file.GetType() == Arc::FileInfo::file_type_file)) {
      // If it is file and we are sure, then just report it.
      files.push_back(file);
    } else {
      // If it is dir then we must list it. But if stat failed or
      // if type is undefined there is still chance it is directory.
      Arc::DataStatus res_ = url->List(files, verb);
      if(!res_) {
        // If listing failed maybe simply report previous result if any.
        if(res) {
          files.push_back(file);
        }
      } else {
        res = res_;
      }
    }
  }
  if (!res) {
    if (files.empty()) {
      logger.msg(Arc::ERROR, std::string(res));
      if (res.Retryable())
        logger.msg(Arc::ERROR, "This seems like a temporary error, please try again later");
      return false;
    }
    logger.msg(Arc::INFO, "Warning: "
               "Failed listing files but some information is obtained");
  }

  files.sort(); // Sort alphabetically by name
  if (show_details) {
    print_details(files, show_urls, show_meta);
  } else {
    for (std::list<Arc::FileInfo>::iterator i = files.begin();
       i != files.end(); i++) {
      std::cout << i->GetName() << std::endl;
      if (show_urls) print_urls(*i);
      if (show_meta) print_meta(*i);
    }
  }
  // Do recursion. Recursion has no sense if listing is forbidden.
  if ((recursion > 0) && (!no_list)) {
    for (std::list<Arc::FileInfo>::iterator i = files.begin();
       i != files.end(); i++) {
      if (i->GetType() == Arc::FileInfo::file_type_dir) {
        Arc::URL suburl = dir_url;
        if(suburl.Protocol() != "file") {
          if (suburl.Path()[suburl.Path().length() - 1] != '/')
            suburl.ChangePath(suburl.Path() + "/" + i->GetName());
          else
            suburl.ChangePath(suburl.Path() + i->GetName());
        } else {
          if (suburl.Path()[suburl.Path().length() - 1] != G_DIR_SEPARATOR)
            suburl.ChangePath(suburl.Path() + G_DIR_SEPARATOR_S + i->GetName());
          else
            suburl.ChangePath(suburl.Path() + i->GetName());
        }
        std::cout << std::endl;
        std::cout << suburl.str() << ":" << std::endl;
        arcls(suburl, usercfg, show_details, show_urls, show_meta,
              no_list, force_list, check_access, recursion - 1, timeout);
        std::cout << std::endl;
      }
    }
  }
  return true;
}