void
privsep_exec_set_args(FILE* fp, ArgList& args)
{
int num_args = args.Count();
for (int i = 0; i < num_args; i++) {
fprintf(fp, "exec-arg<%lu>\n", (unsigned long)strlen(args.GetArg(i)));
fprintf(fp, "%s\n", args.GetArg(i));
}
}
/** Parse the arguments line of an existing .condor.sub file, extracing
the arguments we want to preserve when updating the .condor.sub file.
@param subLine: the arguments line from the .condor.sub file
@param shallowOpts: the condor_submit_dag shallow options
@return 0 if successful, 1 if failed
*/
int
parseArgumentsLine( const MyString &subLine,
SubmitDagShallowOptions &shallowOpts )
{
const char *line = subLine.Value();
const char *start = strchr( line, '"' );
const char *end = strrchr( line, '"' );
MyString arguments;
if ( start && end ) {
arguments = subLine.Substr( start - line, end - line );
} else {
fprintf( stderr, "Missing quotes in arguments line: <%s>\n",
subLine.Value() );
return 1;
}
ArgList arglist;
MyString error;
if ( !arglist.AppendArgsV2Quoted( arguments.Value(),
&error ) ) {
fprintf( stderr, "Error parsing arguments: %s\n", error.Value() );
return 1;
}
for ( int argNum = 0; argNum < arglist.Count(); argNum++ ) {
MyString strArg = arglist.GetArg( argNum );
strArg.lower_case();
(void)parsePreservedArgs( strArg, argNum, arglist.Count(),
arglist.GetStringArray(), shallowOpts);
}
return 0;
}
int
GLExecPrivSepHelper::run_script(ArgList& args,MyString &error_desc)
{
if (!proxy_valid_right_now()) {
dprintf(D_ALWAYS, "GLExecPrivSepHelper::run_script: not invoking glexec since the proxy is not valid!\n");
error_desc += "The job proxy is not valid.";
return INVALID_PROXY_RC;
}
/* Note that set_user_priv is a no-op if condor is running as
non-root (the "usual" mode for invoking glexec) */
priv_state priv_saved = set_user_priv();
FILE* fp = my_popen(args, "r", TRUE);
set_priv(priv_saved);
if (fp == NULL) {
dprintf(D_ALWAYS,
"GLExecPrivSepHelper::run_script: "
"my_popen failure on %s: errno=%d (%s)\n",
args.GetArg(0),
errno,
strerror(errno));
return -1;
}
MyString str;
while (str.readLine(fp, true));
priv_saved = set_user_priv();
int ret = my_pclose(fp);
set_priv(priv_saved);
if (ret != 0) {
str.trim();
dprintf(D_ALWAYS,
"GLExecPrivSepHelper::run_script: %s exited "
"with status %d and following output:\n%s\n",
args.GetArg(0),
ret,
str.Value());
error_desc.formatstr_cat("%s exited with status %d and the following output: %s",
condor_basename(args.GetArg(0)),
ret,
str.Value());
error_desc.replaceString("\n","; ");
}
return ret;
}
void GF_Palette::Assign( const ArgList& inArgs ) {
UtilStr str;
// Mix up the rnd seed
srand( clock() );
// Compile and link the temp exprs. By spec, A-vars are evaluated now
mAVars.Compile( inArgs, 'A', mDict );
mAVars.Evaluate();
inArgs.GetArg( 'H', str ); mH.Compile( str, mDict );
inArgs.GetArg( 'S', str ); mS.Compile( str, mDict );
inArgs.GetArg( 'V', str ); mV.Compile( str, mDict );
mH_I_Dep = mH.IsDependent( "I" );
mS_I_Dep = mS.IsDependent( "I" );
mV_I_Dep = mV.IsDependent( "I" );
}
void ExprArray::Compile( const ArgList& inArgs, long inID, ExpressionDict& ioDict ) {
UtilStr str;
unsigned long i;
// Determine the name of this expression array
i = inID;
mIDStr.Wipe();
while ( i > 0 ) {
mIDStr.Prepend( (char) ( i & 0xFF ) );
i = i >> 8;
}
// Maintain memory heap for arbitrary array size...
mNumExprs = inArgs.GetArraySize( inID );
if ( mNumExprs > mDimNumExprs ) {
if ( mVals )
delete []mVals;
if ( mExprs )
delete []mExprs;
mVals = new float[ mNumExprs + 1 ];
mExprs = new Expression[ mNumExprs + 1 ];
mDimNumExprs = mNumExprs;
}
// Add/Insert the vars to the dict
for ( int i = 0; i < mNumExprs; i++ ) {
str.Assign( mIDStr );
str.Append( (long) i );
mVals[ i ] = 0;
ioDict.AddVar( str, &mVals[ i ] );
}
// Compile each expression array element
for ( int i = 0; i < mNumExprs; i++ ) {
inArgs.GetArg( inID, str, i );
mExprs[ i ].Compile( str, ioDict );
}
}
int
GLExecPrivSepHelper::create_process(const char* path,
ArgList& args,
Env& env,
const char* iwd,
int job_std_fds[3],
const char* std_file_names[3],
int nice_inc,
size_t* core_size_ptr,
int reaper_id,
int dc_job_opts,
FamilyInfo* family_info,
int *,
MyString *error_msg)
{
ASSERT(m_initialized);
if (!proxy_valid_right_now()) {
dprintf(D_ALWAYS, "GLExecPrivSepHelper::create_process: not invoking glexec since the proxy is not valid!\n");
if( error_msg ) {
error_msg->formatstr_cat("The job proxy is invalid.");
}
return -1;
}
// make a copy of std FDs so we're not messing w/ our caller's
// memory
int std_fds[3] = {-1, -1, -1};
ArgList modified_args;
modified_args.AppendArg(m_run_script);
modified_args.AppendArg(m_glexec);
modified_args.AppendArg(m_proxy);
modified_args.AppendArg(m_sandbox);
modified_args.AppendArg(m_wrapper_script);
for (int i = 0; i < 3; i++) {
modified_args.AppendArg((job_std_fds == NULL || job_std_fds[i] == -1) ?
std_file_names[i] : "-");
}
modified_args.AppendArg(path);
for (int i = 1; i < args.Count(); i++) {
modified_args.AppendArg(args.GetArg(i));
}
int glexec_errors = 0;
while(1) {
// setup a UNIX domain socket for communicating with
// condor_glexec_wrapper (see comment above feed_wrapper()
// for details
//
int sock_fds[2];
if (socketpair(PF_UNIX, SOCK_STREAM, 0, sock_fds) == -1)
{
dprintf(D_ALWAYS,
"GLEXEC: socketpair error: %s\n",
strerror(errno));
return false;
}
std_fds[0] = sock_fds[1];
// now create a pipe for receiving diagnostic stdout/stderr from glexec
int glexec_out_fds[2];
if (pipe(glexec_out_fds) < 0) {
dprintf(D_ALWAYS,
"GLEXEC: pipe() error: %s\n",
strerror(errno));
close(sock_fds[0]);
close(sock_fds[1]);
return false;
}
std_fds[1] = glexec_out_fds[1];
std_fds[2] = std_fds[1]; // collect glexec stderr and stdout together
FamilyInfo fi;
FamilyInfo* fi_ptr = (family_info != NULL) ? family_info : &fi;
MyString proxy_path;
proxy_path.formatstr("%s.condor/%s", m_sandbox, m_proxy);
fi_ptr->glexec_proxy = proxy_path.Value();
// At the very least, we need to pass the condor daemon's
// X509_USER_PROXY to condor_glexec_run. Currently, we just
// pass all daemon environment. We do _not_ run
// condor_glexec_run in the job environment, because that
// would be a security risk and would serve no purpose, since
// glexec cleanses the environment anyway.
dc_job_opts &= ~(DCJOBOPT_NO_ENV_INHERIT);
int pid = daemonCore->Create_Process(m_run_script.Value(),
modified_args,
PRIV_USER_FINAL,
reaper_id,
FALSE,
FALSE,
NULL,
iwd,
fi_ptr,
NULL,
std_fds,
NULL,
nice_inc,
NULL,
dc_job_opts,
core_size_ptr,
NULL,
NULL,
error_msg);
// close our handle to glexec's end of the diagnostic output pipe
close(glexec_out_fds[1]);
MyString glexec_error_msg;
int glexec_rc = 0;
int ret_val = feed_wrapper(pid, sock_fds, env, dc_job_opts, job_std_fds, glexec_out_fds[0], &glexec_error_msg, &glexec_rc);
// if not closed in feed_wrapper, close the glexec error pipe now
if( glexec_out_fds[0] != -1 ) {
close(glexec_out_fds[0]);
}
// Unlike the other glexec operations where we handle
// glexec retry inside the helper script,
// condor_glexec_run cannot handle retry for us, because
// it must exec glexec rather than spawning it off in a
// new process. Therefore, we handle here retries in case
// of transient errors.
if( ret_val != 0 ) {
return ret_val; // success
}
bool retry = true;
if( glexec_rc != 202 && glexec_rc != 203 ) {
// Either a non-transient glexec error, or some other
// non-glexec error.
retry = false;
}
else {
// This _could_ be a transient glexec issue, such as a
// communication error with GUMS, so retry up to some
// limit.
glexec_errors += 1;
if( glexec_errors > m_glexec_retries ) {
retry = false;
}
}
if( !retry ) {
// return the most recent glexec error output
if( error_msg ) {
error_msg->formatstr_cat(glexec_error_msg.Value());
}
return 0;
}
// truncated exponential backoff
int delay_rand = 1 + (get_random_int() % glexec_errors) % 100;
int delay = m_glexec_retry_delay * delay_rand;
dprintf(D_ALWAYS,"Glexec exited with status %d; retrying in %d seconds.\n",
glexec_rc, delay );
sleep(delay);
// now try again ...
}
// should never get here
return 0;
}
//
// Because we fork before calling docker, we don't actually
// care if the image is stored locally or not (except to the extent that
// remote image pull violates the principle of least astonishment).
//
int DockerAPI::run(
ClassAd &machineAd,
ClassAd &jobAd,
const std::string & containerName,
const std::string & imageID,
const std::string & command,
const ArgList & args,
const Env & env,
const std::string & sandboxPath,
const std::list<std::string> extraVolumes,
int & pid,
int * childFDs,
CondorError & /* err */ )
{
gc_image(imageID);
//
// We currently assume that the system has been configured so that
// anyone (user) who can run an HTCondor job can also run docker. It's
// also apparently a security worry to run Docker as root, so let's not.
//
ArgList runArgs;
if ( ! add_docker_arg(runArgs))
return -1;
runArgs.AppendArg( "run" );
// Write out a file with the container ID.
// FIXME: The startd can check this to clean up after us.
// This needs to go into a directory that condor user
// can write to.
/*
std::string cidFileName = sandboxPath + "/.cidfile";
runArgs.AppendArg( "--cidfile=" + cidFileName );
*/
// Configure resource limits.
// First cpus
int cpus;
int cpuShare;
if (machineAd.LookupInteger(ATTR_CPUS, cpus)) {
cpuShare = 10 * cpus;
} else {
cpuShare = 10;
}
std::string cpuShareStr;
formatstr(cpuShareStr, "--cpu-shares=%d", cpuShare);
runArgs.AppendArg(cpuShareStr);
// Now memory
int memory; // in Megabytes
if (machineAd.LookupInteger(ATTR_MEMORY, memory)) {
std::string mem;
formatstr(mem, "--memory=%dm", memory);
runArgs.AppendArg(mem);
}
// drop unneeded Linux capabilities
if (param_boolean("DOCKER_DROP_ALL_CAPABILITIES", true /*default*/,
true /*do_log*/, &machineAd, &jobAd)) {
runArgs.AppendArg("--cap-drop=all");
// --no-new-privileges flag appears in docker 1.11
if (DockerAPI::majorVersion > 1 ||
DockerAPI::minorVersion > 10) {
runArgs.AppendArg("--no-new-privileges");
}
}
// Give the container a useful name
std::string hname = makeHostname(&machineAd, &jobAd);
runArgs.AppendArg("--hostname");
runArgs.AppendArg(hname.c_str());
// Now the container name
runArgs.AppendArg( "--name" );
runArgs.AppendArg( containerName );
if ( ! add_env_to_args_for_docker(runArgs, env)) {
dprintf( D_ALWAYS | D_FAILURE, "Failed to pass enviroment to docker.\n" );
return -8;
}
// Map the external sanbox to the internal sandbox.
runArgs.AppendArg( "--volume" );
runArgs.AppendArg( sandboxPath + ":" + sandboxPath );
// Now any extra volumes
for (std::list<std::string>::const_iterator it = extraVolumes.begin(); it != extraVolumes.end(); it++) {
runArgs.AppendArg("--volume");
std::string volume = *it;
runArgs.AppendArg(volume);
}
// Start in the sandbox.
runArgs.AppendArg( "--workdir" );
runArgs.AppendArg( sandboxPath );
// Run with the uid that condor selects for the user
// either a slot user or submitting user or nobody
uid_t uid = 0;
uid_t gid = 0;
// Docker doesn't actually run on Windows, but we compile
// on Windows because...
#ifndef WIN32
uid = get_user_uid();
gid = get_user_gid();
#endif
if ((uid == 0) || (gid == 0)) {
dprintf(D_ALWAYS|D_FAILURE, "Failed to get userid to run docker job\n");
return -9;
}
runArgs.AppendArg("--user");
std::string uidgidarg;
formatstr(uidgidarg, "%d:%d", uid, gid);
runArgs.AppendArg(uidgidarg);
// Run the command with its arguments in the image.
runArgs.AppendArg( imageID );
// If no command given, the default command in the image will run
if (command.length() > 0) {
runArgs.AppendArg( command );
}
runArgs.AppendArgsFromArgList( args );
MyString displayString;
runArgs.GetArgsStringForLogging( & displayString );
dprintf( D_ALWAYS, "Attempting to run: %s\n", displayString.c_str() );
//
// If we run Docker attached, we avoid a race condition where
// 'docker logs --follow' returns before 'docker rm' knows that the
// container is gone (and refuses to remove it). Of course, we
// can't block, so we have a proxy process run attached for us.
//
FamilyInfo fi;
fi.max_snapshot_interval = param_integer( "PID_SNAPSHOT_INTERVAL", 15 );
int childPID = daemonCore->Create_Process( runArgs.GetArg(0), runArgs,
PRIV_CONDOR_FINAL, 1, FALSE, FALSE, NULL, "/",
& fi, NULL, childFDs );
if( childPID == FALSE ) {
dprintf( D_ALWAYS | D_FAILURE, "Create_Process() failed.\n" );
return -1;
}
pid = childPID;
return 0;
}
int
GLExecPrivSepHelper::create_process(const char* path,
ArgList& args,
Env& env,
const char* iwd,
int job_std_fds[3],
const char* std_file_names[3],
int nice_inc,
size_t* core_size_ptr,
int reaper_id,
int dc_job_opts,
FamilyInfo* family_info,
int *,
MyString *error_msg)
{
ASSERT(m_initialized);
if (!proxy_valid_right_now()) {
dprintf(D_ALWAYS, "GLExecPrivSepHelper::create_process: not invoking glexec since the proxy is not valid!\n");
return -1;
}
// make a copy of std FDs so we're not messing w/ our caller's
// memory
int std_fds[3] = {-1, -1, -1};
ArgList modified_args;
modified_args.AppendArg(m_run_script);
modified_args.AppendArg(m_glexec);
modified_args.AppendArg(m_proxy);
modified_args.AppendArg(m_sandbox);
modified_args.AppendArg(m_wrapper_script);
for (int i = 0; i < 3; i++) {
modified_args.AppendArg((job_std_fds == NULL || job_std_fds[i] == -1) ?
std_file_names[i] : "-");
}
modified_args.AppendArg(path);
for (int i = 1; i < args.Count(); i++) {
modified_args.AppendArg(args.GetArg(i));
}
// setup a UNIX domain socket for communicating with
// condor_glexec_wrapper (see comment above feed_wrapper()
// for details
//
int sock_fds[2];
if (socketpair(PF_UNIX, SOCK_STREAM, 0, sock_fds) == -1)
{
dprintf(D_ALWAYS,
"GLEXEC: socketpair error: %s\n",
strerror(errno));
return false;
}
std_fds[0] = sock_fds[1];
// now create a pipe for receiving diagnostic stdout/stderr from glexec
int glexec_out_fds[2];
if (pipe(glexec_out_fds) < 0) {
dprintf(D_ALWAYS,
"GLEXEC: pipe() error: %s\n",
strerror(errno));
close(sock_fds[0]);
close(sock_fds[1]);
return false;
}
std_fds[1] = glexec_out_fds[1];
std_fds[2] = std_fds[1]; // collect glexec stderr and stdout together
FamilyInfo fi;
FamilyInfo* fi_ptr = (family_info != NULL) ? family_info : &fi;
MyString proxy_path;
proxy_path.formatstr("%s.condor/%s", m_sandbox, m_proxy);
fi_ptr->glexec_proxy = proxy_path.Value();
// At the very least, we need to pass the condor daemon's
// X509_USER_PROXY to condor_glexec_run. Currently, we just
// pass all daemon environment. We do _not_ run
// condor_glexec_run in the job environment, because that
// would be a security risk and would serve no purpose, since
// glexec cleanses the environment anyway.
dc_job_opts &= ~(DCJOBOPT_NO_ENV_INHERIT);
int pid = daemonCore->Create_Process(m_run_script.Value(),
modified_args,
PRIV_USER_FINAL,
reaper_id,
FALSE,
NULL,
iwd,
fi_ptr,
NULL,
std_fds,
NULL,
nice_inc,
NULL,
dc_job_opts,
core_size_ptr,
NULL,
NULL,
error_msg);
// close our handle to glexec's end of the diagnostic output pipe
close(glexec_out_fds[1]);
int ret_val = feed_wrapper(pid, sock_fds, env, dc_job_opts, job_std_fds, glexec_out_fds[0], error_msg);
// if not closed in feed_wrapper, close the glexec error pipe now
if( glexec_out_fds[0] != -1 ) {
close(glexec_out_fds[0]);
}
return ret_val;
}
std::string *NordugridJob::buildSubmitRSL()
{
int transfer_exec = TRUE;
std::string *rsl = new std::string;
StringList *stage_list = NULL;
StringList *stage_local_list = NULL;
char *attr_value = NULL;
std::string rsl_suffix;
std::string iwd;
std::string executable;
if ( jobAd->LookupString( ATTR_NORDUGRID_RSL, rsl_suffix ) &&
rsl_suffix[0] == '&' ) {
*rsl = rsl_suffix;
return rsl;
}
if ( jobAd->LookupString( ATTR_JOB_IWD, iwd ) != 1 ) {
errorString = "ATTR_JOB_IWD not defined";
delete rsl;
return NULL;
}
//Start off the RSL
attr_value = param( "FULL_HOSTNAME" );
formatstr( *rsl, "&(savestate=yes)(action=request)(hostname=%s)", attr_value );
free( attr_value );
attr_value = NULL;
//We're assuming all job clasads have a command attribute
jobAd->LookupString( ATTR_JOB_CMD, executable );
jobAd->LookupBool( ATTR_TRANSFER_EXECUTABLE, transfer_exec );
*rsl += "(executable=";
// If we're transferring the executable, strip off the path for the
// remote machine, since it refers to the submit machine.
if ( transfer_exec ) {
*rsl += condor_basename( executable.c_str() );
} else {
*rsl += executable;
}
{
ArgList args;
MyString arg_errors;
MyString rsl_args;
if(!args.AppendArgsFromClassAd(jobAd,&arg_errors)) {
dprintf(D_ALWAYS,"(%d.%d) Failed to read job arguments: %s\n",
procID.cluster, procID.proc, arg_errors.Value());
formatstr(errorString,"Failed to read job arguments: %s\n",
arg_errors.Value());
delete rsl;
return NULL;
}
if(args.Count() != 0) {
if(args.InputWasV1()) {
// In V1 syntax, the user's input _is_ RSL
if(!args.GetArgsStringV1Raw(&rsl_args,&arg_errors)) {
dprintf(D_ALWAYS,
"(%d.%d) Failed to get job arguments: %s\n",
procID.cluster,procID.proc,arg_errors.Value());
formatstr(errorString,"Failed to get job arguments: %s\n",
arg_errors.Value());
delete rsl;
return NULL;
}
}
else {
// In V2 syntax, we convert the ArgList to RSL
for(int i=0;i<args.Count();i++) {
if(i) {
rsl_args += ' ';
}
rsl_args += rsl_stringify(args.GetArg(i));
}
}
*rsl += ")(arguments=";
*rsl += rsl_args;
}
}
// If we're transferring the executable, tell Nordugrid to set the
// execute bit on the transferred executable.
if ( transfer_exec ) {
*rsl += ")(executables=";
*rsl += condor_basename( executable.c_str() );
}
if ( jobAd->LookupString( ATTR_JOB_INPUT, &attr_value ) == 1) {
// only add to list if not NULL_FILE (i.e. /dev/null)
if ( ! nullFile(attr_value) ) {
*rsl += ")(stdin=";
*rsl += condor_basename(attr_value);
}
free( attr_value );
attr_value = NULL;
}
stage_list = buildStageInList();
if ( stage_list->isEmpty() == false ) {
char *file;
stage_list->rewind();
*rsl += ")(inputfiles=";
while ( (file = stage_list->next()) != NULL ) {
*rsl += "(";
*rsl += condor_basename(file);
if ( IsUrl( file ) ) {
formatstr_cat( *rsl, " \"%s\")", file );
} else {
*rsl += " \"\")";
}
}
}
delete stage_list;
stage_list = NULL;
if ( jobAd->LookupString( ATTR_JOB_OUTPUT, &attr_value ) == 1) {
// only add to list if not NULL_FILE (i.e. /dev/null)
if ( ! nullFile(attr_value) ) {
*rsl += ")(stdout=" REMOTE_STDOUT_NAME;
}
free( attr_value );
attr_value = NULL;
}
if ( jobAd->LookupString( ATTR_JOB_ERROR, &attr_value ) == 1) {
// only add to list if not NULL_FILE (i.e. /dev/null)
if ( ! nullFile(attr_value) ) {
*rsl += ")(stderr=" REMOTE_STDERR_NAME;
}
free( attr_value );
}
stage_list = buildStageOutList();
stage_local_list = buildStageOutLocalList( stage_list );
if ( stage_list->isEmpty() == false ) {
char *file;
char *local_file;
stage_list->rewind();
stage_local_list->rewind();
*rsl += ")(outputfiles=";
while ( (file = stage_list->next()) != NULL ) {
local_file = stage_local_list->next();
*rsl += "(";
*rsl += condor_basename(file);
if ( IsUrl( local_file ) ) {
formatstr_cat( *rsl, " \"%s\")", local_file );
} else {
*rsl += " \"\")";
}
}
}
delete stage_list;
stage_list = NULL;
delete stage_local_list;
stage_local_list = NULL;
*rsl += ')';
if ( !rsl_suffix.empty() ) {
*rsl += rsl_suffix;
}
dprintf(D_FULLDEBUG,"*** RSL='%s'\n",rsl->c_str());
return rsl;
}
