int
Starter::execDCStarter( ArgList const &args, Env const *env,
int* std_fds, Stream* s )
{
Stream *inherit_list[] =
{ 0 /*ClassAd update stream (assigned below)*/,
s /*shadow syscall sock*/,
0 /*terminal NULL*/ };
const ArgList* final_args = &args;
const char* final_path = s_path;
Env new_env;
if( env ) {
new_env.MergeFrom( *env );
}
// The starter figures out its execute directory by paraming
// for EXECUTE, which we override in the environment here.
// This way, all the logic about choosing a directory to use
// is in only one place.
ASSERT( executeDir() );
new_env.SetEnv( "_CONDOR_EXECUTE", executeDir() );
env = &new_env;
// Build the affinity string to pass to the starter via env
std::string affinityString;
if (s_claim && s_claim->rip() && s_claim->rip()->get_affinity_set()) {
std::list<int> *l = s_claim->rip()->get_affinity_set();
bool needComma = false;
for (std::list<int>::iterator it = l->begin(); it != l->end(); it++) {
if (needComma) {
formatstr_cat(affinityString, ", %d", *it);
} else {
formatstr_cat(affinityString, "%d ", *it);
needComma = true;
}
}
}
int slotId = s_claim->rip()->r_sub_id;
if (slotId == 0) {
// Isn't a paritionable slot, use the main id
slotId = s_claim->rip()->r_id;
}
std::string affinityKnob;
formatstr(affinityKnob, "_CONDOR_SLOT%d_CPU_AFFINITY", slotId);
if (param_boolean("ASSIGN_CPU_AFFINITY", false)) {
new_env.SetEnv(affinityKnob.c_str(), affinityString.c_str());
new_env.SetEnv("_CONDOR_ENFORCE_CPU_AFFINITY", "true");
dprintf(D_FULLDEBUG, "Setting affinity env to %s = %s\n", affinityKnob.c_str(), affinityString.c_str());
}
ReliSock child_job_update_sock; // child inherits this socket
ASSERT( !s_job_update_sock );
s_job_update_sock = new ReliSock; // parent (yours truly) keeps this socket
ASSERT( s_job_update_sock );
// Connect parent and child sockets together so child can send us
// udpates to the job ClassAd.
if( !s_job_update_sock->connect_socketpair( child_job_update_sock ) ) {
dprintf( D_ALWAYS, "ERROR: Failed to create job ClassAd update socket!\n");
s_pid = 0;
return s_pid;
}
inherit_list[0] = &child_job_update_sock;
// Pass the machine ad to the starter
if (s_claim)
s_claim->writeMachAd( s_job_update_sock );
if( daemonCore->Register_Socket(
s_job_update_sock,
"starter ClassAd update socket",
(SocketHandlercpp)&Starter::receiveJobClassAdUpdate,
"receiveJobClassAdUpdate",
this) < 0 )
{
EXCEPT("Failed to register ClassAd update socket.");
}
#if defined(LINUX)
// see if we should be using glexec to spawn the starter.
// if we are, the cmd, args, env, and stdin to use will be
// modified
ArgList glexec_args;
Env glexec_env;
int glexec_std_fds[3];
if( param_boolean( "GLEXEC_STARTER", false ) ) {
if( ! glexec_starter_prepare( s_path,
s_claim->client()->proxyFile(),
args,
env,
std_fds,
glexec_args,
glexec_env,
glexec_std_fds ) )
{
// something went wrong; prepareForGlexec will
// have already logged it
cleanupAfterGlexec();
return 0;
}
final_path = glexec_args.GetArg(0);
final_args = &glexec_args;
env = &glexec_env;
std_fds = glexec_std_fds;
}
#endif
int reaper_id;
if( s_reaper_id > 0 ) {
reaper_id = s_reaper_id;
} else {
reaper_id = main_reaper;
}
if(IsFulldebug(D_FULLDEBUG)) {
MyString args_string;
final_args->GetArgsStringForDisplay(&args_string);
dprintf( D_FULLDEBUG, "About to Create_Process \"%s\"\n",
args_string.Value() );
}
FamilyInfo fi;
fi.max_snapshot_interval = pid_snapshot_interval;
s_pid = daemonCore->
Create_Process( final_path, *final_args, PRIV_ROOT, reaper_id,
TRUE, env, NULL, &fi, inherit_list, std_fds );
if( s_pid == FALSE ) {
dprintf( D_ALWAYS, "ERROR: exec_starter failed!\n");
s_pid = 0;
}
#if defined(LINUX)
if( param_boolean( "GLEXEC_STARTER", false ) ) {
// if we used glexec to spawn the Starter, we now need to send
// the Starter's environment to our glexec wrapper script so it
// can exec the Starter with all the environment variablew we rely
// on it inheriting
//
if ( !glexec_starter_handle_env(s_pid) ) {
// something went wrong; handleGlexecEnvironment will
// have already logged it
cleanupAfterGlexec();
return 0;
}
}
#endif
return s_pid;
}
int
Starter::execDCStarter( ArgList const &args, Env const *env,
int* std_fds, Stream* s )
{
Stream *inherit_list[] =
{ 0 /*ClassAd update stream (assigned below)*/,
s /*shadow syscall sock*/,
0 /*terminal NULL*/ };
const ArgList* final_args = &args;
const char* final_path = s_path;
Env new_env;
if( env ) {
new_env.MergeFrom( *env );
}
// The starter figures out its execute directory by paraming
// for EXECUTE, which we override in the environment here.
// This way, all the logic about choosing a directory to use
// is in only one place.
ASSERT( executeDir() );
new_env.SetEnv( "_CONDOR_EXECUTE", executeDir() );
// Handle encrypted execute directory
FilesystemRemap fs_remap_obj; // put on stack so destroyed when leave this method
FilesystemRemap* fs_remap = NULL;
// If admin desires encrypted exec dir in config, do it
bool encrypt_execdir = param_boolean_crufty("ENCRYPT_EXECUTE_DIRECTORY",false);
// Or if user wants encrypted exec in job ad, do it
if (!encrypt_execdir && s_claim->ad()) {
s_claim->ad()->LookupBool(ATTR_ENCRYPT_EXECUTE_DIRECTORY,encrypt_execdir);
}
if ( encrypt_execdir ) {
#ifdef LINUX
// On linux, setup a directory $EXECUTE/encryptedX subdirectory
// to serve as an ecryptfs mount point; pass this directory
// down to the condor_starter as if it were $EXECUTE so
// that the starter creates its dir_<pid> directory on the
// ecryptfs filesystem setup by doing an AddEncryptedMapping.
static int unsigned long privdirnum = 0;
TemporaryPrivSentry sentry(PRIV_CONDOR);
s_encrypted_execute_dir.formatstr("%s%cencrypted%lu",executeDir(),
DIR_DELIM_CHAR,privdirnum++);
if( mkdir(encryptedExecuteDir(), 0755) < 0 ) {
dprintf( D_FAILURE|D_ALWAYS,
"Failed to create encrypted dir %s: %s\n",
encryptedExecuteDir(),
strerror(errno) );
return 0;
}
dprintf( D_ALWAYS,
"Created encrypted dir %s\n", encryptedExecuteDir() );
fs_remap = &fs_remap_obj;
if ( fs_remap->AddEncryptedMapping(encryptedExecuteDir()) ) {
// FilesystemRemap object dprintfs out an error message for us
return 0;
}
new_env.SetEnv( "_CONDOR_EXECUTE", encryptedExecuteDir() );
#endif
}
env = &new_env;
// Build the affinity string to pass to the starter via env
std::string affinityString;
if (s_claim && s_claim->rip() && s_claim->rip()->get_affinity_set()) {
std::list<int> *l = s_claim->rip()->get_affinity_set();
bool needComma = false;
for (std::list<int>::iterator it = l->begin(); it != l->end(); it++) {
if (needComma) {
formatstr_cat(affinityString, ", %d", *it);
} else {
formatstr_cat(affinityString, "%d ", *it);
needComma = true;
}
}
}
bool affinityBool = false;
if ( ! s_claim || ! s_claim->ad()) {
affinityBool = param_boolean("ASSIGN_CPU_AFFINITY", false);
} else {
auto_free_ptr assign_cpu_affinity(param("ASSIGN_CPU_AFFINITY"));
if ( ! assign_cpu_affinity.empty()) {
classad::Value value;
if (s_claim->ad()->EvaluateExpr(assign_cpu_affinity.ptr(), value)) {
if ( ! value.IsBooleanValueEquiv(affinityBool)) {
// was an expression, but not a bool, so report it and continue
EXCEPT("ASSIGN_CPU_AFFINITY does not evaluate to a boolean, it is : %s", ClassAdValueToString(value));
}
}
}
}
if (affinityBool) {
new_env.SetEnv("_CONDOR_STARTD_ASSIGNED_AFFINITY", affinityString.c_str());
new_env.SetEnv("_CONDOR_ENFORCE_CPU_AFFINITY", "true");
dprintf(D_ALWAYS, "Setting affinity env to %s\n", affinityString.c_str());
}
ReliSock child_job_update_sock; // child inherits this socket
ASSERT( !s_job_update_sock );
s_job_update_sock = new ReliSock; // parent (yours truly) keeps this socket
ASSERT( s_job_update_sock );
// Connect parent and child sockets together so child can send us
// udpates to the job ClassAd.
if( !s_job_update_sock->connect_socketpair( child_job_update_sock ) ) {
dprintf( D_ALWAYS, "ERROR: Failed to create job ClassAd update socket!\n");
s_pid = 0;
return s_pid;
}
inherit_list[0] = &child_job_update_sock;
// Pass the machine ad to the starter
if (s_claim)
s_claim->writeMachAd( s_job_update_sock );
if( daemonCore->Register_Socket(
s_job_update_sock,
"starter ClassAd update socket",
(SocketHandlercpp)&Starter::receiveJobClassAdUpdate,
"receiveJobClassAdUpdate",
this) < 0 )
{
EXCEPT("Failed to register ClassAd update socket.");
}
#if defined(LINUX)
// see if we should be using glexec to spawn the starter.
// if we are, the cmd, args, env, and stdin to use will be
// modified
ArgList glexec_args;
Env glexec_env;
int glexec_std_fds[3];
if( param_boolean( "GLEXEC_STARTER", false ) ) {
if( ! glexec_starter_prepare( s_path,
s_claim->client()->proxyFile(),
args,
env,
std_fds,
glexec_args,
glexec_env,
glexec_std_fds ) )
{
// something went wrong; prepareForGlexec will
// have already logged it
cleanupAfterGlexec();
return 0;
}
final_path = glexec_args.GetArg(0);
final_args = &glexec_args;
env = &glexec_env;
std_fds = glexec_std_fds;
}
#endif
int reaper_id;
if( s_reaper_id > 0 ) {
reaper_id = s_reaper_id;
} else {
reaper_id = main_reaper;
}
if(IsFulldebug(D_FULLDEBUG)) {
MyString args_string;
final_args->GetArgsStringForDisplay(&args_string);
dprintf( D_FULLDEBUG, "About to Create_Process \"%s\"\n",
args_string.Value() );
}
FamilyInfo fi;
fi.max_snapshot_interval = pid_snapshot_interval;
s_pid = daemonCore->
Create_Process( final_path, *final_args, PRIV_ROOT, reaper_id,
TRUE, TRUE, env, NULL, &fi, inherit_list, std_fds,
NULL, 0, NULL, 0, NULL, NULL, NULL, NULL, fs_remap);
if( s_pid == FALSE ) {
dprintf( D_ALWAYS, "ERROR: exec_starter failed!\n");
s_pid = 0;
}
#if defined(LINUX)
if( param_boolean( "GLEXEC_STARTER", false ) ) {
// if we used glexec to spawn the Starter, we now need to send
// the Starter's environment to our glexec wrapper script so it
// can exec the Starter with all the environment variablew we rely
// on it inheriting
//
if ( !glexec_starter_handle_env(s_pid) ) {
// something went wrong; handleGlexecEnvironment will
// have already logged it
cleanupAfterGlexec();
return 0;
}
}
#endif
return s_pid;
}
