void chili_redirect_start(const char *name)
{
if (!_enabled){
/* Nothing to do, we don't wan't to redirect */
return;
}
if (_build_path(_stdout_name, name) < 0){
return;
}
debug_print("Redirecting to file %s\n", _stdout_name);
_stdout_temp = creat(_stdout_name, S_IRUSR|S_IWUSR);
if (_stdout_temp == -1){
printf("Failed to create file %s due to %s\n",
_stdout_name, strerror(errno));
return;
}
fflush(stdout);
_stdout_copy = dup(1);
dup2(_stdout_temp, 1);
close(_stdout_temp);
_stdout_temp = 0;
}
/*
* Current process is running as the user when this is called.
*/
void
exec_task(slurmd_job_t *job, int i)
{
uint32_t *gtids; /* pointer to arrary of ranks */
int fd, j;
slurmd_task_info_t *task = job->task[i];
char **tmp_env;
if (i == 0)
_make_tmpdir(job);
gtids = xmalloc(job->node_tasks * sizeof(uint32_t));
for (j = 0; j < job->node_tasks; j++)
gtids[j] = job->task[j]->gtid;
job->envtp->sgtids = _uint32_array_to_str(job->node_tasks, gtids);
xfree(gtids);
job->envtp->jobid = job->jobid;
job->envtp->stepid = job->stepid;
job->envtp->nodeid = job->nodeid;
job->envtp->cpus_on_node = job->cpus;
job->envtp->procid = task->gtid;
job->envtp->localid = task->id;
job->envtp->task_pid = getpid();
job->envtp->distribution = job->task_dist;
job->envtp->cpu_bind = xstrdup(job->cpu_bind);
job->envtp->cpu_bind_type = job->cpu_bind_type;
job->envtp->cpu_freq = job->cpu_freq;
job->envtp->mem_bind = xstrdup(job->mem_bind);
job->envtp->mem_bind_type = job->mem_bind_type;
job->envtp->distribution = -1;
job->envtp->ckpt_dir = xstrdup(job->ckpt_dir);
job->envtp->batch_flag = job->batch;
/* Modify copy of job's environment. Do not alter in place or
* concurrent searches of the environment can generate invalid memory
* references. */
job->envtp->env = env_array_copy((const char **) job->env);
setup_env(job->envtp, false);
setenvf(&job->envtp->env, "SLURMD_NODENAME", "%s", conf->node_name);
tmp_env = job->env;
job->env = job->envtp->env;
env_array_free(tmp_env);
job->envtp->env = NULL;
xfree(job->envtp->task_count);
if (task->argv[0] && *task->argv[0] != '/') {
/*
* Normally the client (srun) expands the command name
* to a fully qualified path, but in --multi-prog mode it
* is left up to the server to search the PATH for the
* executable.
*/
task->argv[0] = _build_path(task->argv[0], job->env);
}
if (!job->batch) {
if (interconnect_attach(job->switch_job, &job->env,
job->nodeid, (uint32_t) i, job->nnodes,
job->ntasks, task->gtid) < 0) {
error("Unable to attach to interconnect: %m");
log_fini();
exit(1);
}
if (_setup_mpi(job, i) != SLURM_SUCCESS) {
error("Unable to configure MPI plugin: %m");
log_fini();
exit(1);
}
}
/* task-specific pre-launch activities */
if (spank_user_task (job, i) < 0) {
error ("Failed to invoke task plugin stack");
exit (1);
}
/* task plugin hook */
if (pre_launch(job)) {
error ("Failed task affinity setup");
exit (1);
}
if (conf->task_prolog) {
char *my_prolog;
slurm_mutex_lock(&conf->config_mutex);
my_prolog = xstrdup(conf->task_prolog);
slurm_mutex_unlock(&conf->config_mutex);
_run_script_and_set_env("slurm task_prolog",
my_prolog, job);
xfree(my_prolog);
}
if (job->task_prolog) {
_run_script_and_set_env("user task_prolog",
job->task_prolog, job);
}
if (!job->batch)
pdebug_stop_current(job);
if (job->env == NULL) {
debug("job->env is NULL");
job->env = (char **)xmalloc(sizeof(char *));
job->env[0] = (char *)NULL;
}
if (job->restart_dir) {
info("restart from %s", job->restart_dir);
/* no return on success */
checkpoint_restart_task(job, job->restart_dir, task->gtid);
error("Restart task failed: %m");
exit(errno);
}
if (task->argv[0] == NULL) {
error("No executable program specified for this task");
exit(2);
}
/* Do this last so you don't worry too much about the users
limits including the slurmstepd in with it.
*/
if (set_user_limits(job) < 0) {
debug("Unable to set user limits");
log_fini();
exit(5);
}
execve(task->argv[0], task->argv, job->env);
/*
* print error message and clean up if execve() returns:
*/
if ((errno == ENOENT) &&
((fd = open(task->argv[0], O_RDONLY)) >= 0)) {
char buf[256], *eol;
int sz;
sz = read(fd, buf, sizeof(buf));
if ((sz >= 3) && (strncmp(buf, "#!", 2) == 0)) {
eol = strchr(buf, '\n');
if (eol)
eol[0] = '\0';
else
buf[sizeof(buf)-1] = '\0';
error("execve(): bad interpreter(%s): %m", buf+2);
exit(errno);
}
}
error("execve(): %s: %m", task->argv[0]);
exit(errno);
}
/*
* Current process is running as the user when this is called.
*/
extern void exec_task(stepd_step_rec_t *job, int local_proc_id)
{
uint32_t *gtids; /* pointer to array of ranks */
int fd, j;
stepd_step_task_info_t *task = job->task[local_proc_id];
char **tmp_env;
int saved_errno;
uint32_t node_offset = 0, task_offset = 0;
if (job->node_offset != NO_VAL)
node_offset = job->node_offset;
if (job->pack_task_offset != NO_VAL)
task_offset = job->pack_task_offset;
gtids = xmalloc(job->node_tasks * sizeof(uint32_t));
for (j = 0; j < job->node_tasks; j++)
gtids[j] = job->task[j]->gtid + task_offset;
job->envtp->sgtids = _uint32_array_to_str(job->node_tasks, gtids);
xfree(gtids);
if (job->pack_jobid != NO_VAL)
job->envtp->jobid = job->pack_jobid;
else
job->envtp->jobid = job->jobid;
job->envtp->stepid = job->stepid;
job->envtp->nodeid = job->nodeid + node_offset;
job->envtp->cpus_on_node = job->cpus;
job->envtp->procid = task->gtid + task_offset;
job->envtp->localid = task->id;
job->envtp->task_pid = getpid();
job->envtp->distribution = job->task_dist;
job->envtp->cpu_bind = xstrdup(job->cpu_bind);
job->envtp->cpu_bind_type = job->cpu_bind_type;
job->envtp->cpu_freq_min = job->cpu_freq_min;
job->envtp->cpu_freq_max = job->cpu_freq_max;
job->envtp->cpu_freq_gov = job->cpu_freq_gov;
job->envtp->mem_bind = xstrdup(job->mem_bind);
job->envtp->mem_bind_type = job->mem_bind_type;
job->envtp->distribution = -1;
job->envtp->ckpt_dir = xstrdup(job->ckpt_dir);
job->envtp->batch_flag = job->batch;
job->envtp->uid = job->uid;
job->envtp->user_name = xstrdup(job->user_name);
/*
* Modify copy of job's environment. Do not alter in place or
* concurrent searches of the environment can generate invalid memory
* references.
*/
job->envtp->env = env_array_copy((const char **) job->env);
setup_env(job->envtp, false);
setenvf(&job->envtp->env, "SLURM_JOB_GID", "%d", job->gid);
setenvf(&job->envtp->env, "SLURMD_NODENAME", "%s", conf->node_name);
if (job->tres_bind) {
setenvf(&job->envtp->env, "SLURMD_TRES_BIND", "%s",
job->tres_bind);
}
if (job->tres_freq) {
setenvf(&job->envtp->env, "SLURMD_TRES_FREQ", "%s",
job->tres_freq);
}
tmp_env = job->env;
job->env = job->envtp->env;
env_array_free(tmp_env);
job->envtp->env = NULL;
xfree(job->envtp->task_count);
if (task->argv[0] && *task->argv[0] != '/') {
/*
* Normally the client (srun) expands the command name
* to a fully qualified path, but in --multi-prog mode it
* is left up to the server to search the PATH for the
* executable.
*/
task->argv[0] = _build_path(task->argv[0], job->env, NULL);
}
if (!job->batch && (job->stepid != SLURM_EXTERN_CONT)) {
if (switch_g_job_attach(job->switch_job, &job->env,
job->nodeid, (uint32_t) local_proc_id,
job->nnodes, job->ntasks,
task->gtid) < 0) {
error("Unable to attach to interconnect: %m");
log_fini();
exit(1);
}
if (_setup_mpi(job, local_proc_id) != SLURM_SUCCESS) {
error("Unable to configure MPI plugin: %m");
log_fini();
exit(1);
}
}
/* task-specific pre-launch activities */
/* task plugin hook */
if (task_g_pre_launch(job)) {
error("Failed to invoke task plugins: task_p_pre_launch error");
exit(1);
}
if (!job->batch &&
(job->accel_bind_type || job->tres_bind || job->tres_freq)) {
/*
* Modify copy of job's environment. Do not alter in place or
* concurrent searches of the environment can generate invalid
* memory references.
*
* Also sets GRES frequency as needed.
*/
job->envtp->env = env_array_copy((const char **) job->env);
gres_plugin_step_set_env(&job->envtp->env, job->step_gres_list,
job->accel_bind_type, job->tres_bind,
job->tres_freq, local_proc_id);
tmp_env = job->env;
job->env = job->envtp->env;
env_array_free(tmp_env);
}
if (spank_user_task(job, local_proc_id) < 0) {
error("Failed to invoke spank plugin stack");
exit(1);
}
if (conf->task_prolog) {
char *my_prolog;
slurm_mutex_lock(&conf->config_mutex);
my_prolog = xstrdup(conf->task_prolog);
slurm_mutex_unlock(&conf->config_mutex);
_run_script_and_set_env("slurm task_prolog",
my_prolog, job);
xfree(my_prolog);
}
if (job->task_prolog) {
_run_script_and_set_env("user task_prolog",
job->task_prolog, job);
}
/*
* Set TMPDIR after running prolog scripts, since TMPDIR
* might be set or changed in one of the prolog scripts.
*/
if (local_proc_id == 0)
_make_tmpdir(job);
if (!job->batch)
pdebug_stop_current(job);
if (job->env == NULL) {
debug("job->env is NULL");
job->env = (char **)xmalloc(sizeof(char *));
job->env[0] = (char *)NULL;
}
if (job->restart_dir) {
info("restart from %s", job->restart_dir);
/* no return on success */
checkpoint_restart_task(job, job->restart_dir, task->gtid);
error("Restart task failed: %m");
exit(errno);
}
if (task->argv[0] == NULL) {
error("No executable program specified for this task");
exit(2);
}
/* Do this last so you don't worry too much about the users
limits including the slurmstepd in with it.
*/
if (set_user_limits(job) < 0) {
debug("Unable to set user limits");
log_fini();
exit(5);
}
execve(task->argv[0], task->argv, job->env);
saved_errno = errno;
/*
* print error message and clean up if execve() returns:
*/
if ((errno == ENOENT) &&
((fd = open(task->argv[0], O_RDONLY)) >= 0)) {
char buf[256], *eol;
int sz;
sz = read(fd, buf, sizeof(buf));
if ((sz >= 3) && (xstrncmp(buf, "#!", 2) == 0)) {
buf[sizeof(buf)-1] = '\0';
eol = strchr(buf, '\n');
if (eol)
eol[0] = '\0';
slurm_seterrno(saved_errno);
error("execve(): bad interpreter(%s): %m", buf+2);
exit(errno);
}
}
slurm_seterrno(saved_errno);
error("execve(): %s: %m", task->argv[0]);
exit(errno);
}
static void
_set_exec_names(char *ranks, char *exec_name, int ntasks)
{
char *ptrptr = NULL, *exec_path = NULL;
int low_num, high_num, num, i;
exec_path = _build_path(exec_name);
if ((ranks[0] == '*') && (ranks[1] == '\0')) {
low_num = 0;
high_num = ntasks - 1;
_set_range(low_num, high_num, exec_path, true);
return;
}
ptrptr = ranks;
for (i=0; i<ntasks; i++) {
if (!isdigit(ptrptr[0]))
goto invalid;
num = strtol(ptrptr, &ptrptr, 10);
if ((ptrptr[0] == ',') || (ptrptr[0] == '\0')) {
low_num = MAX(0, num);
high_num = MIN((ntasks-1), num);
_set_range(low_num, high_num, exec_path, false);
} else if (ptrptr[0] == '-') {
low_num = MAX(0, num);
num = strtol(ptrptr+1, &ptrptr, 10);
if ((ptrptr[0] != ',') && (ptrptr[0] != '\0'))
goto invalid;
high_num = MIN((ntasks-1), num);
_set_range(low_num, high_num, exec_path, false);
} else
goto invalid;
if (ptrptr[0] == '\0')
break;
ptrptr++;
}
return;
invalid:
error ("Invalid task range specification (%s) ignored.", ranks);
return;
}
void chili_redirect_print(const char *name, const char* before,
const char* after)
{
int fd;
const int max = 1024;
char buf[max];
int size;
if (!_enabled){
return;
}
if (_build_path(_print_name, name) < 0){
return;
}
if (before){
_write(1, before, strlen(before));
}
fd = open(_print_name, O_RDONLY);
while ((size = read(fd, buf, max)) > 0){
_write(1, buf, size);
}
close(fd);
if (after){
_write(1, after, strlen(after));
}
}
