/*
* check that a cgroup namespace is ready to be used
*
* returned values:
* - XCGROUP_ERROR : not available
* - XCGROUP_SUCCESS : ready to be used
*/
int xcgroup_ns_is_available(xcgroup_ns_t* cgns)
{
int fstatus = 0;
char* value;
size_t s;
xcgroup_t cg;
if (xcgroup_create(cgns, &cg, "/", 0, 0) == XCGROUP_ERROR)
return 0;
if (xcgroup_get_param(&cg, "release_agent",
&value, &s) != XCGROUP_SUCCESS)
fstatus = 0;
else {
xfree(value);
fstatus = 1;
}
xcgroup_destroy(&cg);
return fstatus;
}
static void _prec_extra(jag_prec_t *prec)
{
unsigned long utime, stime, total_rss, total_pgpgin;
char *cpu_time = NULL, *memory_stat = NULL, *ptr;
size_t cpu_time_size = 0, memory_stat_size = 0;
//DEF_TIMERS;
//START_TIMER;
/* info("before"); */
/* print_jag_prec(prec); */
xcgroup_get_param(&task_cpuacct_cg, "cpuacct.stat",
&cpu_time, &cpu_time_size);
sscanf(cpu_time, "%*s %lu %*s %lu", &utime, &stime);
prec->usec = utime;
prec->ssec = stime;
xcgroup_get_param(&task_memory_cg, "memory.stat",
&memory_stat, &memory_stat_size);
/* This number represents the amount of "dirty" private memory
used by the cgroup. From our experience this is slightly
different than what proc presents, but is probably more
accurate on what the user is actually using.
*/
ptr = strstr(memory_stat, "total_rss");
sscanf(ptr, "total_rss %lu", &total_rss);
prec->rss = total_rss / 1024; /* convert from bytes to KB */
/* total_pgmajfault is what is reported in proc, so we use
* the same thing here. */
if ((ptr = strstr(memory_stat, "total_pgmajfault"))) {
sscanf(ptr, "total_pgmajfault %lu", &total_pgpgin);
prec->pages = total_pgpgin;
}
xfree(cpu_time);
xfree(memory_stat);
/* FIXME: Enable when kernel support ready.
*
* "Read" and "Write" from blkio.throttle.io_service_bytes are
* counts of bytes read and written for physical disk I/Os only.
* These counts do not include disk I/Os satisfied from cache.
*/
/* int dev_major; */
/* uint64_t read_bytes, write_bytes, tot_read, tot_write; */
/* char *blkio_bytes, *next_device; */
/* size_t blkio_bytes_size; */
/* xcgroup_get_param(&task_blkio_cg, "blkio.throttle.io_service_bytes", */
/* &blkio_bytes, &blkio_bytes_size); */
/* next_device = blkio_bytes; */
/* tot_read = tot_write = 0; */
/* while ((sscanf(next_device, "%d:", &dev_major)) > 0) { */
/* if ((dev_major > 239) && (dev_major < 255)) */
/* /\* skip experimental device codes *\/ */
/* continue; */
/* next_device = strstr(next_device, "Read"); */
/* sscanf(next_device, "%*s %"PRIu64"", &read_bytes); */
/* next_device = strstr(next_device, "Write"); */
/* sscanf(next_device, "%*s %"PRIu64"", &write_bytes); */
/* tot_read+=read_bytes; */
/* tot_write+=write_bytes; */
/* next_device = strstr(next_device, "Total"); */
/* } */
/* prec->disk_read = (double)tot_read / (double)1048576; */
/* prec->disk_write = (double)tot_write / (double)1048576; */
/* info("after %d %d", total_rss); */
/* print_jag_prec(prec); */
//END_TIMER;
//info("took %s", TIME_STR);
return;
}
/* when cgroups are configured with cpuset, at least
* cpuset.cpus and cpuset.mems must be set or the cgroup
* will not be available at all.
* we duplicate the ancestor configuration in the init step */
static int _xcgroup_cpuset_init(xcgroup_t* cg)
{
int fstatus,i;
char* cpuset_metafiles[] = {
"cpuset.cpus",
"cpuset.mems"
};
char* cpuset_meta;
char* cpuset_conf;
size_t csize;
xcgroup_t acg;
char* acg_name;
char* p;
fstatus = XCGROUP_ERROR;
/* load ancestor cg */
acg_name = (char*) xstrdup(cg->name);
p = rindex(acg_name,'/');
if (p == NULL) {
debug2("task/cgroup: unable to get ancestor path for "
"cpuset cg '%s' : %m",cg->path);
return fstatus;
} else
*p = '\0';
if (xcgroup_load(cg->ns,&acg,acg_name) != XCGROUP_SUCCESS) {
debug2("task/cgroup: unable to load ancestor for "
"cpuset cg '%s' : %m",cg->path);
return fstatus;
}
/* inherits ancestor params */
for (i = 0 ; i < 2 ; i++) {
cpuset_meta = cpuset_metafiles[i];
if (xcgroup_get_param(&acg,cpuset_meta,
&cpuset_conf,&csize)
!= XCGROUP_SUCCESS) {
debug2("task/cgroup: assuming no cpuset cg "
"support for '%s'",acg.path);
xcgroup_destroy(&acg);
return fstatus;
}
if (csize > 0)
cpuset_conf[csize-1]='\0';
if (xcgroup_set_param(cg,cpuset_meta,cpuset_conf)
!= XCGROUP_SUCCESS) {
debug2("task/cgroup: unable to write %s configuration "
"(%s) for cpuset cg '%s'",cpuset_meta,
cpuset_conf,cg->path);
xcgroup_destroy(&acg);
xfree(cpuset_conf);
return fstatus;
}
xfree(cpuset_conf);
}
xcgroup_destroy(&acg);
return XCGROUP_SUCCESS;
}
extern int task_cgroup_cpuset_create(slurmd_job_t *job)
{
int rc;
int fstatus = SLURM_ERROR;
xcgroup_t cpuset_cg;
uint32_t jobid = job->jobid;
uint32_t stepid = job->stepid;
uid_t uid = job->uid;
uid_t gid = job->gid;
char* user_alloc_cores = NULL;
char* job_alloc_cores = NULL;
char* step_alloc_cores = NULL;
char* cpus = NULL;
size_t cpus_size;
char* slurm_cgpath ;
xcgroup_t slurm_cg;
/* create slurm root cg in this cg namespace */
slurm_cgpath = task_cgroup_create_slurm_cg(&cpuset_ns);
if ( slurm_cgpath == NULL ) {
return SLURM_ERROR;
}
/* check that this cgroup has cpus allowed or initialize them */
if (xcgroup_load(&cpuset_ns,&slurm_cg,slurm_cgpath)
!= XCGROUP_SUCCESS) {
error("task/cgroup: unable to load slurm cpuset xcgroup");
xfree(slurm_cgpath);
return SLURM_ERROR;
}
rc = xcgroup_get_param(&slurm_cg,"cpuset.cpus",&cpus,&cpus_size);
if (rc != XCGROUP_SUCCESS || cpus_size == 1) {
/* initialize the cpusets as it was inexistant */
if (_xcgroup_cpuset_init(&slurm_cg) !=
XCGROUP_SUCCESS) {
xfree(slurm_cgpath);
xcgroup_destroy(&slurm_cg);
return SLURM_ERROR;
}
}
xfree(cpus);
/* build user cgroup relative path if not set (should not be) */
if (*user_cgroup_path == '\0') {
if (snprintf(user_cgroup_path, PATH_MAX,
"%s/uid_%u", slurm_cgpath, uid) >= PATH_MAX) {
error("unable to build uid %u cgroup relative "
"path : %m", uid);
xfree(slurm_cgpath);
return SLURM_ERROR;
}
}
xfree(slurm_cgpath);
/* build job cgroup relative path if no set (should not be) */
if (*job_cgroup_path == '\0') {
if (snprintf(job_cgroup_path,PATH_MAX,"%s/job_%u",
user_cgroup_path,jobid) >= PATH_MAX) {
error("task/cgroup: unable to build job %u cpuset "
"cg relative path : %m",jobid);
return SLURM_ERROR;
}
}
/* build job step cgroup relative path (should not be) */
if (*jobstep_cgroup_path == '\0') {
if (stepid == NO_VAL) {
if (snprintf(jobstep_cgroup_path, PATH_MAX,
"%s/step_batch", job_cgroup_path)
>= PATH_MAX) {
error("task/cgroup: unable to build job step"
" %u.batch cpuset cg relative path: %m",
jobid);
return SLURM_ERROR;
}
} else {
if (snprintf(jobstep_cgroup_path, PATH_MAX, "%s/step_%u",
job_cgroup_path, stepid) >= PATH_MAX) {
error("task/cgroup: unable to build job step"
" %u.%u cpuset cg relative path: %m",
jobid, stepid);
return SLURM_ERROR;
}
}
}
/*
* create cpuset root cg and lock it
*
* we will keep the lock until the end to avoid the effect of a release
* agent that would remove an existing cgroup hierarchy while we are
* setting it up. As soon as the step cgroup is created, we can release
* the lock.
* Indeed, consecutive slurm steps could result in cg being removed
* between the next EEXIST instanciation and the first addition of
* a task. The release_agent will have to lock the root cpuset cgroup
* to avoid this scenario.
*/
if (xcgroup_create(&cpuset_ns,&cpuset_cg,"",0,0) != XCGROUP_SUCCESS) {
error("task/cgroup: unable to create root cpuset xcgroup");
return SLURM_ERROR;
}
if (xcgroup_lock(&cpuset_cg) != XCGROUP_SUCCESS) {
xcgroup_destroy(&cpuset_cg);
error("task/cgroup: unable to lock root cpuset cg");
return SLURM_ERROR;
}
/*
* build job and job steps allocated cores lists
*/
debug("task/cgroup: job abstract cores are '%s'",
job->job_alloc_cores);
debug("task/cgroup: step abstract cores are '%s'",
job->step_alloc_cores);
if (xcpuinfo_abs_to_mac(job->job_alloc_cores,
&job_alloc_cores) != XCPUINFO_SUCCESS) {
error("task/cgroup: unable to build job physical cores");
goto error;
}
if (xcpuinfo_abs_to_mac(job->step_alloc_cores,
&step_alloc_cores) != XCPUINFO_SUCCESS) {
error("task/cgroup: unable to build step physical cores");
goto error;
}
debug("task/cgroup: job physical cores are '%s'",
job->job_alloc_cores);
debug("task/cgroup: step physical cores are '%s'",
job->step_alloc_cores);
/*
* create user cgroup in the cpuset ns (it could already exist)
*/
if (xcgroup_create(&cpuset_ns,&user_cpuset_cg,
user_cgroup_path,
getuid(),getgid()) != XCGROUP_SUCCESS) {
goto error;
}
if (xcgroup_instanciate(&user_cpuset_cg) != XCGROUP_SUCCESS) {
xcgroup_destroy(&user_cpuset_cg);
goto error;
}
/*
* check that user's cpuset cgroup is consistant and add the job cores
*/
rc = xcgroup_get_param(&user_cpuset_cg,"cpuset.cpus",&cpus,&cpus_size);
if (rc != XCGROUP_SUCCESS || cpus_size == 1) {
/* initialize the cpusets as it was inexistant */
if (_xcgroup_cpuset_init(&user_cpuset_cg) !=
XCGROUP_SUCCESS) {
xcgroup_delete(&user_cpuset_cg);
xcgroup_destroy(&user_cpuset_cg);
goto error;
}
}
user_alloc_cores = xstrdup(job_alloc_cores);
if (cpus != NULL && cpus_size > 1) {
cpus[cpus_size-1]='\0';
xstrcat(user_alloc_cores,",");
xstrcat(user_alloc_cores,cpus);
}
xcgroup_set_param(&user_cpuset_cg,"cpuset.cpus",user_alloc_cores);
xfree(cpus);
/*
* create job cgroup in the cpuset ns (it could already exist)
*/
if (xcgroup_create(&cpuset_ns,&job_cpuset_cg,
job_cgroup_path,
getuid(),getgid()) != XCGROUP_SUCCESS) {
xcgroup_destroy(&user_cpuset_cg);
goto error;
}
if (xcgroup_instanciate(&job_cpuset_cg) != XCGROUP_SUCCESS) {
xcgroup_destroy(&user_cpuset_cg);
xcgroup_destroy(&job_cpuset_cg);
goto error;
}
if (_xcgroup_cpuset_init(&job_cpuset_cg) != XCGROUP_SUCCESS) {
xcgroup_destroy(&user_cpuset_cg);
xcgroup_destroy(&job_cpuset_cg);
goto error;
}
xcgroup_set_param(&job_cpuset_cg,"cpuset.cpus",job_alloc_cores);
/*
* create step cgroup in the cpuset ns (it should not exists)
* use job's user uid/gid to enable tasks cgroups creation by
* the user inside the step cgroup owned by root
*/
if (xcgroup_create(&cpuset_ns,&step_cpuset_cg,
jobstep_cgroup_path,
uid,gid) != XCGROUP_SUCCESS) {
/* do not delete user/job cgroup as */
/* they can exist for other steps */
xcgroup_destroy(&user_cpuset_cg);
xcgroup_destroy(&job_cpuset_cg);
goto error;
}
if (xcgroup_instanciate(&step_cpuset_cg) != XCGROUP_SUCCESS) {
xcgroup_destroy(&user_cpuset_cg);
xcgroup_destroy(&job_cpuset_cg);
xcgroup_destroy(&step_cpuset_cg);
goto error;
}
if (_xcgroup_cpuset_init(&step_cpuset_cg) != XCGROUP_SUCCESS) {
xcgroup_destroy(&user_cpuset_cg);
xcgroup_destroy(&job_cpuset_cg);
xcgroup_delete(&step_cpuset_cg);
xcgroup_destroy(&step_cpuset_cg);
goto error;
}
xcgroup_set_param(&step_cpuset_cg,"cpuset.cpus",step_alloc_cores);
/* attach the slurmstepd to the step cpuset cgroup */
pid_t pid = getpid();
rc = xcgroup_add_pids(&step_cpuset_cg,&pid,1);
if (rc != XCGROUP_SUCCESS) {
error("task/cgroup: unable to add slurmstepd to cpuset cg '%s'",
step_cpuset_cg.path);
fstatus = SLURM_ERROR;
} else
fstatus = SLURM_SUCCESS;
error:
xcgroup_unlock(&cpuset_cg);
xcgroup_destroy(&cpuset_cg);
xfree(user_alloc_cores);
xfree(job_alloc_cores);
xfree(step_alloc_cores);
return fstatus;
}
extern int init_system_cpuset_cgroup(void)
{
int rc;
int fstatus = SLURM_ERROR;
char* cpus = NULL;
size_t cpus_size;
char* slurm_cgpath;
xcgroup_t slurm_cg;
/* read cgroup configuration */
if (read_slurm_cgroup_conf(&slurm_cgroup_conf))
return SLURM_ERROR;
/* initialize cpuset cgroup namespace */
if (xcgroup_ns_create(&slurm_cgroup_conf, &cpuset_ns, "", "cpuset")
!= XCGROUP_SUCCESS) {
error("system cgroup: unable to create cpuset namespace");
free_slurm_cgroup_conf(&slurm_cgroup_conf);
return SLURM_ERROR;
}
/* create slurm root cg in this cg namespace */
slurm_cgpath = _system_cgroup_create_slurm_cg(&cpuset_ns);
if ( slurm_cgpath == NULL ) {
xcgroup_ns_destroy(&cpuset_ns);
free_slurm_cgroup_conf(&slurm_cgroup_conf);
return SLURM_ERROR;
}
/* check that this cgroup has cpus allowed or initialize them */
if (xcgroup_load(&cpuset_ns, &slurm_cg, slurm_cgpath)
!= XCGROUP_SUCCESS) {
error("system cgroup: unable to load slurm cpuset xcgroup");
xfree(slurm_cgpath);
xcgroup_ns_destroy(&cpuset_ns);
free_slurm_cgroup_conf(&slurm_cgroup_conf);
return SLURM_ERROR;
}
again:
snprintf(cpuset_meta, sizeof(cpuset_meta), "%scpus", cpuset_prefix);
rc = xcgroup_get_param(&slurm_cg, cpuset_meta, &cpus, &cpus_size);
if (rc != XCGROUP_SUCCESS || cpus_size == 1) {
if (!cpuset_prefix_set && (rc != XCGROUP_SUCCESS)) {
cpuset_prefix_set = 1;
cpuset_prefix = "cpuset.";
goto again;
}
/* initialize the cpusets as it was nonexistent */
if (_xcgroup_cpuset_init(&slurm_cg) != XCGROUP_SUCCESS) {
xfree(slurm_cgpath);
xcgroup_destroy(&slurm_cg);
xcgroup_ns_destroy(&cpuset_ns);
free_slurm_cgroup_conf(&slurm_cgroup_conf);
xfree(cpus);
return SLURM_ERROR;
}
}
xcgroup_destroy(&slurm_cg);
xfree(cpus);
/* build system cgroup relative path */
snprintf(system_cgroup_path, PATH_MAX, "%s/system", slurm_cgpath);
xfree(slurm_cgpath);
/* create system cgroup in the cpuset ns */
if (xcgroup_create(&cpuset_ns, &system_cpuset_cg, system_cgroup_path,
getuid(),getgid()) != XCGROUP_SUCCESS) {
goto error;
}
if (xcgroup_instantiate(&system_cpuset_cg) != XCGROUP_SUCCESS) {
goto error;
}
if (_xcgroup_cpuset_init(&system_cpuset_cg) != XCGROUP_SUCCESS) {
goto error;
}
free_slurm_cgroup_conf(&slurm_cgroup_conf);
debug("system cgroup: system cpuset cgroup initialized");
return SLURM_SUCCESS;
error:
xcgroup_unlock(&system_cpuset_cg);
xcgroup_destroy(&system_cpuset_cg);
xcgroup_ns_destroy(&cpuset_ns);
free_slurm_cgroup_conf(&slurm_cgroup_conf);
return fstatus;
}
/* when cgroups are configured with cpuset, at least
* cpuset.cpus and cpuset.mems must be set or the cgroup
* will not be available at all.
* we duplicate the ancestor configuration in the init step */
static int _xcgroup_cpuset_init(xcgroup_t* cg)
{
int fstatus, i;
char* cpuset_metafiles[] = {
"cpus",
"mems"
};
char* cpuset_conf = NULL;
size_t csize = 0;
xcgroup_t acg;
char* acg_name = NULL;
char* p;
fstatus = XCGROUP_ERROR;
/* load ancestor cg */
acg_name = (char*) xstrdup(cg->name);
p = xstrrchr(acg_name, '/');
if (p == NULL) {
debug2("system cgroup: unable to get ancestor path for "
"cpuset cg '%s' : %m", cg->path);
xfree(acg_name);
return fstatus;
} else
*p = '\0';
if (xcgroup_load(cg->ns, &acg, acg_name) != XCGROUP_SUCCESS) {
debug2("system cgroup: unable to load ancestor for "
"cpuset cg '%s' : %m", cg->path);
xfree(acg_name);
return fstatus;
}
xfree(acg_name);
/* inherits ancestor params */
for (i = 0 ; i < 2 ; i++) {
again:
snprintf(cpuset_meta, sizeof(cpuset_meta), "%s%s",
cpuset_prefix, cpuset_metafiles[i]);
if (xcgroup_get_param(&acg ,cpuset_meta,
&cpuset_conf, &csize)
!= XCGROUP_SUCCESS) {
if (!cpuset_prefix_set) {
cpuset_prefix_set = 1;
cpuset_prefix = "cpuset.";
goto again;
}
debug("system cgroup: assuming no cpuset cg "
"support for '%s'",acg.path);
xcgroup_destroy(&acg);
return fstatus;
}
if (csize > 0)
cpuset_conf[csize-1] = '\0';
if (xcgroup_set_param(cg,cpuset_meta, cpuset_conf)
!= XCGROUP_SUCCESS) {
debug("system cgroup: unable to write %s configuration "
"(%s) for cpuset cg '%s'",cpuset_meta,
cpuset_conf, cg->path);
xcgroup_destroy(&acg);
xfree(cpuset_conf);
return fstatus;
}
xfree(cpuset_conf);
}
xcgroup_destroy(&acg);
return XCGROUP_SUCCESS;
}
extern int task_cgroup_cpuset_create(stepd_step_rec_t *job)
{
int rc;
int fstatus = SLURM_ERROR;
xcgroup_t cpuset_cg;
uint32_t jobid = job->jobid;
uint32_t stepid = job->stepid;
uid_t uid = job->uid;
uid_t gid = job->gid;
char* user_alloc_cores = NULL;
char* job_alloc_cores = NULL;
char* step_alloc_cores = NULL;
char cpuset_meta[PATH_MAX];
char* cpus = NULL;
size_t cpus_size;
char* slurm_cgpath;
xcgroup_t slurm_cg;
#ifdef HAVE_NATIVE_CRAY
char expected_usage[32];
#endif
/* create slurm root cg in this cg namespace */
slurm_cgpath = task_cgroup_create_slurm_cg(&cpuset_ns);
if ( slurm_cgpath == NULL ) {
return SLURM_ERROR;
}
/* check that this cgroup has cpus allowed or initialize them */
if (xcgroup_load(&cpuset_ns,&slurm_cg,slurm_cgpath)
!= XCGROUP_SUCCESS) {
error("task/cgroup: unable to load slurm cpuset xcgroup");
xfree(slurm_cgpath);
return SLURM_ERROR;
}
again:
snprintf(cpuset_meta, sizeof(cpuset_meta), "%scpus", cpuset_prefix);
rc = xcgroup_get_param(&slurm_cg, cpuset_meta, &cpus,&cpus_size);
if (rc != XCGROUP_SUCCESS || cpus_size == 1) {
if (!cpuset_prefix_set && (rc != XCGROUP_SUCCESS)) {
cpuset_prefix_set = 1;
cpuset_prefix = "cpuset.";
goto again;
}
/* initialize the cpusets as it was inexistant */
if (_xcgroup_cpuset_init(&slurm_cg) !=
XCGROUP_SUCCESS) {
xfree(slurm_cgpath);
xcgroup_destroy(&slurm_cg);
return SLURM_ERROR;
}
}
xfree(cpus);
/* build user cgroup relative path if not set (should not be) */
if (*user_cgroup_path == '\0') {
if (snprintf(user_cgroup_path, PATH_MAX,
"%s/uid_%u", slurm_cgpath, uid) >= PATH_MAX) {
error("task/cgroup: unable to build uid %u cgroup "
"relative path : %m", uid);
xfree(slurm_cgpath);
return SLURM_ERROR;
}
}
xfree(slurm_cgpath);
/* build job cgroup relative path if no set (should not be) */
if (*job_cgroup_path == '\0') {
if (snprintf(job_cgroup_path,PATH_MAX,"%s/job_%u",
user_cgroup_path,jobid) >= PATH_MAX) {
error("task/cgroup: unable to build job %u cpuset "
"cg relative path : %m",jobid);
return SLURM_ERROR;
}
}
/* build job step cgroup relative path (should not be) */
if (*jobstep_cgroup_path == '\0') {
int cc;
if (stepid == SLURM_BATCH_SCRIPT) {
cc = snprintf(jobstep_cgroup_path, PATH_MAX,
"%s/step_batch", job_cgroup_path);
} else if (stepid == SLURM_EXTERN_CONT) {
cc = snprintf(jobstep_cgroup_path, PATH_MAX,
"%s/step_extern", job_cgroup_path);
} else {
cc = snprintf(jobstep_cgroup_path, PATH_MAX,
"%s/step_%u", job_cgroup_path, stepid);
}
if (cc >= PATH_MAX) {
error("task/cgroup: unable to build job step %u.%u "
"cpuset cg relative path: %m",
jobid, stepid);
return SLURM_ERROR;
}
}
/*
* create cpuset root cg and lock it
*
* we will keep the lock until the end to avoid the effect of a release
* agent that would remove an existing cgroup hierarchy while we are
* setting it up. As soon as the step cgroup is created, we can release
* the lock.
* Indeed, consecutive slurm steps could result in cg being removed
* between the next EEXIST instanciation and the first addition of
* a task. The release_agent will have to lock the root cpuset cgroup
* to avoid this scenario.
*/
if (xcgroup_create(&cpuset_ns,&cpuset_cg,"",0,0) != XCGROUP_SUCCESS) {
error("task/cgroup: unable to create root cpuset xcgroup");
return SLURM_ERROR;
}
if (xcgroup_lock(&cpuset_cg) != XCGROUP_SUCCESS) {
xcgroup_destroy(&cpuset_cg);
error("task/cgroup: unable to lock root cpuset cg");
return SLURM_ERROR;
}
/*
* build job and job steps allocated cores lists
*/
debug("task/cgroup: job abstract cores are '%s'",
job->job_alloc_cores);
debug("task/cgroup: step abstract cores are '%s'",
job->step_alloc_cores);
if (xcpuinfo_abs_to_mac(job->job_alloc_cores,
&job_alloc_cores) != SLURM_SUCCESS) {
error("task/cgroup: unable to build job physical cores");
goto error;
}
if (xcpuinfo_abs_to_mac(job->step_alloc_cores,
&step_alloc_cores) != SLURM_SUCCESS) {
error("task/cgroup: unable to build step physical cores");
goto error;
}
debug("task/cgroup: job physical cores are '%s'",
job_alloc_cores);
debug("task/cgroup: step physical cores are '%s'",
step_alloc_cores);
/*
* create user cgroup in the cpuset ns (it could already exist)
*/
if (xcgroup_create(&cpuset_ns,&user_cpuset_cg,
user_cgroup_path,
getuid(),getgid()) != XCGROUP_SUCCESS) {
goto error;
}
if (xcgroup_instanciate(&user_cpuset_cg) != XCGROUP_SUCCESS) {
xcgroup_destroy(&user_cpuset_cg);
goto error;
}
/*
* check that user's cpuset cgroup is consistant and add the job cores
*/
rc = xcgroup_get_param(&user_cpuset_cg, cpuset_meta, &cpus,&cpus_size);
if (rc != XCGROUP_SUCCESS || cpus_size == 1) {
/* initialize the cpusets as it was inexistant */
if (_xcgroup_cpuset_init(&user_cpuset_cg) !=
XCGROUP_SUCCESS) {
xcgroup_delete(&user_cpuset_cg);
xcgroup_destroy(&user_cpuset_cg);
goto error;
}
}
user_alloc_cores = xstrdup(job_alloc_cores);
if (cpus != NULL && cpus_size > 1) {
cpus[cpus_size-1]='\0';
xstrcat(user_alloc_cores,",");
xstrcat(user_alloc_cores,cpus);
}
xcgroup_set_param(&user_cpuset_cg, cpuset_meta, user_alloc_cores);
xfree(cpus);
/*
* create job cgroup in the cpuset ns (it could already exist)
*/
if (xcgroup_create(&cpuset_ns,&job_cpuset_cg,
job_cgroup_path,
getuid(),getgid()) != XCGROUP_SUCCESS) {
xcgroup_destroy(&user_cpuset_cg);
goto error;
}
if (xcgroup_instanciate(&job_cpuset_cg) != XCGROUP_SUCCESS) {
xcgroup_destroy(&user_cpuset_cg);
xcgroup_destroy(&job_cpuset_cg);
goto error;
}
if (_xcgroup_cpuset_init(&job_cpuset_cg) != XCGROUP_SUCCESS) {
xcgroup_destroy(&user_cpuset_cg);
xcgroup_destroy(&job_cpuset_cg);
goto error;
}
xcgroup_set_param(&job_cpuset_cg, cpuset_meta, job_alloc_cores);
/*
* create step cgroup in the cpuset ns (it should not exists)
* use job's user uid/gid to enable tasks cgroups creation by
* the user inside the step cgroup owned by root
*/
if (xcgroup_create(&cpuset_ns,&step_cpuset_cg,
jobstep_cgroup_path,
uid,gid) != XCGROUP_SUCCESS) {
/* do not delete user/job cgroup as */
/* they can exist for other steps */
xcgroup_destroy(&user_cpuset_cg);
xcgroup_destroy(&job_cpuset_cg);
goto error;
}
if (xcgroup_instanciate(&step_cpuset_cg) != XCGROUP_SUCCESS) {
xcgroup_destroy(&user_cpuset_cg);
xcgroup_destroy(&job_cpuset_cg);
xcgroup_destroy(&step_cpuset_cg);
goto error;
}
if (_xcgroup_cpuset_init(&step_cpuset_cg) != XCGROUP_SUCCESS) {
xcgroup_destroy(&user_cpuset_cg);
xcgroup_destroy(&job_cpuset_cg);
xcgroup_delete(&step_cpuset_cg);
xcgroup_destroy(&step_cpuset_cg);
goto error;
}
xcgroup_set_param(&step_cpuset_cg, cpuset_meta, step_alloc_cores);
/*
* on Cray systems, set the expected usage in bytes.
* This is used by the Cray OOM killer
*/
#ifdef HAVE_NATIVE_CRAY
snprintf(expected_usage, sizeof(expected_usage), "%"PRIu64,
(uint64_t)job->step_mem * 1024 * 1024);
xcgroup_set_param(&step_cpuset_cg, "expected_usage_in_bytes",
expected_usage);
#endif
/* attach the slurmstepd to the step cpuset cgroup */
pid_t pid = getpid();
rc = xcgroup_add_pids(&step_cpuset_cg,&pid,1);
if (rc != XCGROUP_SUCCESS) {
error("task/cgroup: unable to add slurmstepd to cpuset cg '%s'",
step_cpuset_cg.path);
fstatus = SLURM_ERROR;
} else
fstatus = SLURM_SUCCESS;
/* validate the requested cpu frequency and set it */
cpu_freq_cgroup_validate(job, step_alloc_cores);
error:
xcgroup_unlock(&cpuset_cg);
xcgroup_destroy(&cpuset_cg);
xfree(user_alloc_cores);
xfree(job_alloc_cores);
xfree(step_alloc_cores);
return fstatus;
}
