int _slurm_cgroup_init(void)
{
/* initialize user/job/jobstep cgroup relative paths
* and release agent path */
user_cgroup_path[0]='\0';
job_cgroup_path[0]='\0';
jobstep_cgroup_path[0]='\0';
/* initialize freezer cgroup namespace */
if (xcgroup_ns_create(&slurm_cgroup_conf, &freezer_ns, "", "freezer")
!= XCGROUP_SUCCESS) {
error("unable to create freezer cgroup namespace");
return SLURM_ERROR;
}
/* initialize the root freezer cg */
if (xcgroup_create(&freezer_ns, &freezer_cg, "", 0, 0)
!= XCGROUP_SUCCESS) {
error("proctrack/cgroup unable to create root freezer xcgroup");
xcgroup_ns_destroy(&freezer_ns);
return SLURM_ERROR;
}
return SLURM_SUCCESS;
}
extern int task_cgroup_cpuset_init(slurm_cgroup_conf_t *slurm_cgroup_conf)
{
char release_agent_path[PATH_MAX];
/* initialize cpuinfo internal data */
if (xcpuinfo_init() != XCPUINFO_SUCCESS) {
return SLURM_ERROR;
}
/* initialize user/job/jobstep cgroup relative paths */
user_cgroup_path[0]='\0';
job_cgroup_path[0]='\0';
jobstep_cgroup_path[0]='\0';
/* initialize cpuset cgroup namespace */
release_agent_path[0]='\0';
if (snprintf(release_agent_path,PATH_MAX,"%s/release_cpuset",
slurm_cgroup_conf->cgroup_release_agent) >= PATH_MAX) {
error("task/cgroup: unable to build cpuset release agent path");
goto error;
}
if (xcgroup_ns_create(slurm_cgroup_conf, &cpuset_ns, "/cpuset", "",
"cpuset",release_agent_path) !=
XCGROUP_SUCCESS) {
error("task/cgroup: unable to create cpuset namespace");
goto error;
}
/* check that cpuset cgroup namespace is available */
if (! xcgroup_ns_is_available(&cpuset_ns)) {
if (slurm_cgroup_conf->cgroup_automount) {
if (xcgroup_ns_mount(&cpuset_ns)) {
error("task/cgroup: unable to mount cpuset "
"namespace");
goto clean;
}
info("task/cgroup: cpuset namespace is now mounted");
} else {
error("task/cgroup: cpuset namespace not mounted. "
"aborting");
goto clean;
}
}
return SLURM_SUCCESS;
clean:
xcgroup_ns_destroy(&cpuset_ns);
error:
xcpuinfo_fini();
return SLURM_ERROR;
}
extern int task_cgroup_memory_init(slurm_cgroup_conf_t *slurm_cgroup_conf)
{
/* initialize user/job/jobstep cgroup relative paths */
user_cgroup_path[0]='\0';
job_cgroup_path[0]='\0';
jobstep_cgroup_path[0]='\0';
/* initialize memory cgroup namespace */
if (xcgroup_ns_create(slurm_cgroup_conf, &memory_ns, "", "memory")
!= XCGROUP_SUCCESS) {
error("task/cgroup: unable to create memory namespace");
return SLURM_ERROR;
}
allowed_ram_space = slurm_cgroup_conf->allowed_ram_space;
allowed_swap_space = slurm_cgroup_conf->allowed_swap_space;
if ((totalram = (uint64_t) conf->real_memory_size) == 0)
error ("task/cgroup: Unable to get RealMemory size");
max_ram = percent_in_bytes(totalram, slurm_cgroup_conf->max_ram_percent);
max_swap = percent_in_bytes(totalram, slurm_cgroup_conf->max_swap_percent);
max_swap += max_ram;
min_ram_space = slurm_cgroup_conf->min_ram_space * 1024 * 1024;
debug ("task/cgroup/memory: total:%luM allowed:%.4g%%, swap:%.4g%%, "
"max:%.4g%%(%luM) max+swap:%.4g%%(%luM) min:%uM",
(unsigned long) totalram,
allowed_ram_space,
allowed_swap_space,
slurm_cgroup_conf->max_ram_percent,
(unsigned long) (max_ram/(1024*1024)),
slurm_cgroup_conf->max_swap_percent,
(unsigned long) (max_swap/(1024*1024)),
(unsigned) slurm_cgroup_conf->min_ram_space);
/*
* Warning: OOM Killer must be disabled for slurmstepd
* or it would be destroyed if the application use
* more memory than permitted
*
* If an env value is already set for slurmstepd
* OOM killer behavior, keep it, otherwise set the
* -17 value, wich means do not let OOM killer kill it
*
* FYI, setting "export SLURMSTEPD_OOM_ADJ=-17"
* in /etc/sysconfig/slurm would be the same
*/
setenv("SLURMSTEPD_OOM_ADJ","-17",0);
return SLURM_SUCCESS;
}
extern int jobacct_gather_cgroup_memory_init(
slurm_cgroup_conf_t *slurm_cgroup_conf)
{
char release_agent_path[PATH_MAX];
/* initialize user/job/jobstep cgroup relative paths */
user_cgroup_path[0]='\0';
job_cgroup_path[0]='\0';
jobstep_cgroup_path[0]='\0';
/* initialize memory cgroup namespace */
release_agent_path[0]='\0';
if (snprintf(release_agent_path, PATH_MAX, "%s/release_memory",
slurm_cgroup_conf->cgroup_release_agent) >= PATH_MAX) {
error("jobacct_gather/cgroup: unable to build memory release "
"agent path");
goto error;
}
if (xcgroup_ns_create(slurm_cgroup_conf, &memory_ns, "/memory", "",
"memory", release_agent_path) !=
XCGROUP_SUCCESS) {
error("jobacct_gather/cgroup: unable to create memory "
"namespace");
goto error;
}
/* check that memory cgroup namespace is available */
if (!xcgroup_ns_is_available(&memory_ns)) {
if (slurm_cgroup_conf->cgroup_automount) {
if (xcgroup_ns_mount(&memory_ns)) {
error("jobacct_gather/cgroup: unable to mount "
"memory namespace");
goto clean;
}
info("jobacct_gather/cgroup: memory namespace is now "
"mounted");
} else {
error("jobacct_gather/cgroup: memory namespace not "
"mounted. aborting");
goto clean;
}
}
return SLURM_SUCCESS;
clean:
xcgroup_ns_destroy(&memory_ns);
error:
return SLURM_ERROR;
}
extern int task_cgroup_cpuset_init(slurm_cgroup_conf_t *slurm_cgroup_conf)
{
/* initialize user/job/jobstep cgroup relative paths */
user_cgroup_path[0]='\0';
job_cgroup_path[0]='\0';
jobstep_cgroup_path[0]='\0';
/* initialize cpuset cgroup namespace */
if (xcgroup_ns_create(slurm_cgroup_conf, &cpuset_ns, "", "cpuset")
!= XCGROUP_SUCCESS) {
error("task/cgroup: unable to create cpuset namespace");
return SLURM_ERROR;
}
return SLURM_SUCCESS;
}
extern int jobacct_gather_cgroup_memory_init(
slurm_cgroup_conf_t *slurm_cgroup_conf)
{
/* initialize user/job/jobstep cgroup relative paths */
user_cgroup_path[0]='\0';
job_cgroup_path[0]='\0';
jobstep_cgroup_path[0]='\0';
/* initialize memory cgroup namespace */
if (xcgroup_ns_create(slurm_cgroup_conf, &memory_ns, "", "memory")
!= XCGROUP_SUCCESS) {
error("jobacct_gather/cgroup: unable to create memory "
"namespace");
return SLURM_ERROR;
}
return SLURM_SUCCESS;
}
extern int task_cgroup_devices_init(slurm_cgroup_conf_t *slurm_cgroup_conf)
{
uint16_t cpunum;
/* initialize cpuinfo internal data */
if ( xcpuinfo_init() != XCPUINFO_SUCCESS )
return SLURM_ERROR;
/* initialize user/job/jobstep cgroup relative paths */
user_cgroup_path[0] = '\0';
job_cgroup_path[0] = '\0';
jobstep_cgroup_path[0] = '\0';
/* initialize allowed_devices_filename */
cgroup_allowed_devices_file[0] = '\0';
if ( get_procs(&cpunum) != 0 ) {
error("task/cgroup: unable to get a number of CPU");
goto error;
}
(void) gres_plugin_node_config_load(cpunum, conf->node_name, NULL);
strcpy(cgroup_allowed_devices_file,
slurm_cgroup_conf->allowed_devices_file);
if (xcgroup_ns_create(slurm_cgroup_conf, &devices_ns, "", "devices")
!= XCGROUP_SUCCESS ) {
error("task/cgroup: unable to create devices namespace");
goto error;
}
return SLURM_SUCCESS;
error:
xcgroup_ns_destroy(&devices_ns);
xcpuinfo_fini();
return SLURM_ERROR;
}
extern int init_system_cpuset_cgroup(void)
{
int rc;
int fstatus = SLURM_ERROR;
char* cpus = NULL;
size_t cpus_size;
char* slurm_cgpath;
xcgroup_t slurm_cg;
/* read cgroup configuration */
if (read_slurm_cgroup_conf(&slurm_cgroup_conf))
return SLURM_ERROR;
/* initialize cpuset cgroup namespace */
if (xcgroup_ns_create(&slurm_cgroup_conf, &cpuset_ns, "", "cpuset")
!= XCGROUP_SUCCESS) {
error("system cgroup: unable to create cpuset namespace");
free_slurm_cgroup_conf(&slurm_cgroup_conf);
return SLURM_ERROR;
}
/* create slurm root cg in this cg namespace */
slurm_cgpath = _system_cgroup_create_slurm_cg(&cpuset_ns);
if ( slurm_cgpath == NULL ) {
xcgroup_ns_destroy(&cpuset_ns);
free_slurm_cgroup_conf(&slurm_cgroup_conf);
return SLURM_ERROR;
}
/* check that this cgroup has cpus allowed or initialize them */
if (xcgroup_load(&cpuset_ns, &slurm_cg, slurm_cgpath)
!= XCGROUP_SUCCESS) {
error("system cgroup: unable to load slurm cpuset xcgroup");
xfree(slurm_cgpath);
xcgroup_ns_destroy(&cpuset_ns);
free_slurm_cgroup_conf(&slurm_cgroup_conf);
return SLURM_ERROR;
}
again:
snprintf(cpuset_meta, sizeof(cpuset_meta), "%scpus", cpuset_prefix);
rc = xcgroup_get_param(&slurm_cg, cpuset_meta, &cpus, &cpus_size);
if (rc != XCGROUP_SUCCESS || cpus_size == 1) {
if (!cpuset_prefix_set && (rc != XCGROUP_SUCCESS)) {
cpuset_prefix_set = 1;
cpuset_prefix = "cpuset.";
goto again;
}
/* initialize the cpusets as it was nonexistent */
if (_xcgroup_cpuset_init(&slurm_cg) != XCGROUP_SUCCESS) {
xfree(slurm_cgpath);
xcgroup_destroy(&slurm_cg);
xcgroup_ns_destroy(&cpuset_ns);
free_slurm_cgroup_conf(&slurm_cgroup_conf);
xfree(cpus);
return SLURM_ERROR;
}
}
xcgroup_destroy(&slurm_cg);
xfree(cpus);
/* build system cgroup relative path */
snprintf(system_cgroup_path, PATH_MAX, "%s/system", slurm_cgpath);
xfree(slurm_cgpath);
/* create system cgroup in the cpuset ns */
if (xcgroup_create(&cpuset_ns, &system_cpuset_cg, system_cgroup_path,
getuid(),getgid()) != XCGROUP_SUCCESS) {
goto error;
}
if (xcgroup_instantiate(&system_cpuset_cg) != XCGROUP_SUCCESS) {
goto error;
}
if (_xcgroup_cpuset_init(&system_cpuset_cg) != XCGROUP_SUCCESS) {
goto error;
}
free_slurm_cgroup_conf(&slurm_cgroup_conf);
debug("system cgroup: system cpuset cgroup initialized");
return SLURM_SUCCESS;
error:
xcgroup_unlock(&system_cpuset_cg);
xcgroup_destroy(&system_cpuset_cg);
xcgroup_ns_destroy(&cpuset_ns);
free_slurm_cgroup_conf(&slurm_cgroup_conf);
return fstatus;
}
extern int init_system_memory_cgroup(void)
{
int fstatus = SLURM_ERROR;
char* slurm_cgpath;
/* read cgroup configuration */
if (read_slurm_cgroup_conf(&slurm_cgroup_conf))
return SLURM_ERROR;
/* initialize memory cgroup namespace */
if (xcgroup_ns_create(&slurm_cgroup_conf, &memory_ns, "", "memory")
!= XCGROUP_SUCCESS) {
error("system cgroup: unable to create memory namespace");
free_slurm_cgroup_conf(&slurm_cgroup_conf);
return SLURM_ERROR;
}
constrain_kmem_space = slurm_cgroup_conf.constrain_kmem_space;
constrain_ram_space = slurm_cgroup_conf.constrain_ram_space;
constrain_swap_space = slurm_cgroup_conf.constrain_swap_space;
/*
* as the swap space threshold will be configured with a
* mem+swp parameter value, if RAM space is not monitored,
* set allowed RAM space to 100% of the job requested memory.
* It will help to construct the mem+swp value that will be
* used for both mem and mem+swp limit during memcg creation.
*/
if ( constrain_ram_space )
allowed_ram_space = slurm_cgroup_conf.allowed_ram_space;
else
allowed_ram_space = 100.0;
allowed_swap_space = slurm_cgroup_conf.allowed_swap_space;
if ((totalram = (uint64_t) conf->real_memory_size) == 0)
error ("system cgroup: Unable to get RealMemory size");
max_kmem = _percent_in_bytes(totalram, slurm_cgroup_conf.max_kmem_percent);
max_ram = _percent_in_bytes(totalram, slurm_cgroup_conf.max_ram_percent);
max_swap = _percent_in_bytes(totalram, slurm_cgroup_conf.max_swap_percent);
max_swap += max_ram;
min_ram_space = slurm_cgroup_conf.min_ram_space * 1024 * 1024;
debug ("system cgroup: memory: total:%luM allowed:%.4g%%(%s), "
"swap:%.4g%%(%s), max:%.4g%%(%luM) "
"max+swap:%.4g%%(%luM) min:%luM "
"kmem:%.4g%%(%luM %s) min:%luM",
(unsigned long) totalram,
allowed_ram_space,
constrain_ram_space?"enforced":"permissive",
allowed_swap_space,
constrain_swap_space?"enforced":"permissive",
slurm_cgroup_conf.max_ram_percent,
(unsigned long) (max_ram/(1024*1024)),
slurm_cgroup_conf.max_swap_percent,
(unsigned long) (max_swap/(1024*1024)),
(unsigned long) slurm_cgroup_conf.min_ram_space,
slurm_cgroup_conf.max_kmem_percent,
(unsigned long)(max_kmem/(1024*1024)),
constrain_kmem_space?"enforced":"permissive",
(unsigned long) slurm_cgroup_conf.min_kmem_space);
/*
* Warning: OOM Killer must be disabled for slurmstepd
* or it would be destroyed if the application use
* more memory than permitted
*
* If an env value is already set for slurmstepd
* OOM killer behavior, keep it, otherwise set the
* -1000 value, wich means do not let OOM killer kill it
*
* FYI, setting "export SLURMSTEPD_OOM_ADJ=-1000"
* in /etc/sysconfig/slurm would be the same
*/
setenv("SLURMSTEPD_OOM_ADJ", "-1000", 0);
/* create slurm root cg in this cg namespace */
slurm_cgpath = _system_cgroup_create_slurm_cg(&memory_ns);
if ( slurm_cgpath == NULL ) {
xcgroup_ns_destroy(&memory_ns);
free_slurm_cgroup_conf(&slurm_cgroup_conf);
return SLURM_ERROR;
}
/* build system cgroup relative path */
snprintf(system_cgroup_path, PATH_MAX, "%s/system", slurm_cgpath);
xfree(slurm_cgpath);
/* create system cgroup in the cpuset ns */
if (xcgroup_create(&memory_ns, &system_memory_cg,
system_cgroup_path,
getuid(), getgid()) != XCGROUP_SUCCESS) {
goto error;
}
if (xcgroup_instantiate(&system_memory_cg) != XCGROUP_SUCCESS) {
goto error;
}
if ( xcgroup_set_param(&system_memory_cg, "memory.use_hierarchy", "1")
!= XCGROUP_SUCCESS ) {
error("system cgroup: unable to ask for hierarchical accounting"
"of system memcg '%s'", system_memory_cg.path);
goto error;
}
free_slurm_cgroup_conf(&slurm_cgroup_conf);
debug("system cgroup: system memory cgroup initialized");
return SLURM_SUCCESS;
error:
xcgroup_unlock(&system_memory_cg);
xcgroup_destroy(&system_memory_cg);
xcgroup_ns_destroy(&memory_ns);
free_slurm_cgroup_conf(&slurm_cgroup_conf);
return fstatus;
}
extern int task_cgroup_memory_init(slurm_cgroup_conf_t *slurm_cgroup_conf)
{
xcgroup_t memory_cg;
/* initialize user/job/jobstep cgroup relative paths */
user_cgroup_path[0]='\0';
job_cgroup_path[0]='\0';
jobstep_cgroup_path[0]='\0';
/* initialize memory cgroup namespace */
if (xcgroup_ns_create(slurm_cgroup_conf, &memory_ns, "", "memory")
!= XCGROUP_SUCCESS) {
error("task/cgroup: unable to create memory namespace");
return SLURM_ERROR;
}
/* Enable memory.use_hierarchy in the root of the cgroup.
*/
xcgroup_create(&memory_ns, &memory_cg, "", 0, 0);
xcgroup_set_param(&memory_cg, "memory.use_hierarchy","1");
xcgroup_destroy(&memory_cg);
constrain_ram_space = slurm_cgroup_conf->constrain_ram_space;
constrain_swap_space = slurm_cgroup_conf->constrain_swap_space;
/*
* as the swap space threshold will be configured with a
* mem+swp parameter value, if RAM space is not monitored,
* set allowed RAM space to 100% of the job requested memory.
* It will help to construct the mem+swp value that will be
* used for both mem and mem+swp limit during memcg creation.
*/
if ( constrain_ram_space )
allowed_ram_space = slurm_cgroup_conf->allowed_ram_space;
else
allowed_ram_space = 100.0;
allowed_swap_space = slurm_cgroup_conf->allowed_swap_space;
if ((totalram = (uint64_t) conf->real_memory_size) == 0)
error ("task/cgroup: Unable to get RealMemory size");
max_ram = percent_in_bytes(totalram, slurm_cgroup_conf->max_ram_percent);
max_swap = percent_in_bytes(totalram, slurm_cgroup_conf->max_swap_percent);
max_swap += max_ram;
min_ram_space = slurm_cgroup_conf->min_ram_space * 1024 * 1024;
debug ("task/cgroup/memory: total:%luM allowed:%.4g%%(%s), "
"swap:%.4g%%(%s), max:%.4g%%(%luM) max+swap:%.4g%%(%luM) min:%uM",
(unsigned long) totalram,
allowed_ram_space,
constrain_ram_space?"enforced":"permissive",
allowed_swap_space,
constrain_swap_space?"enforced":"permissive",
slurm_cgroup_conf->max_ram_percent,
(unsigned long) (max_ram/(1024*1024)),
slurm_cgroup_conf->max_swap_percent,
(unsigned long) (max_swap/(1024*1024)),
(unsigned) slurm_cgroup_conf->min_ram_space);
/*
* Warning: OOM Killer must be disabled for slurmstepd
* or it would be destroyed if the application use
* more memory than permitted
*
* If an env value is already set for slurmstepd
* OOM killer behavior, keep it, otherwise set the
* -1000 value, wich means do not let OOM killer kill it
*
* FYI, setting "export SLURMSTEPD_OOM_ADJ=-1000"
* in /etc/sysconfig/slurm would be the same
*/
setenv("SLURMSTEPD_OOM_ADJ", "-1000", 0);
return SLURM_SUCCESS;
}
