/*
* in remote mode, read DISPLAY file content and set its value in job's DISPLAY
* environment variable
*/
int slurm_spank_user_init (spank_t sp, int ac, char **av)
{
int status=-1;
uint32_t jobid;
uint32_t stepid;
if ( x11_mode == X11_MODE_NONE )
return 0;
/* get job id */
if ( spank_get_item (sp, S_JOB_ID, &jobid) != ESPANK_SUCCESS )
return status;
/* get job step id */
if ( spank_get_item (sp, S_JOB_STEPID, &stepid) != ESPANK_SUCCESS )
return status;
if ( stepid == SLURM_BATCH_SCRIPT && x11_mode == X11_MODE_BATCH ) {
return _x11_init_remote_batch(sp,jobid,stepid);
}
else if ( x11_mode != X11_MODE_BATCH ) {
return _x11_init_remote_inter(sp,jobid,stepid);
}
}
int slurm_spank_user_init (spank_t sp, int ac, char **av)
{
char socket_name[4096];
uid_t uid;
uint32_t jobid, stepid, nodeid;
spank_get_item(sp, S_JOB_UID, &uid);
spank_get_item(sp, S_JOB_ID, &jobid);
spank_get_item(sp, S_JOB_STEPID, &stepid);
spank_get_item(sp, S_JOB_NODEID, &nodeid);
if (nodeid != 0)
return 0;
/* WARNING - If you change the file name here at all, you must
update the file name in src/aixslurm/internal.c as well.
Even though we have spank set the YOGRT_AIXSLURM_SOCKET variable,
the pmdv4 daemon will NOT pass the variable down to the user's
task. */
snprintf(socket_name, sizeof(socket_name),
"/tmp/.yogrtaixslurm_%d_%u.%u", uid, jobid, stepid);
spank_setenv(sp, "YOGRT_AIXSLURM_SOCKET", socket_name, 1);
helper_pid = start_helper(socket_name, jobid);
return 0;
}
static int lua_spank_table_create (lua_State *L, spank_t sp, int ac, char **av)
{
const char *str;
int i;
lua_newtable (L);
luaL_register (L, NULL, spank_functions);
/* Register spank handle as light userdata inside spank table:
*/
lua_pushlightuserdata (L, sp);
lua_setfield (L, -2, SPANK_REFNAME);
l_spank_context (L);
lua_setfield (L, -2, "context");
if (spank_get_item (sp, S_SLURM_VERSION, &str) == ESPANK_SUCCESS) {
lua_pushstring (L, str);
lua_setfield (L, -2, "slurm_version");
}
lua_newtable (L);
for (i = 1; i < ac; i++) {
lua_pushstring (L, av[i]);
lua_rawseti (L, -2, i);
}
lua_setfield (L, -2, "args");
return (0);
}
static int l_spank_get_exit_status (lua_State *L, spank_t sp)
{
spank_err_t err;
int status;
err = spank_get_item (sp, S_TASK_EXIT_STATUS, &status);
if (err != ESPANK_SUCCESS)
return l_spank_error (L, err);
lua_pushnumber (L, status);
/*
* Now push WEXITSTATUS or nil if !WIFEXITED and
* WTERMSIG, WCOREDUMP or nil if !WIFSIGNALED
*/
if (WIFEXITED (status))
lua_pushnumber (L, WEXITSTATUS (status));
else
lua_pushnil (L);
if (WIFSIGNALED (status)) {
lua_pushnumber (L, WTERMSIG (status));
lua_pushnumber (L, WCOREDUMP (status));
}
else {
lua_pushnil (L);
lua_pushnil (L);
}
/* Returns 4 values: status, exitcode, termsig, coredumped */
return (4);
}
/*
* Called from both srun and slurmd.
*/
int slurm_spank_init (spank_t sp, int ac, char **av)
{
uint32_t jobid;
struct passwd *pw;
uid_t uid;
char buf [1024];
int n;
if (ac!=1) {
slurm_error ("bindtmp: Error module need argument ' lls_mountpoint_prefix ' ");
return (-1);
}
if (!spank_remote (sp))
return (0);
slurm_verbose("bindtmp: av[0] == %s ",av[0] );
lls_mountpoint_prefix=av[0];
spank_get_item (sp, S_JOB_UID, &uid);
pw = getpwuid (uid);
if (!pw) {
slurm_error ("bindtmp: Error looking up uid in /etc/passwd");
return (-1);
}
if (unshare (CLONE_NEWNS) < 0) {
slurm_error ("bindtmp: Error unshare CLONE_NEWNS: %m");
return (-1);
}
if ( spank_get_item (sp, S_JOB_ID, &jobid) != ESPANK_SUCCESS ) {
slurm_error ("bindtmp: Error unable to get jobid");
return (-1);
}
n = snprintf (buf, sizeof (buf), "%s%u",lls_mountpoint_prefix, jobid);
if ((n < 0) || (n > sizeof (buf) - 1)) {
slurm_error ("bindtmp: Error sprintf ");
return (-1);
}
if (mount(buf, "/tmp", NULL, MS_BIND , NULL)!= 0) {
slurm_error ("bindtmp: Could not bind %s to /tmp",buf);
return (-1);
}
return (0);
}
/*
* in remote mode, remove DISPLAY file in order to stop
* ssh -X process initialized by the client
*/
int slurm_spank_exit (spank_t sp, int ac, char **av)
{
uint32_t jobid;
uint32_t stepid;
FILE* f;
char* expc_pattern= X11_LIBEXEC_PROG " -i %u.%u -r 2>/dev/null";
char* expc_cmd;
size_t expc_length;
/* noting to do in local mode */
if (!spank_remote (sp))
return 0;
/* get job id */
if ( spank_get_item (sp, S_JOB_ID, &jobid) != ESPANK_SUCCESS )
return -1;
/* get job step id */
if ( spank_get_item (sp, S_JOB_STEPID, &stepid) != ESPANK_SUCCESS )
return -1;
/* remove DISPLAY reference */
expc_length = strlen(expc_pattern) + 128 ;
expc_cmd = (char*) malloc(expc_length*sizeof(char));
if ( expc_cmd != NULL &&
( snprintf(expc_cmd,expc_length,expc_pattern,jobid,stepid)
>= expc_length ) ) {
ERROR("x11: error while creating remove reference cmd");
}
else {
f = popen(expc_cmd,"r");
if ( f == NULL ) {
ERROR("x11: unable to exec remove"
" cmd '%s'",expc_cmd);
}
else
pclose(f);
}
if ( expc_cmd != NULL )
free(expc_cmd);
return 0;
}
/*
* Generic function to push a spank item with a numeric representation
* on to the lua stack.
*/
static int l_spank_get_item_val (lua_State *L, spank_t sp, spank_item_t item)
{
spank_err_t err;
long val;
err = spank_get_item (sp, item, &val);
if (err != ESPANK_SUCCESS)
return l_spank_error (L, err);
lua_pushnumber (L, val);
return (1);
}
/*
* Generic function to push a spank item with a string representation
* on to the lua stack.
*/
static int
l_spank_get_item_string (lua_State *L, spank_t sp, spank_item_t item)
{
spank_err_t err;
const char *s;
err = spank_get_item (sp, item, &s);
if (err != ESPANK_SUCCESS)
return l_spank_error (L, err);
lua_pushstring (L, s);
return (1);
}
static int
l_spank_id_query (lua_State *L, spank_t sp, spank_item_t item)
{
spank_err_t err;
long rv, id;
id = luaL_checknumber (L, -1);
lua_pop (L, 1);
err = spank_get_item (sp, item, id, &rv);
if (err != ESPANK_SUCCESS)
return l_spank_error (L, err);
lua_pushnumber (L, rv);
return (1);
}
/*
* Copy GID list as array on the Lua stack.
*/
static int l_spank_get_item_gids (lua_State *L, spank_t sp)
{
spank_err_t err;
gid_t *gids;
int i, ngids;
err = spank_get_item (sp, S_JOB_SUPPLEMENTARY_GIDS, &gids, &ngids);
if (err != ESPANK_SUCCESS)
return l_spank_error (L, err);
lua_newtable (L);
for (i = 0; i < ngids; i++) {
lua_pushnumber (L, gids[i]);
lua_rawseti (L, -2, i+1);
}
return (1);
}
/*
* Return S_JOB_ARGV as an array on the lua stack
*/
static int l_spank_get_item_argv (lua_State *L, spank_t sp)
{
spank_err_t err;
const char **av;
int i, ac;
err = spank_get_item (sp, S_JOB_ARGV, &ac, &av);
if (err != ESPANK_SUCCESS)
return l_spank_error (L, err);
lua_newtable (L);
for (i = 0; i < ac; i++) {
lua_pushstring (L, av[i]);
lua_rawseti (L, -2, i+1);
}
return (1);
}
/*
* Copy S_JOB_ENV to a table on the Lua stack.
*/
static int l_spank_get_item_env (lua_State *L, spank_t sp)
{
spank_err_t err;
const char **env;
const char **p;
int t;
err = spank_get_item (sp, S_JOB_ENV, &env);
if (err != ESPANK_SUCCESS)
return l_spank_error (L, err);
lua_newtable (L);
t = lua_gettop (L);
for (p = env; *p != NULL; p++)
set_env_table_entry (L, t, *p);
return (1);
}
static void _iotrace_label(spank_t sp, char *buf, int len)
{
char hostname[128], *p;
uint32_t taskid = 0;
spank_err_t rc;
rc = spank_get_item (sp, S_TASK_GLOBAL_ID, &taskid);
if (rc != ESPANK_SUCCESS)
slurm_error ("iotrace: error fetching taskid: %d", rc);
if (gethostname (hostname, sizeof (hostname)) == 0) {
hostname[sizeof(hostname) - 1] = '\0';
if ((p = strchr (hostname, '.')))
*p = '\0';
} else
strncpy (hostname, "unknown", sizeof(hostname));
snprintf (buf, len, "iotrace-%d@%s", taskid, hostname);
}
/*
* srun call, the client node connects the allocated node(s)
*/
int slurm_spank_local_user_init (spank_t sp, int ac, char **av)
{
int status;
uint32_t jobid;
uint32_t stepid;
job_info_msg_t * job_buffer_ptr;
job_info_t* job_ptr;
/* only handle interactive usage */
if ( x11_mode == X11_MODE_NONE ||
x11_mode == X11_MODE_BATCH )
return 0;
/* check DISPLAY value */
if ( getenv("DISPLAY") == NULL ) {
ERROR("x11: no local DISPLAY defined, skipping",jobid);
return 0;
}
/* get job id */
if ( spank_get_item (sp, S_JOB_ID, &jobid)
!= ESPANK_SUCCESS ) {
status = -1;
goto exit;
}
/* get job step id */
if ( spank_get_item (sp, S_JOB_STEPID, &stepid)
!= ESPANK_SUCCESS ) {
status = -1;
goto exit;
}
/* get job infos */
status = slurm_load_job(&job_buffer_ptr,jobid,SHOW_ALL);
if ( status != 0 ) {
ERROR("x11: unable to get job infos");
status = -3;
goto exit;
}
/* check infos validity */
if ( job_buffer_ptr->record_count != 1 ) {
ERROR("x11: job infos are invalid");
status = -4;
goto clean_exit;
}
job_ptr = job_buffer_ptr->job_array;
/* check allocated nodes var */
if ( job_ptr->nodes == NULL ) {
ERROR("x11: job has no allocated nodes defined");
status = -5;
goto clean_exit;
}
/* connect required nodes */
status = _x11_connect_nodes(job_ptr->nodes,jobid,stepid);
clean_exit:
slurm_free_job_info_msg(job_buffer_ptr);
exit:
return status;
}
static int setup_container_environment(spank_t spank)
{
uid_t job_uid = -1;
gid_t job_gid = -1;
char *job_cwd = NULL;
char *bindpath = NULL;
int argc;
char **argv = NULL;
int i;
setenv("SINGULARITY_MESSAGELEVEL", "1", 0); //Don't overwrite if exists
if (ESPANK_SUCCESS != spank_get_item(spank, S_JOB_UID, &job_uid)) {
slurm_error("spank/%s: Failed to get job's target UID", plugin_name);
return -1;
}
if (INT_MAX_STRING_SIZE <=
snprintf(job_uid_str, INT_MAX_STRING_SIZE, "%u", job_uid)) { // Flawfinder: ignore
slurm_error("spank/%s: Failed to serialize job's UID to string",
plugin_name);
return -1;
}
if (setenv("SINGULARITY_TARGET_UID", job_uid_str, 1) < 0) {
slurm_error("spank/%s: Failed to setenv(\"SINGULARITY_TARGET_UID\")",
plugin_name);
return -1;
}
if (ESPANK_SUCCESS != spank_get_item(spank, S_JOB_GID, &job_gid)) {
slurm_error("spank/%s: Failed to get job's target GID", plugin_name);
return -1;
}
if (INT_MAX_STRING_SIZE <=
snprintf(job_gid_str, INT_MAX_STRING_SIZE, "%u", job_gid)) { // Flawfinder: ignore
slurm_error("spank/%s: Failed to serialize job's GID to string",
plugin_name);
return -1;
}
if (setenv("SINGULARITY_TARGET_GID", job_gid_str, 1) < 0) {
slurm_error("spank/%s: Failed to setenv(\"SINGULARITY_TARGET_GID\")",
plugin_name);
return -1;
}
job_cwd = get_current_dir_name();
if (!job_cwd) {
slurm_error("spank/%s: Failed to determine job's correct PWD: %s",
plugin_name, strerror(errno));
return -1;
}
if (setenv("SINGULARITY_TARGET_PWD", job_cwd, 1) < 0) {
slurm_error("spank/%s: Failed to setenv(\"SINGULARITY_TARGET_PWD\")",
plugin_name);
return -1;
}
/* setenv() makes a copy */
free(job_cwd);
if (!job_image) {
slurm_error("spank/%s: Unable to determine job's image file.",
plugin_name);
return -1;
}
if (setenv("SINGULARITY_IMAGE", job_image, 1) < 0) {
slurm_error("spank/%s: Failed to setenv(\"SINGULARITY_IMAGE\")",
plugin_name);
return -1;
}
if ((job_bindpath) &&
(setenv("SINGULARITY_BINDPATH", job_bindpath, 1) < 0)) {
slurm_error("spank/%s: Failed to setenv(\"SINGULARITY_BINDPATH\")",
plugin_name);
return -1;
}
return 0;
}
