/*
* Convert '%' to '%%" on the msg string on top of the stack.
*/
static const char *l_string_sanitized (lua_State *L)
{
int err;
const char fn[] = "local s = ... return string.gsub(s, '%%', '%%%%')";
err = luaL_loadstring (L, fn);
if (err) {
slurm_error ("spank/lua: loadstring (%s): %s", fn, lua_tostring(L, -1));
return (NULL);
}
/*
* Move string to top of stack:
*/
lua_pushvalue (L, 2);
lua_remove (L, 2);
/*
* Call gsub, throwing away 2nd return value (number of matches),
* leaving modified string on top of stack:
*/
err = lua_pcall (L, 1, 1, 0);
if (err) {
slurm_error ("spank/lua: sanitize msg: %s", lua_tostring (L, -1));
return (NULL);
}
return (lua_tostring (L, -1));
}
static int spank_lua_options_table_register (List script_list, spank_t sp)
{
struct lua_script *script;
ListIterator i = list_iterator_create (script_list);
if (i == NULL) {
slurm_error ("spank/lua: spank_lua_opts_register: Out of memory");
return (-1);
}
while ((script = list_next (i))) {
/*
* Load any options exported via a global spank_options table
*/
if (load_spank_options_table (script, sp) < 0) {
slurm_error ("spank/lua: %s: load_spank_options: %s",
basename (script->path), lua_tostring (script->L, -1));
if (script->fail_on_error)
return (-1);
list_remove(i);
lua_script_destroy (script);
continue;
}
}
list_iterator_destroy (i);
return (0);
}
int slurm_spank_task_init (spank_t sp, int ac, char **av)
{
char nbuf [4096], obuf [4096];
char label [64];
const char *preload = "libplasticfs.so";
const char *lflags = flags ? flags : "";
if (!enabled)
return (0);
/* append to LD_PRELOAD (with a space) */
if (spank_getenv (sp, "LD_PRELOAD", obuf, sizeof (obuf)) == ESPANK_SUCCESS)
snprintf (nbuf, sizeof (nbuf), "%s %s", obuf, preload);
else
strncpy (nbuf, preload, strlen (preload));
if (spank_setenv (sp, "LD_PRELOAD", nbuf, 1) != ESPANK_SUCCESS)
slurm_error ("Failed to set LD_PRELOAD=%s\n", nbuf);
/* prepend to PLASTICFS (with a pipe) */
_iotrace_label (sp, label, sizeof (label));
if (spank_getenv (sp, "PLASTICFS", obuf, sizeof (obuf)) == ESPANK_SUCCESS)
snprintf (nbuf, sizeof (nbuf), "log - %s %s | %s",
label, lflags, obuf);
else
snprintf (nbuf, sizeof (nbuf), "log - %s %s", label, flags);
if (spank_setenv (sp, "PLASTICFS", nbuf, 1) != ESPANK_SUCCESS)
slurm_error ("Failed to set PLASTICFS=%s\n", nbuf);
return (0);
}
int slurm_spank_init(spank_t spank, int ac, char **av)
{
int i;
struct spank_option image_opt,
bind_opt;
memset(&image_opt, '\0', sizeof(image_opt));
image_opt.name = "singularity-image";
image_opt.arginfo = "[path]";
image_opt.usage = "Specify a path to a Singularity image, directory tree, "
"or Docker image";
image_opt.has_arg = 1;
image_opt.val = 0;
image_opt.cb = determine_image;
if (ESPANK_SUCCESS != spank_option_register(spank, &image_opt)) {
slurm_error("spank/%s: Unable to register a new option.",
plugin_name);
return -1;
}
memset(&bind_opt, '\0', sizeof(bind_opt));
bind_opt.name = "singularity-bind";
bind_opt.arginfo = "[path || src:dest],...";
bind_opt.usage = "Specify a user-bind path specification. Can either be "
"a path or a src:dest pair, specifying the bind mount to "
"perform";
bind_opt.has_arg = 1;
bind_opt.val = 0;
bind_opt.cb = determine_bind;
if (ESPANK_SUCCESS != spank_option_register(spank, &bind_opt)) {
slurm_error("spank/%s: Unable to register a new option.",
plugin_name);
return -1;
}
// Make this a no-op except when starting the task.
if (spank_context() == S_CTX_ALLOCATOR || (spank_remote(spank) != 1)) {
return 0;
}
for (i = 0; i < ac; i++) {
if (strncmp ("default_image=", av[i], 14) == 0) {
const char *optarg = av[i] + 14;
job_image = strdup(optarg);
} else {
slurm_error ("spank/%s: Invalid option: %s", av[i], plugin_name);
}
}
return 0;
}
static int lua_spank_call (struct lua_script *s, spank_t sp, const char *fn,
int ac, char **av)
{
struct lua_State *L = s->L;
/*
* Missing functions are not an error
*/
lua_getglobal (L, fn);
if (lua_isnil (L, -1)) {
lua_pop (L, 1);
return (0);
}
/*
* Create spank object to pass to spank functions
*/
lua_spank_table_create (L, sp, ac, av);
if (lua_pcall (L, 1, 1, 0)) {
slurm_error ("spank/lua: %s: %s", fn, lua_tostring (L, -1));
return (s->fail_on_error ? -1 : 0);
}
return lua_script_rc (L);
}
/*
* Called from both srun and slurmd.
*/
int slurm_spank_init_post_opt (spank_t sp, int ac, char **av)
{
int rc = ESPANK_SUCCESS;
if (spank_remote (sp))
return (0);
if (io_style == CACHE_IO) {
slurm_debug("cache_io option");
rc = spank_set_job_env("O_DIRECT", "0", 1);
} else if (io_style == DIRECT_IO) {
slurm_debug("direct_io option");
rc = spank_set_job_env("O_DIRECT", "1", 1);
} else if (getenv("SLURM_CACHE_IO")) {
slurm_debug("cache_io env var");
rc = spank_set_job_env("O_DIRECT", "0", 1);
} else if (getenv("SLURM_DIRECT_IO")) {
slurm_debug("direct_io env var");
rc = spank_set_job_env("O_DIRECT", "1", 1);
}
if (rc != ESPANK_SUCCESS)
slurm_error("spank_setjob_env: %s", spank_strerror(rc));
return (0);
}
static void print_lua_script_error (struct lua_script *script)
{
const char *s = basename (script->path);
const char *err = lua_tostring (script->L, -1);
if (script->fail_on_error)
slurm_error ("spank/lua: Fatal: %s", err);
else
slurm_info ("spank/lua: Disabling %s: %s", s, err);
}
List lua_script_list_create (lua_State *L, const char *pattern)
{
glob_t gl;
size_t i;
List l = NULL;
if (pattern == NULL)
return (NULL);
int rc = glob (pattern, GLOB_ERR, ef, &gl);
switch (rc) {
case 0:
l = list_create ((ListDelF) lua_script_destroy);
for (i = 0; i < gl.gl_pathc; i++) {
struct lua_script * s;
s = lua_script_create (L, gl.gl_pathv[i]);
if (s == NULL) {
slurm_error ("lua_script_create failed for %s.",
gl.gl_pathv[i]);
continue;
}
list_push (l, s);
}
break;
case GLOB_NOMATCH:
break;
case GLOB_NOSPACE:
slurm_error ("spank/lua: glob(3): Out of memory");
case GLOB_ABORTED:
slurm_verbose ("spank/lua: cannot read dir %s: %m", pattern);
break;
default:
slurm_error ("Unknown glob(3) return code = %d", rc);
break;
}
globfree (&gl);
return l;
}
int slurm_spank_init(spank_t sp, int ac, char **av)
{
int i, j, rc = ESPANK_SUCCESS;
for (i=0; spank_option_array[i].name; i++) {
j = spank_option_register(sp, &spank_option_array[i]);
if (j != ESPANK_SUCCESS) {
slurm_error("Could not register Spank option %s",
spank_option_array[i].name);
rc = j;
}
}
return rc;
}
static int _opt_process (int val, const char *optarg, int remote)
{
switch (val) {
case IOTRACE_ENABLE:
enabled = 1;
if (optarg)
flags = strdup (optarg);
break;
default:
slurm_error ("Ignoring unknown iotrace option value %d\n", val);
break;
}
return (0);
}
static void _iotrace_label(spank_t sp, char *buf, int len)
{
char hostname[128], *p;
uint32_t taskid = 0;
spank_err_t rc;
rc = spank_get_item (sp, S_TASK_GLOBAL_ID, &taskid);
if (rc != ESPANK_SUCCESS)
slurm_error ("iotrace: error fetching taskid: %d", rc);
if (gethostname (hostname, sizeof (hostname)) == 0) {
hostname[sizeof(hostname) - 1] = '\0';
if ((p = strchr (hostname, '.')))
*p = '\0';
} else
strncpy (hostname, "unknown", sizeof(hostname));
snprintf (buf, len, "iotrace-%d@%s", taskid, hostname);
}
int load_spank_options_table (struct lua_script *script, spank_t sp)
{
int t;
lua_State *L = script->L;
lua_getglobal (L, "spank_options");
if (lua_isnil (L, -1)) {
lua_pop (L, 1);
return (0);
}
/*
* Iterate through spank_options table, which should
* be a table of spank_option entries
*/
t = lua_gettop (L);
lua_pushnil (L); /* push starting "key" on stack */
while (lua_next (L, t) != 0) {
/*
* If lua_script_option_register() returns 2, then it has
* pushed 2 args on the stack and thus has failed. We
* don't need to return the failure message back to SLURM,
* that has been printed by lua, but we pop the stack and
* return < 0 so that SLURM can detect failure.
*/
if (lua_script_option_register (script, sp, -1) > 1) {
slurm_error ("lua_script_option_register: %s", lua_tostring(L, -1));
lua_pop (L, -1); /* pop everything */
return (-1);
}
/* On success, lua_script_option_register pushes a boolean onto
* the stack, so we need to pop 2 items: the boolean and
* the spank_option table itself. This leaves the 'key' on
* top for lua_next().
*/
lua_pop (L, 2);
}
lua_pop (L, 1); /* pop 'spank_options' table */
return (0);
}
/*
* Called from both srun and slurmd.
*/
int slurm_spank_init (spank_t sp, int ac, char **av)
{
uint32_t jobid;
struct passwd *pw;
uid_t uid;
char buf [1024];
int n;
if (ac!=1) {
slurm_error ("bindtmp: Error module need argument ' lls_mountpoint_prefix ' ");
return (-1);
}
if (!spank_remote (sp))
return (0);
slurm_verbose("bindtmp: av[0] == %s ",av[0] );
lls_mountpoint_prefix=av[0];
spank_get_item (sp, S_JOB_UID, &uid);
pw = getpwuid (uid);
if (!pw) {
slurm_error ("bindtmp: Error looking up uid in /etc/passwd");
return (-1);
}
if (unshare (CLONE_NEWNS) < 0) {
slurm_error ("bindtmp: Error unshare CLONE_NEWNS: %m");
return (-1);
}
if ( spank_get_item (sp, S_JOB_ID, &jobid) != ESPANK_SUCCESS ) {
slurm_error ("bindtmp: Error unable to get jobid");
return (-1);
}
n = snprintf (buf, sizeof (buf), "%s%u",lls_mountpoint_prefix, jobid);
if ((n < 0) || (n > sizeof (buf) - 1)) {
slurm_error ("bindtmp: Error sprintf ");
return (-1);
}
if (mount(buf, "/tmp", NULL, MS_BIND , NULL)!= 0) {
slurm_error ("bindtmp: Could not bind %s to /tmp",buf);
return (-1);
}
return (0);
}
static int l_spank_log_msg (lua_State *L)
{
int level = luaL_checknumber (L, 1);
const char *msg;
msg = l_string_sanitized (L);
if (!msg)
return (0);
if (level == -1) {
slurm_error (msg);
lua_pushnumber (L, -1);
return (1);
}
if (level == 0)
slurm_info (msg);
else if (level == 1)
slurm_verbose (msg);
else if (level == 2)
slurm_debug (msg);
return (0);
}
static int lua_spank_option_callback (int val, const char *optarg, int remote)
{
lua_State *L;
struct lua_script_option *o;
o = list_find_first (
script_option_list,
(ListFindF) s_opt_find,
&val);
if (o == NULL)
return (-1);
if (o->l_function == NULL)
return (0);
L = o->script->L;
lua_getglobal (L, o->l_function);
lua_pushnumber (L, o->l_val);
lua_pushstring (L, optarg);
lua_pushboolean (L, remote);
slurm_debug ("spank/lua: %s: callback %s for option %s optarg=%s",
o->script->path, o->l_function, o->s_opt.name,
optarg ? optarg : "nil");
if (lua_pcall (L, 3, 1, 0) != 0) {
slurm_error ("Failed to call lua callback function %s: %s",
o->l_function, lua_tostring (L, -1));
lua_pop (L, 1);
return (-1);
}
return lua_script_rc (L);
}
int spank_lua_init (spank_t sp, int ac, char *av[])
{
struct spank_lua_options opt;
ListIterator i;
struct lua_script *script;
int rc = 0;
if (ac == 0) {
slurm_error ("spank/lua: Requires at least 1 arg");
return (-1);
}
/*
* Check for spank/lua options in argv
*/
spank_lua_process_args (&ac, &av, &opt);
/*
* dlopen liblua to ensure that symbols from that lib are
* available globally (so lua doesn't fail to dlopen its
* DSOs
*/
if (!dlopen ("liblua.so", RTLD_NOW | RTLD_GLOBAL)) {
slurm_error ("spank/lua: Failed to open liblua.so");
return (-1);
}
global_L = luaL_newstate ();
luaL_openlibs (global_L);
/*
* Create the global SPANK table
*/
SPANK_table_create (global_L);
lua_script_list = lua_script_list_create (global_L, av[0]);
if (lua_script_list == NULL) {
slurm_verbose ("spank/lua: No files found in %s", av[0]);
return (0);
}
/*
* Set up handler for lua_atpanic() so lua doesn't exit() on us.
* This handles errors from outside of protected mode --
* for example when this plugin is processing the global
* spank_options table. The spank_atpanic() function will
* return to the setjmp() point below (thus avoiding Lua's
* call to exit() from its own panic handler). This is basically
* here so that we can use luaL_error() everwhere without
* worrying about the context of the call.
*/
lua_atpanic (global_L, spank_atpanic);
if (setjmp (panicbuf)) {
slurm_error ("spank/lua: PANIC: %s: %s",
av[0], lua_tostring (global_L, -1));
return (-1);
}
i = list_iterator_create (lua_script_list);
while ((script = list_next (i))) {
script->fail_on_error = opt.fail_on_error;
/*
* Load script (luaL_loadfile) and compile it (lua_pcall).
*/
if (luaL_loadfile (script->L, script->path) ||
lua_pcall (script->L, 0, 0, 0)) {
print_lua_script_error (script);
if (opt.fail_on_error)
return (-1);
list_remove(i);
lua_script_destroy (script);
continue;
}
/*
* Don't keep script loaded if the script doesn't have any
* callbacks in the current context.
*/
if (!lua_script_valid_in_context (sp, script)) {
slurm_debug ("%s: no callbacks in this context",
basename (script->path));
list_remove (i);
lua_script_destroy (script);
}
}
list_iterator_destroy (i);
slurm_verbose ("spank/lua: Loaded %d plugins in this context",
list_count (lua_script_list));
return rc;
}
static int setup_container_environment(spank_t spank)
{
uid_t job_uid = -1;
gid_t job_gid = -1;
char *job_cwd = NULL;
char *bindpath = NULL;
int argc;
char **argv = NULL;
int i;
setenv("SINGULARITY_MESSAGELEVEL", "1", 0); //Don't overwrite if exists
if (ESPANK_SUCCESS != spank_get_item(spank, S_JOB_UID, &job_uid)) {
slurm_error("spank/%s: Failed to get job's target UID", plugin_name);
return -1;
}
if (INT_MAX_STRING_SIZE <=
snprintf(job_uid_str, INT_MAX_STRING_SIZE, "%u", job_uid)) { // Flawfinder: ignore
slurm_error("spank/%s: Failed to serialize job's UID to string",
plugin_name);
return -1;
}
if (setenv("SINGULARITY_TARGET_UID", job_uid_str, 1) < 0) {
slurm_error("spank/%s: Failed to setenv(\"SINGULARITY_TARGET_UID\")",
plugin_name);
return -1;
}
if (ESPANK_SUCCESS != spank_get_item(spank, S_JOB_GID, &job_gid)) {
slurm_error("spank/%s: Failed to get job's target GID", plugin_name);
return -1;
}
if (INT_MAX_STRING_SIZE <=
snprintf(job_gid_str, INT_MAX_STRING_SIZE, "%u", job_gid)) { // Flawfinder: ignore
slurm_error("spank/%s: Failed to serialize job's GID to string",
plugin_name);
return -1;
}
if (setenv("SINGULARITY_TARGET_GID", job_gid_str, 1) < 0) {
slurm_error("spank/%s: Failed to setenv(\"SINGULARITY_TARGET_GID\")",
plugin_name);
return -1;
}
job_cwd = get_current_dir_name();
if (!job_cwd) {
slurm_error("spank/%s: Failed to determine job's correct PWD: %s",
plugin_name, strerror(errno));
return -1;
}
if (setenv("SINGULARITY_TARGET_PWD", job_cwd, 1) < 0) {
slurm_error("spank/%s: Failed to setenv(\"SINGULARITY_TARGET_PWD\")",
plugin_name);
return -1;
}
/* setenv() makes a copy */
free(job_cwd);
if (!job_image) {
slurm_error("spank/%s: Unable to determine job's image file.",
plugin_name);
return -1;
}
if (setenv("SINGULARITY_IMAGE", job_image, 1) < 0) {
slurm_error("spank/%s: Failed to setenv(\"SINGULARITY_IMAGE\")",
plugin_name);
return -1;
}
if ((job_bindpath) &&
(setenv("SINGULARITY_BINDPATH", job_bindpath, 1) < 0)) {
slurm_error("spank/%s: Failed to setenv(\"SINGULARITY_BINDPATH\")",
plugin_name);
return -1;
}
return 0;
}
static int ef (const char *p, int eerrno)
{
slurm_error ("spank/lua: glob: %s: %s", p, strerror (eerrno));
return (-1);
}
