static int _slurmd_init(void) { struct rlimit rlim; slurm_ctl_conf_t *cf; struct stat stat_buf; uint32_t cpu_cnt; /* * Process commandline arguments first, since one option may be * an alternate location for the slurm config file. */ _process_cmdline(*conf->argc, *conf->argv); /* * Build nodes table like in slurmctld * This is required by the topology stack * Node tables setup must preceed _read_config() so that the * proper hostname is set. */ slurm_conf_init(conf->conffile); init_node_conf(); /* slurm_select_init() must be called before * build_all_nodeline_info() to be called with proper argument. */ if (slurm_select_init(1) != SLURM_SUCCESS ) return SLURM_FAILURE; build_all_nodeline_info(true); build_all_frontend_info(true); /* * Read global slurm config file, override necessary values from * defaults and command line. */ _read_config(); cpu_cnt = MAX(conf->conf_cpus, conf->block_map_size); if ((gres_plugin_init() != SLURM_SUCCESS) || (gres_plugin_node_config_load(cpu_cnt) != SLURM_SUCCESS)) return SLURM_FAILURE; if (slurm_topo_init() != SLURM_SUCCESS) return SLURM_FAILURE; /* * Get and set slurmd topology information * Build node hash table first to speed up the topo build */ rehash_node(); slurm_topo_build_config(); _set_topo_info(); /* * Check for cpu frequency set capabilities on this node */ cpu_freq_init(conf); _print_conf(); if (slurm_proctrack_init() != SLURM_SUCCESS) return SLURM_FAILURE; if (slurmd_task_init() != SLURM_SUCCESS) return SLURM_FAILURE; if (slurm_auth_init(NULL) != SLURM_SUCCESS) return SLURM_FAILURE; if (spank_slurmd_init() < 0) return SLURM_FAILURE; if (getrlimit(RLIMIT_CPU, &rlim) == 0) { rlim.rlim_cur = rlim.rlim_max; setrlimit(RLIMIT_CPU, &rlim); if (rlim.rlim_max != RLIM_INFINITY) { error("Slurmd process CPU time limit is %d seconds", (int) rlim.rlim_max); } } if (getrlimit(RLIMIT_NOFILE, &rlim) == 0) { rlim.rlim_cur = rlim.rlim_max; setrlimit(RLIMIT_NOFILE, &rlim); } #ifndef NDEBUG if (getrlimit(RLIMIT_CORE, &rlim) == 0) { rlim.rlim_cur = rlim.rlim_max; setrlimit(RLIMIT_CORE, &rlim); } #endif /* !NDEBUG */ /* * Create a context for verifying slurm job credentials */ if (!(conf->vctx = slurm_cred_verifier_ctx_create(conf->pubkey))) return SLURM_FAILURE; if (!strcmp(conf->select_type, "select/serial")) { /* Only cache credential for 5 seconds with select/serial * for shorter cache searches and higher throughput */ slurm_cred_ctx_set(conf->vctx, SLURM_CRED_OPT_EXPIRY_WINDOW, 5); } /* * Create slurmd spool directory if necessary. */ if (_set_slurmd_spooldir() < 0) { error("Unable to initialize slurmd spooldir"); return SLURM_FAILURE; } if (conf->cleanstart) { /* * Need to kill any running slurmd's here */ _kill_old_slurmd(); stepd_cleanup_sockets(conf->spooldir, conf->node_name); _stepd_cleanup_batch_dirs(conf->spooldir, conf->node_name); } if (conf->daemonize) { bool success = false; if (conf->logfile && (conf->logfile[0] == '/')) { char *slash_ptr, *work_dir; work_dir = xstrdup(conf->logfile); slash_ptr = strrchr(work_dir, '/'); if (slash_ptr == work_dir) work_dir[1] = '\0'; else slash_ptr[0] = '\0'; if ((access(work_dir, W_OK) != 0) || (chdir(work_dir) < 0)) { error("Unable to chdir to %s", work_dir); } else success = true; xfree(work_dir); } if (!success) { if ((access(conf->spooldir, W_OK) != 0) || (chdir(conf->spooldir) < 0)) { error("Unable to chdir to %s", conf->spooldir); } else success = true; } if (!success) { if ((access("/var/tmp", W_OK) != 0) || (chdir("/var/tmp") < 0)) { error("chdir(/var/tmp): %m"); return SLURM_FAILURE; } else info("chdir to /var/tmp"); } } /* * Cache the group access list */ cf = slurm_conf_lock(); if (cf->group_info & GROUP_CACHE) init_gids_cache(1); else init_gids_cache(0); slurm_conf_unlock(); if ((devnull = open_cloexec("/dev/null", O_RDWR)) < 0) { error("Unable to open /dev/null: %m"); return SLURM_FAILURE; } /* make sure we have slurmstepd installed */ if (stat(conf->stepd_loc, &stat_buf)) fatal("Unable to find slurmstepd file at %s", conf->stepd_loc); if (!S_ISREG(stat_buf.st_mode)) fatal("slurmstepd not a file at %s", conf->stepd_loc); return SLURM_SUCCESS; }
/* main - slurmctld main function, start various threads and process RPCs */ int main(int argc, char *argv[]) { pthread_attr_t thread_attr; char node_name[128]; void *db_conn = NULL; assoc_init_args_t assoc_init_arg; _init_config(); log_init(argv[0], log_opts, LOG_DAEMON, NULL); if (read_slurmdbd_conf()) exit(1); _parse_commandline(argc, argv); _update_logging(true); _update_nice(); if (slurm_auth_init(NULL) != SLURM_SUCCESS) { fatal("Unable to initialize %s authentication plugin", slurmdbd_conf->auth_type); } if (slurm_acct_storage_init(NULL) != SLURM_SUCCESS) { fatal("Unable to initialize %s accounting storage plugin", slurmdbd_conf->storage_type); } _kill_old_slurmdbd(); if (foreground == 0) _daemonize(); /* * Need to create pidfile here in case we setuid() below * (init_pidfile() exits if it can't initialize pid file). * On Linux we also need to make this setuid job explicitly * able to write a core dump. * This also has to happen after daemon(), which closes all fd's, * so we keep the write lock of the pidfile. */ _init_pidfile(); _become_slurm_user(); if (foreground == 0) _set_work_dir(); log_config(); #ifdef PR_SET_DUMPABLE if (prctl(PR_SET_DUMPABLE, 1) < 0) debug ("Unable to set dumpable to 1"); #endif /* PR_SET_DUMPABLE */ if (xsignal_block(dbd_sigarray) < 0) error("Unable to block signals"); /* Create attached thread for signal handling */ slurm_attr_init(&thread_attr); if (pthread_create(&signal_handler_thread, &thread_attr, _signal_handler, NULL)) fatal("pthread_create %m"); slurm_attr_destroy(&thread_attr); registered_clusters = list_create(NULL); slurm_attr_init(&thread_attr); if (pthread_create(&commit_handler_thread, &thread_attr, _commit_handler, NULL)) fatal("pthread_create %m"); slurm_attr_destroy(&thread_attr); memset(&assoc_init_arg, 0, sizeof(assoc_init_args_t)); /* If we are tacking wckey we need to cache wckeys, if we aren't only cache the users, qos */ assoc_init_arg.cache_level = ASSOC_MGR_CACHE_USER | ASSOC_MGR_CACHE_QOS; if (slurmdbd_conf->track_wckey) assoc_init_arg.cache_level |= ASSOC_MGR_CACHE_WCKEY; db_conn = acct_storage_g_get_connection(NULL, 0, true, NULL); if (assoc_mgr_init(db_conn, &assoc_init_arg, errno) == SLURM_ERROR) { error("Problem getting cache of data"); acct_storage_g_close_connection(&db_conn); goto end_it; } if (gethostname_short(node_name, sizeof(node_name))) fatal("getnodename: %m"); while (1) { if (slurmdbd_conf->dbd_backup && (!strcmp(node_name, slurmdbd_conf->dbd_backup) || !strcmp(slurmdbd_conf->dbd_backup, "localhost"))) { info("slurmdbd running in background mode"); have_control = false; backup = true; /* make sure any locks are released */ acct_storage_g_commit(db_conn, 1); run_dbd_backup(); if (!shutdown_time) assoc_mgr_refresh_lists(db_conn); } else if (slurmdbd_conf->dbd_host && (!strcmp(slurmdbd_conf->dbd_host, node_name) || !strcmp(slurmdbd_conf->dbd_host, "localhost"))) { backup = false; have_control = true; } else { fatal("This host not configured to run SlurmDBD " "(%s != %s | (backup) %s)", node_name, slurmdbd_conf->dbd_host, slurmdbd_conf->dbd_backup); } if (!shutdown_time) { /* Create attached thread to process incoming RPCs */ slurm_attr_init(&thread_attr); if (pthread_create(&rpc_handler_thread, &thread_attr, rpc_mgr, NULL)) fatal("pthread_create error %m"); slurm_attr_destroy(&thread_attr); } if (!shutdown_time) { /* Create attached thread to do usage rollup */ slurm_attr_init(&thread_attr); if (pthread_create(&rollup_handler_thread, &thread_attr, _rollup_handler, db_conn)) fatal("pthread_create error %m"); slurm_attr_destroy(&thread_attr); } /* Daemon is fully operational here */ if (!shutdown_time || primary_resumed) { shutdown_time = 0; info("slurmdbd version %s started", SLURM_VERSION_STRING); if (backup) run_dbd_backup(); } _request_registrations(db_conn); acct_storage_g_commit(db_conn, 1); /* this is only ran if not backup */ if (rollup_handler_thread) pthread_join(rollup_handler_thread, NULL); if (rpc_handler_thread) pthread_join(rpc_handler_thread, NULL); if (backup && primary_resumed) { shutdown_time = 0; info("Backup has given up control"); } if (shutdown_time) break; } /* Daemon termination handled here */ end_it: if (signal_handler_thread) pthread_join(signal_handler_thread, NULL); if (commit_handler_thread) pthread_join(commit_handler_thread, NULL); acct_storage_g_commit(db_conn, 1); acct_storage_g_close_connection(&db_conn); if (slurmdbd_conf->pid_file && (unlink(slurmdbd_conf->pid_file) < 0)) { verbose("Unable to remove pidfile '%s': %m", slurmdbd_conf->pid_file); } FREE_NULL_LIST(registered_clusters); assoc_mgr_fini(NULL); slurm_acct_storage_fini(); slurm_auth_fini(); log_fini(); free_slurmdbd_conf(); exit(0); }
int main (int argc, char **argv) { slurm_addr_t *cli; slurm_addr_t *self; slurm_msg_t *msg; stepd_step_rec_t *job; int ngids; gid_t *gids; int rc = 0; char *launch_params; if (_process_cmdline (argc, argv) < 0) fatal ("Error in slurmstepd command line"); xsignal_block(slurmstepd_blocked_signals); conf = xmalloc(sizeof(*conf)); conf->argv = &argv; conf->argc = &argc; init_setproctitle(argc, argv); if (slurm_select_init(1) != SLURM_SUCCESS ) fatal( "failed to initialize node selection plugin" ); if (slurm_auth_init(NULL) != SLURM_SUCCESS) fatal( "failed to initialize authentication plugin" ); /* Receive job parameters from the slurmd */ _init_from_slurmd(STDIN_FILENO, argv, &cli, &self, &msg, &ngids, &gids); /* Create the stepd_step_rec_t, mostly from info in a * launch_tasks_request_msg_t or a batch_job_launch_msg_t */ if (!(job = _step_setup(cli, self, msg))) { _send_fail_to_slurmd(STDOUT_FILENO); rc = SLURM_FAILURE; goto ending; } job->ngids = ngids; job->gids = gids; /* fork handlers cause mutexes on some global data structures * to be re-initialized after the fork. */ list_install_fork_handlers(); slurm_conf_install_fork_handlers(); /* sets job->msg_handle and job->msgid */ if (msg_thr_create(job) == SLURM_ERROR) { _send_fail_to_slurmd(STDOUT_FILENO); rc = SLURM_FAILURE; goto ending; } _send_ok_to_slurmd(STDOUT_FILENO); _got_ack_from_slurmd(STDIN_FILENO); /* Fancy way of closing stdin that keeps STDIN_FILENO from being * allocated to any random file. The slurmd already opened /dev/null * on STDERR_FILENO for us. */ dup2(STDERR_FILENO, STDIN_FILENO); /* Fancy way of closing stdout that keeps STDOUT_FILENO from being * allocated to any random file. The slurmd already opened /dev/null * on STDERR_FILENO for us. */ dup2(STDERR_FILENO, STDOUT_FILENO); /* slurmstepd is the only daemon that should survive upgrade. If it * had been swapped out before upgrade happened it could easily lead * to SIGBUS at any time after upgrade. Avoid that by locking it * in-memory. */ launch_params = slurm_get_launch_params(); if (launch_params && strstr(launch_params, "slurmstepd_memlock")) { #ifdef _POSIX_MEMLOCK int flags = MCL_CURRENT; if (strstr(launch_params, "slurmstepd_memlock_all")) flags |= MCL_FUTURE; if (mlockall(flags) < 0) info("failed to mlock() slurmstepd pages: %m"); else debug("slurmstepd locked in memory"); #else info("mlockall() system call does not appear to be available"); #endif } xfree(launch_params); /* This does most of the stdio setup, then launches all the tasks, * and blocks until the step is complete */ rc = job_manager(job); return stepd_cleanup(msg, job, cli, self, rc, 0); ending: return stepd_cleanup(msg, job, cli, self, rc, 1); }