/* Check for a file in few direct ways for portability */
static void check(char *dir, char *file, char **locations)
{
char *str;
asprintf(&str, "%s/%s.so", dir, file);
#if defined(HAVE_SYS_STAT_H)
{
struct stat buf;
/* Use stat() */
if (0 == stat(str, &buf)) {
opal_argv_append_nosize(&locations, file);
}
}
#else
{
FILE *fp;
/* Just try to open the file */
if (NULL != (fp = fopen(str, "r"))) {
fclose(fp);
opal_argv_append_nosize(&locations, file);
}
}
#endif /* defined(HAVE_SYS_STAT_H) */
free(str);
}
/*
* Process a single MCA argument.
*/
static int process_arg(const char *param, const char *value,
char ***params, char ***values)
{
int i;
char *new_str;
/* Look to see if we've already got an -mca argument for the same
param. Check against the list of MCA param's that we've
already saved arguments for. */
for (i = 0; NULL != *params && NULL != (*params)[i]; ++i) {
if (0 == strcmp(param, (*params)[i])) {
asprintf(&new_str, "%s,%s", (*values)[i], value);
free((*values)[i]);
(*values)[i] = new_str;
return OPAL_SUCCESS;
}
}
/* If we didn't already have an value for the same param, save
this one away */
opal_argv_append_nosize(params, param);
opal_argv_append_nosize(values, value);
return OPAL_SUCCESS;
}
/**
* overwrite: true/false
* ===================================
* char **str_array_pt = NULL;
char *str1 = "hello";
char *str2 = "haha";
char *str3 = "hello";
opal_argv_append_unique_nosize(&str_array_pt, str1, true);
opal_argv_append_unique_nosize(&str_array_pt, str2, true);
opal_argv_append_unique_nosize(&str_array_pt, str3, true);
while(*str_array_pt)
puts(*str_array_pt++);
*===================================
* hello
haha
*/
int opal_argv_append_unique_nosize(char ***argv, const char *arg, bool overwrite)
{
int i;
/* if the provided array is NULL, then the arg cannot be present,
* so just go ahead and append
*/
if (NULL == *argv) {
return opal_argv_append_nosize(argv, arg);
}
/* see if this arg is already present in the array */
for (i=0; NULL != (*argv)[i]; i++) {
if (0 == strcmp(arg, (*argv)[i])) {
/* already exists - are we authorized to overwrite? */
if (overwrite) {
free((*argv)[i]);
(*argv)[i] = strdup(arg);
}
return OPAL_SUCCESS;
}
}
/* we get here if the arg is not in the array - so add it */
return opal_argv_append_nosize(argv, arg);
}
static void local_getfields(char *dptr, char ***fields)
{
char *ptr, *end;
/* set default */
*fields = NULL;
/* find the beginning */
ptr = dptr;
while ('\0' != *ptr && !isalnum(*ptr)) {
ptr++;
}
if ('\0' == *ptr) {
return;
}
/* working from this point, find the end of each
* alpha-numeric field and store it on the stack.
* Then shift across the white space to the start
* of the next one
*/
end = ptr; /* ptr points to an alnum */
end++; /* look at next character */
while ('\0' != *end) {
/* find the end of this alpha string */
while ('\0' != *end && isalnum(*end)) {
end++;
}
/* terminate it */
*end = '\0';
/* store it on the stack */
opal_argv_append_nosize(fields, ptr);
/* step across any white space */
end++;
while ('\0' != *end && !isalnum(*end)) {
end++;
}
if ('\0' == *end) {
ptr = NULL;
break;
}
ptr = end;
end++;
}
if (NULL != ptr) {
/* have a hanging field */
opal_argv_append_nosize(fields, ptr);
}
}
/**
* Initialization of data structures for running under a debugger
* using the MPICH/TotalView parallel debugger interface. Before the
* spawn we need to check if we are being run under a TotalView-like
* debugger; if so then inform applications via an MCA parameter.
*/
void orte_debugger_init_before_spawn(orte_job_t *jdata)
{
char *env_name;
orte_app_context_t **apps, *app;
orte_std_cntr_t i;
int32_t ljob;
if (!MPIR_being_debugged && !orte_in_parallel_debugger) {
/* not being debugged - check if we want to enable
* later attachment by debugger
*/
if (orte_enable_debug_cospawn_while_running) {
/* setup a timer to wake us up periodically
* to check for debugger attach
*/
ORTE_TIMER_EVENT(orte_debugger_check_rate, 0, check_debugger);
}
return;
}
if (orte_debug_flag) {
opal_output(0, "Info: Spawned by a debugger");
}
/* tell the procs they are being debugged */
apps = (orte_app_context_t**)jdata->apps->addr;
env_name = mca_base_param_environ_variable("orte",
"in_parallel_debugger", NULL);
for (i=0; i < jdata->num_apps; i++) {
opal_setenv(env_name, "1", true, &apps[i]->env);
}
free(env_name);
/* check if we need to co-spawn the debugger daemons */
if ('\0' != MPIR_executable_path[0]) {
/* add debugger info to launch message */
orte_debugger_daemon = OBJ_NEW(orte_job_t);
/* create a jobid for these daemons - this is done solely
* to avoid confusing the rest of the system's bookkeeping
*/
orte_plm_base_create_jobid(orte_debugger_daemon);
/* flag the job as being debugger daemons */
orte_debugger_daemon->controls |= ORTE_JOB_CONTROL_DEBUGGER_DAEMON;
/* unless directed, we do not forward output */
if (!MPIR_forward_output) {
orte_debugger_daemon->controls &= ~ORTE_JOB_CONTROL_FORWARD_OUTPUT;
}
/* add it to the global job pool */
ljob = ORTE_LOCAL_JOBID(orte_debugger_daemon->jobid);
opal_pointer_array_set_item(orte_job_data, ljob, orte_debugger_daemon);
/* create an app_context for the debugger daemon */
app = OBJ_NEW(orte_app_context_t);
app->app = strdup((char*)MPIR_executable_path);
opal_argv_append_nosize(&app->argv, app->app);
build_debugger_args(app);
opal_pointer_array_add(orte_debugger_daemon->apps, &app->super);
orte_debugger_daemon->num_apps = 1;
}
}
bool orte_ifislocal(const char *hostname)
{
int i;
/* see if it matches any of our known aliases */
if (NULL != orte_process_info.aliases) {
for (i=0; NULL != orte_process_info.aliases[i]; i++) {
if (0 == strcmp(hostname, orte_process_info.aliases[i])) {
return true;
}
}
}
/* okay, have to resolve the address - the opal_ifislocal
* function will not attempt to resolve the address if
* told not to do so */
if (opal_ifislocal(hostname)) {
/* add this to our known aliases */
opal_argv_append_nosize(&orte_process_info.aliases, hostname);
return true;
}
/* not me */
return false;
}
static int mca_bml_r2_add_btls( void )
{
int i;
opal_list_t *btls = NULL;
mca_btl_base_selected_module_t* selected_btl;
size_t num_btls = 0;
char **btl_names_argv = NULL;
if(true == mca_bml_r2.btls_added) {
return OMPI_SUCCESS;
}
/* build an array of r2s and r2 modules */
btls = &mca_btl_base_modules_initialized;
num_btls = opal_list_get_size(btls);
mca_bml_r2.num_btl_modules = 0;
mca_bml_r2.num_btl_progress = 0;
mca_bml_r2.btl_modules = (mca_btl_base_module_t **)malloc(sizeof(mca_btl_base_module_t*) * num_btls);
mca_bml_r2.btl_progress = (mca_btl_base_component_progress_fn_t*)malloc(sizeof(mca_btl_base_component_progress_fn_t) * num_btls);
if (NULL == mca_bml_r2.btl_modules ||
NULL == mca_bml_r2.btl_progress) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
for(selected_btl = (mca_btl_base_selected_module_t*)opal_list_get_first(btls);
selected_btl != (mca_btl_base_selected_module_t*)opal_list_get_end(btls);
selected_btl = (mca_btl_base_selected_module_t*)opal_list_get_next(selected_btl)) {
mca_btl_base_module_t *btl = selected_btl->btl_module;
mca_bml_r2.btl_modules[mca_bml_r2.num_btl_modules++] = btl;
for (i = 0; NULL != btl_names_argv && NULL != btl_names_argv[i]; ++i) {
if (0 ==
strcmp(btl_names_argv[i],
btl->btl_component->btl_version.mca_component_name)) {
break;
}
}
if (NULL == btl_names_argv || NULL == btl_names_argv[i]) {
opal_argv_append_nosize(&btl_names_argv,
btl->btl_component->btl_version.mca_component_name);
}
}
if (NULL != btl_names_argv) {
btl_names = opal_argv_join(btl_names_argv, ' ');
opal_argv_free(btl_names_argv);
} else {
btl_names = strdup("no devices available");
}
/* sort r2 list by exclusivity */
qsort(mca_bml_r2.btl_modules,
mca_bml_r2.num_btl_modules,
sizeof(struct mca_btl_base_module_t*),
btl_exclusivity_compare);
mca_bml_r2.btls_added = true;
return OMPI_SUCCESS;
}
int opal_crs_base_cleanup_append(char* filename, bool is_dir)
{
if( NULL == filename ) {
return OPAL_SUCCESS;
}
if( is_dir ) {
opal_output_verbose(15, opal_crs_base_framework.framework_output,
"opal:crs: cleanup_append: Append Dir <%s>\n",
filename);
opal_argv_append_nosize(&cleanup_dir_argv, filename);
} else {
opal_output_verbose(15, opal_crs_base_framework.framework_output,
"opal:crs: cleanup_append: Append File <%s>\n",
filename);
opal_argv_append_nosize(&cleanup_file_argv, filename);
}
return OPAL_SUCCESS;
}
/*
* Append a string to the end of a new or existing argv array.
* =====================================
* char **str_array_pt = NULL;
int argc;
char *str1 = "hello";
char *str2 = "haha";
opal_argv_append(&argc, &str_array_pt, str1);
opal_argv_append(&argc, &str_array_pt, str2);
while(*str_array_pt)
puts(*str_array_pt++);
printf("argc = %d\n", argc);
* =====================================
* hello
haha
argc = 2
*/
int opal_argv_append(int *argc, char ***argv, const char *arg)
{
int rc;
/* add the new element */
if (OPAL_SUCCESS != (rc = opal_argv_append_nosize(argv, arg))) {
return rc;
}
*argc = opal_argv_count(*argv);
return OPAL_SUCCESS;
}
void opal_ifgetaliases(char ***aliases)
{
opal_if_t* intf;
char ipv4[INET_ADDRSTRLEN];
struct sockaddr_in *addr;
#if OPAL_ENABLE_IPV6
char ipv6[INET6_ADDRSTRLEN];
struct sockaddr_in6 *addr6;
#endif
/* set default answer */
*aliases = NULL;
if (OPAL_SUCCESS != mca_base_framework_open(&opal_if_base_framework, 0)) {
return;
}
for (intf = (opal_if_t*)opal_list_get_first(&opal_if_list);
intf != (opal_if_t*)opal_list_get_end(&opal_if_list);
intf = (opal_if_t*)opal_list_get_next(intf)) {
addr = (struct sockaddr_in*) &intf->if_addr;
/* ignore purely loopback interfaces */
if ((intf->if_flags & IFF_LOOPBACK) != 0) {
continue;
}
if (addr->sin_family == AF_INET) {
inet_ntop(AF_INET, &(addr->sin_addr.s_addr), ipv4, INET_ADDRSTRLEN);
opal_argv_append_nosize(aliases, ipv4);
}
#if OPAL_ENABLE_IPV6
else {
addr6 = (struct sockaddr_in6*) &intf->if_addr;
inet_ntop(AF_INET6, &(addr6->sin6_addr), ipv6, INET6_ADDRSTRLEN);
opal_argv_append_nosize(aliases, ipv6);
}
#endif
}
}
static int mca_bml_r2_add_btls( void )
{
int i;
opal_list_t *btls = NULL;
mca_btl_base_selected_module_t* selected_btl;
size_t num_btls = 0;
char **btl_names_argv = NULL;
if(true == mca_bml_r2.btls_added) {
return OMPI_SUCCESS;
}
/* build an array of r2s and r2 modules */
btls = &mca_btl_base_modules_initialized;
num_btls = opal_list_get_size(btls);
mca_bml_r2.num_btl_modules = 0;
mca_bml_r2.num_btl_progress = 0;
mca_bml_r2.btl_modules = (mca_btl_base_module_t **)malloc(sizeof(mca_btl_base_module_t*) * num_btls);
mca_bml_r2.btl_progress = (mca_btl_base_component_progress_fn_t*)malloc(sizeof(mca_btl_base_component_progress_fn_t) * num_btls);
if (NULL == mca_bml_r2.btl_modules ||
NULL == mca_bml_r2.btl_progress) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
OPAL_LIST_FOREACH(selected_btl, btls, mca_btl_base_selected_module_t) {
mca_btl_base_module_t *btl = selected_btl->btl_module;
mca_bml_r2.btl_modules[mca_bml_r2.num_btl_modules++] = btl;
for (i = 0; NULL != btl_names_argv && NULL != btl_names_argv[i]; ++i) {
if (0 ==
strcmp(btl_names_argv[i],
btl->btl_component->btl_version.mca_component_name)) {
break;
}
}
if (NULL == btl_names_argv || NULL == btl_names_argv[i]) {
opal_argv_append_nosize(&btl_names_argv,
btl->btl_component->btl_version.mca_component_name);
}
}
static void
add_extra_includes(const char *includes, const char* includedir)
{
int i;
char **values = opal_argv_split(includes, ' ');
for (i = 0 ; i < opal_argv_count(values) ; ++i) {
char *line, *include_directory;
include_directory = opal_os_path(false, includedir, values[i], NULL);
#if defined(__WINDOWS__)
asprintf(&line, OPAL_INCLUDE_FLAG"\"%s\"", include_directory);
#else
asprintf(&line, OPAL_INCLUDE_FLAG"%s", include_directory);
#endif /* defined(__WINDOWS__) */
opal_argv_append_nosize(&options_data[parse_options_idx].preproc_flags, line);
free(include_directory);
free(line);
}
}
static void build_debugger_args(orte_app_context_t *debugger)
{
int i, j;
char mpir_arg[MPIR_MAX_ARG_LENGTH];
if ('\0' != MPIR_server_arguments[0]) {
j=0;
memset(mpir_arg, 0, MPIR_MAX_ARG_LENGTH);
for (i=0; i < MPIR_MAX_ARG_LENGTH; i++) {
if (MPIR_server_arguments[i] == '\0') {
if (0 < j) {
opal_argv_append_nosize(&debugger->argv, mpir_arg);
memset(mpir_arg, 0, MPIR_MAX_ARG_LENGTH);
j=0;
}
} else {
mpir_arg[j] = MPIR_server_arguments[i];
j++;
}
}
}
}
/*
* We have an open file, and we're pointed at the right topic. So
* read in all the lines in the topic and make a list of them.
*/
static int read_message(char ***array)
{
char *tmp;
int token;
while (1) {
token = opal_show_help_yylex();
switch (token) {
case OPAL_SHOW_HELP_PARSE_MESSAGE:
tmp = strdup(opal_show_help_yytext);
if (NULL == tmp) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
opal_argv_append_nosize(array, tmp);
break;
default:
return OPAL_SUCCESS;
break;
}
}
/* Never get here */
}
static int s1_init(void)
{
PMI_BOOL initialized;
int spawned;
int rc, ret = OPAL_ERROR;
int i, rank, lrank, nrank;
char *pmix_id, tmp[64];
opal_value_t kv;
char *str;
uint32_t ui32;
opal_process_name_t ldr;
char **localranks=NULL;
if (PMI_SUCCESS != (rc = PMI_Initialized(&initialized))) {
OPAL_PMI_ERROR(rc, "PMI_Initialized");
return OPAL_ERROR;
}
if (PMI_TRUE != initialized && PMI_SUCCESS != (rc = PMI_Init(&spawned))) {
OPAL_PMI_ERROR(rc, "PMI_Init");
return OPAL_ERROR;
}
// setup hash table
opal_pmix_base_hash_init();
// Initialize space demands
rc = PMI_KVS_Get_value_length_max(&pmix_vallen_max);
if (PMI_SUCCESS != rc) {
OPAL_PMI_ERROR(rc, "PMI_KVS_Get_value_length_max");
goto err_exit;
}
pmix_vallen_threshold = pmix_vallen_max * 3;
pmix_vallen_threshold >>= 2;
rc = PMI_KVS_Get_name_length_max(&pmix_kvslen_max);
if (PMI_SUCCESS != rc) {
OPAL_PMI_ERROR(rc, "PMI_KVS_Get_name_length_max");
goto err_exit;
}
rc = PMI_KVS_Get_key_length_max(&pmix_keylen_max);
if (PMI_SUCCESS != rc) {
OPAL_PMI_ERROR(rc, "PMI_KVS_Get_key_length_max");
goto err_exit;
}
// Initialize job environment information
pmix_id = (char*)malloc(pmix_vallen_max);
if (pmix_id == NULL) {
ret = OPAL_ERR_OUT_OF_RESOURCE;
goto err_exit;
}
/* Get domain id */
if (PMI_SUCCESS != (rc = PMI_Get_kvs_domain_id(pmix_id, pmix_vallen_max))) {
free(pmix_id);
goto err_exit;
}
/* get our rank */
ret = PMI_Get_rank(&rank);
if( PMI_SUCCESS != ret ) {
OPAL_PMI_ERROR(ret, "PMI_Get_rank");
free(pmix_id);
goto err_exit;
}
/* Slurm PMI provides the job id as an integer followed
* by a '.', followed by essentially a stepid. The first integer
* defines an overall job number. The second integer is the number of
* individual jobs we have run within that allocation. */
s1_pname.jobid = strtoul(pmix_id, &str, 10);
s1_pname.jobid = (s1_pname.jobid << 16) & 0xffff0000;
if (NULL != str) {
ui32 = strtoul(str, NULL, 10);
s1_pname.jobid |= (ui32 & 0x0000ffff);
}
ldr.jobid = s1_pname.jobid;
s1_pname.vpid = rank;
/* store our name in the opal_proc_t so that
* debug messages will make sense - an upper
* layer will eventually overwrite it, but that
* won't do any harm */
opal_proc_set_name(&s1_pname);
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s pmix:s1: assigned tmp name",
OPAL_NAME_PRINT(s1_pname));
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_JOBID);
kv.type = OPAL_UINT32;
kv.data.uint32 = s1_pname.jobid;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* save it */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_RANK);
kv.type = OPAL_UINT32;
kv.data.uint32 = rank;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
pmix_kvs_name = (char*)malloc(pmix_kvslen_max);
if (pmix_kvs_name == NULL) {
ret = OPAL_ERR_OUT_OF_RESOURCE;
goto err_exit;
}
rc = PMI_KVS_Get_my_name(pmix_kvs_name, pmix_kvslen_max);
if (PMI_SUCCESS != rc) {
OPAL_PMI_ERROR(rc, "PMI_KVS_Get_my_name");
goto err_exit;
}
/* get our local proc info to find our local rank */
if (PMI_SUCCESS != (rc = PMI_Get_clique_size(&nlranks))) {
OPAL_PMI_ERROR(rc, "PMI_Get_clique_size");
return rc;
}
/* save the local size */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCAL_SIZE);
kv.type = OPAL_UINT32;
kv.data.uint32 = nlranks;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
lrank = 0;
nrank = 0;
ldr.vpid = rank;
if (0 < nlranks) {
/* now get the specific ranks */
lranks = (int*)calloc(nlranks, sizeof(int));
if (NULL == lranks) {
rc = OPAL_ERR_OUT_OF_RESOURCE;
OPAL_ERROR_LOG(rc);
return rc;
}
if (PMI_SUCCESS != (rc = PMI_Get_clique_ranks(lranks, nlranks))) {
OPAL_PMI_ERROR(rc, "PMI_Get_clique_ranks");
free(lranks);
return rc;
}
/* note the local ldr */
ldr.vpid = lranks[0];
/* save this */
memset(tmp, 0, 64);
for (i=0; i < nlranks; i++) {
(void)snprintf(tmp, 64, "%d", lranks[i]);
opal_argv_append_nosize(&localranks, tmp);
if (rank == lranks[i]) {
lrank = i;
nrank = i;
}
}
str = opal_argv_join(localranks, ',');
opal_argv_free(localranks);
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCAL_PEERS);
kv.type = OPAL_STRING;
kv.data.string = str;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
}
/* save the local leader */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCALLDR);
kv.type = OPAL_UINT64;
kv.data.uint64 = *(uint64_t*)&ldr;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* save our local rank */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCAL_RANK);
kv.type = OPAL_UINT16;
kv.data.uint16 = lrank;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* and our node rank */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_NODE_RANK);
kv.type = OPAL_UINT16;
kv.data.uint16 = nrank;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* get universe size */
ret = PMI_Get_universe_size(&i);
if (PMI_SUCCESS != ret) {
OPAL_PMI_ERROR(ret, "PMI_Get_universe_size");
goto err_exit;
}
/* push this into the dstore for subsequent fetches */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_UNIV_SIZE);
kv.type = OPAL_UINT32;
kv.data.uint32 = i;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* get job size */
ret = PMI_Get_size(&i);
if (PMI_SUCCESS != ret) {
OPAL_PMI_ERROR(ret, "PMI_Get_size");
goto err_exit;
}
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_JOB_SIZE);
kv.type = OPAL_UINT32;
kv.data.uint32 = i;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* get appnum */
ret = PMI_Get_appnum(&i);
if (PMI_SUCCESS != ret) {
OPAL_PMI_ERROR(ret, "PMI_Get_appnum");
goto err_exit;
}
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_APPNUM);
kv.type = OPAL_UINT32;
kv.data.uint32 = i;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* increment the init count */
++pmix_init_count;
return OPAL_SUCCESS;
err_exit:
PMI_Finalize();
return ret;
}
static void launch_daemons(int fd, short args, void *cbdata)
{
orte_app_context_t *app;
orte_node_t *node;
orte_std_cntr_t n;
orte_job_map_t *map;
char *jobid_string = NULL;
char *param;
char **argv = NULL;
int argc;
int rc;
char *tmp;
char** env = NULL;
char *nodelist_flat;
char **nodelist_argv;
char *name_string;
char **custom_strings;
int num_args, i;
char *cur_prefix;
int proc_vpid_index;
bool failed_launch=true;
orte_job_t *daemons;
orte_state_caddy_t *state = (orte_state_caddy_t*)cbdata;
OPAL_OUTPUT_VERBOSE((1, orte_plm_base_framework.framework_output,
"%s plm:slurm: LAUNCH DAEMONS CALLED",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* if we are launching debugger daemons, then just go
* do it - no new daemons will be launched
*/
if (ORTE_FLAG_TEST(state->jdata, ORTE_JOB_FLAG_DEBUGGER_DAEMON)) {
state->jdata->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
ORTE_ACTIVATE_JOB_STATE(state->jdata, ORTE_JOB_STATE_DAEMONS_REPORTED);
OBJ_RELEASE(state);
return;
}
/* start by setting up the virtual machine */
daemons = orte_get_job_data_object(ORTE_PROC_MY_NAME->jobid);
if (ORTE_SUCCESS != (rc = orte_plm_base_setup_virtual_machine(state->jdata))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/* if we don't want to launch, then don't attempt to
* launch the daemons - the user really wants to just
* look at the proposed process map
*/
if (orte_do_not_launch) {
/* set the state to indicate the daemons reported - this
* will trigger the daemons_reported event and cause the
* job to move to the following step
*/
state->jdata->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
ORTE_ACTIVATE_JOB_STATE(state->jdata, ORTE_JOB_STATE_DAEMONS_REPORTED);
OBJ_RELEASE(state);
return;
}
/* Get the map for this job */
if (NULL == (map = daemons->map)) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
rc = ORTE_ERR_NOT_FOUND;
goto cleanup;
}
if (0 == map->num_new_daemons) {
/* set the state to indicate the daemons reported - this
* will trigger the daemons_reported event and cause the
* job to move to the following step
*/
OPAL_OUTPUT_VERBOSE((1, orte_plm_base_framework.framework_output,
"%s plm:slurm: no new daemons to launch",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
state->jdata->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
ORTE_ACTIVATE_JOB_STATE(state->jdata, ORTE_JOB_STATE_DAEMONS_REPORTED);
OBJ_RELEASE(state);
return;
}
/* need integer value for command line parameter */
asprintf(&jobid_string, "%lu", (unsigned long) daemons->jobid);
/*
* start building argv array
*/
argv = NULL;
argc = 0;
/*
* SLURM srun OPTIONS
*/
/* add the srun command */
opal_argv_append(&argc, &argv, "srun");
/* start one orted on each node */
opal_argv_append(&argc, &argv, "--ntasks-per-node=1");
/* alert us if any orteds die during startup */
opal_argv_append(&argc, &argv, "--kill-on-bad-exit");
/* ensure the orteds are not bound to a single processor,
* just in case the TaskAffinity option is set by default.
* This will *not* release the orteds from any cpu-set
* constraint, but will ensure it doesn't get
* bound to only one processor
*/
opal_argv_append(&argc, &argv, "--cpu_bind=none");
/* Append user defined arguments to srun */
if ( NULL != mca_plm_slurm_component.custom_args ) {
custom_strings = opal_argv_split(mca_plm_slurm_component.custom_args, ' ');
num_args = opal_argv_count(custom_strings);
for (i = 0; i < num_args; ++i) {
opal_argv_append(&argc, &argv, custom_strings[i]);
}
opal_argv_free(custom_strings);
}
/* create nodelist */
nodelist_argv = NULL;
for (n=0; n < map->nodes->size; n++ ) {
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(map->nodes, n))) {
continue;
}
/* if the daemon already exists on this node, then
* don't include it
*/
if (ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_DAEMON_LAUNCHED)) {
continue;
}
/* otherwise, add it to the list of nodes upon which
* we need to launch a daemon
*/
opal_argv_append_nosize(&nodelist_argv, node->name);
}
if (0 == opal_argv_count(nodelist_argv)) {
orte_show_help("help-plm-slurm.txt", "no-hosts-in-list", true);
rc = ORTE_ERR_FAILED_TO_START;
goto cleanup;
}
nodelist_flat = opal_argv_join(nodelist_argv, ',');
opal_argv_free(nodelist_argv);
/* if we are using all allocated nodes, then srun doesn't
* require any further arguments
*/
if (map->num_new_daemons < orte_num_allocated_nodes) {
asprintf(&tmp, "--nodes=%lu", (unsigned long)map->num_new_daemons);
opal_argv_append(&argc, &argv, tmp);
free(tmp);
asprintf(&tmp, "--nodelist=%s", nodelist_flat);
opal_argv_append(&argc, &argv, tmp);
free(tmp);
}
/* tell srun how many tasks to run */
asprintf(&tmp, "--ntasks=%lu", (unsigned long)map->num_new_daemons);
opal_argv_append(&argc, &argv, tmp);
free(tmp);
OPAL_OUTPUT_VERBOSE((2, orte_plm_base_framework.framework_output,
"%s plm:slurm: launching on nodes %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), nodelist_flat));
/*
* ORTED OPTIONS
*/
/* add the daemon command (as specified by user) */
orte_plm_base_setup_orted_cmd(&argc, &argv);
/* Add basic orted command line options, including debug flags */
orte_plm_base_orted_append_basic_args(&argc, &argv,
NULL, &proc_vpid_index,
nodelist_flat);
free(nodelist_flat);
/* tell the new daemons the base of the name list so they can compute
* their own name on the other end
*/
rc = orte_util_convert_vpid_to_string(&name_string, map->daemon_vpid_start);
if (ORTE_SUCCESS != rc) {
opal_output(0, "plm_slurm: unable to get daemon vpid as string");
goto cleanup;
}
free(argv[proc_vpid_index]);
argv[proc_vpid_index] = strdup(name_string);
free(name_string);
/* Copy the prefix-directory specified in the
corresponding app_context. If there are multiple,
different prefix's in the app context, complain (i.e., only
allow one --prefix option for the entire slurm run -- we
don't support different --prefix'es for different nodes in
the SLURM plm) */
cur_prefix = NULL;
for (n=0; n < state->jdata->apps->size; n++) {
char * app_prefix_dir;
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(state->jdata->apps, n))) {
continue;
}
app_prefix_dir = NULL;
orte_get_attribute(&app->attributes, ORTE_APP_PREFIX_DIR, (void**)&app_prefix_dir, OPAL_STRING);
/* Check for already set cur_prefix_dir -- if different,
complain */
if (NULL != app_prefix_dir) {
if (NULL != cur_prefix &&
0 != strcmp (cur_prefix, app_prefix_dir)) {
orte_show_help("help-plm-slurm.txt", "multiple-prefixes",
true, cur_prefix, app_prefix_dir);
goto cleanup;
}
/* If not yet set, copy it; iff set, then it's the
* same anyway
*/
if (NULL == cur_prefix) {
cur_prefix = strdup(app_prefix_dir);
OPAL_OUTPUT_VERBOSE((1, orte_plm_base_framework.framework_output,
"%s plm:slurm: Set prefix:%s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
cur_prefix));
}
free(app_prefix_dir);
}
}
/* protect the args in case someone has a script wrapper around srun */
mca_base_cmd_line_wrap_args(argv);
/* setup environment */
env = opal_argv_copy(orte_launch_environ);
if (0 < opal_output_get_verbosity(orte_plm_base_framework.framework_output)) {
param = opal_argv_join(argv, ' ');
OPAL_OUTPUT_VERBOSE((1, orte_plm_base_framework.framework_output,
"%s plm:slurm: final top-level argv:\n\t%s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == param) ? "NULL" : param));
if (NULL != param) free(param);
}
/* exec the daemon(s) */
if (ORTE_SUCCESS != (rc = plm_slurm_start_proc(argc, argv, env, cur_prefix))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/* indicate that the daemons for this job were launched */
state->jdata->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
daemons->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
/* flag that launch was successful, so far as we currently know */
failed_launch = false;
cleanup:
if (NULL != argv) {
opal_argv_free(argv);
}
if (NULL != env) {
opal_argv_free(env);
}
if(NULL != jobid_string) {
free(jobid_string);
}
/* cleanup the caddy */
OBJ_RELEASE(state);
/* check for failed launch - if so, force terminate */
if (failed_launch) {
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
}
}
/* we can only enter this routine if no other allocation
* was found, so we only need to know that finding any
* relative node syntax should generate an immediate error
*/
int orte_util_add_dash_host_nodes(opal_list_t *nodes,
bool *override_oversubscribed,
char ** host_argv)
{
opal_list_item_t* item;
orte_std_cntr_t i, j, k;
int rc;
char **mapped_nodes = NULL, **mini_map;
orte_node_t *node;
/* Accumulate all of the host name mappings */
for (j = 0; j < opal_argv_count(host_argv); ++j) {
mini_map = opal_argv_split(host_argv[j], ',');
if (mapped_nodes == NULL) {
mapped_nodes = mini_map;
} else {
for (k = 0; NULL != mini_map[k]; ++k) {
rc = opal_argv_append_nosize(&mapped_nodes,
mini_map[k]);
if (OPAL_SUCCESS != rc) {
goto cleanup;
}
}
opal_argv_free(mini_map);
}
}
/* Did we find anything? If not, then do nothing */
if (NULL == mapped_nodes) {
return ORTE_SUCCESS;
}
/* go through the names found and
add them to the host list. If they're not unique, then
bump the slots count for each duplicate */
for (i = 0; NULL != mapped_nodes[i]; ++i) {
/* if the specified node contains a relative node syntax,
* this is an error
*/
if ('+' == mapped_nodes[i][0]) {
orte_show_help("help-dash-host.txt", "dash-host:relative-syntax",
true, mapped_nodes[i]);
rc = ORTE_ERR_SILENT;
goto cleanup;
}
/* see if the node is already on the list */
for (item = opal_list_get_first(nodes);
item != opal_list_get_end(nodes);
item = opal_list_get_next(item)) {
node = (orte_node_t*) item;
if (0 == strcmp(node->name, mapped_nodes[i]) ||
(0 == strcmp(node->name, orte_process_info.nodename) &&
(0 == strcmp(mapped_nodes[i], "localhost") || opal_ifislocal(mapped_nodes[i])))) {
++node->slots;
break;
}
}
/* If we didn't find it, add it to the list */
if (item == opal_list_get_end(nodes)) {
node = OBJ_NEW(orte_node_t);
if (NULL == node) {
return ORTE_ERR_OUT_OF_RESOURCE;
}
/* check to see if this is a local name */
if (0 == strcmp(mapped_nodes[i], "localhost") ||
opal_ifislocal(mapped_nodes[i])) {
/* it is local, so use the local nodename to avoid
* later confusion
*/
if (orte_show_resolved_nodenames &&
0 != strcmp(mapped_nodes[i], orte_process_info.nodename)) {
/* add to list of aliases for this node - only add if unique */
opal_argv_append_unique_nosize(&node->alias, mapped_nodes[i]);
}
node->name = strdup(orte_process_info.nodename);
} else {
/* not local - use the given name */
node->name = strdup(mapped_nodes[i]);
}
node->state = ORTE_NODE_STATE_UP;
node->slots_inuse = 0;
node->slots_max = 0;
node->slots = 1;
/* indicate that ORTE should override any oversubscribed conditions
* based on local hardware limits since the user (a) might not have
* provided us any info on the #slots for a node, and (b) the user
* might have been wrong! If we don't check the number of local physical
* processors, then we could be too aggressive on our sched_yield setting
* and cause performance problems.
*/
*override_oversubscribed = true;
opal_list_append(nodes, &node->super);
}
}
rc = ORTE_SUCCESS;
cleanup:
if (NULL != mapped_nodes) {
opal_argv_free(mapped_nodes);
}
return rc;
}
static int parse_cli(char **personality,
int argc, int start, char **argv)
{
int i, j, k;
bool ignore;
char *no_dups[] = {
"grpcomm",
"odls",
"rml",
"routed",
NULL
};
bool takeus = false;
/* see if we are included */
for (i=0; NULL != personality[i]; i++) {
if (0 == strcmp(personality[i], "ompi")) {
takeus = true;
break;
}
}
if (!takeus) {
return ORTE_ERR_TAKE_NEXT_OPTION;
}
for (i = 0; i < (argc-start); ++i) {
if (0 == strcmp("-mca", argv[i]) ||
0 == strcmp("--mca", argv[i]) ) {
/* ignore this one */
if (0 == strcmp(argv[i+1], "mca_base_env_list")) {
i += 2;
continue;
}
/* It would be nice to avoid increasing the length
* of the orted cmd line by removing any non-ORTE
* params. However, this raises a problem since
* there could be OPAL directives that we really
* -do- want the orted to see - it's only the OMPI
* related directives we could ignore. This becomes
* a very complicated procedure, however, since
* the OMPI mca params are not cleanly separated - so
* filtering them out is nearly impossible.
*
* see if this is already present so we at least can
* avoid growing the cmd line with duplicates
*/
ignore = false;
if (NULL != orted_cmd_line) {
for (j=0; NULL != orted_cmd_line[j]; j++) {
if (0 == strcmp(argv[i+1], orted_cmd_line[j])) {
/* already here - if the value is the same,
* we can quitely ignore the fact that they
* provide it more than once. However, some
* frameworks are known to have problems if the
* value is different. We don't have a good way
* to know this, but we at least make a crude
* attempt here to protect ourselves.
*/
if (0 == strcmp(argv[i+2], orted_cmd_line[j+1])) {
/* values are the same */
ignore = true;
break;
} else {
/* values are different - see if this is a problem */
for (k=0; NULL != no_dups[k]; k++) {
if (0 == strcmp(no_dups[k], argv[i+1])) {
/* print help message
* and abort as we cannot know which one is correct
*/
orte_show_help("help-orterun.txt", "orterun:conflicting-params",
true, orte_basename, argv[i+1],
argv[i+2], orted_cmd_line[j+1]);
return ORTE_ERR_BAD_PARAM;
}
}
/* this passed muster - just ignore it */
ignore = true;
break;
}
}
}
}
if (!ignore) {
opal_argv_append_nosize(&orted_cmd_line, argv[i]);
opal_argv_append_nosize(&orted_cmd_line, argv[i+1]);
opal_argv_append_nosize(&orted_cmd_line, argv[i+2]);
}
i += 2;
}
}
return ORTE_SUCCESS;
}
/* the -host option can always be used in both absolute
* and relative mode, so we have to check for pre-existing
* allocations if we are to use relative node syntax
*/
int orte_util_filter_dash_host_nodes(opal_list_t *nodes,
char** host_argv)
{
opal_list_item_t* item;
bool found;
opal_list_item_t *next;
orte_std_cntr_t i, j, k, len_mapped_node=0;
int rc;
char **mapped_nodes = NULL, **mini_map, *cptr;
orte_node_t *node, **nodepool;
int nodeidx;
int num_empty=0;
opal_list_t keep;
bool want_all_empty = false;
/* if the incoming node list is empty, then there
* is nothing to filter!
*/
if (opal_list_is_empty(nodes)) {
return ORTE_SUCCESS;
}
/* setup for relative node syntax */
nodepool = (orte_node_t**)orte_node_pool->addr;
/* Accumulate all of the host name mappings */
for (j = 0; j < opal_argv_count(host_argv); ++j) {
mini_map = opal_argv_split(host_argv[j], ',');
for (k = 0; NULL != mini_map[k]; ++k) {
if ('+' == mini_map[k][0]) {
/* see if we specified empty nodes */
if ('e' == mini_map[k][1] ||
'E' == mini_map[k][1]) {
/* request for empty nodes - do they want
* all of them?
*/
if (NULL != (cptr = strchr(mini_map[k], ':'))) {
/* the colon indicates a specific # are requested */
cptr++; /* step past : */
/* put a marker into the list */
cptr--;
*cptr = '*';
opal_argv_append_nosize(&mapped_nodes, cptr);
} else {
/* add a marker to the list */
opal_argv_append_nosize(&mapped_nodes, "*");
want_all_empty = true;
}
} else if ('n' == mini_map[k][1] ||
'N' == mini_map[k][1]) {
/* they want a specific relative node #, so
* look it up on global pool
*/
nodeidx = strtol(&mini_map[k][2], NULL, 10);
if (nodeidx < 0 ||
nodeidx > (int)orte_node_pool->size) {
/* this is an error */
orte_show_help("help-dash-host.txt", "dash-host:relative-node-out-of-bounds",
true, nodeidx, mini_map[k]);
rc = ORTE_ERR_SILENT;
goto cleanup;
}
/* if the HNP is not allocated, then we need to
* adjust the index as the node pool is offset
* by one
*/
if (!orte_hnp_is_allocated) {
nodeidx++;
}
/* see if that location is filled */
if (NULL == nodepool[nodeidx]) {
/* this is an error */
orte_show_help("help-dash-host.txt", "dash-host:relative-node-not-found",
true, nodeidx, mini_map[k]);
rc = ORTE_ERR_SILENT;
goto cleanup;
}
/* add this node to the list */
opal_argv_append_nosize(&mapped_nodes, nodepool[nodeidx]->name);
} else {
/* invalid relative node syntax */
orte_show_help("help-dash-host.txt", "dash-host:invalid-relative-node-syntax",
true, mini_map[k]);
rc = ORTE_ERR_SILENT;
goto cleanup;
}
} else { /* non-relative syntax - add to list */
if (OPAL_SUCCESS != (rc = opal_argv_append_nosize(&mapped_nodes,
mini_map[k]))) {
goto cleanup;
}
}
}
opal_argv_free(mini_map);
}
/* Did we find anything? If not, then do nothing */
if (NULL == mapped_nodes && 0 == num_empty) {
return ORTE_SUCCESS;
}
/* we found some info - filter what is on the list...
* i.e., go through the list and remove any nodes that
* were -not- included on the -host list.
*
* NOTE: The following logic is based on knowing that
* any node can only be included on the incoming
* nodes list ONCE.
*/
len_mapped_node = opal_argv_count(mapped_nodes);
/* setup a working list so we can put the final list
* of nodes in order. This way, if the user specifies a
* set of nodes, we will use them in the order in which
* they were specifed. Note that empty node requests
* will always be appended to the end
*/
OBJ_CONSTRUCT(&keep, opal_list_t);
for (i = 0; i < len_mapped_node; ++i) {
/* check if we are supposed to add some number of empty
* nodes here
*/
if ('*' == mapped_nodes[i][0]) {
/* if there is a number after the '*', then we are
* to insert a specific # of nodes
*/
if ('\0' == mapped_nodes[i][1]) {
/* take all empty nodes from the list */
num_empty = INT_MAX;
} else {
/* extract number of nodes to take */
num_empty = strtol(&mapped_nodes[i][1], NULL, 10);
}
/* search for empty nodes and take them */
item = opal_list_get_first(nodes);
while (0 < num_empty && item != opal_list_get_end(nodes)) {
next = opal_list_get_next(item); /* save this position */
node = (orte_node_t*)item;
/* see if this node is empty */
if (0 == node->slots_inuse) {
/* check to see if it is specified later */
for (j=i+1; j < len_mapped_node; j++) {
if (0 == strcmp(mapped_nodes[j], node->name)) {
/* specified later - skip this one */
goto skipnode;
}
}
/* remove item from list */
opal_list_remove_item(nodes, item);
/* xfer to keep list */
opal_list_append(&keep, item);
--num_empty;
}
skipnode:
item = next;
}
} else {
/* we are looking for a specific node on the list
* we have a match if one of two conditions is met:
* 1. the node_name and mapped_nodes directly match
* 2. the node_name is the local system name AND
* either the mapped_node is "localhost" OR it
* is a local interface as found by opal_ifislocal
*/
item = opal_list_get_first(nodes);
while (item != opal_list_get_end(nodes)) {
next = opal_list_get_next(item); /* save this position */
node = (orte_node_t*)item;
/* search -host list to see if this one is found */
found = false;
if ((0 == strcmp(node->name, mapped_nodes[i]) ||
(0 == strcmp(node->name, orte_process_info.nodename) &&
(0 == strcmp(mapped_nodes[i], "localhost") || opal_ifislocal(mapped_nodes[i]))))) {
/* remove item from list */
opal_list_remove_item(nodes, item);
/* xfer to keep list */
opal_list_append(&keep, item);
break;
}
item = next;
}
}
/* done with the mapped entry */
free(mapped_nodes[i]);
mapped_nodes[i] = NULL;
}
/* was something specified that was -not- found? */
for (i=0; i < len_mapped_node; i++) {
if (NULL != mapped_nodes[i]) {
orte_show_help("help-dash-host.txt", "not-all-mapped-alloc",
true, mapped_nodes[i]);
rc = ORTE_ERR_SILENT;
goto cleanup;
}
}
/* clear the rest of the nodes list */
while (NULL != (item = opal_list_remove_first(nodes))) {
OBJ_RELEASE(item);
}
/* the nodes list has been cleared - rebuild it in order */
while (NULL != (item = opal_list_remove_first(&keep))) {
opal_list_append(nodes, item);
}
/* did they ask for more than we could provide */
if (!want_all_empty && 0 < num_empty) {
orte_show_help("help-dash-host.txt", "dash-host:not-enough-empty",
true, num_empty);
rc = ORTE_ERR_SILENT;
goto cleanup;
}
rc = ORTE_SUCCESS;
/* done filtering existing list */
cleanup:
for (i=0; i < len_mapped_node; i++) {
if (NULL != mapped_nodes[i]) {
free(mapped_nodes[i]);
mapped_nodes[i] = NULL;
}
}
if (NULL != mapped_nodes) {
free(mapped_nodes);
}
return rc;
}
static int parse_env(char **personality,
char *path,
opal_cmd_line_t *cmd_line,
char **srcenv,
char ***dstenv)
{
int i, j;
char *param;
char *value;
char *env_set_flag;
char **vars;
bool takeus = false;
/* see if we are included */
for (i=0; NULL != personality[i]; i++) {
if (0 == strcmp(personality[i], "ompi")) {
takeus = true;
break;
}
}
if (!takeus) {
return ORTE_ERR_TAKE_NEXT_OPTION;
}
for (i = 0; NULL != srcenv[i]; ++i) {
if (0 == strncmp("OMPI_", srcenv[i], 5)) {
/* check for duplicate in app->env - this
* would have been placed there by the
* cmd line processor. By convention, we
* always let the cmd line override the
* environment
*/
param = strdup(srcenv[i]);
value = strchr(param, '=');
*value = '\0';
value++;
opal_setenv(param, value, false, dstenv);
free(param);
}
}
/* set necessary env variables for external usage from tune conf file*/
int set_from_file = 0;
vars = NULL;
if (OPAL_SUCCESS == mca_base_var_process_env_list_from_file(&vars) &&
NULL != vars) {
for (i=0; NULL != vars[i]; i++) {
value = strchr(vars[i], '=');
/* terminate the name of the param */
*value = '\0';
/* step over the equals */
value++;
/* overwrite any prior entry */
opal_setenv(vars[i], value, true, dstenv);
/* save it for any comm_spawn'd apps */
opal_setenv(vars[i], value, true, &orte_forwarded_envars);
}
set_from_file = 1;
opal_argv_free(vars);
}
/* Did the user request to export any environment variables on the cmd line? */
env_set_flag = getenv("OMPI_MCA_mca_base_env_list");
if (opal_cmd_line_is_taken(cmd_line, "x")) {
if (NULL != env_set_flag) {
orte_show_help("help-orterun.txt", "orterun:conflict-env-set", false);
return ORTE_ERR_FATAL;
}
j = opal_cmd_line_get_ninsts(cmd_line, "x");
for (i = 0; i < j; ++i) {
param = opal_cmd_line_get_param(cmd_line, "x", i, 0);
if (NULL != (value = strchr(param, '='))) {
/* terminate the name of the param */
*value = '\0';
/* step over the equals */
value++;
/* overwrite any prior entry */
opal_setenv(param, value, true, dstenv);
/* save it for any comm_spawn'd apps */
opal_setenv(param, value, true, &orte_forwarded_envars);
} else {
value = getenv(param);
if (NULL != value) {
/* overwrite any prior entry */
opal_setenv(param, value, true, dstenv);
/* save it for any comm_spawn'd apps */
opal_setenv(param, value, true, &orte_forwarded_envars);
} else {
opal_output(0, "Warning: could not find environment variable \"%s\"\n", param);
}
}
}
} else if (NULL != env_set_flag) {
/* if mca_base_env_list was set, check if some of env vars were set via -x from a conf file.
* If this is the case, error out.
*/
if (!set_from_file) {
/* set necessary env variables for external usage */
vars = NULL;
if (OPAL_SUCCESS == mca_base_var_process_env_list(&vars) &&
NULL != vars) {
for (i=0; NULL != vars[i]; i++) {
value = strchr(vars[i], '=');
/* terminate the name of the param */
*value = '\0';
/* step over the equals */
value++;
/* overwrite any prior entry */
opal_setenv(vars[i], value, true, dstenv);
/* save it for any comm_spawn'd apps */
opal_setenv(vars[i], value, true, &orte_forwarded_envars);
}
opal_argv_free(vars);
}
} else {
orte_show_help("help-orterun.txt", "orterun:conflict-env-set", false);
return ORTE_ERR_FATAL;
}
}
/* If the user specified --path, store it in the user's app
environment via the OMPI_exec_path variable. */
if (NULL != path) {
asprintf(&value, "OMPI_exec_path=%s", path);
opal_argv_append_nosize(dstenv, value);
/* save it for any comm_spawn'd apps */
opal_argv_append_nosize(&orte_forwarded_envars, value);
free(value);
}
return ORTE_SUCCESS;
}
/*
* Process one line from the orte_base_user_debugger MCA param and
* look for that debugger in the path. If we find it, fill in
* new_argv.
*/
static int process(char *orig_line, char *basename, opal_cmd_line_t *cmd_line,
int argc, char **argv, char ***new_argv, int num_procs)
{
int ret = ORTE_SUCCESS;
int i, j, count;
char *line = NULL, *tmp = NULL, *full_line = strdup(orig_line);
char **orterun_argv = NULL, **executable_argv = NULL, **line_argv = NULL;
char cwd[OPAL_PATH_MAX];
bool used_num_procs = false;
bool single_app = false;
bool fail_needed_executable = false;
line = full_line;
if (NULL == line) {
ret = ORTE_ERR_OUT_OF_RESOURCE;
goto out;
}
/* Trim off whitespace at the beginning and ending of line */
for (i = 0; '\0' != line[i] && isspace(line[i]); ++line) {
continue;
}
for (i = strlen(line) - 2; i > 0 && isspace(line[i]); ++i) {
line[i] = '\0';
}
if (strlen(line) <= 0) {
ret = ORTE_ERROR;
goto out;
}
/* Get the tail of the command line (i.e., the user executable /
argv) */
opal_cmd_line_get_tail(cmd_line, &i, &executable_argv);
/* Make a new copy of the orterun command line args, without the
orterun token itself, and without the --debug, --debugger, and
-tv flags. */
orterun_argv = opal_argv_copy(argv);
count = opal_argv_count(orterun_argv);
opal_argv_delete(&count, &orterun_argv, 0, 1);
for (i = 0; NULL != orterun_argv[i]; ++i) {
count = opal_argv_count(orterun_argv);
if (0 == strcmp(orterun_argv[i], "-debug") ||
0 == strcmp(orterun_argv[i], "--debug")) {
opal_argv_delete(&count, &orterun_argv, i, 1);
} else if (0 == strcmp(orterun_argv[i], "-tv") ||
0 == strcmp(orterun_argv[i], "--tv")) {
opal_argv_delete(&count, &orterun_argv, i, 1);
} else if (0 == strcmp(orterun_argv[i], "--debugger") ||
0 == strcmp(orterun_argv[i], "-debugger")) {
opal_argv_delete(&count, &orterun_argv, i, 2);
}
}
/* Replace @@ tokens - line should never realistically be bigger
than MAX_INT, so just cast to int to remove compiler warning */
*new_argv = NULL;
line_argv = opal_argv_split(line, ' ');
if (NULL == line_argv) {
ret = ORTE_ERR_NOT_FOUND;
goto out;
}
for (i = 0; NULL != line_argv[i]; ++i) {
if (0 == strcmp(line_argv[i], "@mpirun@") ||
0 == strcmp(line_argv[i], "@orterun@")) {
opal_argv_append_nosize(new_argv, argv[0]);
} else if (0 == strcmp(line_argv[i], "@mpirun_args@") ||
0 == strcmp(line_argv[i], "@orterun_args@")) {
for (j = 0; NULL != orterun_argv && NULL != orterun_argv[j]; ++j) {
opal_argv_append_nosize(new_argv, orterun_argv[j]);
}
} else if (0 == strcmp(line_argv[i], "@np@")) {
asprintf(&tmp, "%d", num_procs);
opal_argv_append_nosize(new_argv, tmp);
free(tmp);
} else if (0 == strcmp(line_argv[i], "@single_app@")) {
/* This token is only a flag; it is not replaced with any
alternate text */
single_app = true;
} else if (0 == strcmp(line_argv[i], "@executable@")) {
/* If we found the executable, paste it in. Otherwise,
this is a possible error. */
if (NULL != executable_argv) {
opal_argv_append_nosize(new_argv, executable_argv[0]);
} else {
fail_needed_executable = true;
}
} else if (0 == strcmp(line_argv[i], "@executable_argv@")) {
/* If we found the tail, paste in the argv. Otherwise,
this is a possible error. */
if (NULL != executable_argv) {
for (j = 1; NULL != executable_argv[j]; ++j) {
opal_argv_append_nosize(new_argv, executable_argv[j]);
}
} else {
fail_needed_executable = true;
}
} else {
/* It wasn't a special token, so just copy it over */
opal_argv_append_nosize(new_argv, line_argv[i]);
}
}
/* Can we find argv[0] in the path? */
getcwd(cwd, OPAL_PATH_MAX);
tmp = opal_path_findv((*new_argv)[0], X_OK, environ, cwd);
if (NULL != tmp) {
free(tmp);
/* Ok, we found a good debugger. Check for some error
conditions. */
tmp = opal_argv_join(argv, ' ');
/* We do not support launching a debugger that requires the
-np value if the user did not specify -np on the command
line. */
if (used_num_procs && 0 == num_procs) {
free(tmp);
tmp = opal_argv_join(orterun_argv, ' ');
orte_show_help("help-orterun.txt", "debugger requires -np",
true, (*new_argv)[0], argv[0], tmp,
(*new_argv)[0]);
/* Fall through to free / fail, below */
}
/* Some debuggers do not support launching MPMD */
else if (single_app && NULL != strchr(tmp, ':')) {
orte_show_help("help-orterun.txt",
"debugger only accepts single app", true,
(*new_argv)[0], (*new_argv)[0]);
/* Fall through to free / fail, below */
}
/* Some debuggers do not use orterun/mpirun, and therefore
must have an executable to run (e.g., cannot use mpirun's
app context file feature). */
else if (fail_needed_executable) {
orte_show_help("help-orterun.txt",
"debugger requires executable", true,
(*new_argv)[0], argv[0], (*new_argv)[0], argv[0],
(*new_argv)[0]);
/* Fall through to free / fail, below */
}
/* Otherwise, we succeeded. Return happiness. */
else {
goto out;
}
free(tmp);
}
/* All done -- didn't find it */
opal_argv_free(*new_argv);
*new_argv = NULL;
ret = ORTE_ERR_NOT_FOUND;
out:
if (NULL != orterun_argv) {
opal_argv_free(orterun_argv);
}
if (NULL != executable_argv) {
opal_argv_free(executable_argv);
}
if (NULL != line_argv) {
opal_argv_free(line_argv);
}
if (NULL != tmp) {
free(tmp);
}
if (NULL != full_line) {
free(full_line);
}
return ret;
}
int orte_daemon(int argc, char *argv[])
{
int ret = 0;
opal_cmd_line_t *cmd_line = NULL;
char *rml_uri;
int i;
opal_buffer_t *buffer;
char hostname[100];
#if OPAL_ENABLE_FT_CR == 1
char *tmp_env_var = NULL;
#endif
/* initialize the globals */
memset(&orted_globals, 0, sizeof(orted_globals));
/* initialize the singleton died pipe to an illegal value so we can detect it was set */
orted_globals.singleton_died_pipe = -1;
/* init the failure orted vpid to an invalid value */
orted_globals.fail = ORTE_VPID_INVALID;
/* setup to check common command line options that just report and die */
cmd_line = OBJ_NEW(opal_cmd_line_t);
if (OPAL_SUCCESS != opal_cmd_line_create(cmd_line, orte_cmd_line_opts)) {
OBJ_RELEASE(cmd_line);
exit(1);
}
mca_base_cmd_line_setup(cmd_line);
if (ORTE_SUCCESS != (ret = opal_cmd_line_parse(cmd_line, false,
argc, argv))) {
char *args = NULL;
args = opal_cmd_line_get_usage_msg(cmd_line);
fprintf(stderr, "Usage: %s [OPTION]...\n%s\n", argv[0], args);
free(args);
OBJ_RELEASE(cmd_line);
return ret;
}
/*
* Since this process can now handle MCA/GMCA parameters, make sure to
* process them.
*/
mca_base_cmd_line_process_args(cmd_line, &environ, &environ);
/* Ensure that enough of OPAL is setup for us to be able to run */
/*
* NOTE: (JJH)
* We need to allow 'mca_base_cmd_line_process_args()' to process command
* line arguments *before* calling opal_init_util() since the command
* line could contain MCA parameters that affect the way opal_init_util()
* functions. AMCA parameters are one such option normally received on the
* command line that affect the way opal_init_util() behaves.
* It is "safe" to call mca_base_cmd_line_process_args() before
* opal_init_util() since mca_base_cmd_line_process_args() does *not*
* depend upon opal_init_util() functionality.
*/
if (OPAL_SUCCESS != opal_init_util(&argc, &argv)) {
fprintf(stderr, "OPAL failed to initialize -- orted aborting\n");
exit(1);
}
/* save the environment for launch purposes. This MUST be
* done so that we can pass it to any local procs we
* spawn - otherwise, those local procs won't see any
* non-MCA envars that were set in the enviro when the
* orted was executed - e.g., by .csh
*/
orte_launch_environ = opal_argv_copy(environ);
/* purge any ess flag set in the environ when we were launched */
opal_unsetenv(OPAL_MCA_PREFIX"ess", &orte_launch_environ);
/* if orte_daemon_debug is set, let someone know we are alive right
* away just in case we have a problem along the way
*/
if (orted_globals.debug) {
gethostname(hostname, 100);
fprintf(stderr, "Daemon was launched on %s - beginning to initialize\n", hostname);
}
/* check for help request */
if (orted_globals.help) {
char *args = NULL;
args = opal_cmd_line_get_usage_msg(cmd_line);
orte_show_help("help-orted.txt", "orted:usage", false,
argv[0], args);
free(args);
return 1;
}
#if defined(HAVE_SETSID)
/* see if we were directed to separate from current session */
if (orted_globals.set_sid) {
setsid();
}
#endif
/* see if they want us to spin until they can connect a debugger to us */
i=0;
while (orted_spin_flag) {
i++;
if (1000 < i) i=0;
}
#if OPAL_ENABLE_FT_CR == 1
/* Mark as a tool program */
(void) mca_base_var_env_name ("opal_cr_is_tool", &tmp_env_var);
opal_setenv(tmp_env_var,
"1",
true, &environ);
free(tmp_env_var);
#endif
/* if mapreduce set, flag it */
if (orted_globals.mapreduce) {
orte_map_reduce = true;
}
/* detach from controlling terminal
* otherwise, remain attached so output can get to us
*/
if(!orte_debug_flag &&
!orte_debug_daemons_flag &&
orted_globals.daemonize) {
opal_daemon_init(NULL);
}
/* Set the flag telling OpenRTE that I am NOT a
* singleton, but am "infrastructure" - prevents setting
* up incorrect infrastructure that only a singleton would
* require.
*/
if (orted_globals.hnp) {
if (ORTE_SUCCESS != (ret = orte_init(&argc, &argv, ORTE_PROC_HNP))) {
ORTE_ERROR_LOG(ret);
return ret;
}
} else {
if (ORTE_SUCCESS != (ret = orte_init(&argc, &argv, ORTE_PROC_DAEMON))) {
ORTE_ERROR_LOG(ret);
return ret;
}
}
/* finalize the OPAL utils. As they are opened again from orte_init->opal_init
* we continue to have a reference count on them. So we have to finalize them twice...
*/
opal_finalize_util();
#if OPAL_HAVE_HWLOC
/* bind ourselves if so directed */
if (NULL != orte_daemon_cores) {
char **cores=NULL, tmp[128];
hwloc_obj_t pu;
hwloc_cpuset_t ours, pucpus, res;
int core;
/* could be a collection of comma-delimited ranges, so
* use our handy utility to parse it
*/
orte_util_parse_range_options(orte_daemon_cores, &cores);
if (NULL != cores) {
ours = hwloc_bitmap_alloc();
hwloc_bitmap_zero(ours);
pucpus = hwloc_bitmap_alloc();
res = hwloc_bitmap_alloc();
for (i=0; NULL != cores[i]; i++) {
core = strtoul(cores[i], NULL, 10);
if (NULL == (pu = opal_hwloc_base_get_pu(opal_hwloc_topology, core, OPAL_HWLOC_LOGICAL))) {
/* turn off the show help forwarding as we won't
* be able to cycle the event library to send
*/
orte_show_help_finalize();
/* the message will now come out locally */
orte_show_help("help-orted.txt", "orted:cannot-bind",
true, orte_process_info.nodename,
orte_daemon_cores);
ret = ORTE_ERR_NOT_SUPPORTED;
goto DONE;
}
hwloc_bitmap_and(pucpus, pu->online_cpuset, pu->allowed_cpuset);
hwloc_bitmap_or(res, ours, pucpus);
hwloc_bitmap_copy(ours, res);
}
/* if the result is all zeros, then don't bind */
if (!hwloc_bitmap_iszero(ours)) {
(void)hwloc_set_cpubind(opal_hwloc_topology, ours, 0);
if (opal_hwloc_report_bindings) {
opal_hwloc_base_cset2mapstr(tmp, sizeof(tmp), opal_hwloc_topology, ours);
opal_output(0, "Daemon %s is bound to cores %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), tmp);
}
}
/* cleanup */
hwloc_bitmap_free(ours);
hwloc_bitmap_free(pucpus);
hwloc_bitmap_free(res);
opal_argv_free(cores);
}
}
#endif
if ((int)ORTE_VPID_INVALID != orted_globals.fail) {
orted_globals.abort=false;
/* some vpid was ordered to fail. The value can be positive
* or negative, depending upon the desired method for failure,
* so need to check both here
*/
if (0 > orted_globals.fail) {
orted_globals.fail = -1*orted_globals.fail;
orted_globals.abort = true;
}
/* are we the specified vpid? */
if ((int)ORTE_PROC_MY_NAME->vpid == orted_globals.fail) {
/* if the user specified we delay, then setup a timer
* and have it kill us
*/
if (0 < orted_globals.fail_delay) {
ORTE_TIMER_EVENT(orted_globals.fail_delay, 0, shutdown_callback, ORTE_SYS_PRI);
} else {
opal_output(0, "%s is executing clean %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
orted_globals.abort ? "abort" : "abnormal termination");
/* do -not- call finalize as this will send a message to the HNP
* indicating clean termination! Instead, just forcibly cleanup
* the local session_dir tree and exit
*/
orte_session_dir_cleanup(ORTE_JOBID_WILDCARD);
/* if we were ordered to abort, do so */
if (orted_globals.abort) {
abort();
}
/* otherwise, return with non-zero status */
ret = ORTE_ERROR_DEFAULT_EXIT_CODE;
goto DONE;
}
}
}
/* insert our contact info into our process_info struct so we
* have it for later use and set the local daemon field to our name
*/
orte_process_info.my_daemon_uri = orte_rml.get_contact_info();
ORTE_PROC_MY_DAEMON->jobid = ORTE_PROC_MY_NAME->jobid;
ORTE_PROC_MY_DAEMON->vpid = ORTE_PROC_MY_NAME->vpid;
/* if I am also the hnp, then update that contact info field too */
if (ORTE_PROC_IS_HNP) {
orte_process_info.my_hnp_uri = orte_rml.get_contact_info();
ORTE_PROC_MY_HNP->jobid = ORTE_PROC_MY_NAME->jobid;
ORTE_PROC_MY_HNP->vpid = ORTE_PROC_MY_NAME->vpid;
}
/* setup the primary daemon command receive function */
orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, ORTE_RML_TAG_DAEMON,
ORTE_RML_PERSISTENT, orte_daemon_recv, NULL);
/* output a message indicating we are alive, our name, and our pid
* for debugging purposes
*/
if (orte_debug_daemons_flag) {
fprintf(stderr, "Daemon %s checking in as pid %ld on host %s\n",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), (long)orte_process_info.pid,
orte_process_info.nodename);
}
/* We actually do *not* want the orted to voluntarily yield() the
processor more than necessary. The orted already blocks when
it is doing nothing, so it doesn't use any more CPU cycles than
it should; but when it *is* doing something, we do not want it
to be unnecessarily delayed because it voluntarily yielded the
processor in the middle of its work.
For example: when a message arrives at the orted, we want the
OS to wake up the orted in a timely fashion (which most OS's
seem good about doing) and then we want the orted to process
the message as fast as possible. If the orted yields and lets
aggressive MPI applications get the processor back, it may be a
long time before the OS schedules the orted to run again
(particularly if there is no IO event to wake it up). Hence,
routed OOB messages (for example) may be significantly delayed
before being delivered to MPI processes, which can be
problematic in some scenarios (e.g., COMM_SPAWN, BTL's that
require OOB messages for wireup, etc.). */
opal_progress_set_yield_when_idle(false);
/* Change the default behavior of libevent such that we want to
continually block rather than blocking for the default timeout
and then looping around the progress engine again. There
should be nothing in the orted that cannot block in libevent
until "something" happens (i.e., there's no need to keep
cycling through progress because the only things that should
happen will happen in libevent). This is a minor optimization,
but what the heck... :-) */
opal_progress_set_event_flag(OPAL_EVLOOP_ONCE);
/* if requested, report my uri to the indicated pipe */
if (orted_globals.uri_pipe > 0) {
orte_job_t *jdata;
orte_proc_t *proc;
orte_node_t *node;
orte_app_context_t *app;
char *tmp, *nptr, *sysinfo;
int32_t ljob;
/* setup the singleton's job */
jdata = OBJ_NEW(orte_job_t);
orte_plm_base_create_jobid(jdata);
ljob = ORTE_LOCAL_JOBID(jdata->jobid);
opal_pointer_array_set_item(orte_job_data, ljob, jdata);
/* must create a map for it (even though it has no
* info in it) so that the job info will be picked
* up in subsequent pidmaps or other daemons won't
* know how to route
*/
jdata->map = OBJ_NEW(orte_job_map_t);
/* setup an app_context for the singleton */
app = OBJ_NEW(orte_app_context_t);
app->app = strdup("singleton");
app->num_procs = 1;
opal_pointer_array_add(jdata->apps, app);
/* setup a proc object for the singleton - since we
* -must- be the HNP, and therefore we stored our
* node on the global node pool, and since the singleton
* -must- be on the same node as us, indicate that
*/
proc = OBJ_NEW(orte_proc_t);
proc->name.jobid = jdata->jobid;
proc->name.vpid = 0;
ORTE_FLAG_SET(proc, ORTE_PROC_FLAG_ALIVE);
proc->state = ORTE_PROC_STATE_RUNNING;
proc->app_idx = 0;
/* obviously, it is on my node */
node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, 0);
proc->node = node;
OBJ_RETAIN(node); /* keep accounting straight */
opal_pointer_array_add(jdata->procs, proc);
jdata->num_procs = 1;
/* and it obviously is on the node */
OBJ_RETAIN(proc);
opal_pointer_array_add(node->procs, proc);
node->num_procs++;
/* and obviously it is one of my local procs */
OBJ_RETAIN(proc);
opal_pointer_array_add(orte_local_children, proc);
jdata->num_local_procs = 1;
/* set the trivial */
proc->local_rank = 0;
proc->node_rank = 0;
proc->app_rank = 0;
proc->state = ORTE_PROC_STATE_RUNNING;
proc->app_idx = 0;
ORTE_FLAG_SET(proc, ORTE_PROC_FLAG_LOCAL);
/* create a string that contains our uri + sysinfo + PMIx server URI */
orte_util_convert_sysinfo_to_string(&sysinfo, orte_local_cpu_type, orte_local_cpu_model);
asprintf(&tmp, "%s[%s]%s", orte_process_info.my_daemon_uri, sysinfo, pmix_server_uri);
free(sysinfo);
/* pass that info to the singleton */
write(orted_globals.uri_pipe, tmp, strlen(tmp)+1); /* need to add 1 to get the NULL */
/* cleanup */
free(tmp);
/* since a singleton spawned us, we need to harvest
* any MCA params from the local environment so
* we can pass them along to any subsequent daemons
* we may start as the result of a comm_spawn
*/
for (i=0; NULL != environ[i]; i++) {
if (0 == strncmp(environ[i], OPAL_MCA_PREFIX, 9)) {
/* make a copy to manipulate */
tmp = strdup(environ[i]);
/* find the equal sign */
nptr = strchr(tmp, '=');
*nptr = '\0';
nptr++;
/* add the mca param to the orted cmd line */
opal_argv_append_nosize(&orted_cmd_line, "-"OPAL_MCA_CMD_LINE_ID);
opal_argv_append_nosize(&orted_cmd_line, &tmp[9]);
opal_argv_append_nosize(&orted_cmd_line, nptr);
free(tmp);
}
}
}
/* if we were given a pipe to monitor for singleton termination, set that up */
if (orted_globals.singleton_died_pipe > 0) {
/* register shutdown handler */
pipe_handler = (opal_event_t*)malloc(sizeof(opal_event_t));
opal_event_set(orte_event_base, pipe_handler,
orted_globals.singleton_died_pipe,
OPAL_EV_READ,
pipe_closed,
pipe_handler);
opal_event_add(pipe_handler, NULL);
}
/* If I have a parent, then save his contact info so
* any messages we send can flow thru him.
*/
orte_parent_uri = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "parent_uri",
"URI for the parent if tree launch is enabled.",
MCA_BASE_VAR_TYPE_STRING, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_CONSTANT,
&orte_parent_uri);
if (NULL != orte_parent_uri) {
orte_process_name_t parent;
/* set the contact info into the hash table */
orte_rml.set_contact_info(orte_parent_uri);
ret = orte_rml_base_parse_uris(orte_parent_uri, &parent, NULL);
if (ORTE_SUCCESS != ret) {
ORTE_ERROR_LOG(ret);
free (orte_parent_uri);
orte_parent_uri = NULL;
goto DONE;
}
/* don't need this value anymore */
free(orte_parent_uri);
orte_parent_uri = NULL;
/* tell the routed module that we have a path
* back to the HNP
*/
if (ORTE_SUCCESS != (ret = orte_routed.update_route(ORTE_PROC_MY_HNP, &parent))) {
ORTE_ERROR_LOG(ret);
goto DONE;
}
/* set the lifeline to point to our parent so that we
* can handle the situation if that lifeline goes away
*/
if (ORTE_SUCCESS != (ret = orte_routed.set_lifeline(&parent))) {
ORTE_ERROR_LOG(ret);
goto DONE;
}
}
/* if we are not the HNP...the only time we will be an HNP
* is if we are launched by a singleton to provide support
* for it
*/
if (!ORTE_PROC_IS_HNP) {
/* send the information to the orted report-back point - this function
* will process the data, but also counts the number of
* orteds that reported back so the launch procedure can continue.
* We need to do this at the last possible second as the HNP
* can turn right around and begin issuing orders to us
*/
buffer = OBJ_NEW(opal_buffer_t);
/* insert our name for rollup purposes */
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, ORTE_PROC_MY_NAME, 1, ORTE_NAME))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
/* for now, always include our contact info, even if we are using
* static ports. Eventually, this will be removed
*/
rml_uri = orte_rml.get_contact_info();
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &rml_uri, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
/* include our node name */
opal_dss.pack(buffer, &orte_process_info.nodename, 1, OPAL_STRING);
/* if requested, include any non-loopback aliases for this node */
if (orte_retain_aliases) {
char **aliases=NULL;
uint8_t naliases, ni;
char hostname[ORTE_MAX_HOSTNAME_SIZE];
/* if we stripped the prefix or removed the fqdn,
* include full hostname as an alias
*/
gethostname(hostname, ORTE_MAX_HOSTNAME_SIZE);
if (strlen(orte_process_info.nodename) < strlen(hostname)) {
opal_argv_append_nosize(&aliases, hostname);
}
opal_ifgetaliases(&aliases);
naliases = opal_argv_count(aliases);
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &naliases, 1, OPAL_UINT8))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
for (ni=0; ni < naliases; ni++) {
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &aliases[ni], 1, OPAL_STRING))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
}
opal_argv_free(aliases);
}
#if OPAL_HAVE_HWLOC
{
char *coprocessors;
/* add the local topology */
if (NULL != opal_hwloc_topology &&
(1 == ORTE_PROC_MY_NAME->vpid || orte_hetero_nodes)) {
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &opal_hwloc_topology, 1, OPAL_HWLOC_TOPO))) {
ORTE_ERROR_LOG(ret);
}
}
/* detect and add any coprocessors */
coprocessors = opal_hwloc_base_find_coprocessors(opal_hwloc_topology);
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &coprocessors, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(ret);
}
/* see if I am on a coprocessor */
coprocessors = opal_hwloc_base_check_on_coprocessor();
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &coprocessors, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(ret);
}
}
#endif
/* send to the HNP's callback - will be routed if routes are available */
if (0 > (ret = orte_rml.send_buffer_nb(ORTE_PROC_MY_HNP, buffer,
ORTE_RML_TAG_ORTED_CALLBACK,
orte_rml_send_callback, NULL))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
}
/* if we are tree-spawning, then we need to capture the MCA params
* from our cmd line so we can pass them along to the daemons we spawn -
* otherwise, only the first layer of daemons will ever see them
*/
if (orted_globals.tree_spawn) {
int j, k;
bool ignore;
char *no_keep[] = {
"orte_hnp_uri",
"orte_ess_jobid",
"orte_ess_vpid",
"orte_ess_num_procs",
"orte_parent_uri",
"mca_base_env_list",
NULL
};
for (i=0; i < argc; i++) {
if (0 == strcmp("-"OPAL_MCA_CMD_LINE_ID, argv[i]) ||
0 == strcmp("--"OPAL_MCA_CMD_LINE_ID, argv[i]) ) {
ignore = false;
/* see if this is something we cannot pass along */
for (k=0; NULL != no_keep[k]; k++) {
if (0 == strcmp(no_keep[k], argv[i+1])) {
ignore = true;
break;
}
}
if (!ignore) {
/* see if this is already present so we at least can
* avoid growing the cmd line with duplicates
*/
if (NULL != orted_cmd_line) {
for (j=0; NULL != orted_cmd_line[j]; j++) {
if (0 == strcmp(argv[i+1], orted_cmd_line[j])) {
/* already here - ignore it */
ignore = true;
break;
}
}
}
if (!ignore) {
opal_argv_append_nosize(&orted_cmd_line, argv[i]);
opal_argv_append_nosize(&orted_cmd_line, argv[i+1]);
opal_argv_append_nosize(&orted_cmd_line, argv[i+2]);
}
}
i += 2;
}
}
}
if (orte_debug_daemons_flag) {
opal_output(0, "%s orted: up and running - waiting for commands!", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
}
ret = ORTE_SUCCESS;
/* loop the event lib until an exit event is detected */
while (orte_event_base_active) {
opal_event_loop(orte_event_base, OPAL_EVLOOP_ONCE);
}
/* ensure all local procs are dead */
orte_odls.kill_local_procs(NULL);
DONE:
/* update the exit status, in case it wasn't done */
ORTE_UPDATE_EXIT_STATUS(ret);
/* cleanup and leave */
orte_finalize();
if (orte_debug_flag) {
fprintf(stderr, "exiting with status %d\n", orte_exit_status);
}
exit(orte_exit_status);
}
/*
* Register MCA parameters
*/
int opal_common_ofacm_base_register(mca_base_component_t *base)
{
int i, j, save;
char **temp = NULL, *string = NULL, *all_cpc_names = NULL;
if (opal_common_ofacm_base_register_was_called) {
return OPAL_SUCCESS;
}
opal_common_ofacm_base_register_was_called = true;
/* Make an MCA parameter to select which connect module to use */
for (i = 0; NULL != all[i]; ++i) {
/* The CPC name "empty" is reserved for "fake" CPC modules */
if (0 != strcmp(all[i]->cbc_name, "empty")) {
opal_argv_append_nosize(&temp, all[i]->cbc_name);
}
}
all_cpc_names = opal_argv_join(temp, ',');
opal_argv_free(temp);
asprintf(&string,
"Method used to select OpenFabrics connections (valid values: %s)",
all_cpc_names);
opal_common_ofacm_cpc_include = NULL;
(void) mca_base_component_var_register(base, "ofacm_cpc_include", string,
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&opal_common_ofacm_cpc_include);
free(string);
asprintf(&string,
"Method used to exclude OpenFabrics connections (valid values: %s)",
all_cpc_names);
opal_common_ofacm_cpc_exclude = NULL;
(void) mca_base_component_var_register(base, "ofacm_cpc_exclude", string,
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&opal_common_ofacm_cpc_exclude);
free(string);
/* Register the name of the file containing the fabric's Service Levels (SL) */
opal_common_ofacm_three_dim_torus = NULL;
(void) mca_base_var_register("ompi", "common", "ofacm", "three_dim_torus",
"The name of the file contating Service Level (SL) data for 3D-Torus cluster",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&opal_common_ofacm_three_dim_torus);
opal_common_ofacm_base_verbose = 0;
(void) mca_base_var_register("ompi", "common", "ofacm", "base_verbose",
"Verbosity level of the OFACM framework",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&opal_common_ofacm_base_verbose);
/* Parse the if_[in|ex]clude paramters to come up with a list of
CPCs that are available */
available = calloc(1, sizeof(all));
/* If we have an "include" list, then find all those CPCs and put
them in available[] */
if (NULL != opal_common_ofacm_cpc_include) {
cpc_explicitly_defined = true;
temp = opal_argv_split(opal_common_ofacm_cpc_include, ',');
for (save = j = 0; NULL != temp[j]; ++j) {
for (i = 0; NULL != all[i]; ++i) {
if (0 == strcmp(temp[j], all[i]->cbc_name)) {
OFACM_VERBOSE(("include: saving %s", all[i]->cbc_name));
available[save++] = all[i];
++num_available;
break;
}
}
if (NULL == all[i]) {
opal_show_help("help-mpi-common-ofacm-cpc-base.txt",
"cpc name not found", true,
"include", opal_proc_local_get()->proc_hostname,
"include", opal_common_ofacm_cpc_include, temp[j],
all_cpc_names);
opal_argv_free(temp);
free(all_cpc_names);
return OPAL_ERR_NOT_FOUND;
}
}
opal_argv_free(temp);
}
/* Otherwise, if we have an "exclude" list, take all the CPCs that
are not in that list and put them in available[] */
else if (NULL != opal_common_ofacm_cpc_exclude) {
cpc_explicitly_defined = true;
temp = opal_argv_split(opal_common_ofacm_cpc_exclude, ',');
/* First: error check -- ensure that all the names are valid */
for (j = 0; NULL != temp[j]; ++j) {
for (i = 0; NULL != all[i]; ++i) {
if (0 == strcmp(temp[j], all[i]->cbc_name)) {
break;
}
}
if (NULL == all[i]) {
opal_show_help("help-mpi-common-ofacm-cpc-base.txt",
"cpc name not found", true,
"exclude", opal_proc_local_get()->proc_hostname,
"exclude", opal_common_ofacm_cpc_exclude, temp[j],
all_cpc_names);
opal_argv_free(temp);
free(all_cpc_names);
return OPAL_ERR_NOT_FOUND;
}
}
/* Now do the exclude */
for (save = i = 0; NULL != all[i]; ++i) {
for (j = 0; NULL != temp[j]; ++j) {
if (0 == strcmp(temp[j], all[i]->cbc_name)) {
break;
}
}
if (NULL == temp[j]) {
OFACM_VERBOSE(("exclude: saving %s", all[i]->cbc_name));
available[save++] = all[i];
++num_available;
}
}
opal_argv_free(temp);
}
/* If there's no include/exclude list, copy all[] into available[] */
else {
OFACM_VERBOSE(("no include or exclude: saving all"));
memcpy(available, all, sizeof(all));
num_available = (sizeof(all) /
sizeof(opal_common_ofacm_base_module_t *)) - 1;
}
/* Call the register function on all the CPCs so that they may
setup any MCA params specific to the connection type */
for (i = 0; NULL != available[i]; ++i) {
if (NULL != available[i]->cbc_register) {
available[i]->cbc_register();
}
}
return OPAL_SUCCESS;
}
int orte_proc_info(void)
{
int idx, i;
char *ptr;
char hostname[OPAL_MAXHOSTNAMELEN];
char **prefixes;
bool match;
if (init) {
return ORTE_SUCCESS;
}
init = true;
OBJ_CONSTRUCT(&orte_process_info.super, opal_proc_t);
orte_process_info.my_hnp_uri = NULL;
mca_base_var_register ("orte", "orte", NULL, "hnp_uri",
"HNP contact info",
MCA_BASE_VAR_TYPE_STRING, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&orte_process_info.my_hnp_uri);
if (NULL != orte_process_info.my_hnp_uri) {
ptr = orte_process_info.my_hnp_uri;
/* the uri value passed to us will have quote marks around it to protect
* the value if passed on the command line. We must remove those
* to have a correct uri string
*/
if ('"' == ptr[0]) {
/* if the first char is a quote, then so will the last one be */
ptr[strlen(ptr)-1] = '\0';
memmove (ptr, ptr + 1, strlen (ptr));
}
}
orte_process_info.my_daemon_uri = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "local_daemon_uri",
"Daemon contact info",
MCA_BASE_VAR_TYPE_STRING, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&orte_process_info.my_daemon_uri);
if (NULL != orte_process_info.my_daemon_uri) {
ptr = orte_process_info.my_daemon_uri;
/* the uri value passed to us may have quote marks around it to protect
* the value if passed on the command line. We must remove those
* to have a correct uri string
*/
if ('"' == ptr[0]) {
/* if the first char is a quote, then so will the last one be */
ptr[strlen(ptr)-1] = '\0';
memmove (ptr, ptr + 1, strlen (ptr) - 1);
}
}
orte_process_info.app_num = 0;
(void) mca_base_var_register ("orte", "orte", NULL, "app_num",
"Index of the app_context that defines this proc",
MCA_BASE_VAR_TYPE_INT, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&orte_process_info.app_num);
/* get the process id */
orte_process_info.pid = getpid();
/* get the nodename */
gethostname(hostname, sizeof(hostname));
/* add this to our list of aliases */
opal_argv_append_nosize(&orte_process_info.aliases, hostname);
// Strip off the FQDN if present, ignore IP addresses
if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(hostname) ) {
if (NULL != (ptr = strchr(hostname, '.'))) {
*ptr = '\0';
/* add this to our list of aliases */
opal_argv_append_nosize(&orte_process_info.aliases, hostname);
}
}
orte_strip_prefix = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "strip_prefix",
"Prefix(es) to match when deciding whether to strip leading characters and zeroes from "
"node names returned by daemons", MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_strip_prefix);
/* we have to strip node names here, if user directs, to ensure that
* the names exchanged in the modex match the names found locally
*/
if (NULL != orte_strip_prefix) {
prefixes = opal_argv_split(orte_strip_prefix, ',');
match = false;
for (i=0; NULL != prefixes[i]; i++) {
if (0 == strncmp(hostname, prefixes[i], strlen(prefixes[i]))) {
/* remove the prefix and leading zeroes */
idx = strlen(prefixes[i]);
while (idx < (int)strlen(hostname) &&
(hostname[idx] <= '0' || '9' < hostname[idx])) {
idx++;
}
if ((int)strlen(hostname) <= idx) {
/* there were no non-zero numbers in the name */
orte_process_info.nodename = strdup(&hostname[strlen(prefixes[i])]);
} else {
orte_process_info.nodename = strdup(&hostname[idx]);
}
/* add this to our list of aliases */
opal_argv_append_nosize(&orte_process_info.aliases, orte_process_info.nodename);
match = true;
break;
}
}
/* if we didn't find a match, then just use the hostname as-is */
if (!match) {
orte_process_info.nodename = strdup(hostname);
}
opal_argv_free(prefixes);
} else {
orte_process_info.nodename = strdup(hostname);
}
/* add "localhost" to our list of aliases */
opal_argv_append_nosize(&orte_process_info.aliases, "localhost");
/* get the number of nodes in the job */
orte_process_info.num_nodes = 1;
(void) mca_base_var_register ("orte", "orte", NULL, "num_nodes",
"Number of nodes in the job",
MCA_BASE_VAR_TYPE_INT, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&orte_process_info.num_nodes);
/* get the number of times this proc has restarted */
orte_process_info.num_restarts = 0;
(void) mca_base_var_register ("orte", "orte", NULL, "num_restarts",
"Number of times this proc has restarted",
MCA_BASE_VAR_TYPE_INT, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&orte_process_info.num_restarts);
orte_process_info.app_rank = 0;
(void) mca_base_var_register ("orte", "orte", NULL, "app_rank",
"Rank of this proc within its app_context",
MCA_BASE_VAR_TYPE_INT, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&orte_process_info.app_rank);
/* get my node rank in case we are using static ports - this won't
* be present for daemons, so don't error out if we don't have it
*/
orte_ess_node_rank = ORTE_NODE_RANK_INVALID;
(void) mca_base_var_register ("orte", "orte", NULL, "ess_node_rank", "Process node rank",
MCA_BASE_VAR_TYPE_INT, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_CONSTANT,
&orte_ess_node_rank);
orte_process_info.my_node_rank = (orte_node_rank_t) orte_ess_node_rank;
/* setup the sync buffer */
orte_process_info.sync_buf = OBJ_NEW(opal_buffer_t);
return ORTE_SUCCESS;
}
static void check_debugger(int fd, short event, void *arg)
{
struct timeval now;
opal_event_t *tmp = (opal_event_t*)arg;
orte_job_t *jdata;
orte_app_context_t *app;
char cwd[OPAL_PATH_MAX];
int rc;
int32_t ljob;
if (MPIR_being_debugged) {
if (orte_debug_flag) {
opal_output(0, "%s Launching debugger %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
MPIR_executable_path);
}
/* a debugger has attached! All the MPIR_Proctable
* data is already available, so we only need to
* check to see if we should spawn any daemons
*/
if ('\0' != MPIR_executable_path[0]) {
/* this will be launched just like a regular job,
* so we do not use the global orte_debugger_daemon
* as this is reserved for co-location upon startup
*/
jdata = OBJ_NEW(orte_job_t);
/* create a jobid for these daemons - this is done solely
* to avoid confusing the rest of the system's bookkeeping
*/
orte_plm_base_create_jobid(jdata);
/* flag the job as being debugger daemons */
jdata->controls |= ORTE_JOB_CONTROL_DEBUGGER_DAEMON;
/* unless directed, we do not forward output */
if (!MPIR_forward_output) {
jdata->controls &= ~ORTE_JOB_CONTROL_FORWARD_OUTPUT;
}
/* set the mapping policy to "pernode" so we only get
* one debugger daemon on each node
*/
jdata->map = OBJ_NEW(orte_job_map_t);
jdata->map->npernode = 1;
/* add it to the global job pool */
ljob = ORTE_LOCAL_JOBID(jdata->jobid);
opal_pointer_array_set_item(orte_job_data, ljob, jdata);
/* create an app_context for the debugger daemon */
app = OBJ_NEW(orte_app_context_t);
app->app = strdup((char*)MPIR_executable_path);
if (OPAL_SUCCESS != (rc = opal_getcwd(cwd, sizeof(cwd)))) {
orte_show_help("help-orterun.txt", "orterun:init-failure",
true, "get the cwd", rc);
OBJ_RELEASE(jdata);
goto RELEASE;
}
app->cwd = strdup(cwd);
app->user_specified_cwd = false;
opal_argv_append_nosize(&app->argv, app->app);
build_debugger_args(app);
opal_pointer_array_add(jdata->apps, &app->super);
jdata->num_apps = 1;
/* now go ahead and spawn this job */
if (ORTE_SUCCESS != (rc = orte_plm.spawn(jdata))) {
ORTE_ERROR_LOG(rc);
}
}
RELEASE:
/* notify the debugger that all is ready */
MPIR_Breakpoint();
} else {
/* reissue the timer to wake us up again */
now.tv_sec = orte_debugger_check_rate;
now.tv_usec = 0;
opal_evtimer_add(tmp, &now);
}
}
/**
* Discover the available resources.
*
* In order to fully support slurm, we need to be able to handle
* node regexp/task_per_node strings such as:
* foo,bar 5,3
* foo 5
* foo[2-10,12,99-105],bar,foobar[3-11] 2(x10),5,100(x16)
*
* @param *regexp A node regular expression from SLURM (i.e. SLURM_NODELIST)
* @param *tasks_per_node A tasks per node expression from SLURM
* (i.e. SLURM_TASKS_PER_NODE)
* @param *nodelist A list which has already been constucted to return
* the found nodes in
*/
static int orte_ras_slurm_discover(char *regexp, char *tasks_per_node,
int cpus_per_task, opal_list_t* nodelist)
{
int i, j, len, ret, count, reps, num_nodes;
char *base, **names = NULL;
char *begptr, *endptr, *orig;
int *slots;
bool found_range = false;
bool more_to_come = false;
orig = base = strdup(regexp);
if (NULL == base) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
return ORTE_ERR_OUT_OF_RESOURCE;
}
OPAL_OUTPUT_VERBOSE((1, orte_ras_base.ras_output,
"%s ras:slurm:allocate:discover: checking nodelist: %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
regexp));
do {
/* Find the base */
len = strlen(base);
for (i = 0; i <= len; ++i) {
if (base[i] == '[') {
/* we found a range. this gets dealt with below */
base[i] = '\0';
found_range = true;
break;
}
if (base[i] == ',') {
/* we found a singleton node, and there are more to come */
base[i] = '\0';
found_range = false;
more_to_come = true;
break;
}
if (base[i] == '\0') {
/* we found a singleton node */
found_range = false;
more_to_come = false;
break;
}
}
if(i == 0) {
/* we found a special character at the beginning of the string */
orte_show_help("help-ras-slurm.txt", "slurm-env-var-bad-value",
1, regexp, tasks_per_node, "SLURM_NODELIST");
ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
free(orig);
return ORTE_ERR_BAD_PARAM;
}
if (found_range) {
/* If we found a range, now find the end of the range */
for (j = i; j < len; ++j) {
if (base[j] == ']') {
base[j] = '\0';
break;
}
}
if (j >= len) {
/* we didn't find the end of the range */
orte_show_help("help-ras-slurm.txt", "slurm-env-var-bad-value",
1, regexp, tasks_per_node, "SLURM_NODELIST");
ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
free(orig);
return ORTE_ERR_BAD_PARAM;
}
ret = orte_ras_slurm_parse_ranges(base, base + i + 1, &names);
if(ORTE_SUCCESS != ret) {
orte_show_help("help-ras-slurm.txt", "slurm-env-var-bad-value",
1, regexp, tasks_per_node, "SLURM_NODELIST");
ORTE_ERROR_LOG(ret);
free(orig);
return ret;
}
if(base[j + 1] == ',') {
more_to_come = true;
base = &base[j + 2];
} else {
more_to_come = false;
}
} else {
/* If we didn't find a range, just add the node */
OPAL_OUTPUT_VERBOSE((1, orte_ras_base.ras_output,
"%s ras:slurm:allocate:discover: found node %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
base));
if(ORTE_SUCCESS != (ret = opal_argv_append_nosize(&names, base))) {
ORTE_ERROR_LOG(ret);
free(orig);
return ret;
}
/* set base equal to the (possible) next base to look at */
base = &base[i + 1];
}
} while(more_to_come);
free(orig);
num_nodes = opal_argv_count(names);
/* Find the number of slots per node */
slots = malloc(sizeof(int) * num_nodes);
if (NULL == slots) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
return ORTE_ERR_OUT_OF_RESOURCE;
}
memset(slots, 0, sizeof(int) * num_nodes);
orig = begptr = strdup(tasks_per_node);
if (NULL == begptr) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
free(slots);
return ORTE_ERR_OUT_OF_RESOURCE;
}
j = 0;
while (begptr) {
count = strtol(begptr, &endptr, 10);
if ((endptr[0] == '(') && (endptr[1] == 'x')) {
reps = strtol((endptr+2), &endptr, 10);
if (endptr[0] == ')') {
endptr++;
}
} else {
reps = 1;
}
/**
* TBP: it seems like it would be an error to have more slot
* descriptions than nodes. Turns out that this valid, and SLURM will
* return such a thing. For instance, if I did:
* srun -A -N 30 -w odin001
* I would get SLURM_NODELIST=odin001 SLURM_TASKS_PER_NODE=4(x30)
* That is, I am allocated 30 nodes, but since I only requested
* one specific node, that's what is in the nodelist.
* I'm not sure this is what users would expect, but I think it is
* more of a SLURM issue than a orte issue, since SLURM is OK with it,
* I'm ok with it
*/
for (i = 0; i < reps && j < num_nodes; i++) {
slots[j++] = count;
}
if (*endptr == ',') {
begptr = endptr + 1;
} else if (*endptr == '\0' || j >= num_nodes) {
break;
} else {
orte_show_help("help-ras-slurm.txt", "slurm-env-var-bad-value", 1,
regexp, tasks_per_node, "SLURM_TASKS_PER_NODE");
ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
free(slots);
free(orig);
return ORTE_ERR_BAD_PARAM;
}
}
free(orig);
/* Convert the argv of node names to a list of node_t's */
for (i = 0; NULL != names && NULL != names[i]; ++i) {
orte_node_t *node;
OPAL_OUTPUT_VERBOSE((1, orte_ras_base.ras_output,
"%s ras:slurm:allocate:discover: adding node %s (%d slot%s)",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
names[i], slots[i], (1 == slots[i]) ? "" : "s"));
node = OBJ_NEW(orte_node_t);
if (NULL == node) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
free(slots);
return ORTE_ERR_OUT_OF_RESOURCE;
}
node->name = strdup(names[i]);
node->state = ORTE_NODE_STATE_UP;
node->slots_inuse = 0;
node->slots_max = 0;
node->slots = slots[i] / cpus_per_task;
opal_list_append(nodelist, &node->super);
}
free(slots);
opal_argv_free(names);
/* All done */
return ret;
}
/*
* Scan all the files in a directory (or path) and invoke a callback
* on each one.
*/
static int dlopen_foreachfile(const char *search_path,
int (*func)(const char *filename, void *data),
void *data)
{
int ret;
DIR *dp = NULL;
char **dirs = NULL;
char **good_files = NULL;
dirs = opal_argv_split(search_path, OPAL_ENV_SEP);
for (int i = 0; NULL != dirs && NULL != dirs[i]; ++i) {
dp = opendir(dirs[i]);
if (NULL == dp) {
ret = OPAL_ERR_IN_ERRNO;
goto error;
}
struct dirent *de;
while (NULL != (de = readdir(dp))) {
/* Make the absolute path name */
char *abs_name = NULL;
asprintf(&abs_name, "%s/%s", dirs[i], de->d_name);
if (NULL == abs_name) {
ret = OPAL_ERR_IN_ERRNO;
goto error;
}
/* Stat the file */
struct stat buf;
if (stat(abs_name, &buf) < 0) {
free(abs_name);
ret = OPAL_ERR_IN_ERRNO;
goto error;
}
/* Skip if not a file */
if (!S_ISREG(buf.st_mode)) {
free(abs_name);
continue;
}
/* Find the suffix */
char *ptr = strrchr(abs_name, '.');
if (NULL != ptr) {
/* Skip libtool files */
if (strcmp(ptr, ".la") == 0 ||
strcmp(ptr, ".lo") == 0) {
continue;
}
*ptr = '\0';
}
/* Have we already found this file? Or already found a
file with the same basename (but different suffix)? */
bool found = false;
for (int j = 0; NULL != good_files &&
NULL != good_files[j]; ++j) {
if (strcmp(good_files[j], abs_name) == 0) {
found = true;
break;
}
}
if (!found) {
opal_argv_append_nosize(&good_files, abs_name);
}
free(abs_name);
}
closedir(dp);
}
dp = NULL;
/* Invoke the callback on all the found files */
if (NULL != good_files) {
for (int i = 0; NULL != good_files[i]; ++i) {
ret = func(good_files[i], data);
if (OPAL_SUCCESS != ret) {
goto error;
}
}
}
ret = OPAL_SUCCESS;
error:
if (NULL != dp) {
closedir(dp);
}
if (NULL != dirs) {
opal_argv_free(dirs);
}
if (NULL != good_files) {
opal_argv_free(good_files);
}
return ret;
}
static int setup_fork(orte_job_t *jdata,
orte_app_context_t *app)
{
int i;
char *param;
bool oversubscribed;
orte_node_t *node;
char **envcpy, **nps, **firstranks;
char *npstring, *firstrankstring;
char *num_app_ctx;
bool takeus = false;
/* see if we are included */
for (i=0; NULL != jdata->personality[i]; i++) {
if (0 == strcmp(jdata->personality[i], "ompi")) {
takeus = true;
break;
}
}
if (!takeus) {
return ORTE_ERR_TAKE_NEXT_OPTION;
}
/* see if the mapper thinks we are oversubscribed */
oversubscribed = false;
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, ORTE_PROC_MY_NAME->vpid))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_ERR_NOT_FOUND;
}
if (ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_OVERSUBSCRIBED)) {
oversubscribed = true;
}
/* setup base environment: copy the current environ and merge
in the app context environ */
if (NULL != app->env) {
/* manually free original context->env to avoid a memory leak */
char **tmp = app->env;
envcpy = opal_environ_merge(orte_launch_environ, app->env);
if (NULL != tmp) {
opal_argv_free(tmp);
}
} else {
envcpy = opal_argv_copy(orte_launch_environ);
}
app->env = envcpy;
/* special case handling for --prefix: this is somewhat icky,
but at least some users do this. :-\ It is possible that
when using --prefix, the user will also "-x PATH" and/or
"-x LD_LIBRARY_PATH", which would therefore clobber the
work that was done in the prior pls to ensure that we have
the prefix at the beginning of the PATH and
LD_LIBRARY_PATH. So examine the context->env and see if we
find PATH or LD_LIBRARY_PATH. If found, that means the
prior work was clobbered, and we need to re-prefix those
variables. */
param = NULL;
orte_get_attribute(&app->attributes, ORTE_APP_PREFIX_DIR, (void**)¶m, OPAL_STRING);
for (i = 0; NULL != param && NULL != app->env && NULL != app->env[i]; ++i) {
char *newenv;
/* Reset PATH */
if (0 == strncmp("PATH=", app->env[i], 5)) {
asprintf(&newenv, "%s/bin:%s", param, app->env[i] + 5);
opal_setenv("PATH", newenv, true, &app->env);
free(newenv);
}
/* Reset LD_LIBRARY_PATH */
else if (0 == strncmp("LD_LIBRARY_PATH=", app->env[i], 16)) {
asprintf(&newenv, "%s/lib:%s", param, app->env[i] + 16);
opal_setenv("LD_LIBRARY_PATH", newenv, true, &app->env);
free(newenv);
}
}
if (NULL != param) {
free(param);
}
/* pass my contact info to the local proc so we can talk */
opal_setenv("OMPI_MCA_orte_local_daemon_uri", orte_process_info.my_daemon_uri, true, &app->env);
/* pass the hnp's contact info to the local proc in case it
* needs it
*/
if (NULL != orte_process_info.my_hnp_uri) {
opal_setenv("OMPI_MCA_orte_hnp_uri", orte_process_info.my_hnp_uri, true, &app->env);
}
/* setup yield schedule - do not override any user-supplied directive! */
if (oversubscribed) {
opal_setenv("OMPI_MCA_mpi_yield_when_idle", "1", false, &app->env);
} else {
opal_setenv("OMPI_MCA_mpi_yield_when_idle", "0", false, &app->env);
}
/* set the app_context number into the environment */
asprintf(¶m, "%ld", (long)app->idx);
opal_setenv("OMPI_MCA_orte_app_num", param, true, &app->env);
free(param);
/* although the total_slots_alloc is the universe size, users
* would appreciate being given a public environmental variable
* that also represents this value - something MPI specific - so
* do that here. Also required by the ompi_attributes code!
*
* AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
* We know - just live with it
*/
asprintf(¶m, "%ld", (long)jdata->total_slots_alloc);
opal_setenv("OMPI_UNIVERSE_SIZE", param, true, &app->env);
free(param);
/* pass the number of nodes involved in this job */
asprintf(¶m, "%ld", (long)(jdata->map->num_nodes));
opal_setenv("OMPI_MCA_orte_num_nodes", param, true, &app->env);
free(param);
/* pass a param telling the child what type and model of cpu we are on,
* if we know it. If hwloc has the value, use what it knows. Otherwise,
* see if we were explicitly given it and use that value.
*/
hwloc_obj_t obj;
char *htmp;
if (NULL != opal_hwloc_topology) {
obj = hwloc_get_root_obj(opal_hwloc_topology);
if (NULL != (htmp = (char*)hwloc_obj_get_info_by_name(obj, "CPUType")) ||
NULL != (htmp = orte_local_cpu_type)) {
opal_setenv("OMPI_MCA_orte_cpu_type", htmp, true, &app->env);
}
if (NULL != (htmp = (char*)hwloc_obj_get_info_by_name(obj, "CPUModel")) ||
NULL != (htmp = orte_local_cpu_model)) {
opal_setenv("OMPI_MCA_orte_cpu_model", htmp, true, &app->env);
}
} else {
if (NULL != orte_local_cpu_type) {
opal_setenv("OMPI_MCA_orte_cpu_type", orte_local_cpu_type, true, &app->env);
}
if (NULL != orte_local_cpu_model) {
opal_setenv("OMPI_MCA_orte_cpu_model", orte_local_cpu_model, true, &app->env);
}
}
/* get shmem's best component name so we can provide a hint to the shmem
* framework. the idea here is to have someone figure out what component to
* select (via the shmem framework) and then have the rest of the
* components in shmem obey that decision. for more details take a look at
* the shmem framework in opal.
*/
if (NULL != (param = opal_shmem_base_best_runnable_component_name())) {
opal_setenv("OMPI_MCA_shmem_RUNTIME_QUERY_hint", param, true, &app->env);
free(param);
}
/* Set an info MCA param that tells the launched processes that
* any binding policy was applied by us (e.g., so that
* MPI_INIT doesn't try to bind itself)
*/
opal_setenv("OMPI_MCA_orte_bound_at_launch", "1", true, &app->env);
/* tell the ESS to avoid the singleton component - but don't override
* anything that may have been provided elsewhere
*/
opal_setenv("OMPI_MCA_ess", "^singleton", false, &app->env);
/* ensure that the spawned process ignores direct launch components,
* but do not overrride anything we were given */
opal_setenv("OMPI_MCA_pmix", "^s1,s2,cray", false, &app->env);
/* since we want to pass the name as separate components, make sure
* that the "name" environmental variable is cleared!
*/
opal_unsetenv("OMPI_MCA_orte_ess_name", &app->env);
asprintf(¶m, "%ld", (long)jdata->num_procs);
opal_setenv("OMPI_MCA_orte_ess_num_procs", param, true, &app->env);
/* although the num_procs is the comm_world size, users
* would appreciate being given a public environmental variable
* that also represents this value - something MPI specific - so
* do that here.
*
* AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
* We know - just live with it
*/
opal_setenv("OMPI_COMM_WORLD_SIZE", param, true, &app->env);
free(param);
/* users would appreciate being given a public environmental variable
* that also represents this value - something MPI specific - so
* do that here.
*
* AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
* We know - just live with it
*/
asprintf(¶m, "%ld", (long)jdata->num_local_procs);
opal_setenv("OMPI_COMM_WORLD_LOCAL_SIZE", param, true, &app->env);
free(param);
/* forcibly set the local tmpdir base to match ours */
opal_setenv("OMPI_MCA_orte_tmpdir_base", orte_process_info.tmpdir_base, true, &app->env);
/* MPI-3 requires we provide some further info to the procs,
* so we pass them as envars to avoid introducing further
* ORTE calls in the MPI layer
*/
asprintf(&num_app_ctx, "%lu", (unsigned long)jdata->num_apps);
/* build some common envars we need to pass for MPI-3 compatibility */
nps = NULL;
firstranks = NULL;
for (i=0; i < jdata->apps->size; i++) {
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, i))) {
continue;
}
opal_argv_append_nosize(&nps, ORTE_VPID_PRINT(app->num_procs));
opal_argv_append_nosize(&firstranks, ORTE_VPID_PRINT(app->first_rank));
}
npstring = opal_argv_join(nps, ' ');
firstrankstring = opal_argv_join(firstranks, ' ');
opal_argv_free(nps);
opal_argv_free(firstranks);
/* add the MPI-3 envars */
opal_setenv("OMPI_NUM_APP_CTX", num_app_ctx, true, &app->env);
opal_setenv("OMPI_FIRST_RANKS", firstrankstring, true, &app->env);
opal_setenv("OMPI_APP_CTX_NUM_PROCS", npstring, true, &app->env);
free(num_app_ctx);
free(firstrankstring);
free(npstring);
return ORTE_SUCCESS;
}
/*
* Register MCA parameters
*/
int ompi_btl_openib_connect_base_register(void)
{
int i, j, save;
char **temp = NULL, *string = NULL, *all_cpc_names = NULL;
char *cpc_include = NULL, *cpc_exclude = NULL;
/* Make an MCA parameter to select which connect module to use */
for (i = 0; NULL != all[i]; ++i) {
/* The CPC name "empty" is reserved for "fake" CPC modules */
if (0 != strcmp(all[i]->cbc_name, "empty")) {
opal_argv_append_nosize(&temp, all[i]->cbc_name);
}
}
all_cpc_names = opal_argv_join(temp, ',');
opal_argv_free(temp);
asprintf(&string,
"Method used to select OpenFabrics connections (valid values: %s)",
all_cpc_names);
mca_base_param_reg_string(&mca_btl_openib_component.super.btl_version,
"cpc_include", string, false, false,
NULL, &cpc_include);
free(string);
asprintf(&string,
"Method used to exclude OpenFabrics connections (valid values: %s)",
all_cpc_names);
mca_base_param_reg_string(&mca_btl_openib_component.super.btl_version,
"cpc_exclude", string, false, false,
NULL, &cpc_exclude);
free(string);
/* Parse the if_[in|ex]clude paramters to come up with a list of
CPCs that are available */
available = calloc(1, sizeof(all));
/* If we have an "include" list, then find all those CPCs and put
them in available[] */
if (NULL != cpc_include) {
mca_btl_openib_component.cpc_explicitly_defined = true;
temp = opal_argv_split(cpc_include, ',');
for (save = j = 0; NULL != temp[j]; ++j) {
for (i = 0; NULL != all[i]; ++i) {
if (0 == strcmp(temp[j], all[i]->cbc_name)) {
opal_output(-1, "include: saving %s", all[i]->cbc_name);
available[save++] = all[i];
++num_available;
break;
}
}
if (NULL == all[i]) {
orte_show_help("help-mpi-btl-openib-cpc-base.txt",
"cpc name not found", true,
"include", orte_process_info.nodename,
"include", cpc_include, temp[j],
all_cpc_names);
opal_argv_free(temp);
free(all_cpc_names);
return OMPI_ERR_NOT_FOUND;
}
}
opal_argv_free(temp);
}
/* Otherwise, if we have an "exclude" list, take all the CPCs that
are not in that list and put them in available[] */
else if (NULL != cpc_exclude) {
mca_btl_openib_component.cpc_explicitly_defined = true;
temp = opal_argv_split(cpc_exclude, ',');
/* First: error check -- ensure that all the names are valid */
for (j = 0; NULL != temp[j]; ++j) {
for (i = 0; NULL != all[i]; ++i) {
if (0 == strcmp(temp[j], all[i]->cbc_name)) {
break;
}
}
if (NULL == all[i]) {
orte_show_help("help-mpi-btl-openib-cpc-base.txt",
"cpc name not found", true,
"exclude", orte_process_info.nodename,
"exclude", cpc_exclude, temp[j],
all_cpc_names);
opal_argv_free(temp);
free(all_cpc_names);
return OMPI_ERR_NOT_FOUND;
}
}
/* Now do the exclude */
for (save = i = 0; NULL != all[i]; ++i) {
for (j = 0; NULL != temp[j]; ++j) {
if (0 == strcmp(temp[j], all[i]->cbc_name)) {
break;
}
}
if (NULL == temp[j]) {
opal_output(-1, "exclude: saving %s", all[i]->cbc_name);
available[save++] = all[i];
++num_available;
}
}
opal_argv_free(temp);
}
/* If there's no include/exclude list, copy all[] into available[] */
else {
opal_output(-1, "no include or exclude: saving all");
memcpy(available, all, sizeof(all));
num_available = (sizeof(all) /
sizeof(ompi_btl_openib_connect_base_module_t *)) - 1;
}
/* Call the register function on all the CPCs so that they may
setup any MCA params specific to the connection type */
for (i = 0; NULL != available[i]; ++i) {
if (NULL != available[i]->cbc_register) {
available[i]->cbc_register();
}
}
return OMPI_SUCCESS;
}
/*
* Parse a single range in a set and add the full names of the nodes
* found to the names argv
*
* @param base The base text of the node name
* @param *ranges A pointer to a single range. (i.e. "1-3" or "5")
* @param ***names An argv array to add the newly discovered nodes to
*/
static int orte_ras_slurm_parse_range(char *base, char *range, char ***names)
{
char *str, temp1[BUFSIZ];
size_t i, j, start, end;
size_t base_len, len, num_len;
size_t str_start, str_end;
size_t num_str_len;
bool found;
int ret;
len = strlen(range);
base_len = strlen(base);
/* Silence compiler warnings; start and end are always assigned
properly, below */
start = end = 0;
/* Look for the beginning of the first number */
for (found = false, i = 0; i < len; ++i) {
if (isdigit((int) range[i])) {
if (!found) {
str_start = i;
start = atoi(range + i);
found = true;
break;
}
}
}
if (!found) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_ERR_NOT_FOUND;
}
/* Look for the end of the first number */
for (found = false, num_str_len = 0; i < len; ++i, ++num_str_len) {
if (!isdigit((int) range[i])) {
break;
}
}
/* Was there no range, just a single number? */
if (i >= len) {
str_end = len;
end = start;
found = true;
}
/* Nope, there was a range. Look for the beginning of the second
number */
else {
str_end = i - 1;
for (; i < len; ++i) {
if (isdigit((int) range[i])) {
end = atoi(range + i);
found = true;
break;
}
}
}
if (!found) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_ERR_NOT_FOUND;
}
/* Make strings for all values in the range */
len = base_len + num_str_len + 32;
str = malloc(len);
if (NULL == str) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
return ORTE_ERR_OUT_OF_RESOURCE;
}
strcpy(str, base);
for (i = start; i <= end; ++i) {
str[base_len] = '\0';
snprintf(temp1, BUFSIZ - 1, "%lu", (long) i);
/* Do we need zero pading? */
if ((num_len = strlen(temp1)) < num_str_len) {
for (j = base_len; j < base_len + (num_str_len - num_len); ++j) {
str[j] = '0';
}
str[j] = '\0';
}
strcat(str, temp1);
ret = opal_argv_append_nosize(names, str);
if(ORTE_SUCCESS != ret) {
ORTE_ERROR_LOG(ret);
free(str);
return ret;
}
}
free(str);
/* All done */
return ORTE_SUCCESS;
}
