static int
create_peer_and_endpoint(int interface,
opal_proc_t *proc,
ptl_process_t *phys_peer,
mca_btl_base_endpoint_t **endpoint)
{
int ret;
size_t size;
ptl_process_t *id;
OPAL_MODEX_RECV(ret, &mca_btl_portals4_component.super.btl_version,
&proc->proc_name, (void**) &id, &size);
if (OPAL_ERR_NOT_FOUND == ret) {
OPAL_OUTPUT_VERBOSE((30, opal_btl_base_framework.framework_output,
"btl/portals4: Portals 4 BTL not available on peer: %s", opal_strerror(ret)));
return ret;
} else if (OPAL_SUCCESS != ret) {
opal_output_verbose(0, opal_btl_base_framework.framework_output,
"btl/portals4: opal_modex_recv failed: %s", opal_strerror(ret));
return ret;
}
if (size < sizeof(ptl_process_t)) { /* no available connection */
return OPAL_ERROR;
}
if ((size % sizeof(ptl_process_t)) != 0) {
opal_output_verbose(0, opal_btl_base_framework.framework_output,
"btl/portals4: invalid format in modex");
return OPAL_ERROR;
}
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"btl/portals4: %d NI(s) declared in the modex", (int) (size/sizeof(ptl_process_t))));
/*
* check if create_endpoint() already created the endpoint.
* if not, create it here.
*/
if (NULL == *endpoint) {
*endpoint = malloc(sizeof(mca_btl_base_endpoint_t));
if (NULL == *endpoint) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
}
/*
* regardless of who created the endpoint, set the rank here
* because we are using logical mapping.
*/
(*endpoint)->ptl_proc.rank = proc->proc_name.vpid;
phys_peer->phys.pid = id[interface].phys.pid;
phys_peer->phys.nid = id[interface].phys.nid;
opal_output_verbose(50, opal_btl_base_framework.framework_output,
"logical: global rank=%d pid=%d nid=%d\n",
proc->proc_name.vpid, phys_peer->phys.pid, phys_peer->phys.nid);
return OPAL_SUCCESS;
}
int
ompi_rte_error_log(const char *file, int line,
const char *func, int ret)
{
opal_output(0, "%s:%d:%s: Error: %s\n", file, line, func, opal_strerror(ret));
return OMPI_SUCCESS;
}
static int parse_args(int argc, char *argv[]) {
int ret;
opal_cmd_line_t cmd_line;
orte_ps_globals_t tmp = { false, /* help */
false, /* verbose */
false, /* parseable */
ORTE_JOBID_WILDCARD, /* jobid */
false, /* nodes */
false, /* daemons */
-1, /* output */
0}; /* pid */
orte_ps_globals = tmp;
/* Parse the command line options */
opal_cmd_line_create(&cmd_line, cmd_line_opts);
mca_base_open();
mca_base_cmd_line_setup(&cmd_line);
ret = opal_cmd_line_parse(&cmd_line, false, argc, argv);
if (OPAL_SUCCESS != ret) {
if (OPAL_ERR_SILENT != ret) {
fprintf(stderr, "%s: command line error (%s)\n", argv[0],
opal_strerror(ret));
}
return ret;
}
/**
* Now start parsing our specific arguments
*/
if (orte_ps_globals.help) {
char *str, *args = NULL;
args = opal_cmd_line_get_usage_msg(&cmd_line);
str = opal_show_help_string("help-orte-ps.txt", "usage", true,
args);
if (NULL != str) {
printf("%s", str);
free(str);
}
free(args);
/* If we show the help message, that should be all we do */
exit(0);
}
/* if the jobid is given, then we need a pid */
if (ORTE_JOBID_WILDCARD != orte_ps_globals.jobid &&
0 == orte_ps_globals.pid) {
orte_show_help("help-orte-ps.txt", "need-vpid", true,
orte_ps_globals.jobid);
return ORTE_ERROR;
}
return ORTE_SUCCESS;
}
/* Quere for the XOOB priority - will be highest in XRC case */
static int xoob_component_query(mca_btl_openib_module_t *openib_btl,
ompi_btl_openib_connect_base_module_t **cpc)
{
int rc;
if (mca_btl_openib_component.num_xrc_qps <= 0) {
opal_output_verbose(5, mca_btl_base_output,
"openib BTL: xoob CPC only supported with XRC receive queues; skipped on %s:%d",
ibv_get_device_name(openib_btl->device->ib_dev),
openib_btl->port_num);
return OMPI_ERR_NOT_SUPPORTED;
}
*cpc = malloc(sizeof(ompi_btl_openib_connect_base_module_t));
if (NULL == *cpc) {
opal_output_verbose(5, mca_btl_base_output,
"openib BTL: xoob CPC system error (malloc failed)");
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* If this btl supports XOOB, then post the RML message. But
ensure to only post it *once*, because another btl may have
come in before this and already posted it. */
if (!rml_recv_posted) {
rc = orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD,
OMPI_RML_TAG_XOPENIB,
ORTE_RML_PERSISTENT,
xoob_rml_recv_cb,
NULL);
if (ORTE_SUCCESS != rc) {
opal_output_verbose(5, mca_btl_base_output,
"openib BTL: xoob CPC system error %d (%s)",
rc, opal_strerror(rc));
return rc;
}
rml_recv_posted = true;
}
(*cpc)->data.cbm_component = &ompi_btl_openib_connect_xoob;
(*cpc)->data.cbm_priority = xoob_priority;
(*cpc)->data.cbm_modex_message = NULL;
(*cpc)->data.cbm_modex_message_len = 0;
(*cpc)->cbm_endpoint_init = NULL;
(*cpc)->cbm_start_connect = xoob_module_start_connect;
(*cpc)->cbm_endpoint_finalize = NULL;
(*cpc)->cbm_finalize = NULL;
(*cpc)->cbm_uses_cts = false;
opal_output_verbose(5, mca_btl_base_output,
"openib BTL: xoob CPC available for use on %s:%d",
ibv_get_device_name(openib_btl->device->ib_dev),
openib_btl->port_num);
return OMPI_SUCCESS;
}
static int
create_endpoint(int interface,
opal_proc_t *proc,
mca_btl_base_endpoint_t **endpoint)
{
int ret;
size_t size;
ptl_process_t *id;
OPAL_MODEX_RECV(ret, &mca_btl_portals4_component.super.btl_version,
&proc->proc_name, (void**) &id, &size);
if (OPAL_ERR_NOT_FOUND == ret) {
OPAL_OUTPUT_VERBOSE((30, opal_btl_base_framework.framework_output,
"btl/portals4: Portals 4 BTL not available on peer: %s", opal_strerror(ret)));
return ret;
} else if (OPAL_SUCCESS != ret) {
opal_output_verbose(0, opal_btl_base_framework.framework_output,
"btl/portals4: opal_modex_recv failed: %s", opal_strerror(ret));
return ret;
}
if (size < sizeof(ptl_process_t)) { /* no available connection */
return OPAL_ERROR;
}
if ((size % sizeof(ptl_process_t)) != 0) {
opal_output_verbose(0, opal_btl_base_framework.framework_output,
"btl/portals4: invalid format in modex");
return OPAL_ERROR;
}
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"btl/portals4: %d NI(s) declared in the modex", (int) (size/sizeof(ptl_process_t))));
*endpoint = malloc(sizeof(mca_btl_base_endpoint_t));
if (NULL == *endpoint) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
(*endpoint)->ptl_proc = id[interface];
return OPAL_SUCCESS;
}
static int parse_args(int argc, char *argv[]) {
int ret;
opal_cmd_line_t cmd_line;
orte_ps_globals_t tmp = { false, /* help */
false, /* verbose */
false, /* parseable */
ORTE_JOBID_WILDCARD, /* jobid */
false, /* nodes */
false, /* daemons */
-1, /* output */
0}; /* pid */
orte_ps_globals = tmp;
/* Parse the command line options */
opal_cmd_line_create(&cmd_line, cmd_line_opts);
mca_base_cmd_line_setup(&cmd_line);
ret = opal_cmd_line_parse(&cmd_line, false, argc, argv);
if (OPAL_SUCCESS != ret) {
if (OPAL_ERR_SILENT != ret) {
fprintf(stderr, "%s: command line error (%s)\n", argv[0],
opal_strerror(ret));
}
return ret;
}
/**
* Now start parsing our specific arguments
*/
if (orte_ps_globals.help) {
char *str, *args = NULL;
args = opal_cmd_line_get_usage_msg(&cmd_line);
str = opal_show_help_string("help-orte-ps.txt", "usage", true,
args);
if (NULL != str) {
printf("%s", str);
free(str);
}
free(args);
/* If we show the help message, that should be all we do */
exit(0);
}
/*
* Since this process can now handle MCA/GMCA parameters, make sure to
* process them.
*/
mca_base_cmd_line_process_args(&cmd_line, &environ, &environ);
return ORTE_SUCCESS;
}
/*
* Parse the command line arguments using the functions command
* line utility functions.
*/
static int parse_args(int argc, char *argv[]) {
int ret;
opal_cmd_line_t cmd_line;
orte_clean_globals_t tmp = { false, false, false };
/* NOTE: There is a bug in the PGI 6.2 series that causes the
compiler to choke when copying structs containing bool members
by value. So do a memcpy here instead. */
memcpy(&orte_clean_globals, &tmp, sizeof(tmp));
/*
* Initialize list of available command line options.
*/
opal_cmd_line_create(&cmd_line, cmd_line_opts);
ret = opal_cmd_line_parse(&cmd_line, false, argc, argv);
if (OPAL_SUCCESS != ret) {
if (OPAL_ERR_SILENT != ret) {
fprintf(stderr, "%s: command line error (%s)\n", argv[0],
opal_strerror(ret));
}
return ret;
}
/**
* Now start parsing our specific arguments
*/
if (orte_clean_globals.help) {
char *str, *args = NULL;
args = opal_cmd_line_get_usage_msg(&cmd_line);
str = opal_show_help_string("help-orte-clean.txt", "usage", true,
args);
if (NULL != str) {
printf("%s", str);
free(str);
}
free(args);
/* If we show the help message, that should be all we do */
exit(0);
}
OBJ_DESTRUCT(&cmd_line);
return ORTE_SUCCESS;
}
int init(int argc, char *argv[])
{
int rc;
opal_cmd_line_t cmd_line;
/* Setup and parse the command line */
init_globals();
opal_cmd_line_create(&cmd_line, cmd_line_init);
mca_base_cmd_line_setup(&cmd_line);
if (OPAL_SUCCESS != (rc = opal_cmd_line_parse(&cmd_line, true,
argc, argv)) ) {
if (OPAL_ERR_SILENT != rc) {
fprintf(stderr, "%s: command line error (%s)\n", argv[0],
opal_strerror(rc));
}
return rc;
}
/*
* Since this process can now handle MCA/GMCA parameters, make sure to
* process them.
*/
mca_base_cmd_line_process_args(&cmd_line, &environ, &environ);
}
static void set(orte_job_t *jobdat,
orte_proc_t *child,
char ***environ_copy,
int write_fd)
{
hwloc_cpuset_t cpuset;
hwloc_obj_t root;
opal_hwloc_topo_data_t *sum;
orte_app_context_t *context;
int rc=ORTE_ERROR;
char *msg, *param;
char *cpu_bitmap;
opal_output_verbose(2, orte_rtc_base_framework.framework_output,
"%s hwloc:set on child %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == child) ? "NULL" : ORTE_NAME_PRINT(&child->name));
if (NULL == jobdat || NULL == child) {
/* nothing for us to do */
opal_output_verbose(2, orte_rtc_base_framework.framework_output,
"%s hwloc:set jobdat %s child %s - nothing to do",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == jobdat) ? "NULL" : ORTE_JOBID_PRINT(jobdat->jobid),
(NULL == child) ? "NULL" : ORTE_NAME_PRINT(&child->name));
return;
}
context = (orte_app_context_t*)opal_pointer_array_get_item(jobdat->apps, child->app_idx);
/* Set process affinity, if given */
cpu_bitmap = NULL;
if (!orte_get_attribute(&child->attributes, ORTE_PROC_CPU_BITMAP, (void**)&cpu_bitmap, OPAL_STRING) ||
NULL == cpu_bitmap || 0 == strlen(cpu_bitmap)) {
/* if the daemon is bound, then we need to "free" this proc */
if (NULL != orte_daemon_cores) {
root = hwloc_get_root_obj(opal_hwloc_topology);
if (NULL == root->userdata) {
orte_rtc_base_send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "incorrectly bound",
orte_process_info.nodename, context->app,
__FILE__, __LINE__);
}
sum = (opal_hwloc_topo_data_t*)root->userdata;
/* bind this proc to all available processors */
rc = hwloc_set_cpubind(opal_hwloc_topology, sum->available, 0);
/* if we got an error and this wasn't a default binding policy, then report it */
if (rc < 0 && OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
if (errno == ENOSYS) {
msg = "hwloc indicates cpu binding not supported";
} else if (errno == EXDEV) {
msg = "hwloc indicates cpu binding cannot be enforced";
} else {
char *tmp;
(void)hwloc_bitmap_list_asprintf(&tmp, sum->available);
asprintf(&msg, "hwloc_set_cpubind returned \"%s\" for bitmap \"%s\"",
opal_strerror(rc), tmp);
free(tmp);
}
if (OPAL_BINDING_REQUIRED(jobdat->map->binding)) {
/* If binding is required, send an error up the pipe (which exits
-- it doesn't return). */
orte_rtc_base_send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"binding generic error",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
} else {
orte_rtc_base_send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
return;
}
}
}
if (0 == rc && opal_hwloc_report_bindings) {
opal_output(0, "MCW rank %d is not bound (or bound to all available processors)", child->name.vpid);
/* avoid reporting it twice */
(void) mca_base_var_env_name ("hwloc_base_report_bindings", ¶m);
opal_unsetenv(param, environ_copy);
free(param);
}
} else {
/* convert the list to a cpuset */
cpuset = hwloc_bitmap_alloc();
if (0 != (rc = hwloc_bitmap_list_sscanf(cpuset, cpu_bitmap))) {
/* See comment above about "This may be a small memory leak" */
asprintf(&msg, "hwloc_bitmap_sscanf returned \"%s\" for the string \"%s\"",
opal_strerror(rc), cpu_bitmap);
if (NULL == msg) {
msg = "failed to convert bitmap list to hwloc bitmap";
}
if (OPAL_BINDING_REQUIRED(jobdat->map->binding) &&
OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
/* If binding is required and a binding directive was explicitly
* given (i.e., we are not binding due to a default policy),
* send an error up the pipe (which exits -- it doesn't return).
*/
orte_rtc_base_send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"binding generic error",
orte_process_info.nodename,
context->app, msg,
__FILE__, __LINE__);
} else {
orte_rtc_base_send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
free(cpu_bitmap);
return;
}
}
/* bind as specified */
rc = hwloc_set_cpubind(opal_hwloc_topology, cpuset, 0);
/* if we got an error and this wasn't a default binding policy, then report it */
if (rc < 0 && OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
char *tmp = NULL;
if (errno == ENOSYS) {
msg = "hwloc indicates cpu binding not supported";
} else if (errno == EXDEV) {
msg = "hwloc indicates cpu binding cannot be enforced";
} else {
asprintf(&msg, "hwloc_set_cpubind returned \"%s\" for bitmap \"%s\"",
opal_strerror(rc), cpu_bitmap);
}
if (OPAL_BINDING_REQUIRED(jobdat->map->binding)) {
/* If binding is required, send an error up the pipe (which exits
-- it doesn't return). */
orte_rtc_base_send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"binding generic error",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
} else {
orte_rtc_base_send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
if (NULL != tmp) {
free(tmp);
free(msg);
}
return;
}
if (NULL != tmp) {
free(tmp);
free(msg);
}
}
if (0 == rc && opal_hwloc_report_bindings) {
char tmp1[1024], tmp2[1024];
hwloc_cpuset_t mycpus;
/* get the cpus we are bound to */
mycpus = hwloc_bitmap_alloc();
if (hwloc_get_cpubind(opal_hwloc_topology,
mycpus,
HWLOC_CPUBIND_PROCESS) < 0) {
opal_output(0, "MCW rank %d is not bound",
child->name.vpid);
} else {
if (OPAL_ERR_NOT_BOUND == opal_hwloc_base_cset2str(tmp1, sizeof(tmp1), opal_hwloc_topology, mycpus)) {
opal_output(0, "MCW rank %d is not bound (or bound to all available processors)", child->name.vpid);
} else {
opal_hwloc_base_cset2mapstr(tmp2, sizeof(tmp2), opal_hwloc_topology, mycpus);
opal_output(0, "MCW rank %d bound to %s: %s",
child->name.vpid, tmp1, tmp2);
}
}
hwloc_bitmap_free(mycpus);
/* avoid reporting it twice */
(void) mca_base_var_env_name ("hwloc_base_report_bindings", ¶m);
opal_unsetenv(param, environ_copy);
free(param);
}
/* set memory affinity policy - if we get an error, don't report
* anything unless the user actually specified the binding policy
*/
rc = opal_hwloc_base_set_process_membind_policy();
if (ORTE_SUCCESS != rc && OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
if (errno == ENOSYS) {
msg = "hwloc indicates memory binding not supported";
} else if (errno == EXDEV) {
msg = "hwloc indicates memory binding cannot be enforced";
} else {
msg = "failed to bind memory";
}
if (OPAL_HWLOC_BASE_MBFA_ERROR == opal_hwloc_base_mbfa) {
/* If binding is required, send an error up the pipe (which exits
-- it doesn't return). */
orte_rtc_base_send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"memory binding error",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
} else {
orte_rtc_base_send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "memory not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
free(cpu_bitmap);
return;
}
}
}
if (NULL != cpu_bitmap) {
free(cpu_bitmap);
}
}
/*
* Init function. Post non-blocking RML receive to accept incoming
* connection requests.
*/
static int oob_component_query(mca_btl_openib_module_t *btl,
ompi_btl_openib_connect_base_module_t **cpc)
{
int rc;
/* If we have the transport_type member, check to ensure we're on
IB (this CPC will not work with iWarp). If we do not have the
transport_type member, then we must be < OFED v1.2, and
therefore we must be IB. */
#if defined(HAVE_STRUCT_IBV_DEVICE_TRANSPORT_TYPE)
if (IBV_TRANSPORT_IB != btl->device->ib_dev->transport_type) {
opal_output_verbose(5, mca_btl_base_output,
"openib BTL: oob CPC only supported on InfiniBand; skipped on %s:%d",
ibv_get_device_name(btl->device->ib_dev),
btl->port_num);
return OMPI_ERR_NOT_SUPPORTED;
}
#endif
if (mca_btl_openib_component.num_xrc_qps > 0) {
opal_output_verbose(5, mca_btl_base_output,
"openib BTL: oob CPC not supported with XRC receive queues, please try xoob CPC; skipped on %s:%d",
ibv_get_device_name(btl->device->ib_dev),
btl->port_num);
return OMPI_ERR_NOT_SUPPORTED;
}
/* If this btl supports OOB, then post the RML message. But
ensure to only post it *once*, because another btl may have
come in before this and already posted it. */
if (!rml_recv_posted) {
rc = orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD,
OMPI_RML_TAG_OPENIB,
ORTE_RML_PERSISTENT,
rml_recv_cb,
NULL);
if (ORTE_SUCCESS != rc) {
opal_output_verbose(5, mca_btl_base_output,
"openib BTL: oob CPC system error %d (%s)",
rc, opal_strerror(rc));
return rc;
}
rml_recv_posted = true;
}
*cpc = malloc(sizeof(ompi_btl_openib_connect_base_module_t));
if (NULL == *cpc) {
orte_rml.recv_cancel(ORTE_NAME_WILDCARD, OMPI_RML_TAG_OPENIB);
rml_recv_posted = false;
opal_output_verbose(5, mca_btl_base_output,
"openib BTL: oob CPC system error (malloc failed)");
return OMPI_ERR_OUT_OF_RESOURCE;
}
(*cpc)->data.cbm_component = &ompi_btl_openib_connect_oob;
(*cpc)->data.cbm_priority = oob_priority;
(*cpc)->data.cbm_modex_message = NULL;
(*cpc)->data.cbm_modex_message_len = 0;
(*cpc)->cbm_endpoint_init = NULL;
(*cpc)->cbm_start_connect = oob_module_start_connect;
(*cpc)->cbm_endpoint_finalize = NULL;
(*cpc)->cbm_finalize = NULL;
(*cpc)->cbm_uses_cts = false;
opal_output_verbose(5, mca_btl_base_output,
"openib BTL: oob CPC available for use on %s:%d",
ibv_get_device_name(btl->device->ib_dev),
btl->port_num);
return OMPI_SUCCESS;
}
int
main(int argc, char *argv[])
{
int exit_status = 0, ret, flags = 0, i;
int exec_argc = 0, user_argc = 0;
char **exec_argv = NULL, **user_argv = NULL;
char *exec_command, *base_argv0 = NULL;
bool disable_flags = true;
bool real_flag = false;
if (OPAL_SUCCESS != (ret = opal_init_util())) {
return ret;
}
/****************************************************
*
* Setup compiler information
*
****************************************************/
base_argv0 = opal_basename(argv[0]);
#if defined(EXEEXT)
if( 0 != strlen(EXEEXT) ) {
char extension[] = EXEEXT;
char* temp = strstr( base_argv0, extension );
char* old_match = temp;
while( NULL != temp ) {
old_match = temp;
temp = strstr( temp + 1, extension );
}
/* Only if there was a match of .exe, erase the last occurence of .exe */
if ( NULL != old_match ) {
*old_match = '\0';
}
}
#endif /* defined(EXEEXT) */
if (OPAL_SUCCESS != (ret = data_init(base_argv0))) {
fprintf(stderr, "Error parsing data file %s: %s\n", base_argv0, opal_strerror(ret));
return ret;
}
for (i = 1 ; i < argc && user_data_idx < 0 ; ++i) {
user_data_idx = find_options_index(argv[i]);
}
/* if we didn't find a match, look for the NULL (base case) options */
if (user_data_idx < 0) {
user_data_idx = default_data_idx;
}
/* if we still didn't find a match, abort */
if (user_data_idx < 0) {
char *flat = opal_argv_join(argv, ' ');
opal_show_help("help-opal-wrapper.txt", "no-options-support", true,
base_argv0, flat, NULL);
free(flat);
exit(1);
}
/* compiler */
load_env_data(options_data[user_data_idx].project_short, options_data[user_data_idx].compiler_env, &options_data[user_data_idx].compiler);
/* preprocessor flags */
load_env_data_argv(options_data[user_data_idx].project_short, "CPPFLAGS", &options_data[user_data_idx].preproc_flags);
/* compiler flags */
load_env_data_argv(options_data[user_data_idx].project_short, options_data[user_data_idx].compiler_flags_env,
&options_data[user_data_idx].comp_flags);
/* linker flags */
load_env_data_argv(options_data[user_data_idx].project_short, "LDFLAGS", &options_data[user_data_idx].link_flags);
/* libs */
load_env_data_argv(options_data[user_data_idx].project_short, "LIBS", &options_data[user_data_idx].libs);
/****************************************************
*
* Sanity Checks
*
****************************************************/
if (NULL != options_data[user_data_idx].req_file) {
/* make sure the language is supported */
if (0 == strcmp(options_data[user_data_idx].req_file, "not supported")) {
opal_show_help("help-opal-wrapper.txt", "no-language-support", true,
options_data[user_data_idx].language, base_argv0, NULL);
exit_status = 1;
goto cleanup;
}
if (options_data[user_data_idx].req_file[0] != '\0') {
char *filename;
struct stat buf;
filename = opal_os_path( false, options_data[user_data_idx].path_libdir, options_data[user_data_idx].req_file, NULL );
if (0 != stat(filename, &buf)) {
opal_show_help("help-opal-wrapper.txt", "file-not-found", true,
base_argv0, options_data[user_data_idx].req_file, options_data[user_data_idx].language, NULL);
}
}
}
/****************************************************
*
* Parse user flags
*
****************************************************/
flags = COMP_WANT_COMMAND|COMP_WANT_PREPROC|
COMP_WANT_COMPILE|COMP_WANT_LINK;
user_argv = opal_argv_copy(argv + 1);
user_argc = opal_argv_count(user_argv);
for (i = 0 ; i < user_argc ; ++i) {
if (0 == strncmp(user_argv[i], "-showme", strlen("-showme")) ||
0 == strncmp(user_argv[i], "--showme", strlen("--showme")) ||
0 == strncmp(user_argv[i], "-show", strlen("-show")) ||
0 == strncmp(user_argv[i], "--show", strlen("--show"))) {
bool done_now = false;
/* check for specific things we want to see. First three
still invoke all the building routines. Last set want
to parse out certain flags, so we don't go through the
normal build routine - skip to cleanup. */
if (0 == strncmp(user_argv[i], "-showme:command", strlen("-showme:command")) ||
0 == strncmp(user_argv[i], "--showme:command", strlen("--showme:command"))) {
flags = COMP_WANT_COMMAND;
/* we know what we want, so don't process any more args */
done_now = true;
} else if (0 == strncmp(user_argv[i], "-showme:compile", strlen("-showme:compile")) ||
0 == strncmp(user_argv[i], "--showme:compile", strlen("--showme:compile"))) {
flags = COMP_WANT_PREPROC|COMP_WANT_COMPILE;
/* we know what we want, so don't process any more args */
done_now = true;
} else if (0 == strncmp(user_argv[i], "-showme:link", strlen("-showme:link")) ||
0 == strncmp(user_argv[i], "--showme:link", strlen("--showme:link"))) {
flags = COMP_WANT_COMPILE|COMP_WANT_LINK;
/* we know what we want, so don't process any more args */
done_now = true;
} else if (0 == strncmp(user_argv[i], "-showme:incdirs", strlen("-showme:incdirs")) ||
0 == strncmp(user_argv[i], "--showme:incdirs", strlen("--showme:incdirs"))) {
print_flags(options_data[user_data_idx].preproc_flags, OPAL_INCLUDE_FLAG);
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:libdirs", strlen("-showme:libdirs")) ||
0 == strncmp(user_argv[i], "--showme:libdirs", strlen("--showme:libdirs"))) {
print_flags(options_data[user_data_idx].link_flags, OPAL_LIBDIR_FLAG);
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:libs", strlen("-showme:libs")) ||
0 == strncmp(user_argv[i], "--showme:libs", strlen("--showme:libs"))) {
print_flags(options_data[user_data_idx].libs, "-l");
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:version", strlen("-showme:version")) ||
0 == strncmp(user_argv[i], "--showme:version", strlen("--showme:version"))) {
opal_show_help("help-opal-wrapper.txt", "version", false,
argv[0], options_data[user_data_idx].project, options_data[user_data_idx].version, options_data[user_data_idx].language, NULL);
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:", strlen("-showme:")) ||
0 == strncmp(user_argv[i], "--showme:", strlen("--showme:"))) {
opal_show_help("help-opal-wrapper.txt", "usage", true,
argv[0], options_data[user_data_idx].project, NULL);
goto cleanup;
}
flags |= (COMP_DRY_RUN|COMP_SHOW_ERROR);
/* remove element from user_argv */
opal_argv_delete(&user_argc, &user_argv, i, 1);
--i;
if (done_now) {
disable_flags = false;
break;
}
} else if (0 == strcmp(user_argv[i], "-c")) {
flags &= ~COMP_WANT_LINK;
real_flag = true;
} else if (0 == strcmp(user_argv[i], "-E") ||
0 == strcmp(user_argv[i], "-M")) {
flags &= ~(COMP_WANT_COMPILE | COMP_WANT_LINK);
real_flag = true;
} else if (0 == strcmp(user_argv[i], "-S")) {
flags &= ~COMP_WANT_LINK;
real_flag = true;
} else if (0 == strcmp(user_argv[i], "-lpmpi")) {
flags |= COMP_WANT_PMPI;
/* remove element from user_argv */
opal_argv_delete(&user_argc, &user_argv, i, 1);
--i;
} else if ('-' != user_argv[i][0]) {
disable_flags = false;
flags |= COMP_SHOW_ERROR;
real_flag = true;
} else {
/* if the option flag is one that we use to determine
which set of compiler data to use, don't count it as a
real option */
if (find_options_index(user_argv[i]) < 0) {
real_flag = true;
}
}
}
/* clear out the want_flags if we got no arguments not starting
with a - (dash) and -showme wasn't given OR -showme was given
and we had at least one more non-showme argument that started
with a - (dash) and no other non-dash arguments. Some examples:
opal_wrapper : clear our flags
opal_wrapper -v : clear our flags
opal_wrapper -E a.c : don't clear our flags
opal_wrapper a.c : don't clear our flags
opal_wrapper -showme : don't clear our flags
opal_wrapper -showme -v : clear our flags
opal_wrapper -showme -E a.c : don't clear our flags
opal_wrapper -showme a.c : don't clear our flags
*/
if (disable_flags && !((flags & COMP_DRY_RUN) && !real_flag)) {
flags &= ~(COMP_WANT_PREPROC|COMP_WANT_COMPILE|COMP_WANT_LINK);
}
#if !OMPI_ENABLE_MPI_PROFILING
/* sanity check */
if (flags & COMP_WANT_PMPI) {
opal_show_help("help-opal-wrapper.txt", "no-profiling-support", true,
argv[0], NULL);
}
#endif
/****************************************************
*
* Assemble the command line
*
****************************************************/
/* compiler (may be multiple arguments, so split) */
if (flags & COMP_WANT_COMMAND) {
exec_argv = opal_argv_split(options_data[user_data_idx].compiler, ' ');
exec_argc = opal_argv_count(exec_argv);
} else {
exec_argv = (char **) malloc(sizeof(char*));
exec_argv[0] = NULL;
exec_argc = 0;
}
/* Per https://svn.open-mpi.org/trac/ompi/ticket/2201, add all the
user arguments before anything else. */
opal_argv_insert(&exec_argv, exec_argc, user_argv);
exec_argc = opal_argv_count(exec_argv);
/* preproc flags */
if (flags & COMP_WANT_PREPROC) {
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].preproc_flags);
exec_argc = opal_argv_count(exec_argv);
}
/* compiler flags */
if (flags & COMP_WANT_COMPILE) {
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].comp_flags);
/* Deal with languages like Fortran 90 that have special
places and flags for modules or whatever */
if (options_data[user_data_idx].module_option != NULL) {
char *line;
asprintf(&line, "%s%s", options_data[user_data_idx].module_option, options_data[user_data_idx].path_libdir);
opal_argv_append_nosize(&exec_argv, line);
free(line);
}
exec_argc = opal_argv_count(exec_argv);
}
/* link flags and libs */
if (flags & COMP_WANT_LINK) {
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].link_flags);
exec_argc = opal_argv_count(exec_argv);
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].libs);
exec_argc = opal_argv_count(exec_argv);
}
/****************************************************
*
* Execute the command
*
****************************************************/
if (flags & COMP_DRY_RUN) {
exec_command = opal_argv_join(exec_argv, ' ');
printf("%s\n", exec_command);
} else {
char *tmp;
#if 0
exec_command = opal_argv_join(exec_argv, ' ');
printf("command: %s\n", exec_command);
#endif
tmp = opal_path_findv(exec_argv[0], 0, environ, NULL);
if (NULL == tmp) {
opal_show_help("help-opal-wrapper.txt", "no-compiler-found", true,
exec_argv[0], NULL);
errno = 0;
exit_status = 1;
} else {
int status;
free(exec_argv[0]);
exec_argv[0] = tmp;
ret = opal_few(exec_argv, &status);
exit_status = WIFEXITED(status) ? WEXITSTATUS(status) :
(WIFSIGNALED(status) ? WTERMSIG(status) :
(WIFSTOPPED(status) ? WSTOPSIG(status) : 255));
if( (OPAL_SUCCESS != ret) || ((0 != exit_status) && (flags & COMP_SHOW_ERROR)) ) {
char* exec_command = opal_argv_join(exec_argv, ' ');
if( OPAL_SUCCESS != ret ) {
opal_show_help("help-opal-wrapper.txt", "spawn-failed", true,
exec_argv[0], strerror(status), exec_command, NULL);
} else {
#if 0
opal_show_help("help-opal-wrapper.txt", "compiler-failed", true,
exec_argv[0], exit_status, exec_command, NULL);
#endif
}
free(exec_command);
}
}
}
/****************************************************
*
* Cleanup
*
****************************************************/
cleanup:
opal_argv_free(exec_argv);
opal_argv_free(user_argv);
if (NULL != base_argv0) free(base_argv0);
if (OPAL_SUCCESS != (ret = data_finalize())) {
return ret;
}
if (OPAL_SUCCESS != (ret = opal_finalize_util())) {
return ret;
}
return exit_status;
}
static int do_child(orte_app_context_t* context,
orte_proc_t *child,
char **environ_copy,
orte_job_t *jobdat, int write_fd,
orte_iof_base_io_conf_t opts)
{
int i, rc;
sigset_t sigs;
long fd, fdmax = sysconf(_SC_OPEN_MAX);
char *param, *msg;
if (orte_forward_job_control) {
/* Set a new process group for this child, so that a
SIGSTOP can be sent to it without being sent to the
orted. */
setpgid(0, 0);
}
/* Setup the pipe to be close-on-exec */
fcntl(write_fd, F_SETFD, FD_CLOEXEC);
if (NULL != child) {
/* setup stdout/stderr so that any error messages that we
may print out will get displayed back at orterun.
NOTE: Definitely do this AFTER we check contexts so
that any error message from those two functions doesn't
come out to the user. IF we didn't do it in this order,
THEN a user who gives us a bad executable name or
working directory would get N error messages, where
N=num_procs. This would be very annoying for large
jobs, so instead we set things up so that orterun
always outputs a nice, single message indicating what
happened
*/
if (ORTE_SUCCESS != (i = orte_iof_base_setup_child(&opts,
&environ_copy))) {
ORTE_ERROR_LOG(i);
send_error_show_help(write_fd, 1,
"help-orte-odls-default.txt",
"iof setup failed",
orte_process_info.nodename, context->app);
/* Does not return */
}
#if OPAL_HAVE_HWLOC
{
hwloc_cpuset_t cpuset;
hwloc_obj_t root;
opal_hwloc_topo_data_t *sum;
/* Set process affinity, if given */
if (NULL == child->cpu_bitmap) {
/* if the daemon is bound, then we need to "free" this proc */
if (NULL != orte_daemon_cores) {
root = hwloc_get_root_obj(opal_hwloc_topology);
if (NULL == root->userdata) {
send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "incorrectly bound",
orte_process_info.nodename, context->app,
__FILE__, __LINE__);
}
sum = (opal_hwloc_topo_data_t*)root->userdata;
/* bind this proc to all available processors */
hwloc_set_cpubind(opal_hwloc_topology, sum->available, 0);
}
if (opal_hwloc_report_bindings) {
opal_output(0, "MCW rank %d is not bound (or bound to all available processors)", child->name.vpid);
/* avoid reporting it twice */
(void) mca_base_var_env_name ("hwloc_base_report_bindings", ¶m);
opal_unsetenv(param, &environ_copy);
free(param);
}
} else {
if (0 == strlen(child->cpu_bitmap)) {
/* this proc is not bound */
if (opal_hwloc_report_bindings) {
opal_output(0, "MCW rank %d is not bound (or bound to all available processors)", child->name.vpid);
/* avoid reporting it twice */
(void) mca_base_var_env_name ("hwloc_base_report_bindings", ¶m);
opal_unsetenv(param, &environ_copy);
free(param);
}
/* if the daemon is bound, then we need to "free" this proc */
if (NULL != orte_daemon_cores) {
root = hwloc_get_root_obj(opal_hwloc_topology);
if (NULL == root->userdata) {
send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "incorrectly bound",
orte_process_info.nodename, context->app,
__FILE__, __LINE__);
}
sum = (opal_hwloc_topo_data_t*)root->userdata;
/* bind this proc to all available processors */
hwloc_set_cpubind(opal_hwloc_topology, sum->available, 0);
}
/* provide a nice string representation of what we bound to */
(void) mca_base_var_env_name ("orte_base_applied_binding", ¶m);
opal_setenv(param, child->cpu_bitmap, true, &environ_copy);
free(param);
goto PROCEED;
}
/* convert the list to a cpu bitmap */
cpuset = hwloc_bitmap_alloc();
if (0 != (rc = hwloc_bitmap_list_sscanf(cpuset, child->cpu_bitmap))) {
/* See comment above about "This may be a small memory leak" */
asprintf(&msg, "hwloc_bitmap_sscanf returned \"%s\" for the string \"%s\"",
opal_strerror(rc), child->cpu_bitmap);
if (NULL == msg) {
msg = "failed to convert bitmap list to hwloc bitmap";
}
if (OPAL_BINDING_REQUIRED(jobdat->map->binding) &&
OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
/* If binding is required and a binding directive was explicitly
* given (i.e., we are not binding due to a default policy),
* send an error up the pipe (which exits -- it doesn't return).
*/
send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"binding generic error",
orte_process_info.nodename,
context->app, msg,
__FILE__, __LINE__);
} else {
send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
goto PROCEED;
}
}
/* bind as specified */
rc = hwloc_set_cpubind(opal_hwloc_topology, cpuset, 0);
/* if we got an error and this wasn't a default binding policy, then report it */
if (rc < 0 && OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
char *tmp = NULL;
if (errno == ENOSYS) {
msg = "hwloc indicates cpu binding not supported";
} else if (errno == EXDEV) {
msg = "hwloc indicates cpu binding cannot be enforced";
} else {
asprintf(&msg, "hwloc_set_cpubind returned \"%s\" for bitmap \"%s\"",
opal_strerror(rc), child->cpu_bitmap);
}
if (OPAL_BINDING_REQUIRED(jobdat->map->binding)) {
/* If binding is required, send an error up the pipe (which exits
-- it doesn't return). */
send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"binding generic error",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
} else {
send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
if (NULL != tmp) {
free(tmp);
free(msg);
}
goto PROCEED;
}
if (NULL != tmp) {
free(tmp);
free(msg);
}
}
if (0 == rc && opal_hwloc_report_bindings) {
char tmp1[1024], tmp2[1024];
hwloc_cpuset_t mycpus;
/* get the cpus we are bound to */
mycpus = hwloc_bitmap_alloc();
if (hwloc_get_cpubind(opal_hwloc_topology,
mycpus,
HWLOC_CPUBIND_PROCESS) < 0) {
opal_output(0, "MCW rank %d is not bound",
child->name.vpid);
} else {
if (OPAL_ERR_NOT_BOUND == opal_hwloc_base_cset2str(tmp1, sizeof(tmp1), opal_hwloc_topology, mycpus)) {
opal_output(0, "MCW rank %d is not bound (or bound to all available processors)", child->name.vpid);
} else {
opal_hwloc_base_cset2mapstr(tmp2, sizeof(tmp2), opal_hwloc_topology, mycpus);
opal_output(0, "MCW rank %d bound to %s: %s",
child->name.vpid, tmp1, tmp2);
}
}
hwloc_bitmap_free(mycpus);
/* avoid reporting it twice */
(void) mca_base_var_env_name ("hwloc_base_report_bindings", ¶m);
opal_unsetenv(param, &environ_copy);
free(param);
}
/* set memory affinity policy - if we get an error, don't report
* anything unless the user actually specified the binding policy
*/
rc = opal_hwloc_base_set_process_membind_policy();
if (ORTE_SUCCESS != rc && OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
if (errno == ENOSYS) {
msg = "hwloc indicates memory binding not supported";
} else if (errno == EXDEV) {
msg = "hwloc indicates memory binding cannot be enforced";
} else {
msg = "failed to bind memory";
}
if (OPAL_HWLOC_BASE_MBFA_ERROR == opal_hwloc_base_mbfa) {
/* If binding is required, send an error up the pipe (which exits
-- it doesn't return). */
send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"memory binding error",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
} else {
send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "memory not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
goto PROCEED;
}
}
}
}
#endif
} else if (!(ORTE_JOB_CONTROL_FORWARD_OUTPUT & jobdat->controls)) {
/* tie stdin/out/err/internal to /dev/null */
int fdnull;
for (i=0; i < 3; i++) {
fdnull = open("/dev/null", O_RDONLY, 0);
if (fdnull > i && i != write_fd) {
dup2(fdnull, i);
}
close(fdnull);
}
fdnull = open("/dev/null", O_RDONLY, 0);
if (fdnull > opts.p_internal[1]) {
dup2(fdnull, opts.p_internal[1]);
}
close(fdnull);
}
#if OPAL_HAVE_HWLOC
PROCEED:
#endif
/* if the user requested it, set the system resource limits */
if (OPAL_SUCCESS != (rc = opal_util_init_sys_limits(&msg))) {
send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"set limit",
orte_process_info.nodename, context->app,
__FILE__, __LINE__, msg);
}
/* ensure we only do this once */
(void) mca_base_var_env_name("opal_set_max_sys_limits", ¶m);
opal_unsetenv(param, &environ_copy);
free(param);
/* close all file descriptors w/ exception of stdin/stdout/stderr,
the pipe used for the IOF INTERNAL messages, and the pipe up to
the parent. */
for(fd=3; fd<fdmax; fd++) {
if (fd != opts.p_internal[1] && fd != write_fd) {
close(fd);
}
}
if (context->argv == NULL) {
context->argv = malloc(sizeof(char*)*2);
context->argv[0] = strdup(context->app);
context->argv[1] = NULL;
}
/* Set signal handlers back to the default. Do this close to
the exev() because the event library may (and likely will)
reset them. If we don't do this, the event library may
have left some set that, at least on some OS's, don't get
reset via fork() or exec(). Hence, the launched process
could be unkillable (for example). */
set_handler_default(SIGTERM);
set_handler_default(SIGINT);
set_handler_default(SIGHUP);
set_handler_default(SIGPIPE);
set_handler_default(SIGCHLD);
/* Unblock all signals, for many of the same reasons that we
set the default handlers, above. This is noticable on
Linux where the event library blocks SIGTERM, but we don't
want that blocked by the launched process. */
sigprocmask(0, 0, &sigs);
sigprocmask(SIG_UNBLOCK, &sigs, 0);
/* Exec the new executable */
if (10 < opal_output_get_verbosity(orte_odls_base_framework.framework_output)) {
int jout;
opal_output(0, "%s STARTING %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), context->app);
for (jout=0; NULL != context->argv[jout]; jout++) {
opal_output(0, "%s\tARGV[%d]: %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), jout, context->argv[jout]);
}
for (jout=0; NULL != environ_copy[jout]; jout++) {
opal_output(0, "%s\tENVIRON[%d]: %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), jout, environ_copy[jout]);
}
}
execve(context->app, context->argv, environ_copy);
send_error_show_help(write_fd, 1,
"help-orte-odls-default.txt", "execve error",
orte_process_info.nodename, context->app, strerror(errno));
/* Does not return */
}
int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
{
int ret;
ompi_proc_t** procs;
size_t nprocs;
char *error = NULL;
bool timing = false;
int param, value;
struct timeval ompistart, ompistop;
char *event_val = NULL;
opal_paffinity_base_cpu_set_t mask;
bool proc_bound;
#if 0
/* see comment below about sched_yield */
int num_processors;
#endif
bool orte_setup = false;
bool paffinity_enabled = false;
/* Setup enough to check get/set MCA params */
if (ORTE_SUCCESS != (ret = opal_init_util())) {
error = "ompi_mpi_init: opal_init_util failed";
goto error;
}
/* _After_ opal_init_util() but _before_ orte_init(), we need to
set an MCA param that tells libevent that it's ok to use any
mechanism in libevent that is available on this platform (e.g.,
epoll and friends). Per opal/event/event.s, we default to
select/poll -- but we know that MPI processes won't be using
pty's with the event engine, so it's ok to relax this
constraint and let any fd-monitoring mechanism be used. */
ret = mca_base_param_reg_string_name("opal", "event_include",
"Internal orted MCA param: tell opal_init() to use a specific mechanism in libevent",
false, false, "all", &event_val);
if (ret >= 0) {
/* We have to explicitly "set" the MCA param value here
because libevent initialization will re-register the MCA
param and therefore override the default. Setting the value
here puts the desired value ("all") in different storage
that is not overwritten if/when the MCA param is
re-registered. This is unless the user has specified a different
value for this MCA parameter. Make sure we check to see if the
default is specified before forcing "all" in case that is not what
the user desires. Note that we do *NOT* set this value as an
environment variable, just so that it won't be inherited by
any spawned processes and potentially cause unintented
side-effects with launching ORTE tools... */
if (0 == strcmp("all", event_val)) {
mca_base_param_set_string(ret, "all");
}
}
if( NULL != event_val ) {
free(event_val);
event_val = NULL;
}
/* check to see if we want timing information */
param = mca_base_param_reg_int_name("ompi", "timing",
"Request that critical timing loops be measured",
false, false, 0, &value);
if (value != 0) {
timing = true;
gettimeofday(&ompistart, NULL);
}
/* Setup ORTE - note that we are not a tool */
if (ORTE_SUCCESS != (ret = orte_init(ORTE_NON_TOOL))) {
error = "ompi_mpi_init: orte_init failed";
goto error;
}
orte_setup = true;
/* check for timing request - get stop time and report elapsed time if so */
if (timing && 0 == ORTE_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init [%ld]: time from start to completion of orte_init %ld usec",
(long)ORTE_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
/* Figure out the final MPI thread levels. If we were not
compiled for support for MPI threads, then don't allow
MPI_THREAD_MULTIPLE. Set this stuff up here early in the
process so that other components can make decisions based on
this value. */
ompi_mpi_thread_requested = requested;
if (OMPI_HAVE_THREAD_SUPPORT == 0) {
ompi_mpi_thread_provided = *provided = MPI_THREAD_SINGLE;
ompi_mpi_main_thread = NULL;
} else if (OMPI_ENABLE_MPI_THREADS == 1) {
ompi_mpi_thread_provided = *provided = requested;
ompi_mpi_main_thread = opal_thread_get_self();
} else {
if (MPI_THREAD_MULTIPLE == requested) {
ompi_mpi_thread_provided = *provided = MPI_THREAD_SERIALIZED;
} else {
ompi_mpi_thread_provided = *provided = requested;
}
ompi_mpi_main_thread = opal_thread_get_self();
}
ompi_mpi_thread_multiple = (ompi_mpi_thread_provided ==
MPI_THREAD_MULTIPLE);
/* Once we've joined the RTE, see if any MCA parameters were
passed to the MPI level */
if (OMPI_SUCCESS != (ret = ompi_mpi_register_params())) {
error = "mca_mpi_register_params() failed";
goto error;
}
/* if it hasn't already been done, setup process affinity.
* First check to see if a slot list was
* specified. If so, use it. If no slot list was specified,
* that's not an error -- just fall through and try the next
* paffinity scheme.
*/
ret = opal_paffinity_base_get(&mask);
if (OPAL_SUCCESS == ret) {
/* paffinity is supported - check for binding */
OPAL_PAFFINITY_PROCESS_IS_BOUND(mask, &proc_bound);
if (proc_bound) {
/* someone external set it - indicate it is set
* so that we know
*/
paffinity_enabled = true;
} else {
/* the system is capable of doing processor affinity, but it
* has not yet been set - see if a slot_list was given
*/
if (NULL != opal_paffinity_base_slot_list) {
/* It's an error if multiple paffinity schemes were specified */
if (opal_paffinity_alone) {
ret = OMPI_ERR_BAD_PARAM;
error = "Multiple processor affinity schemes specified (can only specify one)";
goto error;
}
ret = opal_paffinity_base_slot_list_set((long)ORTE_PROC_MY_NAME->vpid, opal_paffinity_base_slot_list);
if (OPAL_ERR_NOT_FOUND != ret) {
error = "opal_paffinity_base_slot_list_set() returned an error";
goto error;
}
paffinity_enabled = true;
} else if (opal_paffinity_alone) {
/* no slot_list, but they asked for paffinity */
int phys_cpu;
orte_node_rank_t nrank;
if (ORTE_NODE_RANK_INVALID == (nrank = orte_ess.get_node_rank(ORTE_PROC_MY_NAME))) {
error = "Could not get node rank - cannot set processor affinity";
goto error;
}
OPAL_PAFFINITY_CPU_ZERO(mask);
phys_cpu = opal_paffinity_base_get_physical_processor_id(nrank);
if (0 > phys_cpu) {
error = "Could not get physical processor id - cannot set processor affinity";
goto error;
}
OPAL_PAFFINITY_CPU_SET(phys_cpu, mask);
ret = opal_paffinity_base_set(mask);
if (OPAL_SUCCESS != ret) {
error = "Setting processor affinity failed";
goto error;
}
paffinity_enabled = true;
}
}
}
/* If we were able to set processor affinity, try setting up
memory affinity */
if (!opal_maffinity_setup && paffinity_enabled) {
if (OPAL_SUCCESS == opal_maffinity_base_open() &&
OPAL_SUCCESS == opal_maffinity_base_select()) {
opal_maffinity_setup = true;
}
}
/* initialize datatypes. This step should be done early as it will
* create the local convertor and local arch used in the proc
* init.
*/
if (OMPI_SUCCESS != (ret = ompi_ddt_init())) {
error = "ompi_ddt_init() failed";
goto error;
}
/* Initialize OMPI procs */
if (OMPI_SUCCESS != (ret = ompi_proc_init())) {
error = "mca_proc_init() failed";
goto error;
}
/* initialize ops. This has to be done *after* ddt_init, but
befor mca_coll_base_open, since come collective modules
(e.g. the hierarchical) need them in the query function
*/
if (OMPI_SUCCESS != (ret = ompi_op_init())) {
error = "ompi_op_init() failed";
goto error;
}
/* Open up MPI-related MCA components */
if (OMPI_SUCCESS != (ret = mca_allocator_base_open())) {
error = "mca_allocator_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_rcache_base_open())) {
error = "mca_rcache_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_mpool_base_open())) {
error = "mca_mpool_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_pml_base_open())) {
error = "mca_pml_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_coll_base_open())) {
error = "mca_coll_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = ompi_osc_base_open())) {
error = "ompi_osc_base_open() failed";
goto error;
}
#if OPAL_ENABLE_FT == 1
if (OMPI_SUCCESS != (ret = ompi_crcp_base_open())) {
error = "ompi_crcp_base_open() failed";
goto error;
}
#endif
/* In order to reduce the common case for MPI apps (where they
don't use MPI-2 IO or MPI-1 topology functions), the io and
topo frameworks are initialized lazily, at the first use of
relevant functions (e.g., MPI_FILE_*, MPI_CART_*, MPI_GRAPH_*),
so they are not opened here. */
/* Select which MPI components to use */
if (OMPI_SUCCESS !=
(ret = mca_mpool_base_init(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_MPI_THREADS))) {
error = "mca_mpool_base_init() failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = mca_pml_base_select(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_MPI_THREADS))) {
error = "mca_pml_base_select() failed";
goto error;
}
/* select buffered send allocator component to be used */
ret=mca_pml_base_bsend_init(OMPI_ENABLE_MPI_THREADS);
if( OMPI_SUCCESS != ret ) {
error = "mca_pml_base_bsend_init() failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = mca_coll_base_find_available(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_MPI_THREADS))) {
error = "mca_coll_base_find_available() failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = ompi_osc_base_find_available(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_MPI_THREADS))) {
error = "ompi_osc_base_find_available() failed";
goto error;
}
#if OPAL_ENABLE_FT == 1
if (OMPI_SUCCESS != (ret = ompi_crcp_base_select() ) ) {
error = "ompi_crcp_base_select() failed";
goto error;
}
#endif
/* io and topo components are not selected here -- see comment
above about the io and topo frameworks being loaded lazily */
/* Initialize each MPI handle subsystem */
/* initialize requests */
if (OMPI_SUCCESS != (ret = ompi_request_init())) {
error = "ompi_request_init() failed";
goto error;
}
/* initialize info */
if (OMPI_SUCCESS != (ret = ompi_info_init())) {
error = "ompi_info_init() failed";
goto error;
}
/* initialize error handlers */
if (OMPI_SUCCESS != (ret = ompi_errhandler_init())) {
error = "ompi_errhandler_init() failed";
goto error;
}
/* initialize error codes */
if (OMPI_SUCCESS != (ret = ompi_mpi_errcode_init())) {
error = "ompi_mpi_errcode_init() failed";
goto error;
}
/* initialize internal error codes */
if (OMPI_SUCCESS != (ret = ompi_errcode_intern_init())) {
error = "ompi_errcode_intern_init() failed";
goto error;
}
/* initialize groups */
if (OMPI_SUCCESS != (ret = ompi_group_init())) {
error = "ompi_group_init() failed";
goto error;
}
/* initialize communicators */
if (OMPI_SUCCESS != (ret = ompi_comm_init())) {
error = "ompi_comm_init() failed";
goto error;
}
/* initialize file handles */
if (OMPI_SUCCESS != (ret = ompi_file_init())) {
error = "ompi_file_init() failed";
goto error;
}
/* initialize windows */
if (OMPI_SUCCESS != (ret = ompi_win_init())) {
error = "ompi_win_init() failed";
goto error;
}
/* initialize attribute meta-data structure for comm/win/dtype */
if (OMPI_SUCCESS != (ret = ompi_attr_init())) {
error = "ompi_attr_init() failed";
goto error;
}
/* check for timing request - get stop time and report elapsed time if so */
if (timing && 0 == ORTE_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time from completion of orte_init to modex %ld usec",
(long)ORTE_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
/* exchange connection info - this function also acts as a barrier
* as it will not return until the exchange is complete
*/
if (OMPI_SUCCESS != (ret = orte_grpcomm.modex(NULL))) {
error = "orte_grpcomm_modex failed";
goto error;
}
if (timing && 0 == ORTE_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time to execute modex %ld usec",
(long)ORTE_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
/* identify the architectures of remote procs and setup
* their datatype convertors, if required
*/
if (OMPI_SUCCESS != (ret = ompi_proc_set_arch())) {
error = "ompi_proc_set_arch failed";
goto error;
}
/* If thread support was enabled, then setup OPAL to allow for
them. */
if ((OMPI_ENABLE_PROGRESS_THREADS == 1) ||
(*provided != MPI_THREAD_SINGLE)) {
opal_set_using_threads(true);
}
/* start PML/BTL's */
ret = MCA_PML_CALL(enable(true));
if( OMPI_SUCCESS != ret ) {
error = "PML control failed";
goto error;
}
/* add all ompi_proc_t's to PML */
if (NULL == (procs = ompi_proc_world(&nprocs))) {
error = "ompi_proc_world() failed";
goto error;
}
ret = MCA_PML_CALL(add_procs(procs, nprocs));
free(procs);
if( OMPI_SUCCESS != ret ) {
error = "PML add procs failed";
goto error;
}
MCA_PML_CALL(add_comm(&ompi_mpi_comm_world.comm));
MCA_PML_CALL(add_comm(&ompi_mpi_comm_self.comm));
/*
* Dump all MCA parameters if requested
*/
if (ompi_mpi_show_mca_params) {
ompi_show_all_mca_params(ompi_mpi_comm_world.comm.c_my_rank,
nprocs,
orte_process_info.nodename);
}
/* Do we need to wait for a debugger? */
ompi_wait_for_debugger();
/* check for timing request - get stop time and report elapsed
time if so, then start the clock again */
if (timing && 0 == ORTE_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time from modex thru complete oob wireup %ld usec",
(long)ORTE_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
/* wait for everyone to reach this point */
if (OMPI_SUCCESS != (ret = orte_grpcomm.barrier())) {
error = "orte_grpcomm_barrier failed";
goto error;
}
/* check for timing request - get stop time and report elapsed
time if so, then start the clock again */
if (timing && 0 == ORTE_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time to execute barrier %ld usec",
(long)ORTE_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
#if OMPI_ENABLE_PROGRESS_THREADS == 0
/* Start setting up the event engine for MPI operations. Don't
block in the event library, so that communications don't take
forever between procs in the dynamic code. This will increase
CPU utilization for the remainder of MPI_INIT when we are
blocking on ORTE-level events, but may greatly reduce non-TCP
latency. */
opal_progress_set_event_flag(OPAL_EVLOOP_NONBLOCK);
#endif
/* wire up the mpi interface, if requested. Do this after the
non-block switch for non-TCP performance. Do before the
polling change as anyone with a complex wire-up is going to be
using the oob. */
if (OMPI_SUCCESS != (ret = ompi_init_preconnect_mpi())) {
error = "ompi_mpi_do_preconnect_all() failed";
goto error;
}
/* Setup the publish/subscribe (PUBSUB) framework */
if (OMPI_SUCCESS != (ret = ompi_pubsub_base_open())) {
error = "ompi_pubsub_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = ompi_pubsub_base_select())) {
error = "ompi_pubsub_base_select() failed";
goto error;
}
/* Setup the dynamic process management (DPM) framework */
if (OMPI_SUCCESS != (ret = ompi_dpm_base_open())) {
error = "ompi_dpm_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = ompi_dpm_base_select())) {
error = "ompi_dpm_base_select() failed";
goto error;
}
/* Init coll for the comms. This has to be after dpm_base_select,
(since dpm.mark_dyncomm is not set in the communicator creation
function else), but before dpm.dyncom_init, since this function
might require collective for the CID allocation. */
if (OMPI_SUCCESS !=
(ret = mca_coll_base_comm_select(MPI_COMM_WORLD))) {
error = "mca_coll_base_comm_select(MPI_COMM_WORLD) failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = mca_coll_base_comm_select(MPI_COMM_SELF))) {
error = "mca_coll_base_comm_select(MPI_COMM_SELF) failed";
goto error;
}
/* Check whether we have been spawned or not. We introduce that
at the very end, since we need collectives, datatypes, ptls
etc. up and running here.... */
if (OMPI_SUCCESS != (ret = ompi_dpm.dyn_init())) {
error = "ompi_comm_dyn_init() failed";
goto error;
}
/*
* Startup the Checkpoint/Restart Mech.
* Note: Always do this so tools don't hang when
* in a non-checkpointable build
*/
if (OMPI_SUCCESS != (ret = ompi_cr_init())) {
error = "ompi_cr_init";
goto error;
}
/* Undo OPAL calling opal_progress_event_users_increment() during
opal_init, to get better latency when not using TCP. Do
this *after* dyn_init, as dyn init uses lots of ORTE
communication and we don't want to hinder the performance of
that code. */
opal_progress_event_users_decrement();
/* see if yield_when_idle was specified - if so, use it */
param = mca_base_param_find("mpi", NULL, "yield_when_idle");
mca_base_param_lookup_int(param, &value);
if (value < 0) {
/* if no info is provided, just default to conservative */
opal_progress_set_yield_when_idle(true);
} else {
/* info was provided, so set idle accordingly */
opal_progress_set_yield_when_idle(value == 0 ? false : true);
}
param = mca_base_param_find("mpi", NULL, "event_tick_rate");
mca_base_param_lookup_int(param, &value);
/* negative value means use default - just don't do anything */
if (value >= 0) {
opal_progress_set_event_poll_rate(value);
}
/* At this point, we are fully configured and in MPI mode. Any
communication calls here will work exactly like they would in
the user's code. Setup the connections between procs and warm
them up with simple sends, if requested */
error:
if (ret != OMPI_SUCCESS) {
const char *err_msg = opal_strerror(ret);
/* If ORTE was not setup yet, don't use orte_show_help */
if (orte_setup) {
orte_show_help("help-mpi-runtime",
"mpi_init:startup:internal-failure", true,
"MPI_INIT", "MPI_INIT", error, err_msg, ret);
} else {
opal_show_help("help-mpi-runtime",
"mpi_init:startup:internal-failure", true,
"MPI_INIT", "MPI_INIT", error, err_msg, ret);
}
return ret;
}
/* Initialize the registered datarep list to be empty */
OBJ_CONSTRUCT(&ompi_registered_datareps, opal_list_t);
/* Initialize the arrays used to store the F90 types returned by the
* MPI_Type_create_f90_XXX functions.
*/
OBJ_CONSTRUCT( &ompi_mpi_f90_integer_hashtable, opal_hash_table_t);
opal_hash_table_init(&ompi_mpi_f90_integer_hashtable, 16 /* why not? */);
OBJ_CONSTRUCT( &ompi_mpi_f90_real_hashtable, opal_hash_table_t);
opal_hash_table_init(&ompi_mpi_f90_real_hashtable, FLT_MAX_10_EXP);
OBJ_CONSTRUCT( &ompi_mpi_f90_complex_hashtable, opal_hash_table_t);
opal_hash_table_init(&ompi_mpi_f90_complex_hashtable, FLT_MAX_10_EXP);
/* All done. Wasn't that simple? */
ompi_mpi_initialized = true;
/* check for timing request - get stop time and report elapsed time if so */
if (timing && 0 == ORTE_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time from barrier p to complete mpi_init %ld usec",
(long)ORTE_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
}
return MPI_SUCCESS;
}
static int _shmem_init(int argc, char **argv, int requested, int *provided)
{
int ret = OSHMEM_SUCCESS;
char *error = NULL;
oshmem_mpi_thread_requested = requested;
oshmem_mpi_thread_provided = requested;
/* Register the OSHMEM layer's MCA parameters */
if (OSHMEM_SUCCESS != (ret = oshmem_shmem_register_params())) {
error = "oshmem_info_register: oshmem_register_params failed";
goto error;
}
/* Setting verbosity for macros like SHMEM_API_VERBOSE, SHMEM_API_ERROR.
* We need to set it right after registering mca verbosity variables
*/
shmem_api_logger_output = opal_output_open(NULL);
opal_output_set_verbosity(shmem_api_logger_output,
oshmem_shmem_api_verbose);
/* initialize info */
if (OSHMEM_SUCCESS != (ret = oshmem_info_init())) {
error = "oshmem_info_init() failed";
goto error;
}
/* initialize proc */
if (OSHMEM_SUCCESS != (ret = oshmem_proc_init())) {
error = "oshmem_proc_init() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = oshmem_op_init())) {
error = "oshmem_op_init() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_spml_base_framework, MCA_BASE_OPEN_DEFAULT))) {
error = "mca_spml_base_open() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_scoll_base_framework, MCA_BASE_OPEN_DEFAULT))) {
error = "mca_scoll_base_open() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_spml_base_select(OPAL_ENABLE_PROGRESS_THREADS, 1))) {
error = "mca_spml_base_select() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_scoll_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1))) {
error = "mca_scoll_base_find_available() failed";
goto error;
}
/* Initialize each SHMEM handle subsystem */
/* Initialize requests */
if (OSHMEM_SUCCESS != (ret = oshmem_request_init())) {
error = "oshmem_request_init() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = oshmem_proc_group_init())) {
error = "oshmem_proc_group_init() failed";
goto error;
}
/* start SPML/BTL's */
ret = MCA_SPML_CALL(enable(true));
if (OSHMEM_SUCCESS != ret) {
error = "SPML control failed";
goto error;
}
ret =
MCA_SPML_CALL(add_procs(oshmem_group_all->proc_array, oshmem_group_all->proc_count));
if (OSHMEM_SUCCESS != ret) {
error = "SPML add procs failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_sshmem_base_framework, MCA_BASE_OPEN_DEFAULT))) {
error = "mca_sshmem_base_open() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_sshmem_base_select())) {
error = "mca_sshmem_base_select() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_memheap_base_framework, MCA_BASE_OPEN_DEFAULT))) {
error = "mca_memheap_base_open() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_memheap_base_select())) {
error = "mca_memheap_base_select() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_atomic_base_framework, MCA_BASE_OPEN_DEFAULT))) {
error = "mca_atomic_base_open() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_atomic_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1))) {
error = "mca_atomic_base_find_available() failed";
goto error;
}
/* This call should be done after memheap initialization */
if (OSHMEM_SUCCESS != (ret = mca_scoll_enable())) {
error = "mca_scoll_enable() failed";
goto error;
}
(*provided) = oshmem_mpi_thread_provided;
oshmem_mpi_thread_multiple = (oshmem_mpi_thread_provided == SHMEM_THREAD_MULTIPLE) ? true : false;
error: if (ret != OSHMEM_SUCCESS) {
const char *err_msg = opal_strerror(ret);
opal_show_help("help-shmem-runtime.txt",
"shmem_init:startup:internal-failure",
true,
"SHMEM_INIT",
"SHMEM_INIT",
error,
err_msg,
ret);
return ret;
}
return ret;
}
static int __shmem_init(int argc, char **argv, int requested, int *provided)
{
int ret = OSHMEM_SUCCESS;
char *error = NULL;
if (OSHMEM_SUCCESS != (ret = oshmem_proc_init())) {
error = "oshmem_proc_init() failed";
goto error;
}
/* We need to do this anyway.
* This place requires to be reviewed and more elegant way is expected
*/
ompi_proc_local_proc = (ompi_proc_t*) oshmem_proc_local_proc;
if (OSHMEM_SUCCESS != (ret = oshmem_group_cache_list_init())) {
error = "oshmem_group_cache_list_init() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = oshmem_op_init())) {
error = "oshmem_op_init() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_spml_base_framework, MCA_BASE_OPEN_DEFAULT))) {
error = "mca_spml_base_open() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_scoll_base_framework, MCA_BASE_OPEN_DEFAULT))) {
error = "mca_scoll_base_open() failed";
goto error;
}
if (OSHMEM_SUCCESS
!= (ret = mca_spml_base_select(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "mca_spml_base_select() failed";
goto error;
}
if (OSHMEM_SUCCESS
!= (ret =
mca_scoll_base_find_available(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "mca_scoll_base_find_available() failed";
goto error;
}
/* Initialize each SHMEM handle subsystem */
/* Initialize requests */
if (OSHMEM_SUCCESS != (ret = oshmem_request_init())) {
error = "oshmem_request_init() failed";
goto error;
}
/* identify the architectures of remote procs and setup
* their datatype convertors, if required
*/
if (OSHMEM_SUCCESS != (ret = oshmem_proc_set_arch())) {
error = "oshmem_proc_set_arch failed";
goto error;
}
/* start SPML/BTL's */
ret = MCA_SPML_CALL(enable(true));
if (OSHMEM_SUCCESS != ret) {
error = "SPML control failed";
goto error;
}
/* There is issue with call add_proc twice so
* we need to use btl info got from PML add_procs() before call of SPML add_procs()
*/
{
ompi_proc_t** procs = NULL;
size_t nprocs = 0;
procs = ompi_proc_world(&nprocs);
while (nprocs--) {
oshmem_group_all->proc_array[nprocs]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML] =
procs[nprocs]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
}
free(procs);
}
ret =
MCA_SPML_CALL(add_procs(oshmem_group_all->proc_array, oshmem_group_all->proc_count));
if (OSHMEM_SUCCESS != ret) {
error = "SPML add procs failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_memheap_base_framework, MCA_BASE_OPEN_DEFAULT))) {
error = "mca_memheap_base_open() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_memheap_base_select())) {
error = "mca_select_base_select() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_atomic_base_framework, MCA_BASE_OPEN_DEFAULT))) {
error = "mca_atomic_base_open() failed";
goto error;
}
if (OSHMEM_SUCCESS
!= (ret =
mca_atomic_base_find_available(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "mca_atomic_base_find_available() failed";
goto error;
}
/* This call should be done after memheap initialization */
if (OSHMEM_SUCCESS != (ret = mca_scoll_enable())) {
error = "mca_scoll_enable() failed";
goto error;
}
if (OSHMEM_SUCCESS != shmem_lock_init()) {
error = "shmem_lock_init() failed";
goto error;
}
error:
if (ret != OSHMEM_SUCCESS) {
const char *err_msg = opal_strerror(ret);
orte_show_help("help-shmem-runtime.txt",
"shmem_init:startup:internal-failure",
true,
"SHMEM_INIT",
"SHMEM_INIT",
error,
err_msg,
ret);
return ret;
}
return ret;
}
static int parse_args(int argc, char *argv[])
{
int i, ret, len;
opal_cmd_line_t cmd_line;
char **app_env = NULL, **global_env = NULL;
opal_restart_globals.help = false;
opal_restart_globals.verbose = false;
opal_restart_globals.snapshot_ref = NULL;
opal_restart_globals.snapshot_loc = NULL;
opal_restart_globals.snapshot_metadata = NULL;
opal_restart_globals.snapshot_cache = NULL;
opal_restart_globals.snapshot_compress = NULL;
opal_restart_globals.snapshot_compress_postfix = NULL;
opal_restart_globals.output = 0;
/* Parse the command line options */
opal_cmd_line_create(&cmd_line, cmd_line_opts);
mca_base_open();
mca_base_cmd_line_setup(&cmd_line);
ret = opal_cmd_line_parse(&cmd_line, false, false, argc, argv);
if (OPAL_SUCCESS != ret) {
if (OPAL_ERR_SILENT != ret) {
fprintf(stderr, "%s: command line error (%s)\n", argv[0],
opal_strerror(ret));
}
return 1;
}
if (opal_restart_globals.help ) {
char *str, *args = NULL;
args = opal_cmd_line_get_usage_msg(&cmd_line);
str = opal_show_help_string("help-opal-restart.txt", "usage", true,
args);
if (NULL != str) {
printf("%s", str);
free(str);
}
free(args);
/* If we show the help message, that should be all we do */
exit(0);
}
/**
* Put all of the MCA arguments in the environment
*/
mca_base_cmd_line_process_args(&cmd_line, &app_env, &global_env);
len = opal_argv_count(app_env);
for(i = 0; i < len; ++i) {
putenv(app_env[i]);
}
len = opal_argv_count(global_env);
for(i = 0; i < len; ++i) {
putenv(global_env[i]);
}
/**
* Now start parsing our specific arguments
*/
/* get the remaining bits */
opal_cmd_line_get_tail(&cmd_line, &argc, &argv);
if ( NULL == opal_restart_globals.snapshot_ref ||
0 >= strlen(opal_restart_globals.snapshot_ref) ) {
opal_show_help("help-opal-restart.txt", "invalid_filename", true,
"<none provided>");
return OPAL_ERROR;
}
/* If we have arguments after the command, then assume they
* need to be grouped together.
* Useful in the 'mca crs self' instance.
*/
if(argc > 0) {
opal_restart_globals.snapshot_ref = strdup(opal_argv_join(argv, ' '));
}
return OPAL_SUCCESS;
}
static void
rml_oob_recv_route_callback(int status,
struct orte_process_name_t* peer,
struct iovec* iov,
int count,
orte_rml_tag_t tag,
void *cbdata)
{
orte_rml_oob_msg_header_t *hdr =
(orte_rml_oob_msg_header_t*) iov[0].iov_base;
int real_tag;
int ret;
orte_process_name_t next, origin;
struct iovec *new_iov;
/* BWB -- propogate errors here... */
assert(status >= 0);
ORTE_RML_OOB_MSG_HEADER_NTOH(*hdr);
origin = hdr->origin;
next = orte_routed.get_route(&hdr->destination);
if (next.vpid == ORTE_VPID_INVALID) {
opal_output(0, "%s:route_callback tried routing message from %s to %s:%d, can't find route",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&origin),
ORTE_NAME_PRINT(&hdr->destination),
hdr->tag);
opal_backtrace_print(stderr);
orte_errmgr.abort(ORTE_ERROR_DEFAULT_EXIT_CODE, NULL);
return;
}
if (OPAL_EQUAL == orte_util_compare_name_fields(ORTE_NS_CMP_ALL, &next, ORTE_PROC_MY_NAME)) {
opal_output(0, "%s:route_callback trying to get message from %s to %s:%d, routing loop",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&origin),
ORTE_NAME_PRINT(&hdr->destination),
hdr->tag);
opal_backtrace_print(stderr);
orte_errmgr.abort(ORTE_ERROR_DEFAULT_EXIT_CODE, NULL);
}
if (OPAL_EQUAL == orte_util_compare_name_fields(ORTE_NS_CMP_ALL, &next, &hdr->destination)) {
real_tag = hdr->tag;
} else {
real_tag = ORTE_RML_TAG_RML_ROUTE;
}
OPAL_OUTPUT_VERBOSE((1, orte_rml_base_output,
"%s routing message from %s for %s to %s (tag: %d)",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&hdr->origin),
ORTE_NAME_PRINT(&hdr->destination),
ORTE_NAME_PRINT(&next),
hdr->tag));
ORTE_RML_OOB_MSG_HEADER_HTON(*hdr);
/* NTH: fix potential race condition. oob may modify iov before
the oob send completes */
new_iov = (struct iovec*) calloc (count, sizeof (struct iovec));
if (NULL == new_iov) {
opal_output (0, "%s:route_callback malloc error!", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
opal_backtrace_print(stderr);
orte_errmgr.abort(ORTE_ERROR_DEFAULT_EXIT_CODE, NULL);
}
memcpy (new_iov, iov, count * sizeof (struct iovec));
ret = orte_rml_oob_module.active_oob->oob_send_nb(&next,
&origin,
new_iov,
count,
real_tag,
0,
rml_oob_recv_route_send_callback,
NULL);
if (ORTE_SUCCESS != ret) {
if (ORTE_ERR_ADDRESSEE_UNKNOWN == ret) {
/* no route -- queue and hope we find a route */
orte_rml_oob_queued_msg_t *qmsg = OBJ_NEW(orte_rml_oob_queued_msg_t);
OPAL_OUTPUT_VERBOSE((1, orte_rml_base_output,
"%s: no OOB information for %s. Queuing for later.",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&next)));
ORTE_RML_OOB_MSG_HEADER_NTOH(*hdr);
qmsg->payload[0].iov_base = (IOVBASE_TYPE*) malloc(iov[0].iov_len);
if (NULL == qmsg->payload[0].iov_base) abort();
qmsg->payload[0].iov_len = iov[0].iov_len;
memcpy(qmsg->payload[0].iov_base, iov[0].iov_base, iov[0].iov_len);
OPAL_THREAD_LOCK(&orte_rml_oob_module.queued_lock);
opal_list_append(&orte_rml_oob_module.queued_routing_messages,
&qmsg->super);
if (1 == opal_list_get_size(&orte_rml_oob_module.queued_routing_messages)) {
opal_event_evtimer_add(orte_rml_oob_module.timer_event,
&orte_rml_oob_module.timeout);
}
OPAL_THREAD_UNLOCK(&orte_rml_oob_module.queued_lock);
} else {
opal_output(0,
"%s failed to send message to %s: %s (rc = %d)",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&next),
opal_strerror(ret),
ret);
orte_errmgr.abort(ORTE_ERROR_DEFAULT_EXIT_CODE, NULL);
}
}
}
int main(int argc, char *argv[])
{
int ret = 0;
opal_cmd_line_t *cmd_line = NULL;
char *rml_uri;
#if OPAL_ENABLE_FT_CR == 1
char * tmp_env_var = NULL;
#endif
/* init enough of opal to process cmd lines */
if (OPAL_SUCCESS != opal_init_util(&argc, &argv)) {
fprintf(stderr, "OPAL failed to initialize -- orted aborting\n");
exit(1);
}
/* setup to check common command line options that just report and die */
cmd_line = OBJ_NEW(opal_cmd_line_t);
opal_cmd_line_create(cmd_line, orte_server_cmd_line_opts);
mca_base_cmd_line_setup(cmd_line);
if (OPAL_SUCCESS != (ret = opal_cmd_line_parse(cmd_line, false, false,
argc, argv))) {
if (OPAL_ERR_SILENT != ret) {
fprintf(stderr, "%s: command line error (%s)\n", argv[0],
opal_strerror(ret));
}
return 1;
}
/* check for help request */
if (help) {
char *str, *args = NULL;
args = opal_cmd_line_get_usage_msg(cmd_line);
str = opal_show_help_string("help-orte-server.txt",
"orteserver:usage", false,
argv[0], args);
if (NULL != str) {
printf("%s", str);
free(str);
}
free(args);
/* If we show the help message, that should be all we do */
return 0;
}
/*
* Since this process can now handle MCA/GMCA parameters, make sure to
* process them.
*/
mca_base_cmd_line_process_args(cmd_line, &environ, &environ);
/* if debug is set, then set orte_debug_flag so that the data server
* code will output
*/
if (debug) {
putenv(OPAL_MCA_PREFIX"orte_debug=1");
}
/* detach from controlling terminal
* otherwise, remain attached so output can get to us
*/
if(debug == false &&
no_daemonize == false) {
opal_daemon_init(NULL);
}
#if OPAL_ENABLE_FT_CR == 1
/* Disable the checkpoint notification routine for this
* tool. As we will never need to checkpoint this tool.
* Note: This must happen before opal_init().
*/
opal_cr_set_enabled(false);
/* Select the none component, since we don't actually use a checkpointer */
(void) mca_base_var_env_name("crs", &tmp_env_var);
opal_setenv(tmp_env_var,
"none",
true, &environ);
free(tmp_env_var);
tmp_env_var = NULL;
/* Mark as a tool program */
(void) mca_base_var_env_name("opal_cr_is_tool", &tmp_env_var);
opal_setenv(tmp_env_var,
"1",
true, &environ);
free(tmp_env_var);
#endif
/* don't want session directories */
orte_create_session_dirs = false;
/* Perform the standard init, but flag that we are an HNP */
if (ORTE_SUCCESS != (ret = orte_init(&argc, &argv, ORTE_PROC_HNP))) {
fprintf(stderr, "orte-server: failed to initialize -- aborting\n");
exit(1);
}
/* report out our URI, if we were requested to do so, using syntax
* proposed in an email thread by Jeff Squyres
*/
if (NULL != report_uri) {
orte_oob_base_get_addr(&rml_uri);
if (0 == strcmp(report_uri, "-")) {
/* if '-', then output to stdout */
printf("%s\n", rml_uri);
} else if (0 == strcmp(report_uri, "+")) {
/* if '+', output to stderr */
fprintf(stderr, "%s\n", rml_uri);
} else {
/* treat it as a filename and output into it */
FILE *fp;
fp = fopen(report_uri, "w");
if (NULL == fp) {
fprintf(stderr, "orte-server: failed to open designated file %s -- aborting\n", report_uri);
orte_finalize();
exit(1);
}
fprintf(fp, "%s\n", rml_uri);
fclose(fp);
}
free(rml_uri);
}
/* setup the data server to listen for commands */
if (ORTE_SUCCESS != (ret = orte_data_server_init())) {
fprintf(stderr, "orte-server: failed to start data server -- aborting\n");
orte_finalize();
exit(1);
}
/* setup to listen for commands sent specifically to me */
orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, ORTE_RML_TAG_DAEMON,
ORTE_RML_NON_PERSISTENT, orte_daemon_recv, NULL);
/* Set signal handlers to catch kill signals so we can properly clean up
* after ourselves.
*/
opal_event_set(orte_event_base, &term_handler, SIGTERM, OPAL_EV_SIGNAL,
shutdown_callback, NULL);
opal_event_add(&term_handler, NULL);
opal_event_set(orte_event_base, &int_handler, SIGINT, OPAL_EV_SIGNAL,
shutdown_callback, NULL);
opal_event_add(&int_handler, NULL);
/* We actually do *not* want the server to voluntarily yield() the
processor more than necessary. The server already blocks when
it is doing nothing, so it doesn't use any more CPU cycles than
it should; but when it *is* doing something, we do not want it
to be unnecessarily delayed because it voluntarily yielded the
processor in the middle of its work.
For example: when a message arrives at the server, we want the
OS to wake up the server in a timely fashion (which most OS's
seem good about doing) and then we want the server to process
the message as fast as possible. If the server yields and lets
aggressive MPI applications get the processor back, it may be a
long time before the OS schedules the server to run again
(particularly if there is no IO event to wake it up). Hence,
publish and lookup (for example) may be significantly delayed
before being delivered to MPI processes, which can be
problematic in some scenarios (e.g., COMM_SPAWN). */
opal_progress_set_yield_when_idle(false);
/* Change the default behavior of libevent such that we want to
continually block rather than blocking for the default timeout
and then looping around the progress engine again. There
should be nothing in the server that cannot block in libevent
until "something" happens (i.e., there's no need to keep
cycling through progress because the only things that should
happen will happen in libevent). This is a minor optimization,
but what the heck... :-) */
opal_progress_set_event_flag(OPAL_EVLOOP_ONCE);
if (debug) {
opal_output(0, "%s orte-server: up and running!", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
}
/* wait to hear we are done */
while (orte_event_base_active) {
opal_event_loop(orte_event_base, OPAL_EVLOOP_ONCE);
}
ORTE_ACQUIRE_OBJECT(orte_event_base_active);
/* should never get here, but if we do... */
/* Finalize and clean up ourselves */
orte_finalize();
return orte_exit_status;
}
/*
* Create an ompi_btl_usnic_proc_t and initialize it with modex info
* and an empty array of endpoints.
*/
static ompi_btl_usnic_proc_t *create_proc(ompi_proc_t *ompi_proc)
{
ompi_btl_usnic_proc_t *proc = NULL;
size_t size;
int rc;
/* Create the proc if it doesn't already exist */
proc = OBJ_NEW(ompi_btl_usnic_proc_t);
if (NULL == proc) {
return NULL;
}
/* Initialize number of peers */
proc->proc_endpoint_count = 0;
proc->proc_ompi = ompi_proc;
/* query for the peer address info */
rc = ompi_modex_recv(&mca_btl_usnic_component.super.btl_version,
ompi_proc, (void*)&proc->proc_modex,
&size);
if (OMPI_SUCCESS != rc) {
opal_show_help("help-mpi-btl-usnic.txt", "internal error during init",
true,
ompi_process_info.nodename,
"<none>", 0,
"ompi_modex_recv() failed", __FILE__, __LINE__,
opal_strerror(rc));
OBJ_RELEASE(proc);
return NULL;
}
if ((size % sizeof(ompi_btl_usnic_addr_t)) != 0) {
char msg[1024];
snprintf(msg, sizeof(msg),
"sizeof(modex for peer %s data) == %d, expected multiple of %d",
OMPI_NAME_PRINT(&ompi_proc->proc_name),
(int) size, (int) sizeof(ompi_btl_usnic_addr_t));
opal_show_help("help-mpi-btl-usnic.txt", "internal error during init",
true,
ompi_process_info.nodename,
"<none>", 0,
"invalid modex data", __FILE__, __LINE__,
msg);
OBJ_RELEASE(proc);
return NULL;
}
proc->proc_modex_count = size / sizeof(ompi_btl_usnic_addr_t);
if (0 == proc->proc_modex_count) {
proc->proc_endpoints = NULL;
OBJ_RELEASE(proc);
return NULL;
}
proc->proc_modex_claimed = (bool*)
calloc(proc->proc_modex_count, sizeof(bool));
if (NULL == proc->proc_modex_claimed) {
OMPI_ERROR_LOG(OMPI_ERR_OUT_OF_RESOURCE);
OBJ_RELEASE(proc);
return NULL;
}
proc->proc_endpoints = (mca_btl_base_endpoint_t**)
calloc(proc->proc_modex_count, sizeof(mca_btl_base_endpoint_t*));
if (NULL == proc->proc_endpoints) {
OMPI_ERROR_LOG(OMPI_ERR_OUT_OF_RESOURCE);
OBJ_RELEASE(proc);
return NULL;
}
return proc;
}
int
main(int argc, char *argv[])
{
int exit_status = 0, ret, flags = 0, i;
int exec_argc = 0, user_argc = 0;
char **exec_argv = NULL, **user_argv = NULL;
char *exec_command, *base_argv0 = NULL;
bool disable_flags = true;
bool real_flag = false;
if (OPAL_SUCCESS != (ret = opal_init_util(&argc, &argv))) {
return ret;
}
/****************************************************
*
* Setup compiler information
*
****************************************************/
base_argv0 = opal_basename(argv[0]);
#if defined(EXEEXT)
if( 0 != strlen(EXEEXT) ) {
char extension[] = EXEEXT;
char* temp = strstr( base_argv0, extension );
char* old_match = temp;
while( NULL != temp ) {
old_match = temp;
temp = strstr( temp + 1, extension );
}
/* Only if there was a match of .exe, erase the last occurence of .exe */
if ( NULL != old_match ) {
*old_match = '\0';
}
}
#endif /* defined(EXEEXT) */
if (OPAL_SUCCESS != (ret = data_init(base_argv0))) {
fprintf(stderr, "Error parsing data file %s: %s\n", base_argv0, opal_strerror(ret));
return ret;
}
for (i = 1 ; i < argc && user_data_idx < 0 ; ++i) {
user_data_idx = find_options_index(argv[i]);
}
/* if we didn't find a match, look for the NULL (base case) options */
if (user_data_idx < 0) {
user_data_idx = default_data_idx;
}
/* if we still didn't find a match, abort */
if (user_data_idx < 0) {
char *flat = opal_argv_join(argv, ' ');
opal_show_help("help-opal-wrapper.txt", "no-options-support", true,
base_argv0, flat, NULL);
free(flat);
exit(1);
}
/* compiler */
load_env_data(options_data[user_data_idx].project_short, options_data[user_data_idx].compiler_env, &options_data[user_data_idx].compiler);
/* preprocessor flags */
load_env_data_argv(options_data[user_data_idx].project_short, "CPPFLAGS", &options_data[user_data_idx].preproc_flags);
/* compiler flags */
load_env_data_argv(options_data[user_data_idx].project_short, options_data[user_data_idx].compiler_flags_env,
&options_data[user_data_idx].comp_flags);
/* linker flags */
load_env_data_argv(options_data[user_data_idx].project_short, "LDFLAGS", &options_data[user_data_idx].link_flags);
/* libs */
load_env_data_argv(options_data[user_data_idx].project_short, "LIBS", &options_data[user_data_idx].libs);
/****************************************************
*
* Sanity Checks
*
****************************************************/
if (NULL != options_data[user_data_idx].req_file) {
/* make sure the language is supported */
if (0 == strcmp(options_data[user_data_idx].req_file, "not supported")) {
opal_show_help("help-opal-wrapper.txt", "no-language-support", true,
options_data[user_data_idx].language, base_argv0, NULL);
exit_status = 1;
goto cleanup;
}
if (options_data[user_data_idx].req_file[0] != '\0') {
char *filename;
struct stat buf;
filename = opal_os_path( false, options_data[user_data_idx].path_libdir, options_data[user_data_idx].req_file, NULL );
if (0 != stat(filename, &buf)) {
opal_show_help("help-opal-wrapper.txt", "file-not-found", true,
base_argv0, options_data[user_data_idx].req_file, options_data[user_data_idx].language, NULL);
}
}
}
/****************************************************
*
* Parse user flags
*
****************************************************/
flags = COMP_WANT_COMMAND|COMP_WANT_PREPROC|
COMP_WANT_COMPILE|COMP_WANT_LINK;
user_argv = opal_argv_copy(argv + 1);
user_argc = opal_argv_count(user_argv);
for (i = 0 ; i < user_argc ; ++i) {
if (0 == strncmp(user_argv[i], "-showme", strlen("-showme")) ||
0 == strncmp(user_argv[i], "--showme", strlen("--showme")) ||
0 == strncmp(user_argv[i], "-show", strlen("-show")) ||
0 == strncmp(user_argv[i], "--show", strlen("--show"))) {
bool done_now = false;
/* check for specific things we want to see. First three
still invoke all the building routines. Last set want
to parse out certain flags, so we don't go through the
normal build routine - skip to cleanup. */
if (0 == strncmp(user_argv[i], "-showme:command", strlen("-showme:command")) ||
0 == strncmp(user_argv[i], "--showme:command", strlen("--showme:command"))) {
flags = COMP_WANT_COMMAND;
/* we know what we want, so don't process any more args */
done_now = true;
} else if (0 == strncmp(user_argv[i], "-showme:compile", strlen("-showme:compile")) ||
0 == strncmp(user_argv[i], "--showme:compile", strlen("--showme:compile"))) {
flags = COMP_WANT_PREPROC|COMP_WANT_COMPILE;
/* we know what we want, so don't process any more args */
done_now = true;
} else if (0 == strncmp(user_argv[i], "-showme:link", strlen("-showme:link")) ||
0 == strncmp(user_argv[i], "--showme:link", strlen("--showme:link"))) {
flags = COMP_WANT_COMPILE|COMP_WANT_LINK;
/* we know what we want, so don't process any more args */
done_now = true;
} else if (0 == strncmp(user_argv[i], "-showme:incdirs", strlen("-showme:incdirs")) ||
0 == strncmp(user_argv[i], "--showme:incdirs", strlen("--showme:incdirs"))) {
print_flags(options_data[user_data_idx].preproc_flags, OPAL_INCLUDE_FLAG);
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:libdirs", strlen("-showme:libdirs")) ||
0 == strncmp(user_argv[i], "--showme:libdirs", strlen("--showme:libdirs"))) {
print_flags(options_data[user_data_idx].link_flags, OPAL_LIBDIR_FLAG);
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:libs", strlen("-showme:libs")) ||
0 == strncmp(user_argv[i], "--showme:libs", strlen("--showme:libs"))) {
print_flags(options_data[user_data_idx].libs, "-l");
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:version", strlen("-showme:version")) ||
0 == strncmp(user_argv[i], "--showme:version", strlen("--showme:version"))) {
char * str;
str = opal_show_help_string("help-opal-wrapper.txt",
"version", false,
argv[0], options_data[user_data_idx].project, options_data[user_data_idx].version, options_data[user_data_idx].language, NULL);
if (NULL != str) {
printf("%s", str);
free(str);
}
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:help", strlen("-showme:help")) ||
0 == strncmp(user_argv[i], "--showme:help", strlen("--showme:help"))) {
char *str;
str = opal_show_help_string("help-opal-wrapper.txt", "usage",
false, argv[0],
options_data[user_data_idx].project,
NULL);
if (NULL != str) {
printf("%s", str);
free(str);
}
exit_status = 0;
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:", strlen("-showme:")) ||
0 == strncmp(user_argv[i], "--showme:", strlen("--showme:"))) {
fprintf(stderr, "%s: unrecognized option: %s\n", argv[0],
user_argv[i]);
fprintf(stderr, "Type '%s --showme:help' for usage.\n",
argv[0]);
exit_status = 1;
goto cleanup;
}
flags |= (COMP_DRY_RUN|COMP_SHOW_ERROR);
/* remove element from user_argv */
opal_argv_delete(&user_argc, &user_argv, i, 1);
--i;
if (done_now) {
disable_flags = false;
break;
}
} else if (0 == strcmp(user_argv[i], "-c")) {
flags &= ~COMP_WANT_LINK;
real_flag = true;
} else if (0 == strcmp(user_argv[i], "-E") ||
0 == strcmp(user_argv[i], "-M")) {
flags &= ~(COMP_WANT_COMPILE | COMP_WANT_LINK);
real_flag = true;
} else if (0 == strcmp(user_argv[i], "-S")) {
flags &= ~COMP_WANT_LINK;
real_flag = true;
} else if (0 == strcmp(user_argv[i], "-lpmpi")) {
flags |= COMP_WANT_PMPI;
/* remove element from user_argv */
opal_argv_delete(&user_argc, &user_argv, i, 1);
--i;
} else if (0 == strcmp(user_argv[i], "-static") ||
0 == strcmp(user_argv[i], "--static") ||
0 == strcmp(user_argv[i], "-Bstatic") ||
0 == strcmp(user_argv[i], "-Wl,-static") ||
0 == strcmp(user_argv[i], "-Wl,--static") ||
0 == strcmp(user_argv[i], "-Wl,-Bstatic")) {
flags |= COMP_WANT_STATIC;
} else if (0 == strcmp(user_argv[i], "-dynamic") ||
0 == strcmp(user_argv[i], "--dynamic") ||
0 == strcmp(user_argv[i], "-Bdynamic") ||
0 == strcmp(user_argv[i], "-Wl,-dynamic") ||
0 == strcmp(user_argv[i], "-Wl,--dynamic") ||
0 == strcmp(user_argv[i], "-Wl,-Bdynamic")) {
flags &= ~COMP_WANT_STATIC;
} else if (0 == strcmp(user_argv[i], "--openmpi:linkall")) {
/* This is an intentionally undocummented wrapper compiler
switch. It should only be used by Open MPI developers
-- not end users. It will cause mpicc to use the
static library list, even if we're compiling
dynamically (i.e., it'll specifically -lopen-rte and
-lopen-pal (and all their dependent libs)). We provide
this flag for test MPI applications that also invoke
ORTE and/or OPAL function calls.
On some systems (e.g., OS X), if the top-level
application calls ORTE/OPAL functions and you don't -l
ORTE and OPAL, then the functions won't be resolved at
link time (i.e., the implicit library dependencies of
libmpi won't be pulled in at link time), and therefore
the link will fail. This flag will cause the wrapper
to explicitly list the ORTE and OPAL libs on the
underlying compiler command line, so the application
will therefore link properly. */
flags |= COMP_WANT_LINKALL;
/* remove element from user_argv */
opal_argv_delete(&user_argc, &user_argv, i, 1);
} else if ('-' != user_argv[i][0]) {
disable_flags = false;
flags |= COMP_SHOW_ERROR;
real_flag = true;
} else {
/* if the option flag is one that we use to determine
which set of compiler data to use, don't count it as a
real option */
if (find_options_index(user_argv[i]) < 0) {
real_flag = true;
}
}
}
/* clear out the want_flags if we got no arguments not starting
with a - (dash) and -showme wasn't given OR -showme was given
and we had at least one more non-showme argument that started
with a - (dash) and no other non-dash arguments. Some examples:
opal_wrapper : clear our flags
opal_wrapper -v : clear our flags
opal_wrapper -E a.c : don't clear our flags
opal_wrapper a.c : don't clear our flags
opal_wrapper -showme : don't clear our flags
opal_wrapper -showme -v : clear our flags
opal_wrapper -showme -E a.c : don't clear our flags
opal_wrapper -showme a.c : don't clear our flags
*/
if (disable_flags && !((flags & COMP_DRY_RUN) && !real_flag)) {
flags &= ~(COMP_WANT_PREPROC|COMP_WANT_COMPILE|COMP_WANT_LINK);
}
/****************************************************
*
* Assemble the command line
*
****************************************************/
/* compiler (may be multiple arguments, so split) */
if (flags & COMP_WANT_COMMAND) {
exec_argv = opal_argv_split(options_data[user_data_idx].compiler, ' ');
exec_argc = opal_argv_count(exec_argv);
} else {
exec_argv = (char **) malloc(sizeof(char*));
exec_argv[0] = NULL;
exec_argc = 0;
}
/* This error would normally not happen unless the user edits the
wrapper data files manually */
if (NULL == exec_argv) {
opal_show_help("help-opal-wrapper.txt", "no-compiler-specified", true);
return 1;
}
if (flags & COMP_WANT_COMPILE) {
opal_argv_insert(&exec_argv, exec_argc,
options_data[user_data_idx].comp_flags_prefix);
exec_argc = opal_argv_count(exec_argv);
}
/* Per https://svn.open-mpi.org/trac/ompi/ticket/2201, add all the
user arguments before anything else. */
opal_argv_insert(&exec_argv, exec_argc, user_argv);
exec_argc = opal_argv_count(exec_argv);
/* preproc flags */
if (flags & COMP_WANT_PREPROC) {
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].preproc_flags);
exec_argc = opal_argv_count(exec_argv);
}
/* compiler flags */
if (flags & COMP_WANT_COMPILE) {
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].comp_flags);
exec_argc = opal_argv_count(exec_argv);
}
/* link flags and libs */
if (flags & COMP_WANT_LINK) {
bool have_static_lib;
bool have_dyn_lib;
bool use_static_libs;
char *filename1, *filename2;
struct stat buf;
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].link_flags);
exec_argc = opal_argv_count(exec_argv);
/* Are we linking statically? If so, decide what libraries to
list. It depends on two factors:
1. Was --static (etc.) specified?
2. Does OMPI have static, dynamic, or both libraries installed?
Here's a matrix showing what we'll do in all 6 cases:
What's installed --static no --static
---------------- ---------- -----------
ompi .so libs -lmpi -lmpi
ompi .a libs all all
ompi both libs all -lmpi
*/
filename1 = opal_os_path( false, options_data[user_data_idx].path_libdir, options_data[user_data_idx].static_lib_file, NULL );
if (0 == stat(filename1, &buf)) {
have_static_lib = true;
} else {
have_static_lib = false;
}
filename2 = opal_os_path( false, options_data[user_data_idx].path_libdir, options_data[user_data_idx].dyn_lib_file, NULL );
if (0 == stat(filename2, &buf)) {
have_dyn_lib = true;
} else {
have_dyn_lib = false;
}
/* Determine which set of libs to use: dynamic or static. Be
pedantic to make the code easy to read. */
if (flags & COMP_WANT_LINKALL) {
/* If --openmpi:linkall was specified, list all the libs
(i.e., the static libs) if they're available, either in
static or dynamic form. */
if (have_static_lib || have_dyn_lib) {
use_static_libs = true;
} else {
fprintf(stderr, "The linkall option has failed as we were unable to find either static or dynamic libs\n"
"Files looked for:\n Static: %s\n Dynamic: %s\n",
filename1, filename2);
free(filename1);
free(filename2);
exit(1);
}
} else if (flags & COMP_WANT_STATIC) {
/* If --static (or something like it) was specified, if we
have the static libs, then use them. Otherwise, use
the dynamic libs. */
if (have_static_lib) {
use_static_libs = true;
} else {
use_static_libs = false;
}
} else {
/* If --static (or something like it) was NOT specified
(or if --dyanic, or something like it, was specified),
if we have the dynamic libs, then use them. Otherwise,
use the static libs. */
if (have_dyn_lib) {
use_static_libs = false;
} else {
use_static_libs = true;
}
}
free(filename1);
free(filename2);
if (use_static_libs) {
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].libs_static);
} else {
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].libs);
}
exec_argc = opal_argv_count(exec_argv);
}
/****************************************************
*
* Execute the command
*
****************************************************/
if (flags & COMP_DRY_RUN) {
exec_command = opal_argv_join(exec_argv, ' ');
printf("%s\n", exec_command);
} else {
char *tmp;
#if 0
exec_command = opal_argv_join(exec_argv, ' ');
printf("command: %s\n", exec_command);
#endif
tmp = opal_path_findv(exec_argv[0], 0, environ, NULL);
if (NULL == tmp) {
opal_show_help("help-opal-wrapper.txt", "no-compiler-found", true,
exec_argv[0], NULL);
errno = 0;
exit_status = 1;
} else {
int status;
free(exec_argv[0]);
exec_argv[0] = tmp;
ret = opal_few(exec_argv, &status);
exit_status = WIFEXITED(status) ? WEXITSTATUS(status) :
(WIFSIGNALED(status) ? WTERMSIG(status) :
(WIFSTOPPED(status) ? WSTOPSIG(status) : 255));
if( (OPAL_SUCCESS != ret) || ((0 != exit_status) && (flags & COMP_SHOW_ERROR)) ) {
char* exec_command = opal_argv_join(exec_argv, ' ');
if( OPAL_SUCCESS != ret ) {
opal_show_help("help-opal-wrapper.txt", "spawn-failed", true,
exec_argv[0], strerror(status), exec_command, NULL);
} else {
#if 0
opal_show_help("help-opal-wrapper.txt", "compiler-failed", true,
exec_argv[0], exit_status, exec_command, NULL);
#endif
}
free(exec_command);
}
}
}
/****************************************************
*
* Cleanup
*
****************************************************/
cleanup:
opal_argv_free(exec_argv);
opal_argv_free(user_argv);
if (NULL != base_argv0) free(base_argv0);
if (OPAL_SUCCESS != (ret = data_finalize())) {
return ret;
}
if (OPAL_SUCCESS != (ret = opal_finalize_util())) {
return ret;
}
return exit_status;
}
static int parse_args(int argc, char *argv[]) {
int i, ret, len;
opal_cmd_line_t cmd_line;
char **app_env = NULL, **global_env = NULL;
char * tmp_env_var = NULL;
char *argv0 = NULL;
memset(&opal_checkpoint_globals, 0, sizeof(opal_checkpoint_globals_t));
opal_checkpoint_globals.snapshot_name = NULL;
opal_checkpoint_globals.snapshot_loc = NULL;
/* Parse the command line options */
opal_cmd_line_create(&cmd_line, cmd_line_opts);
mca_base_open();
mca_base_cmd_line_setup(&cmd_line);
ret = opal_cmd_line_parse(&cmd_line, true, false, argc, argv);
if (OPAL_SUCCESS != ret) {
if (OPAL_ERR_SILENT != ret) {
fprintf(stderr, "%s: command line error (%s)\n", argv[0],
opal_strerror(ret));
}
return 1;
}
if (opal_checkpoint_globals.help) {
char *str, *args = NULL;
args = opal_cmd_line_get_usage_msg(&cmd_line);
str = opal_show_help_string("help-opal-checkpoint.txt", "usage", true,
args);
if (NULL != str) {
printf("%s", str);
free(str);
}
free(args);
/* If we show the help message, that should be all we do */
exit(0);
}
/**
* Put all of the MCA arguments in the environment
*/
mca_base_cmd_line_process_args(&cmd_line, &app_env, &global_env);
len = opal_argv_count(app_env);
for(i = 0; i < len; ++i) {
putenv(app_env[i]);
}
len = opal_argv_count(global_env);
for(i = 0; i < len; ++i) {
putenv(global_env[i]);
}
(void) mca_base_var_env_name("opal_cr_is_tool", &tmp_env_var);
opal_setenv(tmp_env_var,
"1",
true, &environ);
free(tmp_env_var);
tmp_env_var = NULL;
/**
* Now start parsing our specific arguments
*/
if( NULL == opal_checkpoint_globals.snapshot_name )
opal_checkpoint_globals.snapshot_name = strdup("");
if( NULL == opal_checkpoint_globals.snapshot_loc ) {
opal_checkpoint_globals.snapshot_loc = strdup("");
}
/* get the remaining bits */
argv0 = strdup(argv[0]);
opal_cmd_line_get_tail(&cmd_line, &argc, &argv);
if (0 == argc) {
fprintf(stderr, "%s: Nothing to do\n", argv0);
fprintf(stderr, "Type '%s --help' for usage.\n", argv0);
free(argv0);
return OPAL_ERROR;
}
free(argv0);
opal_checkpoint_globals.pid = atoi(argv[0]);
if ( 0 >= opal_checkpoint_globals.pid ) {
opal_show_help("help-opal-checkpoint.txt", "invalid_pid", true,
opal_checkpoint_globals.pid);
return OPAL_ERROR;
}
return OPAL_SUCCESS;
}
int main(int argc, char *argv[])
{
int rc, i, j;
opal_cmd_line_t cmd_line;
char *param, *value;
orte_job_t *jdata=NULL;
orte_app_context_t *app;
char *uri, *ptr;
/* Setup and parse the command line */
memset(&myglobals, 0, sizeof(myglobals));
/* find our basename (the name of the executable) so that we can
use it in pretty-print error messages */
myglobals.basename = opal_basename(argv[0]);
opal_cmd_line_create(&cmd_line, cmd_line_init);
mca_base_cmd_line_setup(&cmd_line);
if (OPAL_SUCCESS != (rc = opal_cmd_line_parse(&cmd_line, true,
argc, argv)) ) {
if (OPAL_ERR_SILENT != rc) {
fprintf(stderr, "%s: command line error (%s)\n", argv[0],
opal_strerror(rc));
}
return rc;
}
/* print version if requested. Do this before check for help so
that --version --help works as one might expect. */
if (myglobals.version) {
char *str;
str = opal_info_make_version_str("all",
OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
OPAL_RELEASE_VERSION,
OPAL_GREEK_VERSION,
OPAL_REPO_REV);
if (NULL != str) {
fprintf(stdout, "%s %s\n\nReport bugs to %s\n",
myglobals.basename, str, PACKAGE_BUGREPORT);
free(str);
}
exit(0);
}
/* check if we are running as root - if we are, then only allow
* us to proceed if the allow-run-as-root flag was given. Otherwise,
* exit with a giant warning flag
*/
if (0 == geteuid() && !myglobals.run_as_root) {
fprintf(stderr, "--------------------------------------------------------------------------\n");
if (myglobals.help) {
fprintf(stderr, "%s cannot provide the help message when run as root\n", myglobals.basename);
} else {
/* show_help is not yet available, so print an error manually */
fprintf(stderr, "%s has detected an attempt to run as root.\n", myglobals.basename);
}
fprintf(stderr, " This is *strongly* discouraged as any mistake (e.g., in defining TMPDIR) or bug can\n");
fprintf(stderr, "result in catastrophic damage to the OS file system, leaving\n");
fprintf(stderr, "your system in an unusable state.\n\n");
fprintf(stderr, "You can override this protection by adding the --allow-run-as-root\n");
fprintf(stderr, "option to your cmd line. However, we reiterate our strong advice\n");
fprintf(stderr, "against doing so - please do so at your own risk.\n");
fprintf(stderr, "--------------------------------------------------------------------------\n");
exit(1);
}
/*
* Since this process can now handle MCA/GMCA parameters, make sure to
* process them.
* NOTE: It is "safe" to call mca_base_cmd_line_process_args() before
* opal_init_util() since mca_base_cmd_line_process_args() does *not*
* depend upon opal_init_util() functionality.
*/
if (OPAL_SUCCESS != mca_base_cmd_line_process_args(&cmd_line, &environ, &environ)) {
exit(1);
}
/* Need to initialize OPAL so that install_dirs are filled in */
if (OPAL_SUCCESS != opal_init(&argc, &argv)) {
exit(1);
}
/* Check for help request */
if (myglobals.help) {
char *str, *args = NULL;
char *project_name = NULL;
if (0 == strcmp(myglobals.basename, "mpirun")) {
project_name = "Open MPI";
} else {
project_name = "OpenRTE";
}
args = opal_cmd_line_get_usage_msg(&cmd_line);
str = opal_show_help_string("help-orterun.txt", "orterun:usage", false,
myglobals.basename, project_name, OPAL_VERSION,
myglobals.basename, args,
PACKAGE_BUGREPORT);
if (NULL != str) {
printf("%s", str);
free(str);
}
free(args);
/* If someone asks for help, that should be all we do */
exit(0);
}
/* flag that I am the HNP */
orte_process_info.proc_type = ORTE_PROC_HNP;
/* Setup MCA params */
orte_register_params();
/* specify the DVM state machine */
opal_setenv("OMPI_MCA_state", "dvm", true, &environ);
/* Intialize our Open RTE environment */
if (ORTE_SUCCESS != (rc = orte_init(&argc, &argv, ORTE_PROC_HNP))) {
/* cannot call ORTE_ERROR_LOG as it could be the errmgr
* never got loaded!
*/
return rc;
}
/* finalize OPAL. As it was opened again from orte_init->opal_init
* we continue to have a reference count on it. So we have to finalize it twice...
*/
opal_finalize();
/* check for request to report uri */
uri = orte_rml.get_contact_info();
if (NULL != myglobals.report_uri) {
FILE *fp;
if (0 == strcmp(myglobals.report_uri, "-")) {
/* if '-', then output to stdout */
printf("VMURI: %s\n", uri);
} else if (0 == strcmp(myglobals.report_uri, "+")) {
/* if '+', output to stderr */
fprintf(stderr, "VMURI: %s\n", uri);
} else if (0 == strncasecmp(myglobals.report_uri, "file:", strlen("file:"))) {
ptr = strchr(myglobals.report_uri, ':');
++ptr;
fp = fopen(ptr, "w");
if (NULL == fp) {
orte_show_help("help-orterun.txt", "orterun:write_file", false,
myglobals.basename, "pid", ptr);
exit(0);
}
fprintf(fp, "%s\n", uri);
fclose(fp);
} else {
fp = fopen(myglobals.report_uri, "w");
if (NULL == fp) {
orte_show_help("help-orterun.txt", "orterun:write_file", false,
myglobals.basename, "pid", myglobals.report_uri);
exit(0);
}
fprintf(fp, "%s\n", uri);
fclose(fp);
}
free(uri);
} else {
printf("VMURI: %s\n", uri);
}
/* get the daemon job object - was created by ess/hnp component */
if (NULL == (jdata = orte_get_job_data_object(ORTE_PROC_MY_NAME->jobid))) {
orte_show_help("help-orterun.txt", "bad-job-object", true,
myglobals.basename);
exit(0);
}
/* also should have created a daemon "app" */
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, 0))) {
orte_show_help("help-orterun.txt", "bad-app-object", true,
myglobals.basename);
exit(0);
}
/* Did the user specify a prefix, or want prefix by default? */
if (opal_cmd_line_is_taken(&cmd_line, "prefix") || want_prefix_by_default) {
size_t param_len;
/* if both the prefix was given and we have a prefix
* given above, check to see if they match
*/
if (opal_cmd_line_is_taken(&cmd_line, "prefix") &&
NULL != myglobals.prefix) {
/* if they don't match, then that merits a warning */
param = strdup(opal_cmd_line_get_param(&cmd_line, "prefix", 0, 0));
/* ensure we strip any trailing '/' */
if (0 == strcmp(OPAL_PATH_SEP, &(param[strlen(param)-1]))) {
param[strlen(param)-1] = '\0';
}
value = strdup(myglobals.prefix);
if (0 == strcmp(OPAL_PATH_SEP, &(value[strlen(value)-1]))) {
value[strlen(value)-1] = '\0';
}
if (0 != strcmp(param, value)) {
orte_show_help("help-orterun.txt", "orterun:app-prefix-conflict",
true, myglobals.basename, value, param);
/* let the global-level prefix take precedence since we
* know that one is being used
*/
free(param);
param = strdup(myglobals.prefix);
}
free(value);
} else if (NULL != myglobals.prefix) {
param = myglobals.prefix;
} else if (opal_cmd_line_is_taken(&cmd_line, "prefix")){
/* must be --prefix alone */
param = strdup(opal_cmd_line_get_param(&cmd_line, "prefix", 0, 0));
} else {
/* --enable-orterun-prefix-default was given to orterun */
param = strdup(opal_install_dirs.prefix);
}
if (NULL != param) {
/* "Parse" the param, aka remove superfluous path_sep. */
param_len = strlen(param);
while (0 == strcmp (OPAL_PATH_SEP, &(param[param_len-1]))) {
param[param_len-1] = '\0';
param_len--;
if (0 == param_len) {
orte_show_help("help-orterun.txt", "orterun:empty-prefix",
true, myglobals.basename, myglobals.basename);
return ORTE_ERR_FATAL;
}
}
orte_set_attribute(&app->attributes, ORTE_APP_PREFIX_DIR, ORTE_ATTR_GLOBAL, param, OPAL_STRING);
free(param);
}
}
/* Did the user specify a hostfile. Need to check for both
* hostfile and machine file.
* We can only deal with one hostfile per app context, otherwise give an error.
*/
if (0 < (j = opal_cmd_line_get_ninsts(&cmd_line, "hostfile"))) {
if(1 < j) {
orte_show_help("help-orterun.txt", "orterun:multiple-hostfiles",
true, myglobals.basename, NULL);
return ORTE_ERR_FATAL;
} else {
value = opal_cmd_line_get_param(&cmd_line, "hostfile", 0, 0);
orte_set_attribute(&app->attributes, ORTE_APP_HOSTFILE, ORTE_ATTR_LOCAL, value, OPAL_STRING);
}
}
if (0 < (j = opal_cmd_line_get_ninsts(&cmd_line, "machinefile"))) {
if(1 < j || orte_get_attribute(&app->attributes, ORTE_APP_HOSTFILE, NULL, OPAL_STRING)) {
orte_show_help("help-orterun.txt", "orterun:multiple-hostfiles",
true, myglobals.basename, NULL);
return ORTE_ERR_FATAL;
} else {
value = opal_cmd_line_get_param(&cmd_line, "machinefile", 0, 0);
orte_set_attribute(&app->attributes, ORTE_APP_HOSTFILE, ORTE_ATTR_LOCAL, value, OPAL_STRING);
}
}
/* Did the user specify any hosts? */
if (0 < (j = opal_cmd_line_get_ninsts(&cmd_line, "host"))) {
char **targ=NULL, *tval;
for (i = 0; i < j; ++i) {
value = opal_cmd_line_get_param(&cmd_line, "host", i, 0);
opal_argv_append_nosize(&targ, value);
}
tval = opal_argv_join(targ, ',');
orte_set_attribute(&app->attributes, ORTE_APP_DASH_HOST, ORTE_ATTR_LOCAL, tval, OPAL_STRING);
opal_argv_free(targ);
free(tval);
}
OBJ_DESTRUCT(&cmd_line);
/* setup to listen for commands sent specifically to me, even though I would probably
* be the one sending them! Unfortunately, since I am a participating daemon,
* there are times I need to send a command to "all daemons", and that means *I* have
* to receive it too
*/
orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, ORTE_RML_TAG_DAEMON,
ORTE_RML_PERSISTENT, orte_daemon_recv, NULL);
/* override the notify_completed state so we can send a message
* back to anyone who submits a job to us telling them the job
* completed */
if (ORTE_SUCCESS != (rc = orte_state.set_job_state_callback(ORTE_JOB_STATE_NOTIFY_COMPLETED, notify_requestor))) {
ORTE_ERROR_LOG(rc);
ORTE_UPDATE_EXIT_STATUS(rc);
exit(orte_exit_status);
}
/* spawn the DVM - we skip the initial steps as this
* isn't a user-level application */
ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_ALLOCATE);
/* loop the event lib until an exit event is detected */
while (orte_event_base_active) {
opal_event_loop(orte_event_base, OPAL_EVLOOP_ONCE);
}
/* cleanup and leave */
orte_finalize();
if (orte_debug_flag) {
fprintf(stderr, "exiting with status %d\n", orte_exit_status);
}
exit(orte_exit_status);
}
/*
* Back-end function to actually tell the child to fork the command
*/
static int send_command(orcm_notifier_base_severity_t severity, int errcode,
char *msg)
{
/* csel = Command, Severity, Errcode, string Length */
int rc, csel[4];
char *errmsg = NULL;
csel[0] = CMD_EXEC;
csel[1] = severity;
csel[2] = errcode;
csel[3] = strlen(msg);
/* Write the severity, errcode, and string length */
if (ORCM_SUCCESS !=
(rc = orcm_notifier_command_write_fd(mca_notifier_command_component.to_child[1],
sizeof(csel), csel))) {
errmsg = "write";
goto error;
}
/* Now write the message itself */
if (ORCM_SUCCESS !=
(rc = orcm_notifier_command_write_fd(mca_notifier_command_component.to_child[1],
csel[3] + 1, msg))) {
errmsg = "write";
goto error;
}
/* Now read back the grandchild's exit status from the child:
0 = 0/1 indicating whether the grandchild exited or not
1 = 0/1 indicating whether the grandchild timed out/was killed or not
2 = exit status returned by waitpid() (only relevant if exited==1) */
if (ORCM_SUCCESS !=
(rc = orcm_notifier_command_read_fd(mca_notifier_command_component.to_parent[0],
sizeof(int) * 3, csel))) {
errmsg = "read";
goto error;
}
/* Did the grandchild exit? */
if (0 == csel[0]) {
orte_show_help("help-orcm-notifier-command.txt",
"grandchild did not exit", true,
orte_process_info.nodename,
mca_notifier_command_component.cmd,
mca_notifier_command_component.timeout);
return ORCM_ERROR;
}
/* Did the grandchild timeout? */
if (1 == csel[1]) {
orte_show_help("help-orcm-notifier-command.txt",
"grandchild timeout", true,
orte_process_info.nodename,
mca_notifier_command_component.cmd,
mca_notifier_command_component.timeout,
WIFEXITED(csel[0]) ? "Exit status" : "Signal",
WIFEXITED(csel[0]) ? WEXITSTATUS(csel[0]) : WTERMSIG(csel[0]));
return ORCM_ERR_TIMEOUT;
}
/* The grandchild exited in less than the timeout -- yay. Did it
exit cleanly? */
if (WIFEXITED(csel[1]) && 0 == WEXITSTATUS(csel[1])) {
return ORCM_SUCCESS;
}
/* Nope -- didn't exit cleanly, so print a warning. */
orte_show_help("help-orcm-notifier-command.txt",
"grandchild fail", true, orte_process_info.nodename,
mca_notifier_command_component.cmd,
WIFEXITED(csel[0]) ? "Exit status" : "Signal",
WIFEXITED(csel[0]) ? WEXITSTATUS(csel[0]) : WTERMSIG(csel[0]));
return ORCM_ERROR;
error:
orte_show_help("help-orcm-notifier-command.txt",
"system call fail", true, orte_process_info.nodename,
errmsg, opal_strerror(rc), rc);
return rc;
}
/*
* Create an ompi_btl_usnic_proc_t and initialize it with modex info
* and an empty array of endpoints.
*
* Returns OMPI_ERR_UNREACH if we can't reach the peer (i.e., we can't
* find their modex data).
*/
static int create_proc(ompi_proc_t *ompi_proc,
ompi_btl_usnic_proc_t **usnic_proc)
{
ompi_btl_usnic_proc_t *proc = NULL;
size_t size;
int rc;
*usnic_proc = NULL;
/* Create the proc if it doesn't already exist */
proc = OBJ_NEW(ompi_btl_usnic_proc_t);
if (NULL == proc) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* Initialize number of peers */
proc->proc_endpoint_count = 0;
proc->proc_ompi = ompi_proc;
/* query for the peer address info */
rc = ompi_modex_recv(&mca_btl_usnic_component.super.btl_version,
ompi_proc, (void*)&proc->proc_modex,
&size);
/* If this proc simply doesn't have this key, then they're not
running the usnic BTL -- just ignore them. Otherwise, show an
error message. */
if (OPAL_ERR_DATA_VALUE_NOT_FOUND == rc) {
OBJ_RELEASE(proc);
return OMPI_ERR_UNREACH;
} else if (OMPI_SUCCESS != rc) {
opal_show_help("help-mpi-btl-usnic.txt",
"internal error during init",
true,
ompi_process_info.nodename,
"<none>", 0,
"ompi_modex_recv() failed", __FILE__, __LINE__,
opal_strerror(rc));
OBJ_RELEASE(proc);
return OMPI_ERROR;
}
if ((size % sizeof(ompi_btl_usnic_addr_t)) != 0) {
char msg[1024];
snprintf(msg, sizeof(msg),
"sizeof(modex for peer %s data) == %d, expected multiple of %d",
OMPI_NAME_PRINT(&ompi_proc->proc_name),
(int) size, (int) sizeof(ompi_btl_usnic_addr_t));
opal_show_help("help-mpi-btl-usnic.txt", "internal error during init",
true,
ompi_process_info.nodename,
"<none>", 0,
"invalid modex data", __FILE__, __LINE__,
msg);
OBJ_RELEASE(proc);
return OMPI_ERR_VALUE_OUT_OF_BOUNDS;
}
proc->proc_modex_count = size / sizeof(ompi_btl_usnic_addr_t);
if (0 == proc->proc_modex_count) {
proc->proc_endpoints = NULL;
OBJ_RELEASE(proc);
return OMPI_ERR_UNREACH;
}
proc->proc_modex_claimed = (bool*)
calloc(proc->proc_modex_count, sizeof(bool));
if (NULL == proc->proc_modex_claimed) {
OMPI_ERROR_LOG(OMPI_ERR_OUT_OF_RESOURCE);
OBJ_RELEASE(proc);
return OMPI_ERR_OUT_OF_RESOURCE;
}
proc->proc_endpoints = (mca_btl_base_endpoint_t**)
calloc(proc->proc_modex_count, sizeof(mca_btl_base_endpoint_t*));
if (NULL == proc->proc_endpoints) {
OMPI_ERROR_LOG(OMPI_ERR_OUT_OF_RESOURCE);
OBJ_RELEASE(proc);
return OMPI_ERR_OUT_OF_RESOURCE;
}
*usnic_proc = proc;
return OMPI_SUCCESS;
}
static int parse_args(int argc, char *argv[]) {
int i, ret, len, exit_status = ORTE_SUCCESS ;
opal_cmd_line_t cmd_line;
char **app_env = NULL, **global_env = NULL;
char * tmp_env_var = NULL;
char *argv0 = NULL;
/* Init structure */
memset(&orte_migrate_globals, 0, sizeof(orte_migrate_globals_t));
orte_migrate_globals.help = false;
orte_migrate_globals.pid = -1;
orte_migrate_globals.verbose = false;
orte_migrate_globals.verbose_level = 0;
orte_migrate_globals.status = false;
orte_migrate_globals.output = -1;
orte_migrate_globals.off_nodes = NULL;
orte_migrate_globals.off_procs = NULL;
orte_migrate_globals.onto_nodes = NULL;
#if OPAL_ENABLE_FT_CR == 0
/* Warn and exit if not configured with Migrate/Restart */
{
char *str, *args = NULL;
args = opal_cmd_line_get_usage_msg(&cmd_line);
str = opal_show_help_string("help-orte-migrate.txt", "usage-no-cr",
true, args);
if (NULL != str) {
printf("%s", str);
free(str);
}
free(args);
exit_status = ORTE_ERROR;
goto cleanup;
}
#endif
/* Parse the command line options */
opal_cmd_line_create(&cmd_line, cmd_line_opts);
mca_base_open();
mca_base_cmd_line_setup(&cmd_line);
ret = opal_cmd_line_parse(&cmd_line, false, argc, argv);
if (OPAL_SUCCESS != ret) {
if (OPAL_ERR_SILENT != ret) {
fprintf(stderr, "%s: command line error (%s)\n", argv[0],
opal_strerror(ret));
}
exit_status = 1;
goto cleanup;
}
if (orte_migrate_globals.help) {
char *str, *args = NULL;
args = opal_cmd_line_get_usage_msg(&cmd_line);
str = opal_show_help_string("help-orte-migrate.txt", "usage", true,
args);
if (NULL != str) {
printf("%s", str);
free(str);
}
free(args);
/* If we show the help message, that should be all we do */
exit(0);
}
/**
* Put all of the MCA arguments in the environment
*/
mca_base_cmd_line_process_args(&cmd_line, &app_env, &global_env);
len = opal_argv_count(app_env);
for(i = 0; i < len; ++i) {
putenv(app_env[i]);
}
len = opal_argv_count(global_env);
for(i = 0; i < len; ++i) {
putenv(global_env[i]);
}
tmp_env_var = mca_base_param_env_var("opal_cr_is_tool");
opal_setenv(tmp_env_var,
"1",
true, &environ);
free(tmp_env_var);
tmp_env_var = NULL;
/**
* Now start parsing our specific arguments
*/
/* get the remaining bits */
argv0 = strdup(argv[0]);
opal_cmd_line_get_tail(&cmd_line, &argc, &argv);
if (NULL == orte_migrate_globals.off_nodes &&
NULL == orte_migrate_globals.off_procs) {
fprintf(stderr, "%s: Nothing to do\n", argv0);
fprintf(stderr, "Type '%s --help' for usage.\n", argv0);
exit_status = 1;
goto cleanup;
}
if(orte_migrate_globals.verbose_level < 0 ) {
orte_migrate_globals.verbose_level = 0;
}
if(orte_migrate_globals.verbose_level > 0) {
orte_migrate_globals.verbose = true;
}
/*
* If the user did not supply an hnp jobid, then they must
* supply the PID of MPIRUN
*/
if(0 >= argc ) {
fprintf(stderr, "%s: Nothing to do\n", argv[0]);
fprintf(stderr, "Type '%s --help' for usage.\n", argv[0]);
exit_status = ORTE_ERROR;
goto cleanup;
}
orte_migrate_globals.pid = atoi(argv[0]);
if ( 0 >= orte_migrate_globals.pid ) {
opal_show_help("help-orte-migrate.txt", "invalid_pid", true,
orte_migrate_globals.pid);
exit_status = ORTE_ERROR;
goto cleanup;
}
if(orte_migrate_globals.verbose) {
orte_migrate_globals.status = true;
}
if(orte_migrate_globals.verbose) {
pretty_print_migration();
}
cleanup:
if (NULL != argv0) {
free(argv0);
}
return exit_status;
}
static int parse_args(int argc, char *argv[])
{
int i, ret, len;
opal_cmd_line_t cmd_line;
char **app_env = NULL, **global_env = NULL;
char * tmp_env_var = NULL;
char *argv0 = NULL;
orte_restart_globals_t tmp = { false, /* help */
NULL, /* filename */
NULL, /* appfile */
false, /* verbose */
false, /* forked */
-1, /* seq_number */
NULL, /* hostfile */
-1, /* output*/
false, /* info only */
false, /* app only */
false, /* showme */
NULL}; /* mpirun_opts */
orte_restart_globals = tmp;
#if OPAL_ENABLE_CRDEBUG == 1
orte_restart_globals.enable_crdebug = false;
#endif
#if OPAL_ENABLE_FT_CR == 0
/* Warn and exit if not configured with Checkpoint/Restart */
{
char *str, *args = NULL;
args = opal_cmd_line_get_usage_msg(&cmd_line);
str = opal_show_help_string("help-orte-restart.txt", "usage-no-cr",
true, args);
if (NULL != str) {
printf("%s", str);
free(str);
}
free(args);
return ORTE_ERROR;
}
#endif
/* Parse the command line options */
opal_cmd_line_create(&cmd_line, cmd_line_opts);
mca_base_open();
mca_base_cmd_line_setup(&cmd_line);
ret = opal_cmd_line_parse(&cmd_line, true, argc, argv);
if (OPAL_SUCCESS != ret) {
if (OPAL_ERR_SILENT != ret) {
fprintf(stderr, "%s: command line error (%s)\n", argv[0],
opal_strerror(ret));
}
return 1;
}
if (orte_restart_globals.help) {
char *str, *args = NULL;
args = opal_cmd_line_get_usage_msg(&cmd_line);
str = opal_show_help_string("help-orte-restart.txt", "usage", true,
args);
if (NULL != str) {
printf("%s", str);
free(str);
}
free(args);
/* If we show the help message, that should be all we do */
exit(0);
}
/**
* Put all of the MCA arguments in the environment
*/
mca_base_cmd_line_process_args(&cmd_line, &app_env, &global_env);
len = opal_argv_count(app_env);
for(i = 0; i < len; ++i) {
putenv(app_env[i]);
}
len = opal_argv_count(global_env);
for(i = 0; i < len; ++i) {
putenv(global_env[i]);
}
(void) mca_base_var_env_name("opal_cr_is_tool", &tmp_env_var);
opal_setenv(tmp_env_var,
"1",
true, &environ);
free(tmp_env_var);
tmp_env_var = NULL;
/**
* Now start parsing our specific arguments
*/
/* get the remaining bits */
argv0 = strdup(argv[0]);
opal_cmd_line_get_tail(&cmd_line, &argc, &argv);
if (0 == argc) {
fprintf(stderr, "%s: Nothing to do\n", argv0);
fprintf(stderr, "Type '%s --help' for usge.\n", argv0);
free(argv0);
return ORTE_ERROR;
}
free(argv0);
orte_restart_globals.snapshot_ref = strdup(argv[0]);
if ( NULL == orte_restart_globals.snapshot_ref ||
0 >= strlen(orte_restart_globals.snapshot_ref) ) {
opal_show_help("help-orte-restart.txt", "invalid_filename", true,
"<none provided>");
return ORTE_ERROR;
}
/* If we have arguments after the command, then assume they
* need to be grouped together.
*/
if(argc > 1) {
orte_restart_globals.snapshot_ref = strdup(opal_argv_join(argv, ' '));
}
return ORTE_SUCCESS;
}
static int parse_args(int argc, char *argv[])
{
int ret;
opal_cmd_line_t cmd_line;
orcm_osub_globals_t tmp = { false, /* help */
false, /* verbose */
-1, /* output */
'\0', /* account */
'\0', /* name */
-1, /* gid */
0, /* max_nodes */
0, /* max_pes */
1, /* min_nodes */
1, /* min_pes */
'\0', /* starttime */
'\0', /* walltime */
false, /* exclusive */
false, /* interactive */
'\0', /* nodefile */
'\0'}; /* resources */
orcm_osub_globals = tmp;
/* Parse the command line options */
opal_cmd_line_create(&cmd_line, cmd_line_opts);
mca_base_cmd_line_setup(&cmd_line);
ret = opal_cmd_line_parse(&cmd_line, false, argc, argv);
if (OPAL_SUCCESS != ret) {
if (OPAL_ERR_SILENT != ret) {
fprintf(stderr, "%s: command line error (%s)\n", argv[0],
opal_strerror(ret));
}
return ret;
}
/**
* Now start parsing our specific arguments
*/
if (orcm_osub_globals.help) {
char *str, *args = NULL;
args = opal_cmd_line_get_usage_msg(&cmd_line);
str = opal_show_help_string("help-osub.txt", "usage", true,
args);
if (NULL != str) {
printf("%s", str);
free(str);
}
free(args);
/* If we show the help message, that should be all we do */
exit(0);
}
/* if user hasn't supplied a group to run under, use effective gid of caller */
/* TODO: double check if user is in group */
/* do we also need to support the name as well as id? */
if (-1 == orcm_osub_globals.gid) {
orcm_osub_globals.gid = getgid();
}
if (orcm_osub_globals.max_nodes < orcm_osub_globals.min_nodes) {
orcm_osub_globals.max_nodes = orcm_osub_globals.min_nodes;
}
if (orcm_osub_globals.max_pes < orcm_osub_globals.min_pes) {
orcm_osub_globals.max_pes = orcm_osub_globals.min_pes;
}
/*
* Since this process can now handle MCA/GMCA parameters, make sure to
* process them.
*/
mca_base_cmd_line_process_args(&cmd_line, &environ, &environ);
return ORTE_SUCCESS;
}
/*
* Create an opal_btl_usnic_proc_t and initialize it with modex info
* and an empty array of endpoints.
*
* Returns OPAL_ERR_UNREACH if we can't reach the peer (i.e., we can't
* find their modex data).
*/
static int create_proc(opal_proc_t *opal_proc,
opal_btl_usnic_proc_t **usnic_proc)
{
opal_btl_usnic_proc_t *proc = NULL;
size_t size;
int rc;
*usnic_proc = NULL;
/* Create the proc if it doesn't already exist */
proc = OBJ_NEW(opal_btl_usnic_proc_t);
if (NULL == proc) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
/* Initialize number of peers */
proc->proc_endpoint_count = 0;
proc->proc_opal = opal_proc;
/* query for the peer address info */
OPAL_MODEX_RECV(rc, &mca_btl_usnic_component.super.btl_version,
opal_proc, (uint8_t**)&proc->proc_modex, &size);
/* If this proc simply doesn't have this key, then they're not
running the usnic BTL -- just ignore them. Otherwise, show an
error message. */
if (OPAL_ERR_DATA_VALUE_NOT_FOUND == rc) {
OBJ_RELEASE(proc);
return OPAL_ERR_UNREACH;
} else if (OPAL_SUCCESS != rc) {
opal_show_help("help-mpi-btl-usnic.txt",
"internal error during init",
true,
opal_process_info.nodename,
"<none>", "<none>",
"opal_modex_recv() failed", __FILE__, __LINE__,
opal_strerror(rc));
OBJ_RELEASE(proc);
return OPAL_ERROR;
}
if ((size % sizeof(opal_btl_usnic_addr_t)) != 0) {
char msg[1024];
snprintf(msg, sizeof(msg),
"sizeof(modex for peer %s data) == %d, expected multiple of %d",
OPAL_NAME_PRINT(opal_proc->proc_name),
(int) size, (int) sizeof(opal_btl_usnic_addr_t));
opal_show_help("help-mpi-btl-usnic.txt", "internal error during init",
true,
opal_process_info.nodename,
"<none>", 0,
"invalid modex data", __FILE__, __LINE__,
msg);
OBJ_RELEASE(proc);
return OPAL_ERR_VALUE_OUT_OF_BOUNDS;
}
proc->proc_modex_count = size / sizeof(opal_btl_usnic_addr_t);
if (0 == proc->proc_modex_count) {
proc->proc_endpoints = NULL;
OBJ_RELEASE(proc);
return OPAL_ERR_UNREACH;
}
/* Sanity check: ensure that the remote proc agrees with this proc
on whether we're doing UDP or not. Note that all endpoints on
the remote proc will have the same "use_udp" value, so we only
need to check one of them. */
if (proc->proc_modex[0].use_udp !=
mca_btl_usnic_component.use_udp) {
opal_show_help("help-mpi-btl-usnic.txt",
"transport mismatch",
true,
opal_process_info.nodename,
proc->proc_opal->proc_hostname);
OBJ_RELEASE(proc);
return OPAL_ERR_BAD_PARAM;
}
proc->proc_modex_claimed = (bool*)
calloc(proc->proc_modex_count, sizeof(bool));
if (NULL == proc->proc_modex_claimed) {
OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);
OBJ_RELEASE(proc);
return OPAL_ERR_OUT_OF_RESOURCE;
}
proc->proc_endpoints = (mca_btl_base_endpoint_t**)
calloc(proc->proc_modex_count, sizeof(mca_btl_base_endpoint_t*));
if (NULL == proc->proc_endpoints) {
OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);
OBJ_RELEASE(proc);
return OPAL_ERR_OUT_OF_RESOURCE;
}
*usnic_proc = proc;
return OPAL_SUCCESS;
}
int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
{
int ret;
ompi_proc_t** procs;
size_t nprocs;
char *error = NULL;
struct timeval ompistart, ompistop;
bool rte_setup = false;
ompi_rte_collective_t *coll;
char *cmd=NULL, *av=NULL;
/* bitflag of the thread level support provided. To be used
* for the modex in order to work in heterogeneous environments. */
uint8_t threadlevel_bf;
/* Indicate that we have *started* MPI_INIT*. MPI_FINALIZE has
something sorta similar in a static local variable in
ompi_mpi_finalize(). */
ompi_mpi_init_started = true;
/* Setup enough to check get/set MCA params */
if (OPAL_SUCCESS != (ret = opal_init_util(&argc, &argv))) {
error = "ompi_mpi_init: opal_init_util failed";
goto error;
}
/* Register MCA variables */
if (OPAL_SUCCESS != (ret = ompi_register_mca_variables())) {
error = "ompi_mpi_init: ompi_register_mca_variables failed";
goto error;
}
if (OPAL_SUCCESS != (ret = opal_arch_set_fortran_logical_size(sizeof(ompi_fortran_logical_t)))) {
error = "ompi_mpi_init: opal_arch_set_fortran_logical_size failed";
goto error;
}
/* _After_ opal_init_util() but _before_ orte_init(), we need to
set an MCA param that tells libevent that it's ok to use any
mechanism in libevent that is available on this platform (e.g.,
epoll and friends). Per opal/event/event.s, we default to
select/poll -- but we know that MPI processes won't be using
pty's with the event engine, so it's ok to relax this
constraint and let any fd-monitoring mechanism be used. */
ret = mca_base_var_find("opal", "event", "*", "event_include");
if (ret >= 0) {
char *allvalue = "all";
/* We have to explicitly "set" the MCA param value here
because libevent initialization will re-register the MCA
param and therefore override the default. Setting the value
here puts the desired value ("all") in different storage
that is not overwritten if/when the MCA param is
re-registered. This is unless the user has specified a different
value for this MCA parameter. Make sure we check to see if the
default is specified before forcing "all" in case that is not what
the user desires. Note that we do *NOT* set this value as an
environment variable, just so that it won't be inherited by
any spawned processes and potentially cause unintented
side-effects with launching RTE tools... */
mca_base_var_set_value(ret, allvalue, 4, MCA_BASE_VAR_SOURCE_DEFAULT, NULL);
}
if (ompi_enable_timing && 0 == OMPI_PROC_MY_NAME->vpid) {
gettimeofday(&ompistart, NULL);
}
/* if we were not externally started, then we need to setup
* some envars so the MPI_INFO_ENV can get the cmd name
* and argv (but only if the user supplied a non-NULL argv!), and
* the requested thread level
*/
if (NULL == getenv("OMPI_COMMAND") && NULL != argv && NULL != argv[0]) {
asprintf(&cmd, "OMPI_COMMAND=%s", argv[0]);
putenv(cmd);
}
if (NULL == getenv("OMPI_ARGV") && 1 < argc) {
char *tmp;
tmp = opal_argv_join(&argv[1], ' ');
asprintf(&av, "OMPI_ARGV=%s", tmp);
free(tmp);
putenv(av);
}
/* open the rte framework */
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_rte_base_framework, 0))) {
error = "ompi_rte_base_open() failed";
goto error;
}
/* no select is required as this is a static framework */
/* Setup RTE - note that we are an MPI process */
if (OMPI_SUCCESS != (ret = ompi_rte_init(NULL, NULL))) {
error = "ompi_mpi_init: ompi_rte_init failed";
goto error;
}
rte_setup = true;
/* check for timing request - get stop time and report elapsed time if so */
if (ompi_enable_timing && 0 == OMPI_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init [%ld]: time from start to completion of rte_init %ld usec",
(long)OMPI_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
#if OPAL_HAVE_HWLOC
/* if hwloc is available but didn't get setup for some
* reason, do so now
*/
if (NULL == opal_hwloc_topology) {
if (OPAL_SUCCESS != (ret = opal_hwloc_base_get_topology())) {
error = "Topology init";
goto error;
}
}
#endif
/* Register the default errhandler callback - RTE will ignore if it
* doesn't support this capability
*/
ompi_rte_register_errhandler(ompi_errhandler_runtime_callback,
OMPI_RTE_ERRHANDLER_LAST);
/* Figure out the final MPI thread levels. If we were not
compiled for support for MPI threads, then don't allow
MPI_THREAD_MULTIPLE. Set this stuff up here early in the
process so that other components can make decisions based on
this value. */
ompi_mpi_thread_level(requested, provided);
/* determine the bitflag belonging to the threadlevel_support provided */
memset ( &threadlevel_bf, 0, sizeof(uint8_t));
OMPI_THREADLEVEL_SET_BITFLAG ( ompi_mpi_thread_provided, threadlevel_bf );
/* add this bitflag to the modex */
if ( OMPI_SUCCESS != (ret = ompi_modex_send_string("MPI_THREAD_LEVEL", &threadlevel_bf, sizeof(uint8_t)))) {
error = "ompi_mpi_init: modex send thread level";
goto error;
}
/* initialize datatypes. This step should be done early as it will
* create the local convertor and local arch used in the proc
* init.
*/
if (OMPI_SUCCESS != (ret = ompi_datatype_init())) {
error = "ompi_datatype_init() failed";
goto error;
}
/* Initialize OMPI procs */
if (OMPI_SUCCESS != (ret = ompi_proc_init())) {
error = "mca_proc_init() failed";
goto error;
}
/* Initialize the op framework. This has to be done *after*
ddt_init, but befor mca_coll_base_open, since some collective
modules (e.g., the hierarchical coll component) may need ops in
their query function. */
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_op_base_framework, 0))) {
error = "ompi_op_base_open() failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = ompi_op_base_find_available(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "ompi_op_base_find_available() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = ompi_op_init())) {
error = "ompi_op_init() failed";
goto error;
}
/* Open up MPI-related MCA components */
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_allocator_base_framework, 0))) {
error = "mca_allocator_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_rcache_base_framework, 0))) {
error = "mca_rcache_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_mpool_base_framework, 0))) {
error = "mca_mpool_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_bml_base_framework, 0))) {
error = "mca_bml_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_pml_base_framework, 0))) {
error = "mca_pml_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_coll_base_framework, 0))) {
error = "mca_coll_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_osc_base_framework, 0))) {
error = "ompi_osc_base_open() failed";
goto error;
}
#if OPAL_ENABLE_FT_CR == 1
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_crcp_base_framework, 0))) {
error = "ompi_crcp_base_open() failed";
goto error;
}
#endif
/* In order to reduce the common case for MPI apps (where they
don't use MPI-2 IO or MPI-1 topology functions), the io and
topo frameworks are initialized lazily, at the first use of
relevant functions (e.g., MPI_FILE_*, MPI_CART_*, MPI_GRAPH_*),
so they are not opened here. */
/* Select which MPI components to use */
if (OMPI_SUCCESS !=
(ret = mca_mpool_base_init(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "mca_mpool_base_init() failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = mca_pml_base_select(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "mca_pml_base_select() failed";
goto error;
}
/* check for timing request - get stop time and report elapsed time if so */
if (ompi_enable_timing && 0 == OMPI_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time from completion of rte_init to modex %ld usec",
(long)OMPI_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
/* exchange connection info - this function also acts as a barrier
* as it will not return until the exchange is complete
*/
coll = OBJ_NEW(ompi_rte_collective_t);
coll->id = ompi_process_info.peer_modex;
coll->active = true;
if (OMPI_SUCCESS != (ret = ompi_rte_modex(coll))) {
error = "rte_modex failed";
goto error;
}
/* wait for modex to complete - this may be moved anywhere in mpi_init
* so long as it occurs prior to calling a function that needs
* the modex info!
*/
OMPI_WAIT_FOR_COMPLETION(coll->active);
OBJ_RELEASE(coll);
if (ompi_enable_timing && 0 == OMPI_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time to execute modex %ld usec",
(long)OMPI_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
/* select buffered send allocator component to be used */
if( OMPI_SUCCESS !=
(ret = mca_pml_base_bsend_init(OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "mca_pml_base_bsend_init() failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = mca_coll_base_find_available(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "mca_coll_base_find_available() failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = ompi_osc_base_find_available(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "ompi_osc_base_find_available() failed";
goto error;
}
#if OPAL_ENABLE_FT_CR == 1
if (OMPI_SUCCESS != (ret = ompi_crcp_base_select() ) ) {
error = "ompi_crcp_base_select() failed";
goto error;
}
#endif
/* io and topo components are not selected here -- see comment
above about the io and topo frameworks being loaded lazily */
/* Initialize each MPI handle subsystem */
/* initialize requests */
if (OMPI_SUCCESS != (ret = ompi_request_init())) {
error = "ompi_request_init() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = ompi_message_init())) {
error = "ompi_message_init() failed";
goto error;
}
/* initialize info */
if (OMPI_SUCCESS != (ret = ompi_info_init())) {
error = "ompi_info_init() failed";
goto error;
}
/* initialize error handlers */
if (OMPI_SUCCESS != (ret = ompi_errhandler_init())) {
error = "ompi_errhandler_init() failed";
goto error;
}
/* initialize error codes */
if (OMPI_SUCCESS != (ret = ompi_mpi_errcode_init())) {
error = "ompi_mpi_errcode_init() failed";
goto error;
}
/* initialize internal error codes */
if (OMPI_SUCCESS != (ret = ompi_errcode_intern_init())) {
error = "ompi_errcode_intern_init() failed";
goto error;
}
/* initialize groups */
if (OMPI_SUCCESS != (ret = ompi_group_init())) {
error = "ompi_group_init() failed";
goto error;
}
/* initialize communicators */
if (OMPI_SUCCESS != (ret = ompi_comm_init())) {
error = "ompi_comm_init() failed";
goto error;
}
/* initialize file handles */
if (OMPI_SUCCESS != (ret = ompi_file_init())) {
error = "ompi_file_init() failed";
goto error;
}
/* initialize windows */
if (OMPI_SUCCESS != (ret = ompi_win_init())) {
error = "ompi_win_init() failed";
goto error;
}
/* initialize attribute meta-data structure for comm/win/dtype */
if (OMPI_SUCCESS != (ret = ompi_attr_init())) {
error = "ompi_attr_init() failed";
goto error;
}
/* identify the architectures of remote procs and setup
* their datatype convertors, if required
*/
if (OMPI_SUCCESS != (ret = ompi_proc_complete_init())) {
error = "ompi_proc_complete_init failed";
goto error;
}
/* If thread support was enabled, then setup OPAL to allow for
them. */
if ((OMPI_ENABLE_PROGRESS_THREADS == 1) ||
(*provided != MPI_THREAD_SINGLE)) {
opal_set_using_threads(true);
}
/* start PML/BTL's */
ret = MCA_PML_CALL(enable(true));
if( OMPI_SUCCESS != ret ) {
error = "PML control failed";
goto error;
}
/* add all ompi_proc_t's to PML */
if (NULL == (procs = ompi_proc_world(&nprocs))) {
error = "ompi_proc_world() failed";
goto error;
}
ret = MCA_PML_CALL(add_procs(procs, nprocs));
free(procs);
/* If we got "unreachable", then print a specific error message.
Otherwise, if we got some other failure, fall through to print
a generic message. */
if (OMPI_ERR_UNREACH == ret) {
opal_show_help("help-mpi-runtime",
"mpi_init:startup:pml-add-procs-fail", true);
error = NULL;
goto error;
} else if (OMPI_SUCCESS != ret) {
error = "PML add procs failed";
goto error;
}
MCA_PML_CALL(add_comm(&ompi_mpi_comm_world.comm));
MCA_PML_CALL(add_comm(&ompi_mpi_comm_self.comm));
/*
* Dump all MCA parameters if requested
*/
if (ompi_mpi_show_mca_params) {
ompi_show_all_mca_params(ompi_mpi_comm_world.comm.c_my_rank,
nprocs,
ompi_process_info.nodename);
}
/* Do we need to wait for a debugger? */
ompi_rte_wait_for_debugger();
/* check for timing request - get stop time and report elapsed
time if so, then start the clock again */
if (ompi_enable_timing && 0 == OMPI_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time from modex to first barrier %ld usec",
(long)OMPI_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
/* wait for everyone to reach this point */
coll = OBJ_NEW(ompi_rte_collective_t);
coll->id = ompi_process_info.peer_init_barrier;
coll->active = true;
if (OMPI_SUCCESS != (ret = ompi_rte_barrier(coll))) {
error = "rte_barrier failed";
goto error;
}
/* wait for barrier to complete */
OMPI_WAIT_FOR_COMPLETION(coll->active);
OBJ_RELEASE(coll);
/* check for timing request - get stop time and report elapsed
time if so, then start the clock again */
if (ompi_enable_timing && 0 == OMPI_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time to execute barrier %ld usec",
(long)OMPI_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
#if OMPI_ENABLE_PROGRESS_THREADS == 0
/* Start setting up the event engine for MPI operations. Don't
block in the event library, so that communications don't take
forever between procs in the dynamic code. This will increase
CPU utilization for the remainder of MPI_INIT when we are
blocking on RTE-level events, but may greatly reduce non-TCP
latency. */
opal_progress_set_event_flag(OPAL_EVLOOP_NONBLOCK);
#endif
/* wire up the mpi interface, if requested. Do this after the
non-block switch for non-TCP performance. Do before the
polling change as anyone with a complex wire-up is going to be
using the oob. */
if (OMPI_SUCCESS != (ret = ompi_init_preconnect_mpi())) {
error = "ompi_mpi_do_preconnect_all() failed";
goto error;
}
/* Setup the publish/subscribe (PUBSUB) framework */
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_pubsub_base_framework, 0))) {
error = "mca_pubsub_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = ompi_pubsub_base_select())) {
error = "ompi_pubsub_base_select() failed";
goto error;
}
/* Setup the dynamic process management (DPM) framework */
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_dpm_base_framework, 0))) {
error = "ompi_dpm_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = ompi_dpm_base_select())) {
error = "ompi_dpm_base_select() failed";
goto error;
}
/* Determine the overall threadlevel support of all processes
in MPI_COMM_WORLD. This has to be done before calling
coll_base_comm_select, since some of the collective components
e.g. hierarch, might create subcommunicators. The threadlevel
requested by all processes is required in order to know
which cid allocation algorithm can be used. */
if ( OMPI_SUCCESS !=
( ret = ompi_comm_cid_init ())) {
error = "ompi_mpi_init: ompi_comm_cid_init failed";
goto error;
}
/* Init coll for the comms. This has to be after dpm_base_select,
(since dpm.mark_dyncomm is not set in the communicator creation
function else), but before dpm.dyncom_init, since this function
might require collective for the CID allocation. */
if (OMPI_SUCCESS !=
(ret = mca_coll_base_comm_select(MPI_COMM_WORLD))) {
error = "mca_coll_base_comm_select(MPI_COMM_WORLD) failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = mca_coll_base_comm_select(MPI_COMM_SELF))) {
error = "mca_coll_base_comm_select(MPI_COMM_SELF) failed";
goto error;
}
/* Check whether we have been spawned or not. We introduce that
at the very end, since we need collectives, datatypes, ptls
etc. up and running here.... */
if (OMPI_SUCCESS != (ret = ompi_dpm.dyn_init())) {
error = "ompi_comm_dyn_init() failed";
goto error;
}
/*
* Startup the Checkpoint/Restart Mech.
* Note: Always do this so tools don't hang when
* in a non-checkpointable build
*/
if (OMPI_SUCCESS != (ret = ompi_cr_init())) {
error = "ompi_cr_init";
goto error;
}
/* Undo OPAL calling opal_progress_event_users_increment() during
opal_init, to get better latency when not using TCP. Do
this *after* dyn_init, as dyn init uses lots of RTE
communication and we don't want to hinder the performance of
that code. */
opal_progress_event_users_decrement();
/* see if yield_when_idle was specified - if so, use it */
opal_progress_set_yield_when_idle(ompi_mpi_yield_when_idle);
/* negative value means use default - just don't do anything */
if (ompi_mpi_event_tick_rate >= 0) {
opal_progress_set_event_poll_rate(ompi_mpi_event_tick_rate);
}
/* At this point, we are fully configured and in MPI mode. Any
communication calls here will work exactly like they would in
the user's code. Setup the connections between procs and warm
them up with simple sends, if requested */
if (OMPI_SUCCESS != (ret = ompi_mpiext_init())) {
error = "ompi_mpiext_init";
goto error;
}
/* Fall through */
error:
if (ret != OMPI_SUCCESS) {
/* Only print a message if one was not already printed */
if (NULL != error) {
const char *err_msg = opal_strerror(ret);
/* If RTE was not setup yet, don't use opal_show_help */
if (rte_setup) {
opal_show_help("help-mpi-runtime",
"mpi_init:startup:internal-failure", true,
"MPI_INIT", "MPI_INIT", error, err_msg, ret);
} else {
opal_show_help("help-mpi-runtime",
"mpi_init:startup:internal-failure", true,
"MPI_INIT", "MPI_INIT", error, err_msg, ret);
}
}
return ret;
}
/* Initialize the registered datarep list to be empty */
OBJ_CONSTRUCT(&ompi_registered_datareps, opal_list_t);
/* Initialize the arrays used to store the F90 types returned by the
* MPI_Type_create_f90_XXX functions.
*/
OBJ_CONSTRUCT( &ompi_mpi_f90_integer_hashtable, opal_hash_table_t);
opal_hash_table_init(&ompi_mpi_f90_integer_hashtable, 16 /* why not? */);
OBJ_CONSTRUCT( &ompi_mpi_f90_real_hashtable, opal_hash_table_t);
opal_hash_table_init(&ompi_mpi_f90_real_hashtable, FLT_MAX_10_EXP);
OBJ_CONSTRUCT( &ompi_mpi_f90_complex_hashtable, opal_hash_table_t);
opal_hash_table_init(&ompi_mpi_f90_complex_hashtable, FLT_MAX_10_EXP);
/* All done. Wasn't that simple? */
ompi_mpi_initialized = true;
/* check for timing request - get stop time and report elapsed time if so */
if (ompi_enable_timing && 0 == OMPI_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time from barrier to complete mpi_init %ld usec",
(long)OMPI_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
}
return MPI_SUCCESS;
}
static int parse_args(int argc, char *argv[]) {
int i, ret, len, exit_status = ORTE_SUCCESS ;
opal_cmd_line_t cmd_line;
char **app_env = NULL, **global_env = NULL;
char * tmp_env_var = NULL;
char *argv0 = NULL;
/* Init structure */
memset(&orte_checkpoint_globals, 0, sizeof(orte_checkpoint_globals_t));
orte_checkpoint_globals.help = false;
orte_checkpoint_globals.pid = -1;
orte_checkpoint_globals.verbose = false;
orte_checkpoint_globals.verbose_level = 0;
orte_checkpoint_globals.req_hnp = ORTE_JOBID_INVALID;
orte_checkpoint_globals.nowait = false;
orte_checkpoint_globals.status = false;
orte_checkpoint_globals.output = -1;
orte_checkpoint_globals.ckpt_status = ORTE_SNAPC_CKPT_STATE_NONE;
orte_checkpoint_globals.list_only = false;
#if OPAL_ENABLE_CRDEBUG == 1
orte_checkpoint_globals.enable_crdebug = false;
#endif
orte_checkpoint_globals.options = OBJ_NEW(opal_crs_base_ckpt_options_t);
orte_checkpoint_globals.term = false;
orte_checkpoint_globals.stop = false;
#if OPAL_ENABLE_CRDEBUG == 1
orte_checkpoint_globals.attach_debugger = false;
orte_checkpoint_globals.detach_debugger = false;
#endif
#if OPAL_ENABLE_FT_CR == 0
/* Warn and exit if not configured with Checkpoint/Restart */
{
char *str, *args = NULL;
args = opal_cmd_line_get_usage_msg(&cmd_line);
str = opal_show_help_string("help-orte-checkpoint.txt", "usage-no-cr",
true, args);
if (NULL != str) {
printf("%s", str);
free(str);
}
free(args);
exit_status = ORTE_ERROR;
goto cleanup;
}
#endif
/* Parse the command line options */
opal_cmd_line_create(&cmd_line, cmd_line_opts);
mca_base_open();
mca_base_cmd_line_setup(&cmd_line);
ret = opal_cmd_line_parse(&cmd_line, true, argc, argv);
if (OPAL_SUCCESS != ret) {
if (OPAL_ERR_SILENT != ret) {
fprintf(stderr, "%s: command line error (%s)\n", argv[0],
opal_strerror(ret));
}
exit_status = 1;
goto cleanup;
}
if (orte_checkpoint_globals.help) {
char *str, *args = NULL;
args = opal_cmd_line_get_usage_msg(&cmd_line);
str = opal_show_help_string("help-orte-checkpoint.txt", "usage", true,
args);
if (NULL != str) {
printf("%s", str);
free(str);
}
free(args);
/* If we show the help message, that should be all we do */
exit(0);
}
/**
* Put all of the MCA arguments in the environment
*/
mca_base_cmd_line_process_args(&cmd_line, &app_env, &global_env);
len = opal_argv_count(app_env);
for(i = 0; i < len; ++i) {
putenv(app_env[i]);
}
len = opal_argv_count(global_env);
for(i = 0; i < len; ++i) {
putenv(global_env[i]);
}
(void) mca_base_var_env_name("opal_cr_is_tool", &tmp_env_var);
opal_setenv(tmp_env_var,
"1",
true, &environ);
free(tmp_env_var);
tmp_env_var = NULL;
/**
* Now start parsing our specific arguments
*/
/* get the remaining bits */
argv0 = strdup(argv[0]);
opal_cmd_line_get_tail(&cmd_line, &argc, &argv);
if(orte_checkpoint_globals.list_only ) {
exit_status = ORTE_SUCCESS;
goto cleanup;
}
if (0 >= argc && ORTE_JOBID_INVALID == orte_checkpoint_globals.req_hnp) {
fprintf(stderr, "%s: Nothing to do\n", argv0);
fprintf(stderr, "Type '%s --help' for usage.\n", argv0);
exit_status = 1;
goto cleanup;
}
orte_checkpoint_globals.options->term = orte_checkpoint_globals.term;
orte_checkpoint_globals.options->stop = orte_checkpoint_globals.stop;
#if OPAL_ENABLE_CRDEBUG == 1
orte_checkpoint_globals.options->attach_debugger = orte_checkpoint_globals.attach_debugger;
orte_checkpoint_globals.options->detach_debugger = orte_checkpoint_globals.detach_debugger;
#endif
if(orte_checkpoint_globals.verbose_level < 0 ) {
orte_checkpoint_globals.verbose_level = 0;
}
if(orte_checkpoint_globals.verbose_level > 0) {
orte_checkpoint_globals.verbose = true;
}
/*
* If the user did not supply an hnp jobid, then they must
* supply the PID of MPIRUN
*/
if(0 >= argc &&
ORTE_JOBID_INVALID != orte_checkpoint_globals.req_hnp) {
exit_status = ORTE_SUCCESS;
goto cleanup;
}
orte_checkpoint_globals.pid = atoi(argv[0]);
if ( 0 >= orte_checkpoint_globals.pid ) {
opal_show_help("help-orte-checkpoint.txt", "invalid_pid", true,
orte_checkpoint_globals.pid);
exit_status = ORTE_ERROR;
goto cleanup;
}
/*
* JJH: No wait is currently not implemented or tested
*/
if(orte_checkpoint_globals.nowait) {
orte_checkpoint_globals.nowait = false;
opal_show_help("help-orte-checkpoint.txt", "not_impl",
true,
"Disconnected checkpoint");
}
if(orte_checkpoint_globals.verbose) {
orte_checkpoint_globals.status = true;
}
cleanup:
if (NULL != argv0) {
free(argv0);
}
return exit_status;
}