int ompi_mtl_mx_module_init(){
mx_param_t mx_param;
mx_return_t mx_return;
int32_t nic, ep;
/* setup params */
mx_param.key = MX_PARAM_UNEXP_QUEUE_MAX;
mx_param.val.unexp_queue_max = ompi_mtl_mx.mx_unexp_queue_max;
/* get a local endpoint */
nic = ompi_mtl_mx.mx_board_num;
if (nic < 0) {
nic = MX_ANY_NIC;
}
ep = ompi_mtl_mx.mx_endpoint_num;
if (ep < 0) {
ep = MX_ANY_ENDPOINT;
}
mx_return = mx_open_endpoint(nic,
ep,
ompi_mtl_mx.mx_filter,
NULL,
0,
&ompi_mtl_mx.mx_endpoint);
if(mx_return != MX_SUCCESS) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_open_endpoint (error %s)\n", mx_strerror(mx_return));
return OMPI_ERROR;
}
/* get the endpoint address */
mx_return = mx_get_endpoint_addr( ompi_mtl_mx.mx_endpoint,
&ompi_mtl_mx.mx_endpoint_addr);
if(mx_return != MX_SUCCESS) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_get_endpoint_addr (error %s)\n", mx_strerror(mx_return));
return OMPI_ERROR;
}
mx_return = mx_decompose_endpoint_addr( ompi_mtl_mx.mx_endpoint_addr, &(ompi_mtl_mx.mx_addr.nic_id),
&(ompi_mtl_mx.mx_addr.endpoint_id) );
if(mx_return != MX_SUCCESS) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_decompose_endpoint_addr (error %s)\n", mx_strerror(mx_return));
return OMPI_ERROR;
}
opal_output_verbose(10, ompi_mtl_base_framework.framework_output,
"mtl:mx: local nic %d, endpoint %d, got nic %d, ep %d\n", nic, ep,
(int)ompi_mtl_mx.mx_addr.nic_id,
ompi_mtl_mx.mx_addr.endpoint_id);
ompi_modex_send( &mca_mtl_mx_component.super.mtl_version,
&ompi_mtl_mx.mx_addr,
sizeof(mca_mtl_mx_addr_t));
/* register the mtl mx progress function */
opal_progress_register(ompi_mtl_mx_progress);
return OMPI_SUCCESS;
}
static int mca_btl_ud_modex_send(void)
{
int rc;
size_t i;
size_t size;
mca_btl_ud_addr_t* addrs = NULL;
size = mca_btl_ofud_component.num_btls * sizeof(mca_btl_ud_addr_t);
if(size != 0) {
addrs = (mca_btl_ud_addr_t*)malloc(size);
if(NULL == addrs) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
for(i = 0; i < mca_btl_ofud_component.num_btls; i++) {
mca_btl_ud_module_t* btl = &mca_btl_ofud_component.ud_btls[i];
addrs[i] = btl->addr;
BTL_VERBOSE((0, "modex_send QP num %x, LID = %x",
addrs[i].qp_num, addrs[i].lid));
}
}
rc = ompi_modex_send(
&mca_btl_ofud_component.super.btl_version, addrs, size);
if(NULL != addrs) {
free(addrs);
}
return rc;
}
static int send_ep_address(void)
{
mxm_error_t error;
void *address;
size_t addrlen;
int rc;
addrlen = 0;
mxm_ep_get_address(ompi_pml_yalla.mxm_ep, NULL, &addrlen);
address = alloca(addrlen);
error = mxm_ep_get_address(ompi_pml_yalla.mxm_ep, address, &addrlen);
if (MXM_OK != error) {
PML_YALLA_ERROR("Failed to get EP address");
return OMPI_ERROR;
}
rc = ompi_modex_send(&mca_pml_yalla_component.pmlm_version, address, addrlen);
if (OMPI_SUCCESS != rc) {
PML_YALLA_ERROR("Open MPI couldn't distribute EP connection details");
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
/**************************
* usNIC BTL-specific functions to hide differences between master and
* v1.8
**************************/
void usnic_compat_modex_send(int *rc,
mca_base_component_t *component,
struct opal_btl_usnic_modex_t *modexes,
size_t size)
{
*rc = ompi_modex_send(component, modexes, size);
}
static int
mca_btl_udapl_modex_send(void)
{
int rc;
size_t i;
size_t size;
mca_btl_udapl_addr_t *addrs = NULL;
size = sizeof(mca_btl_udapl_addr_t) *
mca_btl_udapl_component.udapl_num_btls;
if (0 != size) {
addrs = (mca_btl_udapl_addr_t*)malloc(size);
if (NULL == addrs) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
memset(addrs, 0, size);
for (i = 0; i < mca_btl_udapl_component.udapl_num_btls; i++) {
mca_btl_udapl_module_t* btl = mca_btl_udapl_component.udapl_btls[i];
addrs[i] = btl->udapl_addr;
}
}
rc = ompi_modex_send(
&mca_btl_udapl_component.super.btl_version, addrs, size);
if (NULL != addrs) {
free (addrs);
}
return rc;
}
static int mca_btl_base_vader_modex_send (void)
{
struct vader_modex_t modex;
modex.seg_id = mca_btl_vader_component.my_seg_id;
modex.segment_base = mca_btl_vader_component.my_segment;
return ompi_modex_send(&mca_btl_vader_component.super.btl_version,
&modex, sizeof (modex));
}
int ompi_mtl_mx_module_init(){
mx_param_t mx_param;
mx_return_t mx_return;
/* setup params */
mx_param.key = MX_PARAM_UNEXP_QUEUE_MAX;
mx_param.val.unexp_queue_max = ompi_mtl_mx.mx_unexp_queue_max;
/* get a local endpoint */
mx_return = mx_open_endpoint(MX_ANY_NIC,
MX_ANY_ENDPOINT,
ompi_mtl_mx.mx_filter,
NULL,
0,
&ompi_mtl_mx.mx_endpoint);
if(mx_return != MX_SUCCESS) {
opal_output(ompi_mtl_base_output, "Error in mx_open_endpoint (error %s)\n", mx_strerror(mx_return));
return OMPI_ERROR;
}
/* get the endpoint address */
mx_return = mx_get_endpoint_addr( ompi_mtl_mx.mx_endpoint,
&ompi_mtl_mx.mx_endpoint_addr);
if(mx_return != MX_SUCCESS) {
opal_output(ompi_mtl_base_output, "Error in mx_get_endpoint_addr (error %s)\n", mx_strerror(mx_return));
return OMPI_ERROR;
}
mx_return = mx_decompose_endpoint_addr( ompi_mtl_mx.mx_endpoint_addr, &(ompi_mtl_mx.mx_addr.nic_id),
&(ompi_mtl_mx.mx_addr.endpoint_id) );
if(mx_return != MX_SUCCESS) {
opal_output(ompi_mtl_base_output, "Error in mx_decompose_endpoint_addr (error %s)\n", mx_strerror(mx_return));
return OMPI_ERROR;
}
ompi_modex_send( &mca_mtl_mx_component.super.mtl_version,
&ompi_mtl_mx.mx_addr,
sizeof(mca_mtl_mx_addr_t));
/* register the mtl mx progress function */
opal_progress_register(ompi_mtl_mx_progress);
return OMPI_SUCCESS;
}
int
ompi_common_portals_initialize(ptl_handle_ni_t *ni_handle, bool *accel)
{
int ret, max_interfaces;
ptl_process_id_t ptl_process_id;
ptl_interface_t ni_iface = PTL_IFACE_DEFAULT;
*accel = false;
/*
* If we use the YOD launcher we can use the default interface
* otherwise we need to use the SeaStar Bridged interface (for CNL/APRUN)
*/
ni_iface = IFACE_FROM_BRIDGE_AND_NALID(PTL_BRIDGE_UK,PTL_IFACE_SS);
/*
* Initialize Portals interface
*/
ret = PtlInit(&max_interfaces);
if (PTL_OK != ret) {
opal_output(0, "PtlInit failed, returning %d\n", ret);
return OMPI_ERR_NOT_AVAILABLE;
}
/*
* Initialize a network device
*/
ret = PtlNIInit(ni_iface, /* interface to initialize */
PTL_PID_ANY, /* let library assign our pid */
NULL, /* no desired limits */
NULL, /* actual limits */
ni_handle /* our interface handle */
);
if (PTL_OK != ret && PTL_IFACE_DUP != ret) {
opal_output(0, "PtlNIInit failed, returning %d (%s : %d)\n",
ret, __FILE__, __LINE__);
return OMPI_ERROR;
}
ret = PtlGetId(*ni_handle ,&ptl_process_id);
if(PTL_OK != ret) {
opal_output(0, "PtlGetId failed, returning %d\n", ret);
return OMPI_ERROR;
}
/* publish my nid/pid info */
ret = ompi_modex_send(&portals_component,
&ptl_process_id, sizeof(ptl_process_id_t));
if (OMPI_SUCCESS != ret) {
return ret;
}
return OMPI_SUCCESS;
}
static int mca_btl_sctp_component_exchange(void)
{
int rc=0;
size_t i=0;
size_t size = mca_btl_sctp_component.sctp_num_btls * sizeof(mca_btl_sctp_addr_t);
if(mca_btl_sctp_component.sctp_num_btls != 0) {
mca_btl_sctp_addr_t *addrs = (mca_btl_sctp_addr_t *)malloc(size);
for(i=0; i<mca_btl_sctp_component.sctp_num_btls; i++) {
struct mca_btl_sctp_module_t* btl = mca_btl_sctp_component.sctp_btls[i];
addrs[i].addr_inet = btl->sctp_ifaddr.sin_addr;
addrs[i].addr_port = mca_btl_sctp_component.sctp_listen_port;
addrs[i].addr_inuse = 0;
}
rc = ompi_modex_send(&mca_btl_sctp_component.super.btl_version, addrs, size);
free(addrs);
}
return rc;
}
int ompi_mtl_psm_module_init(int local_rank, int num_local_procs) {
psm_error_t err;
psm_ep_t ep; /* endpoint handle */
psm_mq_t mq;
psm_epid_t epid; /* unique lid+port identifier */
psm_uuid_t unique_job_key;
struct psm_ep_open_opts ep_opt;
unsigned long long *uu = (unsigned long long *) unique_job_key;
char *generated_key;
char env_string[256];
generated_key = getenv("OMPI_MCA_orte_precondition_transports");
memset(uu, 0, sizeof(psm_uuid_t));
if (!generated_key || (strlen(generated_key) != 33) ||
sscanf(generated_key, "%016llx-%016llx", &uu[0], &uu[1]) != 2)
{
opal_show_help("help-mtl-psm.txt",
"no uuid present", true,
generated_key ? "could not be parsed from" :
"not present in", ompi_process_info.nodename);
return OMPI_ERROR;
}
/* Handle our own errors for opening endpoints */
psm_error_register_handler(ompi_mtl_psm.ep, ompi_mtl_psm_errhandler);
/* Setup MPI_LOCALRANKID and MPI_LOCALNRANKS so PSM can allocate hardware
* contexts correctly.
*/
snprintf(env_string, sizeof(env_string), "%d", local_rank);
setenv("MPI_LOCALRANKID", env_string, 0);
snprintf(env_string, sizeof(env_string), "%d", num_local_procs);
setenv("MPI_LOCALNRANKS", env_string, 0);
/* Setup the endpoint options. */
bzero((void*) &ep_opt, sizeof(ep_opt));
ep_opt.timeout = ompi_mtl_psm.connect_timeout * 1e9;
ep_opt.unit = ompi_mtl_psm.ib_unit;
ep_opt.affinity = PSM_EP_OPEN_AFFINITY_SKIP; /* do not let PSM set affinity */
ep_opt.shm_mbytes = -1; /* Choose PSM defaults */
ep_opt.sendbufs_num = -1; /* Choose PSM defaults */
#if PSM_VERNO >= 0x0101
ep_opt.network_pkey = ompi_mtl_psm.ib_pkey;
#endif
#if PSM_VERNO >= 0x0107
ep_opt.port = ompi_mtl_psm.ib_port;
ep_opt.outsl = ompi_mtl_psm.ib_service_level;
#endif
#if PSM_VERNO >= 0x010d
ep_opt.service_id = ompi_mtl_psm.ib_service_id;
ep_opt.path_res_type = ompi_mtl_psm.path_res_type;
#endif
/* Open PSM endpoint */
err = psm_ep_open(unique_job_key, &ep_opt, &ep, &epid);
if (err) {
opal_show_help("help-mtl-psm.txt",
"unable to open endpoint", true,
psm_error_get_string(err));
return OMPI_ERROR;
}
/* Future errors are handled by the default error handler */
psm_error_register_handler(ompi_mtl_psm.ep, PSM_ERRHANDLER_DEFAULT);
err = psm_mq_init(ep,
0xffff000000000000ULL,
NULL,
0,
&mq);
if (err) {
opal_show_help("help-mtl-psm.txt",
"psm init", true,
psm_error_get_string(err));
return OMPI_ERROR;
}
ompi_mtl_psm.ep = ep;
ompi_mtl_psm.epid = epid;
ompi_mtl_psm.mq = mq;
if (OMPI_SUCCESS !=
ompi_modex_send( &mca_mtl_psm_component.super.mtl_version,
&ompi_mtl_psm.epid,
sizeof(psm_epid_t))) {
opal_output(0, "Open MPI couldn't send PSM epid to head node process");
return OMPI_ERROR;
}
/* register the psm progress function */
opal_progress_register(ompi_mtl_psm_progress);
return OMPI_SUCCESS;
}
int
ompi_common_portals_initialize(ptl_handle_ni_t *ni_handle, bool *accel)
{
int ret;
ptl_process_id_t info;
if (OPAL_THREAD_ADD32(&usage_count, 1) > 1) return OMPI_SUCCESS;
/* if the environment variables for the utcp implementation are
already set, assume the user is running without the full Open
RTE and is doing RTE testing for a more tightly-coupled
platform (like, say, Red Storm). Otherwise, be nice and use
the modex to setup everything for the user */
if (NULL == getenv("PTL_MY_RID")) {
setup_utcp_params = true;
} else {
setup_utcp_params = false;
}
if (setup_utcp_params) {
/* Find our contact information and post to registry. Don't
initialize Portals until we have everyone's contact
information. */
unsigned int nid;
p3tcp_my_nid(ptl_ifname, &nid);
info.nid = htonl(nid);
info.pid = htonl((ptl_pid_t) getpid());
} else {
/* Initialize Portals and publish our assigned contact
information */
int max_interfaces;
unsigned int nptl_procs, rank;
ret = PtlInit(&max_interfaces);
if (PTL_OK != ret) {
opal_output(0, "%5d: PtlInit failed, returning %d\n",
getpid(), ret);
return OMPI_ERR_NOT_AVAILABLE;
}
init_called = true;
/* tell the UTCP runtime code to read the env variables */
PtlSetRank(PTL_INVALID_HANDLE, -1, -1);
/* Initialize a network device */
ret = PtlNIInit(PTL_IFACE_DEFAULT, /* interface to initialize */
PTL_PID_ANY, /* let library assign our pid */
NULL, /* no desired limits */
NULL, /* no need to have limits around */
&active_ni_h /* our interface handle */
);
if (PTL_OK != ret) {
opal_output(0, "%5d: PtlNIInit failed, returning %d\n",
getpid(), ret);
return OMPI_ERR_FATAL;
}
ret = PtlGetRank(active_ni_h, &rank, &nptl_procs);
if (ret != PTL_OK) {
opal_output(0, "%5d, PtlGetRank() returned %d",
getpid(), ret);
return OMPI_ERR_FATAL;
}
ret = PtlGetRankId(active_ni_h, rank, &info);
if (ret != PTL_OK) {
opal_output(0, "%5d, PtlGetRank(rank=%d) returned %d",
getpid(), rank, ret);
return OMPI_ERR_FATAL;
}
}
ret = ompi_modex_send(&portals_component,
&info, sizeof(ptl_process_id_t));
if (OMPI_SUCCESS != ret) {
return ret;
}
return OMPI_SUCCESS;
}
