static mxm_error_t ompi_mtl_mxm_create_ep(mxm_h ctx, mxm_ep_h *ep, unsigned ptl_bitmap, int lr, uint32_t jobid, uint64_t mxlr, int nlps) { mxm_error_t err; #if MXM_API < MXM_VERSION(1,5) mxm_ep_opts_t ep_opt; struct sockaddr_mxm_local_proc sa_bind_self; struct sockaddr_mxm_ib_local sa_bind_rdma; struct sockaddr_mxm_shm_proc sa_bind_shm; mxm_fill_ep_opts(&ep_opt); sa_bind_self.sa_family = AF_MXM_LOCAL_PROC; sa_bind_self.context_id = lr; sa_bind_rdma.sa_family = AF_MXM_IB_LOCAL; sa_bind_rdma.lid = 0; sa_bind_rdma.pkey = 0; sa_bind_rdma.qp_num = 0; sa_bind_rdma.sl = 0; sa_bind_shm.sa_family = AF_MXM_SHM_PROC; sa_bind_shm.jobid = jobid; sa_bind_shm.process_id = lr; sa_bind_shm.context_id = mxlr; sa_bind_shm.num_procs = nlps; ep_opt.ptl_bind_addr[MXM_PTL_SELF] = (ptl_bitmap & MXM_BIT(MXM_PTL_SELF)) ? (struct sockaddr*) &sa_bind_self : NULL; ep_opt.ptl_bind_addr[MXM_PTL_RDMA] = (ptl_bitmap & MXM_BIT(MXM_PTL_RDMA)) ? (struct sockaddr*) &sa_bind_rdma : NULL; ep_opt.ptl_bind_addr[MXM_PTL_SHM] = (ptl_bitmap & MXM_BIT(MXM_PTL_SHM)) ? (struct sockaddr*) &sa_bind_shm : NULL; MXM_VERBOSE(1, "MXM version is old, consider to upgrade"); err = mxm_ep_create(ctx, &ep_opt, ep); #elif MXM_API < MXM_VERSION(2,0) mxm_ep_opts_t *ep_opts; err = mxm_config_read_ep_opts(&ep_opts); if (err != MXM_OK) { MXM_ERROR("Failed to parse MXM configuration"); return err; } ep_opts->job_id = jobid; ep_opts->local_rank = lr; ep_opts->num_local_procs = nlps; err = mxm_ep_create(ctx, ep_opts, ep); mxm_config_free(ep_opts); #else mxm_ep_opts_t *ep_opts; err = mxm_config_read_ep_opts(&ep_opts); if (err != MXM_OK) { MXM_ERROR("Failed to parse MXM configuration"); return err; } err = mxm_ep_create(ctx, ep_opts, ep); mxm_config_free_ep_opts(ep_opts); #endif return err; }
int ompi_mtl_mxm_module_init(void) { #if MXM_API < MXM_VERSION(2,0) ompi_mtl_mxm_ep_conn_info_t ep_info; #endif void *ep_address; size_t ep_address_len; mxm_error_t err; uint32_t jobid; uint64_t mxlr; ompi_proc_t **procs; unsigned ptl_bitmap; size_t totps, proc; int lr, nlps; int rc; mxlr = 0; lr = -1; jobid = ompi_mtl_mxm_get_job_id(); if (0 == jobid) { MXM_ERROR("Failed to generate jobid"); return OMPI_ERROR; } if (NULL == (procs = ompi_proc_world(&totps))) { MXM_ERROR("Unable to obtain process list"); return OMPI_ERROR; } if (totps < (size_t)ompi_mtl_mxm.mxm_np) { MXM_VERBOSE(1, "MXM support will be disabled because of total number " "of processes (%lu) is less than the minimum set by the " "mtl_mxm_np MCA parameter (%u)", totps, ompi_mtl_mxm.mxm_np); return OMPI_ERR_NOT_SUPPORTED; } MXM_VERBOSE(1, "MXM support enabled"); if (ORTE_NODE_RANK_INVALID == (lr = ompi_process_info.my_node_rank)) { MXM_ERROR("Unable to obtain local node rank"); return OMPI_ERROR; } nlps = ompi_process_info.num_local_peers + 1; for (proc = 0; proc < totps; proc++) { if (OPAL_PROC_ON_LOCAL_NODE(procs[proc]->proc_flags)) { mxlr = max(mxlr, procs[proc]->proc_name.vpid); } } /* Setup the endpoint options and local addresses to bind to. */ #if MXM_API < MXM_VERSION(2,0) ptl_bitmap = ompi_mtl_mxm.mxm_ctx_opts->ptl_bitmap; #else ptl_bitmap = 0; #endif /* Open MXM endpoint */ err = ompi_mtl_mxm_create_ep(ompi_mtl_mxm.mxm_context, &ompi_mtl_mxm.ep, ptl_bitmap, lr, jobid, mxlr, nlps); if (MXM_OK != err) { opal_show_help("help-mtl-mxm.txt", "unable to create endpoint", true, mxm_error_string(err)); return OMPI_ERROR; } /* * Get address for each PTL on this endpoint, and share it with other ranks. */ #if MXM_API < MXM_VERSION(2,0) if ((ptl_bitmap & MXM_BIT(MXM_PTL_SELF)) && OMPI_SUCCESS != ompi_mtl_mxm_get_ep_address(&ep_info, MXM_PTL_SELF)) { return OMPI_ERROR; } if ((ptl_bitmap & MXM_BIT(MXM_PTL_RDMA)) && OMPI_SUCCESS != ompi_mtl_mxm_get_ep_address(&ep_info, MXM_PTL_RDMA)) { return OMPI_ERROR; } if ((ptl_bitmap & MXM_BIT(MXM_PTL_SHM)) && OMPI_SUCCESS != ompi_mtl_mxm_get_ep_address(&ep_info, MXM_PTL_SHM)) { return OMPI_ERROR; } ep_address = &ep_info; ep_address_len = sizeof(ep_info); #else rc = ompi_mtl_mxm_get_ep_address(&ep_address, &ep_address_len); if (OMPI_SUCCESS != rc) { return rc; } #endif rc = ompi_mtl_mxm_send_ep_address(ep_address, ep_address_len); if (OMPI_SUCCESS != rc) { MXM_ERROR("Modex session failed."); return rc; } #if MXM_API >= MXM_VERSION(2,0) free(ep_address); #endif /* Register the MXM progress function */ opal_progress_register(ompi_mtl_mxm_progress); #if MXM_API >= MXM_VERSION(2,0) if (ompi_mtl_mxm.using_mem_hooks) { opal_mem_hooks_register_release(ompi_mtl_mxm_mem_release_cb, NULL); } #endif return OMPI_SUCCESS; }
static int ompi_mtl_mxm_component_register(void) { mca_base_component_t*c; c = &mca_mtl_mxm_component.super.mtl_version; ompi_mtl_mxm.verbose = 0; (void) mca_base_component_var_register(c, "verbose", "Verbose level of the MXM component", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &ompi_mtl_mxm.verbose); #if MXM_API > MXM_VERSION(2,0) ompi_mtl_mxm.mxm_np = 0; #else ompi_mtl_mxm.mxm_np = 128; #endif (void) mca_base_component_var_register(c, "np", "[integer] Minimal number of MPI processes in a single job " "required to activate the MXM transport", MCA_BASE_VAR_TYPE_INT, NULL,0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mxm.mxm_np); param_priority = 30; (void) mca_base_component_var_register (c, "priority", "Priority of the MXM MTL component", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, ¶m_priority); #if MXM_API >= MXM_VERSION(3,1) { unsigned long cur_ver = mxm_get_version(); ompi_mtl_mxm.bulk_connect = 0; if (cur_ver < MXM_VERSION(3,2)) { ompi_mtl_mxm.bulk_disconnect = 0; } else { ompi_mtl_mxm.bulk_disconnect = 1; } (void) mca_base_component_var_register(c, "bulk_connect", "[integer] use bulk connect", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mxm.bulk_connect); (void) mca_base_component_var_register(c, "bulk_disconnect", "[integer] use bulk disconnect", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mxm.bulk_disconnect); if (cur_ver < MXM_VERSION(3,2) && (ompi_mtl_mxm.bulk_connect || ompi_mtl_mxm.bulk_disconnect)) { ompi_mtl_mxm.bulk_connect = 0; ompi_mtl_mxm.bulk_disconnect = 0; MXM_VERBOSE(1, "WARNING: OMPI runs with %s version of MXM that is less than 3.2, " "so bulk connect/disconnect cannot work properly and will be turn off.", ompi_mtl_mxm.runtime_version); } } #endif return OMPI_SUCCESS; }