int
ompi_osc_portals4_free(struct ompi_win_t *win)
{
ompi_osc_portals4_module_t *module =
(ompi_osc_portals4_module_t*) win->w_osc_module;
int ret = OMPI_SUCCESS;
/* synchronize */
module->comm->c_coll.coll_barrier(module->comm,
module->comm->c_coll.coll_barrier_module);
/* cleanup */
PtlMEUnlink(module->control_me_h);
PtlMEUnlink(module->data_me_h);
PtlMDRelease(module->md_h);
PtlMDRelease(module->req_md_h);
PtlCTFree(module->ct_h);
if (NULL != module->disp_units) free(module->disp_units);
ompi_comm_free(&module->comm);
if (NULL != module->free_after) free(module->free_after);
if (!opal_list_is_empty(&module->outstanding_locks)) {
ret = OMPI_ERR_RMA_SYNC;
}
OBJ_DESTRUCT(&module->outstanding_locks);
free(module);
return ret;
}
int
ompi_mtl_portals4_flowctl_fini(void)
{
PtlPTFree(ompi_mtl_portals4.ni_h, ompi_mtl_portals4.flowctl_idx);
PtlCTFree(ompi_mtl_portals4.flowctl.trigger_ct_h);
PtlMEUnlink(ompi_mtl_portals4.flowctl.trigger_me_h);
PtlCTFree(ompi_mtl_portals4.flowctl.alert_ct_h);
PtlMEUnlink(ompi_mtl_portals4.flowctl.alert_me_h);
PtlCTFree(ompi_mtl_portals4.flowctl.fanin_ct_h);
PtlMEUnlink(ompi_mtl_portals4.flowctl.fanin_me_h);
PtlCTFree(ompi_mtl_portals4.flowctl.fanout_ct_h);
PtlMEUnlink(ompi_mtl_portals4.flowctl.fanout_me_h);
return OMPI_SUCCESS;
}
int
ompi_coll_portals4_ibarrier_intra_fini(ompi_coll_portals4_request_t *request)
{
int ret;
/* cleanup */
ret = PtlMEUnlink(request->me_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlMEUnlink failed: %d\n",
__FILE__, __LINE__, ret);
return OMPI_ERROR;
}
ret = PtlCTFree(request->ct_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlCTFree failed: %d\n",
__FILE__, __LINE__, ret);
return OMPI_ERROR;
}
OPAL_THREAD_LOCK(&ompi_request_lock);
ompi_request_complete(&request->super, true);
OPAL_THREAD_UNLOCK(&ompi_request_lock);
return OMPI_SUCCESS;
}
int MPID_nem_ptl_rptl_ptfini(ptl_pt_index_t pt_index)
{
int i;
int ret = PTL_OK;
struct rptl *rptl;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_PTL_RPTL_PTFINI);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_PTL_RPTL_PTFINI);
/* find the right rptl */
for (rptl = rptl_info.rptl_list; rptl && rptl->data.pt != pt_index; rptl = rptl->next);
assert(rptl);
/* free control portals that were created */
if (rptl->control.pt != PTL_PT_ANY) {
for (i = 0; i < rptl_info.world_size * 2; i++) {
ret = PtlMEUnlink(rptl->control.me[i]);
RPTLU_ERR_POP(ret, "Error unlinking control buffers\n");
}
MPIU_Free(rptl->control.me);
}
MPL_DL_DELETE(rptl_info.rptl_list, rptl);
MPIU_Free(rptl);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_PTL_RPTL_PTFINI);
return ret;
fn_fail:
goto fn_exit;
}
int
mca_btl_portals4_deregister_mem(mca_btl_base_module_t *btl_base,
mca_btl_base_registration_handle_t *handle)
{
int ret;
struct mca_btl_portals4_module_t *portals4_btl = (struct mca_btl_portals4_module_t*) btl_base;
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"mca_btl_portals4_deregister_mem NI=%d handle=%p key=%ld me_h=%d\n",
portals4_btl->interface_num, (void *)handle, handle->key, handle->me_h));
if (!PtlHandleIsEqual(handle->me_h, PTL_INVALID_HANDLE)) {
ret = PtlMEUnlink(handle->me_h);
if (PTL_OK != ret) {
opal_output_verbose(1, opal_btl_base_framework.framework_output,
"%s:%d: PtlMEUnlink failed: %d\n",__FILE__, __LINE__, ret);
return OPAL_ERROR;
}
handle->me_h = PTL_INVALID_HANDLE;
}
free(handle);
return OPAL_SUCCESS;
}
static void
mca_btl_portals4_frag_eager_destructor(mca_btl_portals4_frag_t* frag)
{
if (PTL_INVALID_HANDLE != frag->me_h) {
PtlMEUnlink(frag->me_h);
frag->me_h = PTL_INVALID_HANDLE;
}
}
void mca_btl_portals4_free_module(mca_btl_portals4_module_t *portals4_btl)
{
int ret;
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"mca_btl_portals4_free_module portals_outstanding_ops=%d\n", portals4_btl->portals_outstanding_ops));
/* sanity check */
assert(portals4_btl->portals_outstanding_ops >= 0);
/* finalize all communication */
while (portals4_btl->portals_outstanding_ops > 0) {
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"mca_btl_portals4_free_module portals_outstanding_ops: %d",
portals4_btl->portals_outstanding_ops));
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "Call to mca_btl_portals4_component_progress (3)\n"));
mca_btl_portals4_component_progress();
}
if (!PtlHandleIsEqual(portals4_btl->send_md_h, PTL_INVALID_HANDLE)) {
PtlMDRelease(portals4_btl->send_md_h);
portals4_btl->send_md_h = PTL_INVALID_HANDLE;
}
if (!PtlHandleIsEqual(portals4_btl->zero_md_h, PTL_INVALID_HANDLE)) {
PtlMDRelease(portals4_btl->zero_md_h);
portals4_btl->zero_md_h = PTL_INVALID_HANDLE;
}
if (!PtlHandleIsEqual(portals4_btl->long_overflow_me_h, PTL_INVALID_HANDLE)) {
PtlMEUnlink(portals4_btl->long_overflow_me_h);
portals4_btl->long_overflow_me_h = PTL_INVALID_HANDLE;
}
if ((ptl_pt_index_t) ~0UL != mca_btl_portals4_module.recv_idx) {
PtlPTFree(portals4_btl->portals_ni_h, portals4_btl->recv_idx);
portals4_btl->recv_idx= (ptl_pt_index_t) ~0UL;
}
if (PTL_EQ_NONE != portals4_btl->recv_eq_h) {
ret = PtlEQFree(portals4_btl->recv_eq_h);
if (PTL_OK != ret) OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "Error freeing EQ recv: %d", ret));
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "PtlEQFree: recv_eq_h=%d portals4_btl=%p",
portals4_btl->recv_eq_h, (void*)portals4_btl));
portals4_btl->recv_eq_h = PTL_EQ_NONE;
}
if (!PtlHandleIsEqual(portals4_btl->portals_ni_h, PTL_INVALID_HANDLE)) {
ret = PtlNIFini(portals4_btl->portals_ni_h);
if (PTL_OK != ret) OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "Error returned by PtlNIFini: %d\n", ret));
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "PtlNIFini: portals_ni_h=%d portals4_btl=%p",
portals4_btl->portals_ni_h, (void*)portals4_btl));
portals4_btl->portals_ni_h = PTL_INVALID_HANDLE;
}
ret = mca_btl_portals4_recv_disable(portals4_btl);
if (PTL_OK != ret) OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "Error freeing recv list: %d", ret));
}
static int
cleanup_sync_handles(ompi_coll_portals4_request_t *request)
{
int ret, line;
int ptl_ret;
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
"coll:portals4:cleanup_sync_handles enter rank %d", request->u.scatter.my_rank));
/**********************************/
/* Cleanup Sync Handles */
/**********************************/
do {
ret = PtlMEUnlink(request->u.scatter.sync_meh);
if (PTL_IN_USE == ret) {
opal_output(ompi_coll_base_framework.framework_output,
"%s:%4d: sync_meh still in use (ret=%d, rank %2d)",
__FILE__, __LINE__, ret, request->u.scatter.my_rank);
continue;
}
if (PTL_OK != ret) {
ret = OMPI_ERROR;
line = __LINE__;
goto err_hdlr;
}
} while (ret == PTL_IN_USE);
ret = PtlCTFree(request->u.scatter.sync_cth);
if (PTL_OK != ret) {
ptl_ret = ret;
ret = OMPI_ERROR;
line = __LINE__;
goto err_hdlr;
}
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
"coll:portals4:cleanup_sync_handles exit rank %d", request->u.scatter.my_rank));
return OMPI_SUCCESS;
err_hdlr:
opal_output(ompi_coll_base_framework.framework_output,
"%s:%4d:%4d\tError occurred (ptl_ret=%d) ret=%d, rank %2d",
__FILE__, __LINE__, line, ptl_ret, ret, request->u.scatter.my_rank);
return ret;
}
static int
ompi_mtl_portals4_recv_short_block_free(ompi_mtl_portals4_recv_short_block_t *block)
{
if (PTL_INVALID_HANDLE != block->me_h) {
PtlMEUnlink(block->me_h);
block->me_h = PTL_INVALID_HANDLE;
}
if (NULL != block->start) {
free(block->start);
block->start = NULL;
}
OBJ_RELEASE(block);
return OMPI_SUCCESS;
}
int
ompi_mtl_portals4_cancel(struct mca_mtl_base_module_t* mtl,
mca_mtl_request_t *mtl_request,
int flag)
{
ompi_mtl_portals4_base_request_t *base_request =
(ompi_mtl_portals4_base_request_t*) mtl_request;
int ret;
switch (base_request->type) {
case portals4_req_isend:
/* can't cancel sends yet */
break;
case portals4_req_recv:
{
ompi_mtl_portals4_recv_request_t *recvreq =
(ompi_mtl_portals4_recv_request_t*) base_request;
/* Cancel receive requests if not yet matched (otherwise,
they are guaranteed to complete and don't need to be
cancelled). If the me_h is already INVALID, that means
that not only has matching occurred, but the
communication end event has been seen. If MEUnlink
fails, that means that either something bad has
happened or the ME is in use (meaning no cancel). Need
to drain queue to make sure there isn't a pending
receive completion event... */
ompi_mtl_portals4_progress();
if (PTL_INVALID_HANDLE != recvreq->me_h) {
ret = PtlMEUnlink(recvreq->me_h);
if (OPAL_UNLIKELY(PTL_OK == ret)) {
recvreq->super.super.ompi_req->req_status._cancelled = true;
recvreq->super.super.completion_callback(&recvreq->super.super);
}
}
}
break;
default:
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
int
ompi_mtl_portals4_finalize(struct mca_mtl_base_module_t *mtl)
{
opal_progress_unregister(ompi_mtl_portals4_progress);
while (0 != ompi_mtl_portals4_progress()) { }
ompi_mtl_portals4_recv_short_fini(&ompi_mtl_portals4);
PtlMEUnlink(ompi_mtl_portals4.long_overflow_me_h);
PtlMDRelease(ompi_mtl_portals4.zero_md_h);
PtlPTFree(ompi_mtl_portals4.ni_h, ompi_mtl_portals4.read_idx);
PtlPTFree(ompi_mtl_portals4.ni_h, ompi_mtl_portals4.send_idx);
PtlEQFree(ompi_mtl_portals4.eq_h);
PtlNIFini(ompi_mtl_portals4.ni_h);
PtlFini();
return OMPI_SUCCESS;
}
int MPID_nem_ptl_pkt_cancel_send_resp_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
MPIDI_msg_sz_t *buflen, MPID_Request **rreqp)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *sreq;
MPIDI_nem_ptl_pkt_cancel_send_resp_t *resp_pkt = (MPIDI_nem_ptl_pkt_cancel_send_resp_t *)pkt;
int i, ret;
MPID_Request_get_ptr(resp_pkt->sender_req_id, sreq);
if (resp_pkt->ack) {
MPIR_STATUS_SET_CANCEL_BIT(sreq->status, TRUE);
/* remove/free any remaining get MEs and handles */
for (i = 0; i < REQ_PTL(sreq)->num_gets; i++) {
ret = PtlMEUnlink(REQ_PTL(sreq)->get_me_p[i]);
MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeunlink", "**ptlmeunlink %s", MPID_nem_ptl_strerror(ret));
mpi_errno = MPID_Request_complete(sreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
}
if (REQ_PTL(sreq)->get_me_p)
MPIU_Free(REQ_PTL(sreq)->get_me_p);
MPIU_DBG_MSG(CH3_OTHER,TYPICAL,"message cancelled");
} else {
MPIR_STATUS_SET_CANCEL_BIT(sreq->status, FALSE);
MPIU_DBG_MSG(CH3_OTHER,TYPICAL,"unable to cancel message");
}
mpi_errno = MPID_Request_complete(sreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
*rreqp = NULL;
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_ptl_poll_finalize(void)
{
int mpi_errno = MPI_SUCCESS;
int i;
int ret;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_PTL_POLL_FINALIZE);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_PTL_POLL_FINALIZE);
for (i = 0; i < NUM_OVERFLOW_ME; ++i) {
if (overflow_me_handle[i] != PTL_INVALID_HANDLE) {
ret = PtlMEUnlink(overflow_me_handle[i]);
MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeunlink", "**ptlmeunlink %s", MPID_nem_ptl_strerror(ret));
}
MPL_free(overflow_buf[i]);
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_PTL_POLL_FINALIZE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int
ompi_mtl_portals4_finalize(struct mca_mtl_base_module_t *mtl)
{
opal_progress_unregister(ompi_mtl_portals4_progress);
while (0 != ompi_mtl_portals4_progress()) { }
#if OMPI_MTL_PORTALS4_FLOW_CONTROL
ompi_mtl_portals4_flowctl_fini();
#endif
ompi_mtl_portals4_recv_short_fini();
PtlMEUnlink(ompi_mtl_portals4.long_overflow_me_h);
PtlMDRelease(ompi_mtl_portals4.zero_md_h);
#if OPAL_PORTALS4_MAX_MD_SIZE < OPAL_PORTALS4_MAX_VA_SIZE
{
int i;
int num_mds = ompi_mtl_portals4_get_num_mds();
for (i = 0 ; i < num_mds ; ++i) {
PtlMDRelease(ompi_mtl_portals4.send_md_hs[i]);
}
free(ompi_mtl_portals4.send_md_hs);
}
#else
PtlMDRelease(ompi_mtl_portals4.send_md_h);
#endif
PtlPTFree(ompi_mtl_portals4.ni_h, ompi_mtl_portals4.read_idx);
PtlPTFree(ompi_mtl_portals4.ni_h, ompi_mtl_portals4.recv_idx);
PtlEQFree(ompi_mtl_portals4.send_eq_h);
PtlEQFree(ompi_mtl_portals4.recv_eq_h);
PtlNIFini(ompi_mtl_portals4.ni_h);
PtlFini();
return OMPI_SUCCESS;
}
mca_btl_base_descriptor_t*
mca_btl_portals_prepare_dst(struct mca_btl_base_module_t* btl_base,
struct mca_btl_base_endpoint_t* peer,
mca_mpool_base_registration_t* registration,
struct ompi_convertor_t* convertor,
size_t reserve,
size_t* size)
{
mca_btl_portals_frag_t* frag;
ptl_md_t md;
ptl_handle_me_t me_h;
int ret;
ptrdiff_t lb;
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
/* reserve space in the event queue for rdma operations immediately */
while (OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, 1) >
mca_btl_portals_module.portals_max_outstanding_ops) {
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
mca_btl_portals_component_progress();
}
OMPI_BTL_PORTALS_FRAG_ALLOC_USER(&mca_btl_portals_module.super, frag, ret);
if(NULL == frag) {
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
return NULL;
}
ompi_ddt_type_lb(convertor->pDesc, &lb);
frag->segments[0].seg_len = *size;
frag->segments[0].seg_addr.pval = convertor->pBaseBuf + lb + convertor->bConverted;
frag->segments[0].seg_key.key64 =
OPAL_THREAD_ADD64(&(mca_btl_portals_module.portals_rdma_key), 1);
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->base.des_dst = frag->segments;
frag->base.des_dst_cnt = 1;
frag->base.des_flags = 0;
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"rdma dest posted for frag 0x%x, callback 0x%x, bits %lld",
frag, frag->base.des_cbfunc, frag->segments[0].seg_key.key64));
/* create a match entry */
ret = PtlMEAttach(mca_btl_portals_module.portals_ni_h,
OMPI_BTL_PORTALS_RDMA_TABLE_ID,
*((mca_btl_base_endpoint_t*) peer),
frag->segments[0].seg_key.key64, /* match */
0, /* ignore */
PTL_UNLINK,
PTL_INS_AFTER,
&me_h);
if (PTL_OK != ret) {
opal_output(mca_btl_portals_component.portals_output,
"Error creating rdma dest ME: %d", ret);
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
OMPI_BTL_PORTALS_FRAG_RETURN_USER(&mca_btl_portals_module.super, frag);
return NULL;
}
/* setup the memory descriptor. */
md.start = frag->segments[0].seg_addr.pval;
md.length = frag->segments[0].seg_len;
md.threshold = PTL_MD_THRESH_INF;
md.max_size = 0;
md.options = PTL_MD_OP_PUT | PTL_MD_OP_GET | PTL_MD_EVENT_START_DISABLE;
md.user_ptr = frag; /* keep a pointer to ourselves */
md.eq_handle = mca_btl_portals_module.portals_eq_handles[OMPI_BTL_PORTALS_EQ_SEND];
ret = PtlMDAttach(me_h,
md,
PTL_UNLINK,
&(frag->md_h));
if (PTL_OK != ret) {
opal_output(mca_btl_portals_component.portals_output,
"Error creating rdma dest MD: %d", ret);
PtlMEUnlink(me_h);
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
OMPI_BTL_PORTALS_FRAG_RETURN_USER(&mca_btl_portals_module.super, frag);
return NULL;
}
return &frag->base;
}
mca_btl_base_descriptor_t*
mca_btl_portals_prepare_src(struct mca_btl_base_module_t* btl_base,
struct mca_btl_base_endpoint_t* peer,
mca_mpool_base_registration_t* registration,
struct ompi_convertor_t* convertor,
size_t reserve,
size_t* size)
{
mca_btl_portals_frag_t* frag;
size_t max_data = *size;
struct iovec iov;
uint32_t iov_count = 1;
int ret;
assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
if (0 != reserve || 0 != ompi_convertor_need_buffers(convertor)) {
frag = (mca_btl_portals_frag_t*)
mca_btl_portals_alloc(btl_base, max_data + reserve);
if (NULL == frag) {
return NULL;
}
if (max_data + reserve > frag->size) {
max_data = frag->size - reserve;
}
iov.iov_len = max_data;
iov.iov_base = (unsigned char*) frag->segments[0].seg_addr.pval + reserve;
ret = ompi_convertor_pack(convertor, &iov, &iov_count,
&max_data );
*size = max_data;
if ( ret < 0 ) {
return NULL;
}
frag->segments[0].seg_len = max_data + reserve;
frag->base.des_src_cnt = 1;
} else {
/* no need to pack - rdma operation out of user's buffer */
ptl_md_t md;
ptl_handle_me_t me_h;
/* reserve space in the event queue for rdma operations immediately */
while (OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, 1) >
mca_btl_portals_module.portals_max_outstanding_ops) {
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
mca_btl_portals_component_progress();
}
OMPI_BTL_PORTALS_FRAG_ALLOC_USER(&mca_btl_portals_module.super, frag, ret);
if(NULL == frag){
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
return NULL;
}
iov.iov_len = max_data;
iov.iov_base = NULL;
ompi_convertor_pack(convertor, &iov, &iov_count, &max_data );
frag->segments[0].seg_len = max_data;
frag->segments[0].seg_addr.pval = iov.iov_base;
frag->segments[0].seg_key.key64 =
OPAL_THREAD_ADD64(&(mca_btl_portals_module.portals_rdma_key), 1);
frag->base.des_src_cnt = 1;
/* either a put or get. figure out which later */
OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
"rdma src posted for frag 0x%x, callback 0x%x, bits %lld",
frag, frag->base.des_cbfunc, frag->segments[0].seg_key.key64));
/* create a match entry */
ret = PtlMEAttach(mca_btl_portals_module.portals_ni_h,
OMPI_BTL_PORTALS_RDMA_TABLE_ID,
*((mca_btl_base_endpoint_t*) peer),
frag->segments[0].seg_key.key64, /* match */
0, /* ignore */
PTL_UNLINK,
PTL_INS_AFTER,
&me_h);
if (PTL_OK != ret) {
opal_output(mca_btl_portals_component.portals_output,
"Error creating rdma src ME: %d", ret);
OMPI_BTL_PORTALS_FRAG_RETURN_USER(&mca_btl_portals_module.super, frag);
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
return NULL;
}
/* setup the memory descriptor */
md.start = frag->segments[0].seg_addr.pval;
md.length = frag->segments[0].seg_len;
md.threshold = PTL_MD_THRESH_INF;
md.max_size = 0;
md.options = PTL_MD_OP_PUT | PTL_MD_OP_GET | PTL_MD_EVENT_START_DISABLE;
md.user_ptr = frag; /* keep a pointer to ourselves */
md.eq_handle = mca_btl_portals_module.portals_eq_handles[OMPI_BTL_PORTALS_EQ_SEND];
ret = PtlMDAttach(me_h,
md,
PTL_UNLINK,
&(frag->md_h));
if (PTL_OK != ret) {
opal_output(mca_btl_portals_component.portals_output,
"Error creating rdma src MD: %d", ret);
PtlMEUnlink(me_h);
OMPI_BTL_PORTALS_FRAG_RETURN_USER(&mca_btl_portals_module.super, frag);
OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
return NULL;
}
}
frag->base.des_src = frag->segments;
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
frag->base.des_flags = 0;
return &frag->base;
}
static inline int
ompi_mtl_portals4_short_isend(mca_pml_base_send_mode_t mode,
void *start, int length, int contextid, int tag,
int localrank,
ptl_process_t ptl_proc,
ompi_mtl_portals4_isend_request_t *ptl_request)
{
int ret;
ptl_match_bits_t match_bits;
ptl_me_t me;
ptl_hdr_data_t hdr_data;
MTL_PORTALS4_SET_SEND_BITS(match_bits, contextid, localrank, tag,
MTL_PORTALS4_SHORT_MSG);
MTL_PORTALS4_SET_HDR_DATA(hdr_data, ptl_request->opcount, length,
(MCA_PML_BASE_SEND_SYNCHRONOUS == mode) ? 1 : 0);
if (MCA_PML_BASE_SEND_SYNCHRONOUS == mode) {
me.start = NULL;
me.length = 0;
me.ct_handle = PTL_CT_NONE;
me.min_free = 0;
me.uid = ompi_mtl_portals4.uid;
me.options =
PTL_ME_OP_PUT |
PTL_ME_USE_ONCE |
PTL_ME_EVENT_LINK_DISABLE |
PTL_ME_EVENT_UNLINK_DISABLE;
me.match_id = ptl_proc;
me.match_bits = hdr_data;
me.ignore_bits = 0;
ret = PtlMEAppend(ompi_mtl_portals4.ni_h,
ompi_mtl_portals4.read_idx,
&me,
PTL_PRIORITY_LIST,
ptl_request,
&ptl_request->me_h);
if (OPAL_UNLIKELY(PTL_OK != ret)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlMEAppend failed: %d",
__FILE__, __LINE__, ret);
return ompi_mtl_portals4_get_error(ret);
}
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Send %lu short sync send with hdr_data 0x%lx (0x%lx)",
ptl_request->opcount, hdr_data, match_bits));
} else {
ptl_request->event_count = 1;
ptl_request->me_h = PTL_INVALID_HANDLE;
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Send %lu short send with hdr_data 0x%lx (0x%lx)",
ptl_request->opcount, hdr_data, match_bits));
}
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Send %lu, start: %p",
ptl_request->opcount, start));
ret = PtlPut(ompi_mtl_portals4.send_md_h,
(ptl_size_t) start,
length,
PTL_ACK_REQ,
ptl_proc,
ompi_mtl_portals4.recv_idx,
match_bits,
0,
ptl_request,
hdr_data);
if (OPAL_UNLIKELY(PTL_OK != ret)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlPut failed: %d",
__FILE__, __LINE__, ret);
if (MCA_PML_BASE_SEND_SYNCHRONOUS == mode) {
PtlMEUnlink(ptl_request->me_h);
}
return ompi_mtl_portals4_get_error(ret);
}
return OMPI_SUCCESS;
}
static inline int
ompi_mtl_portals4_long_isend(void *start, size_t length, int contextid, int tag,
int localrank,
ptl_process_t ptl_proc,
ompi_mtl_portals4_isend_request_t *ptl_request)
{
int ret;
ptl_match_bits_t match_bits;
ptl_me_t me;
ptl_hdr_data_t hdr_data;
ptl_size_t put_length;
MTL_PORTALS4_SET_SEND_BITS(match_bits, contextid, localrank, tag,
MTL_PORTALS4_LONG_MSG);
MTL_PORTALS4_SET_HDR_DATA(hdr_data, ptl_request->opcount, length, 0);
me.start = start;
me.length = length;
me.ct_handle = PTL_CT_NONE;
me.min_free = 0;
me.uid = ompi_mtl_portals4.uid;
me.options =
PTL_ME_OP_GET |
PTL_ME_USE_ONCE |
PTL_ME_EVENT_LINK_DISABLE |
PTL_ME_EVENT_UNLINK_DISABLE;
me.match_id = ptl_proc;
me.match_bits = hdr_data;
me.ignore_bits = 0;
ret = PtlMEAppend(ompi_mtl_portals4.ni_h,
ompi_mtl_portals4.read_idx,
&me,
PTL_PRIORITY_LIST,
ptl_request,
&ptl_request->me_h);
if (OPAL_UNLIKELY(PTL_OK != ret)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlMEAppend failed: %d",
__FILE__, __LINE__, ret);
return ompi_mtl_portals4_get_error(ret);
}
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Send %lu long send with hdr_data 0x%lx (0x%lx)",
ptl_request->opcount, hdr_data, match_bits));
put_length = (rndv == ompi_mtl_portals4.protocol) ?
(ptl_size_t) ompi_mtl_portals4.eager_limit : (ptl_size_t) length;
ret = PtlPut(ompi_mtl_portals4.send_md_h,
(ptl_size_t) start,
put_length,
PTL_ACK_REQ,
ptl_proc,
ompi_mtl_portals4.recv_idx,
match_bits,
0,
ptl_request,
hdr_data);
if (OPAL_UNLIKELY(PTL_OK != ret)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlPut failed: %d",
__FILE__, __LINE__, ret);
PtlMEUnlink(ptl_request->me_h);
return ompi_mtl_portals4_get_error(ret);
}
return OMPI_SUCCESS;
}
static inline int
ompi_mtl_portals4_callback(ptl_event_t *ev,
ompi_mtl_portals4_base_request_t* ptl_base_request,
bool *complete)
{
int retval = OMPI_SUCCESS, ret, val, add = 1;
ompi_mtl_portals4_isend_request_t* ptl_request =
(ompi_mtl_portals4_isend_request_t*) ptl_base_request;
#if OMPI_MTL_PORTALS4_FLOW_CONTROL
if (OPAL_UNLIKELY(ev->ni_fail_type == PTL_NI_PT_DISABLED)) {
ompi_mtl_portals4_pending_request_t *pending =
ptl_request->pending;
OPAL_OUTPUT_VERBOSE((10, ompi_mtl_base_framework.framework_output,
"send %lu hit flow control (%d)",
ptl_request->opcount, ev->type));
/* BWB: FIX ME: this is a hack.. */
if (pending->fc_notified) {
return OMPI_SUCCESS;
}
pending->fc_notified = 1;
if (!PtlHandleIsEqual(ptl_request->me_h, PTL_INVALID_HANDLE)) {
ret = PtlMEUnlink(ptl_request->me_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: send callback PtlMEUnlink returned %d",
__FILE__, __LINE__, ret);
}
}
opal_list_append(&ompi_mtl_portals4.flowctl.pending_sends,
&pending->super.super);
OPAL_THREAD_ADD32(&ompi_mtl_portals4.flowctl.send_slots, 1);
ompi_mtl_portals4_flowctl_trigger();
return OMPI_SUCCESS;
}
#endif
if (OPAL_UNLIKELY(ev->ni_fail_type != PTL_NI_OK)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: send callback ni_fail_type: %d",
__FILE__, __LINE__, ev->ni_fail_type);
*complete = true;
return OMPI_ERROR;
}
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"send %lu got event of type %d",
ptl_request->opcount, ev->type));
if ((PTL_EVENT_ACK == ev->type) &&
(PTL_PRIORITY_LIST == ev->ptl_list) &&
(eager == ompi_mtl_portals4.protocol) &&
(!PtlHandleIsEqual(ptl_request->me_h, PTL_INVALID_HANDLE))) {
/* long expected messages with the eager protocol won't see a
get event to complete the message. Give them an extra
count to cause the message to complete with just the SEND
and ACK events and remove the ME. (we wait for the counter
to reach 3 events, but short messages start the counter at
1, so they don't need to enter this path) */
ret = PtlMEUnlink(ptl_request->me_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: send callback PtlMEUnlink returned %d",
__FILE__, __LINE__, ret);
}
add++;
}
val = OPAL_THREAD_ADD32((int32_t*)&ptl_request->event_count, add);
assert(val <= 3);
if (val == 3) {
if (NULL != ptl_request->buffer_ptr) {
free(ptl_request->buffer_ptr);
}
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output, "send %lu completed",
ptl_request->opcount));
*complete = true;
#if OMPI_MTL_PORTALS4_FLOW_CONTROL
OPAL_THREAD_ADD32(&ompi_mtl_portals4.flowctl.send_slots, 1);
opal_free_list_return (&ompi_mtl_portals4.flowctl.pending_fl,
&ptl_request->pending->super);
if (OPAL_UNLIKELY(0 != opal_list_get_size(&ompi_mtl_portals4.flowctl.pending_sends))) {
ompi_mtl_portals4_pending_list_progress();
}
#endif
}
return retval;
}
int
ompi_coll_portals4_barrier_intra(struct ompi_communicator_t *comm,
mca_coll_base_module_t *module)
{
mca_coll_portals4_module_t *portals4_module = (mca_coll_portals4_module_t*) module;
int ret, i, dim, hibit, mask, num_msgs;
int size = ompi_comm_size(comm);
int rank = ompi_comm_rank(comm);
ptl_ct_event_t ct;
ptl_handle_ct_t ct_h;
ptl_handle_me_t me_h;
ptl_me_t me;
size_t count;
ptl_match_bits_t match_bits;
ptl_handle_md_t md_h;
void *base;
ompi_coll_portals4_get_md(0, &md_h, &base);
count = opal_atomic_add_size_t(&portals4_module->barrier_count, 1);
ret = PtlCTAlloc(mca_coll_portals4_component.ni_h,
&ct_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlCTAlloc failed: %d\n",
__FILE__, __LINE__, ret);
return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
}
COLL_PORTALS4_SET_BITS(match_bits, ompi_comm_get_cid(comm),
0, COLL_PORTALS4_BARRIER, count);
/* Build "tree" out of hypercube */
dim = comm->c_cube_dim;
hibit = opal_hibit(rank, dim);
--dim;
/* receive space */
me.start = NULL;
me.length = 0;
me.ct_handle = ct_h;
me.min_free = 0;
me.uid = mca_coll_portals4_component.uid;
me.options = PTL_ME_OP_PUT | PTL_ME_EVENT_SUCCESS_DISABLE |
PTL_ME_EVENT_LINK_DISABLE | PTL_ME_EVENT_UNLINK_DISABLE |
PTL_ME_EVENT_CT_COMM | PTL_ME_EVENT_CT_OVERFLOW;
me.match_id.phys.nid = PTL_NID_ANY;
me.match_id.phys.pid = PTL_PID_ANY;
me.match_bits = match_bits;
me.ignore_bits = 0;
ret = PtlMEAppend(mca_coll_portals4_component.ni_h,
mca_coll_portals4_component.pt_idx,
&me,
PTL_PRIORITY_LIST,
NULL,
&me_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlMEAppend failed: %d\n",
__FILE__, __LINE__, ret);
return OMPI_ERROR;
}
/* calculate number of children to receive from */
num_msgs = ompi_coll_portals4_get_nchildren(dim + 1, hibit, rank, size);
/* send to parent when children have sent to us */
if (rank > 0) {
int parent = rank & ~(1 << hibit);
ret = PtlTriggeredPut(md_h,
0,
0,
PTL_NO_ACK_REQ,
ompi_coll_portals4_get_peer(comm, parent),
mca_coll_portals4_component.pt_idx,
match_bits,
0,
NULL,
0,
ct_h,
num_msgs);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlTriggeredPut failed: %d\n",
__FILE__, __LINE__, ret);
return OMPI_ERROR;
}
/* we'll need to wait for the parent response before the next set of comms */
num_msgs++;
}
/* send to children when parent (or all children if root) has sent to us */
for (i = hibit + 1, mask = 1 << i; i <= dim; ++i, mask <<= 1) {
int peer = rank | mask;
if (peer < size) {
ret = PtlTriggeredPut(md_h,
0,
0,
PTL_NO_ACK_REQ,
ompi_coll_portals4_get_peer(comm, peer),
mca_coll_portals4_component.pt_idx,
match_bits,
0,
NULL,
0,
ct_h,
num_msgs);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlTriggeredPut failed: %d\n",
__FILE__, __LINE__, ret);
return OMPI_ERROR;
}
}
}
/* Wait for all incoming messages */
ret = PtlCTWait(ct_h, num_msgs, &ct);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlCTWait failed: %d\n",
__FILE__, __LINE__, ret);
return OMPI_ERROR;
}
/* cleanup */
ret = PtlMEUnlink(me_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlMEUnlink failed: %d\n",
__FILE__, __LINE__, ret);
return OMPI_ERROR;
}
ret = PtlCTFree(ct_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlCTFree failed: %d\n",
__FILE__, __LINE__, ret);
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
int
ompi_mtl_portals_isend(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct ompi_convertor_t *convertor,
mca_pml_base_send_mode_t mode,
bool blocking,
mca_mtl_request_t *mtl_request)
{
int ret;
ptl_match_bits_t match_bits;
ptl_md_t md;
ptl_handle_md_t md_h;
ptl_handle_me_t me_h;
ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, dest );
mca_mtl_base_endpoint_t *endpoint = (mca_mtl_base_endpoint_t*) ompi_proc->proc_pml;
ompi_mtl_portals_request_t *ptl_request =
(ompi_mtl_portals_request_t*) mtl_request;
size_t buflen;
assert(mtl == &ompi_mtl_portals.base);
ret = ompi_mtl_datatype_pack(convertor, &md.start, &buflen,
&(ptl_request->free_after));
if (OMPI_SUCCESS != ret) return ret;
md.length = buflen;
ptl_request->event_callback = ompi_mtl_portals_send_progress;
if ((MCA_PML_BASE_SEND_READY == mode)) {
/* ready send (length doesn't matter) or short non-sync send.
Eagerly send data and don't wait for completion */
PTL_SET_SEND_BITS(match_bits, comm->c_contextid,
comm->c_my_rank,
tag, PTL_READY_MSG);
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output,
"ready send bits: 0x%016llx\n",
match_bits));
md.threshold = 1;
md.options = PTL_MD_EVENT_START_DISABLE;
md.user_ptr = ptl_request;
md.eq_handle = ompi_mtl_portals.ptl_eq_h;
ret = PtlMDBind(ompi_mtl_portals.ptl_ni_h,
md,
PTL_UNLINK,
&(md_h));
if (OMPI_SUCCESS != ret) {
if (ptl_request->free_after) free(md.start);
return ompi_common_portals_error_ptl_to_ompi(ret);
}
ptl_request->event_callback = ompi_mtl_portals_send_progress_no_ack;
ret = PtlPut(md_h,
PTL_NO_ACK_REQ,
endpoint->ptl_proc,
OMPI_MTL_PORTALS_SEND_TABLE_ID,
0,
match_bits,
0,
0);
if (OMPI_SUCCESS != ret) {
PtlMDUnlink(md_h);
if (ptl_request->free_after) free(md.start);
return ompi_common_portals_error_ptl_to_ompi(ret);
}
} else if (md.length > ompi_mtl_portals.eager_limit) {
/* it's a long message - same protocol for all send modes
other than ready */
PTL_SET_SEND_BITS(match_bits, comm->c_contextid,
comm->c_my_rank,
tag, PTL_LONG_MSG);
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output,
"long send bits: 0x%016llx (%d)\n",
match_bits, dest));
md.threshold = 2; /* send, {ack, get} */
md.options = PTL_MD_OP_GET | PTL_MD_EVENT_START_DISABLE;
md.user_ptr = ptl_request;
md.eq_handle = ompi_mtl_portals.ptl_eq_h;
ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h,
OMPI_MTL_PORTALS_READ_TABLE_ID,
endpoint->ptl_proc,
(ptl_match_bits_t)(uintptr_t) ptl_request,
0,
PTL_UNLINK,
PTL_INS_AFTER,
&me_h);
if (OMPI_SUCCESS != ret) {
if (ptl_request->free_after) free(md.start);
return ompi_common_portals_error_ptl_to_ompi(ret);
}
ret = PtlMDAttach(me_h,
md,
PTL_UNLINK,
&(md_h));
if (OMPI_SUCCESS != ret) {
PtlMEUnlink(me_h);
if (ptl_request->free_after) free(md.start);
return ompi_common_portals_error_ptl_to_ompi(ret);
}
ret = PtlPut(md_h,
PTL_ACK_REQ,
endpoint->ptl_proc,
OMPI_MTL_PORTALS_SEND_TABLE_ID,
0,
match_bits,
0,
(ptl_hdr_data_t)(uintptr_t) ptl_request);
if (OMPI_SUCCESS != ret) {
PtlMDUnlink(md_h);
if (ptl_request->free_after) free(md.start);
return ompi_common_portals_error_ptl_to_ompi(ret);
}
} else if (MCA_PML_BASE_SEND_SYNCHRONOUS == mode) {
/* short synchronous message */
PTL_SET_SEND_BITS(match_bits, comm->c_contextid,
comm->c_my_rank,
tag, PTL_SHORT_MSG);
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output,
"short ssend bits: 0x%016llx (%d)\n",
match_bits, dest));
md.threshold = 2; /* send, {ack, put} */
md.options = PTL_MD_OP_PUT | PTL_MD_EVENT_START_DISABLE;
md.user_ptr = ptl_request;
md.eq_handle = ompi_mtl_portals.ptl_eq_h;
ret = PtlMEAttach(ompi_mtl_portals.ptl_ni_h,
OMPI_MTL_PORTALS_ACK_TABLE_ID,
endpoint->ptl_proc,
(ptl_match_bits_t)(uintptr_t) ptl_request,
0,
PTL_UNLINK,
PTL_INS_AFTER,
&me_h);
if (OMPI_SUCCESS != ret) {
if (ptl_request->free_after) free(md.start);
return ompi_common_portals_error_ptl_to_ompi(ret);
}
ret = PtlMDAttach(me_h,
md,
PTL_UNLINK,
&(md_h));
if (OMPI_SUCCESS != ret) {
PtlMEUnlink(me_h);
if (ptl_request->free_after) free(md.start);
return ompi_common_portals_error_ptl_to_ompi(ret);
}
ret = PtlPut(md_h,
PTL_ACK_REQ,
endpoint->ptl_proc,
OMPI_MTL_PORTALS_SEND_TABLE_ID,
0,
match_bits,
0,
(ptl_hdr_data_t)(uintptr_t) ptl_request);
if (OMPI_SUCCESS != ret) {
PtlMDUnlink(md_h);
if (ptl_request->free_after) free(md.start);
return ompi_common_portals_error_ptl_to_ompi(ret);
}
} else {
/* short send message */
PTL_SET_SEND_BITS(match_bits, comm->c_contextid,
comm->c_my_rank,
tag, PTL_SHORT_MSG);
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output,
"short send bits: 0x%016llx\n",
match_bits));
md.threshold = 1;
md.options = PTL_MD_EVENT_START_DISABLE;
md.user_ptr = ptl_request;
md.eq_handle = ompi_mtl_portals.ptl_eq_h;
ret = PtlMDBind(ompi_mtl_portals.ptl_ni_h,
md,
PTL_UNLINK,
&(md_h));
if (OMPI_SUCCESS != ret) {
if (ptl_request->free_after) free(md.start);
return ompi_common_portals_error_ptl_to_ompi(ret);
}
ptl_request->event_callback = ompi_mtl_portals_send_progress_no_ack;
ret = PtlPut(md_h,
PTL_NO_ACK_REQ,
endpoint->ptl_proc,
OMPI_MTL_PORTALS_SEND_TABLE_ID,
0,
match_bits,
0,
0);
if (OMPI_SUCCESS != ret) {
PtlMDUnlink(md_h);
if (ptl_request->free_after) free(md.start);
return ompi_common_portals_error_ptl_to_ompi(ret);
}
}
return OMPI_SUCCESS;
}
static int
portals4_close(void)
{
int ret;
OBJ_DESTRUCT(&mca_coll_portals4_component.requests);
if (!PtlHandleIsEqual(mca_coll_portals4_component.zero_md_h, PTL_INVALID_HANDLE)) {
ret = PtlMDRelease(mca_coll_portals4_component.zero_md_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlMDRelease failed: %d\n",
__FILE__, __LINE__, ret);
}
}
mca_coll_portals4_component.zero_md_h = PTL_INVALID_HANDLE;
if (!PtlHandleIsEqual(mca_coll_portals4_component.data_md_h, PTL_INVALID_HANDLE)) {
ret = PtlMDRelease(mca_coll_portals4_component.data_md_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlMDRelease failed: %d\n",
__FILE__, __LINE__, ret);
}
}
mca_coll_portals4_component.data_md_h = PTL_INVALID_HANDLE;
if (!PtlHandleIsEqual(mca_coll_portals4_component.finish_me_h, PTL_INVALID_HANDLE)) {
ret = PtlMEUnlink(mca_coll_portals4_component.finish_me_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlMEUnlink failed: %d\n",
__FILE__, __LINE__, ret);
}
}
if (!PtlHandleIsEqual(mca_coll_portals4_component.unex_me_h, PTL_INVALID_HANDLE)) {
ret = PtlMEUnlink(mca_coll_portals4_component.unex_me_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlMEUnlink failed: %d\n",
__FILE__, __LINE__, ret);
}
}
if (mca_coll_portals4_component.finish_pt_idx >= 0) {
ret = PtlPTFree(mca_coll_portals4_component.ni_h, mca_coll_portals4_component.finish_pt_idx);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlPTFree failed: %d\n",
__FILE__, __LINE__, ret);
}
}
if (mca_coll_portals4_component.pt_idx >= 0) {
ret = PtlPTFree(mca_coll_portals4_component.ni_h, mca_coll_portals4_component.pt_idx);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlPTFree failed: %d\n",
__FILE__, __LINE__, ret);
}
}
if (!PtlHandleIsEqual(mca_coll_portals4_component.eq_h, PTL_INVALID_HANDLE)) {
ret = PtlEQFree(mca_coll_portals4_component.eq_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlEQFree failed: %d\n",
__FILE__, __LINE__, ret);
}
}
if (!PtlHandleIsEqual(mca_coll_portals4_component.ni_h, PTL_INVALID_HANDLE)) {
ret = PtlNIFini(mca_coll_portals4_component.ni_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlNIFini failed: %d\n",
__FILE__, __LINE__, ret);
}
PtlFini();
}
opal_progress_unregister(portals4_progress);
return OMPI_SUCCESS;
}
static int
component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
struct ompi_communicator_t *comm, struct ompi_info_t *info,
int flavor, int *model)
{
ompi_osc_portals4_module_t *module = NULL;
int ret = OMPI_ERROR;
int tmp;
ptl_md_t md;
ptl_me_t me;
char *name;
if (MPI_WIN_FLAVOR_SHARED == flavor) return OMPI_ERR_NOT_SUPPORTED;
/* create module structure */
module = (ompi_osc_portals4_module_t*)
calloc(1, sizeof(ompi_osc_portals4_module_t));
if (NULL == module) return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
/* fill in the function pointer part */
memcpy(module, &ompi_osc_portals4_module_template,
sizeof(ompi_osc_base_module_t));
/* fill in our part */
if (MPI_WIN_FLAVOR_ALLOCATE == flavor) {
module->free_after = *base = malloc(size);
if (NULL == *base) goto error;
} else {
module->free_after = NULL;
}
ret = ompi_comm_dup(comm, &module->comm);
if (OMPI_SUCCESS != ret) goto error;
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"portals4 component creating window with id %d",
ompi_comm_get_cid(module->comm));
asprintf(&name, "portals4 window %d", ompi_comm_get_cid(module->comm));
ompi_win_set_name(win, name);
free(name);
/* share everyone's displacement units. Only do an allgather if
strictly necessary, since it requires O(p) state. */
tmp = disp_unit;
ret = module->comm->c_coll.coll_bcast(&tmp, 1, MPI_INT, 0,
module->comm,
module->comm->c_coll.coll_bcast_module);
if (OMPI_SUCCESS != ret) {
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"%s:%d: MPI_Bcast failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
tmp = (tmp == disp_unit) ? 1 : 0;
ret = module->comm->c_coll.coll_allreduce(MPI_IN_PLACE, &tmp, 1, MPI_INT, MPI_LAND,
module->comm, module->comm->c_coll.coll_allreduce_module);
if (OMPI_SUCCESS != ret) goto error;
if (tmp == 1) {
module->disp_unit = disp_unit;
module->disp_units = NULL;
} else {
module->disp_unit = -1;
module->disp_units = malloc(sizeof(int) * ompi_comm_size(module->comm));
ret = module->comm->c_coll.coll_allgather(&disp_unit, 1, MPI_INT,
module->disp_units, 1, MPI_INT,
module->comm,
module->comm->c_coll.coll_allgather_module);
if (OMPI_SUCCESS != ret) goto error;
}
module->ni_h = mca_osc_portals4_component.matching_ni_h;
module->pt_idx = mca_osc_portals4_component.matching_pt_idx;
ret = PtlCTAlloc(module->ni_h, &(module->ct_h));
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"%s:%d: PtlCTAlloc failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
md.start = 0;
md.length = PTL_SIZE_MAX;
md.options = PTL_MD_EVENT_SUCCESS_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK;
md.eq_handle = mca_osc_portals4_component.matching_eq_h;
md.ct_handle = module->ct_h;
ret = PtlMDBind(module->ni_h, &md, &module->md_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"%s:%d: PtlMDBind failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
md.start = 0;
md.length = PTL_SIZE_MAX;
md.options = PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK;
md.eq_handle = mca_osc_portals4_component.matching_eq_h;
md.ct_handle = module->ct_h;
ret = PtlMDBind(module->ni_h, &md, &module->req_md_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"%s:%d: PtlMDBind failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
if (MPI_WIN_FLAVOR_DYNAMIC == flavor) {
me.start = 0;
me.length = PTL_SIZE_MAX;
} else {
me.start = *base;
me.length = size;
}
me.ct_handle = PTL_CT_NONE;
me.uid = PTL_UID_ANY;
me.options = PTL_ME_OP_PUT | PTL_ME_OP_GET | PTL_ME_NO_TRUNCATE | PTL_ME_EVENT_SUCCESS_DISABLE;
me.match_id.phys.nid = PTL_NID_ANY;
me.match_id.phys.pid = PTL_PID_ANY;
me.match_bits = module->comm->c_contextid;
me.ignore_bits = 0;
ret = PtlMEAppend(module->ni_h,
module->pt_idx,
&me,
PTL_PRIORITY_LIST,
NULL,
&module->data_me_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"%s:%d: PtlMEAppend failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
me.start = &module->state;
me.length = sizeof(module->state);
me.ct_handle = PTL_CT_NONE;
me.uid = PTL_UID_ANY;
me.options = PTL_ME_OP_PUT | PTL_ME_OP_GET | PTL_ME_NO_TRUNCATE | PTL_ME_EVENT_SUCCESS_DISABLE;
me.match_id.phys.nid = PTL_NID_ANY;
me.match_id.phys.pid = PTL_PID_ANY;
me.match_bits = module->comm->c_contextid | OSC_PORTALS4_MB_CONTROL;
me.ignore_bits = 0;
ret = PtlMEAppend(module->ni_h,
module->pt_idx,
&me,
PTL_PRIORITY_LIST,
NULL,
&module->control_me_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"%s:%d: PtlMEAppend failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
module->opcount = 0;
module->match_bits = module->comm->c_contextid;
module->atomic_max = (check_config_value_equal("accumulate_ordering", info, "none")) ?
mca_osc_portals4_component.matching_atomic_max :
MIN(mca_osc_portals4_component.matching_atomic_max,
mca_osc_portals4_component.matching_atomic_ordered_size);
module->fetch_atomic_max = (check_config_value_equal("accumulate_ordering", info, "none")) ?
mca_osc_portals4_component.matching_fetch_atomic_max :
MIN(mca_osc_portals4_component.matching_fetch_atomic_max,
mca_osc_portals4_component.matching_atomic_ordered_size);
module->zero = 0;
module->one = 1;
module->start_group = NULL;
module->post_group = NULL;
module->state.post_count = 0;
module->state.complete_count = 0;
if (check_config_value_bool("no_locks", info)) {
module->state.lock = LOCK_ILLEGAL;
} else {
module->state.lock = LOCK_UNLOCKED;
}
OBJ_CONSTRUCT(&module->outstanding_locks, opal_list_t);
module->passive_target_access_epoch = false;
#if OPAL_ASSEMBLY_ARCH == OPAL_AMD64 || OPAL_ASSEMBLY_ARCH == OPAL_IA32
*model = MPI_WIN_UNIFIED;
#else
*model = MPI_WIN_SEPARATE;
#endif
win->w_osc_module = &module->super;
PtlAtomicSync();
/* Make sure that everyone's ready to receive. */
module->comm->c_coll.coll_barrier(module->comm,
module->comm->c_coll.coll_barrier_module);
return OMPI_SUCCESS;
error:
/* BWB: FIX ME: This is all wrong... */
if (0 != module->ct_h) PtlCTFree(module->ct_h);
if (0 != module->data_me_h) PtlMEUnlink(module->data_me_h);
if (0 != module->req_md_h) PtlMDRelease(module->req_md_h);
if (0 != module->md_h) PtlMDRelease(module->md_h);
if (NULL != module->comm) ompi_comm_free(&module->comm);
if (NULL != module) free(module);
return ret;
}
static int
portals4_init_interface(void)
{
unsigned int ret;
ptl_md_t md;
ptl_me_t me;
/* create event queues */
ret = PtlEQAlloc(ompi_mtl_portals4.ni_h,
ompi_mtl_portals4.send_queue_size,
&ompi_mtl_portals4.send_eq_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlEQAlloc failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
ret = PtlEQAlloc(ompi_mtl_portals4.ni_h,
ompi_mtl_portals4.recv_queue_size,
&ompi_mtl_portals4.recv_eq_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlEQAlloc failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
/* Create send and long message (read) portal table entries */
ret = PtlPTAlloc(ompi_mtl_portals4.ni_h,
PTL_PT_ONLY_USE_ONCE |
PTL_PT_ONLY_TRUNCATE |
PTL_PT_FLOWCTRL,
ompi_mtl_portals4.recv_eq_h,
REQ_RECV_TABLE_ID,
&ompi_mtl_portals4.recv_idx);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlPTAlloc failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
if (ompi_mtl_portals4.recv_idx != REQ_RECV_TABLE_ID) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlPTAlloc did not allocate the requested PT: %d\n",
__FILE__, __LINE__, ompi_mtl_portals4.recv_idx);
goto error;
}
ret = PtlPTAlloc(ompi_mtl_portals4.ni_h,
PTL_PT_ONLY_USE_ONCE |
PTL_PT_ONLY_TRUNCATE,
ompi_mtl_portals4.send_eq_h,
REQ_READ_TABLE_ID,
&ompi_mtl_portals4.read_idx);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlPTAlloc failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
if (ompi_mtl_portals4.read_idx != REQ_READ_TABLE_ID) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlPTAlloc did not allocate the requested PT: %d\n",
__FILE__, __LINE__, ompi_mtl_portals4.read_idx);
goto error;
}
/* bind zero-length md for sending acks */
md.start = NULL;
md.length = 0;
md.options = 0;
md.eq_handle = PTL_EQ_NONE;
md.ct_handle = PTL_CT_NONE;
ret = PtlMDBind(ompi_mtl_portals4.ni_h,
&md,
&ompi_mtl_portals4.zero_md_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlMDBind failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
/* Bind MD across all memory */
md.start = 0;
md.length = PTL_SIZE_MAX;
md.options = 0;
md.eq_handle = ompi_mtl_portals4.send_eq_h;
md.ct_handle = PTL_CT_NONE;
ret = PtlMDBind(ompi_mtl_portals4.ni_h,
&md,
&ompi_mtl_portals4.send_md_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlMDBind failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
/* Handle long overflows */
me.start = NULL;
me.length = 0;
me.ct_handle = PTL_CT_NONE;
me.min_free = 0;
me.uid = ompi_mtl_portals4.uid;
me.options = PTL_ME_OP_PUT |
PTL_ME_EVENT_LINK_DISABLE |
PTL_ME_EVENT_COMM_DISABLE |
PTL_ME_EVENT_UNLINK_DISABLE;
if (ompi_mtl_portals4.use_logical) {
me.match_id.rank = PTL_RANK_ANY;
} else {
me.match_id.phys.nid = PTL_NID_ANY;
me.match_id.phys.pid = PTL_PID_ANY;
}
me.match_bits = MTL_PORTALS4_LONG_MSG;
me.ignore_bits = MTL_PORTALS4_CONTEXT_MASK |
MTL_PORTALS4_SOURCE_MASK |
MTL_PORTALS4_TAG_MASK;
ret = PtlMEAppend(ompi_mtl_portals4.ni_h,
ompi_mtl_portals4.recv_idx,
&me,
PTL_OVERFLOW_LIST,
NULL,
&ompi_mtl_portals4.long_overflow_me_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlMEAppend failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
/* attach short unex recv blocks */
ret = ompi_mtl_portals4_recv_short_init();
if (OMPI_SUCCESS != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: short receive block initialization failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
ompi_mtl_portals4.opcount = 0;
#if OPAL_ENABLE_DEBUG
ompi_mtl_portals4.recv_opcount = 0;
#endif
#if OMPI_MTL_PORTALS4_FLOW_CONTROL
ret = ompi_mtl_portals4_flowctl_init();
if (OMPI_SUCCESS != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: ompi_mtl_portals4_flowctl_init failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
#endif
return OMPI_SUCCESS;
error:
if (!PtlHandleIsEqual(ompi_mtl_portals4.long_overflow_me_h, PTL_INVALID_HANDLE)) {
PtlMEUnlink(ompi_mtl_portals4.long_overflow_me_h);
}
if (!PtlHandleIsEqual(ompi_mtl_portals4.zero_md_h, PTL_INVALID_HANDLE)) {
PtlMDRelease(ompi_mtl_portals4.zero_md_h);
}
if (!PtlHandleIsEqual(ompi_mtl_portals4.send_md_h, PTL_INVALID_HANDLE)) {
PtlMDRelease(ompi_mtl_portals4.send_md_h);
}
if (ompi_mtl_portals4.read_idx != (ptl_pt_index_t) ~0UL) {
PtlPTFree(ompi_mtl_portals4.ni_h, ompi_mtl_portals4.read_idx);
}
if (ompi_mtl_portals4.recv_idx != (ptl_pt_index_t) ~0UL) {
PtlPTFree(ompi_mtl_portals4.ni_h, ompi_mtl_portals4.recv_idx);
}
if (!PtlHandleIsEqual(ompi_mtl_portals4.send_eq_h, PTL_INVALID_HANDLE)) {
PtlEQFree(ompi_mtl_portals4.send_eq_h);
}
if (!PtlHandleIsEqual(ompi_mtl_portals4.recv_eq_h, PTL_INVALID_HANDLE)) {
PtlEQFree(ompi_mtl_portals4.recv_eq_h);
}
return OMPI_ERROR;
}
