static int setup_alarm(uint32_t epoch) { int ret = OMPI_SUCCESS; size_t i; /* setup trigger */ if (ompi_mtl_portals4.flowctl.i_am_root) { ret = PtlTriggeredPut(ompi_mtl_portals4.zero_md_h, 0, 0, PTL_NO_ACK_REQ, ompi_mtl_portals4.flowctl.me, ompi_mtl_portals4.flowctl_idx, MTL_PORTALS4_FLOWCTL_ALERT, 0, NULL, 0, ompi_mtl_portals4.flowctl.trigger_ct_h, (epoch * ompi_mtl_portals4.flowctl.num_procs) + 1); if (OPAL_UNLIKELY(PTL_OK != ret)) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: PtlTriggeredPut failed: %d\n", __FILE__, __LINE__, ret); goto cleanup; } } /* setup the alert broadcast tree */ for (i = 0 ; i < ompi_mtl_portals4.flowctl.num_children ; ++i) { ret = PtlTriggeredPut(ompi_mtl_portals4.zero_md_h, 0, 0, PTL_NO_ACK_REQ, ompi_mtl_portals4.flowctl.children[i], ompi_mtl_portals4.flowctl_idx, MTL_PORTALS4_FLOWCTL_ALERT, 0, NULL, 0, ompi_mtl_portals4.flowctl.alert_ct_h, epoch + 1); if (OPAL_UNLIKELY(PTL_OK != ret)) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: PtlTriggeredPut failed: %d\n", __FILE__, __LINE__, ret); goto cleanup; } } cleanup: return ret; }
int ompi_coll_portals4_barrier_intra(struct ompi_communicator_t *comm, mca_coll_base_module_t *module) { mca_coll_portals4_module_t *portals4_module = (mca_coll_portals4_module_t*) module; int ret, i, dim, hibit, mask, num_msgs; int size = ompi_comm_size(comm); int rank = ompi_comm_rank(comm); ptl_ct_event_t ct; ptl_handle_ct_t ct_h; ptl_handle_me_t me_h; ptl_me_t me; size_t count; ptl_match_bits_t match_bits; ptl_handle_md_t md_h; void *base; ompi_coll_portals4_get_md(0, &md_h, &base); count = opal_atomic_add_size_t(&portals4_module->barrier_count, 1); ret = PtlCTAlloc(mca_coll_portals4_component.ni_h, &ct_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: PtlCTAlloc failed: %d\n", __FILE__, __LINE__, ret); return OMPI_ERR_TEMP_OUT_OF_RESOURCE; } COLL_PORTALS4_SET_BITS(match_bits, ompi_comm_get_cid(comm), 0, COLL_PORTALS4_BARRIER, count); /* Build "tree" out of hypercube */ dim = comm->c_cube_dim; hibit = opal_hibit(rank, dim); --dim; /* receive space */ me.start = NULL; me.length = 0; me.ct_handle = ct_h; me.min_free = 0; me.uid = mca_coll_portals4_component.uid; me.options = PTL_ME_OP_PUT | PTL_ME_EVENT_SUCCESS_DISABLE | PTL_ME_EVENT_LINK_DISABLE | PTL_ME_EVENT_UNLINK_DISABLE | PTL_ME_EVENT_CT_COMM | PTL_ME_EVENT_CT_OVERFLOW; me.match_id.phys.nid = PTL_NID_ANY; me.match_id.phys.pid = PTL_PID_ANY; me.match_bits = match_bits; me.ignore_bits = 0; ret = PtlMEAppend(mca_coll_portals4_component.ni_h, mca_coll_portals4_component.pt_idx, &me, PTL_PRIORITY_LIST, NULL, &me_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: PtlMEAppend failed: %d\n", __FILE__, __LINE__, ret); return OMPI_ERROR; } /* calculate number of children to receive from */ num_msgs = ompi_coll_portals4_get_nchildren(dim + 1, hibit, rank, size); /* send to parent when children have sent to us */ if (rank > 0) { int parent = rank & ~(1 << hibit); ret = PtlTriggeredPut(md_h, 0, 0, PTL_NO_ACK_REQ, ompi_coll_portals4_get_peer(comm, parent), mca_coll_portals4_component.pt_idx, match_bits, 0, NULL, 0, ct_h, num_msgs); if (PTL_OK != ret) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: PtlTriggeredPut failed: %d\n", __FILE__, __LINE__, ret); return OMPI_ERROR; } /* we'll need to wait for the parent response before the next set of comms */ num_msgs++; } /* send to children when parent (or all children if root) has sent to us */ for (i = hibit + 1, mask = 1 << i; i <= dim; ++i, mask <<= 1) { int peer = rank | mask; if (peer < size) { ret = PtlTriggeredPut(md_h, 0, 0, PTL_NO_ACK_REQ, ompi_coll_portals4_get_peer(comm, peer), mca_coll_portals4_component.pt_idx, match_bits, 0, NULL, 0, ct_h, num_msgs); if (PTL_OK != ret) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: PtlTriggeredPut failed: %d\n", __FILE__, __LINE__, ret); return OMPI_ERROR; } } } /* Wait for all incoming messages */ ret = PtlCTWait(ct_h, num_msgs, &ct); if (PTL_OK != ret) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: PtlCTWait failed: %d\n", __FILE__, __LINE__, ret); return OMPI_ERROR; } /* cleanup */ ret = PtlMEUnlink(me_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: PtlMEUnlink failed: %d\n", __FILE__, __LINE__, ret); return OMPI_ERROR; } ret = PtlCTFree(ct_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: PtlCTFree failed: %d\n", __FILE__, __LINE__, ret); return OMPI_ERROR; } return OMPI_SUCCESS; }
static int ompi_coll_portals4_scatter_intra_linear_top(const void *sbuf, int scount, struct ompi_datatype_t *sdtype, void *rbuf, int rcount, struct ompi_datatype_t *rdtype, int root, struct ompi_communicator_t *comm, ompi_coll_portals4_request_t *request, mca_coll_base_module_t *module) { mca_coll_portals4_module_t *portals4_module = (mca_coll_portals4_module_t*) module; int ret, line; ptl_ct_event_t ct; ptl_ct_event_t sync_incr_event; int8_t i_am_root; int32_t expected_rtrs = 0; int32_t expected_puts = 0; int32_t expected_acks = 0; int32_t expected_ops = 0; int32_t expected_chained_rtrs = 0; int32_t expected_chained_acks = 0; OPAL_OUTPUT((ompi_coll_base_framework.framework_output, "coll:portals4:scatter_intra_linear_top enter rank %d", request->u.scatter.my_rank)); request->type = OMPI_COLL_PORTALS4_TYPE_SCATTER; request->u.scatter.scatter_buf = NULL; request->u.scatter.scatter_mdh = PTL_INVALID_HANDLE; request->u.scatter.scatter_cth = PTL_INVALID_HANDLE; request->u.scatter.scatter_meh = PTL_INVALID_HANDLE; request->u.scatter.sync_mdh = PTL_INVALID_HANDLE; request->u.scatter.sync_cth = PTL_INVALID_HANDLE; request->u.scatter.sync_meh = PTL_INVALID_HANDLE; request->u.scatter.my_rank = ompi_comm_rank(comm); request->u.scatter.size = ompi_comm_size(comm); request->u.scatter.root_rank = root; request->u.scatter.sbuf = sbuf; request->u.scatter.rbuf = rbuf; request->u.scatter.pack_src_buf = sbuf; request->u.scatter.pack_src_count = scount; request->u.scatter.pack_src_dtype = sdtype; ompi_datatype_get_extent(request->u.scatter.pack_src_dtype, &request->u.scatter.pack_src_lb, &request->u.scatter.pack_src_extent); ompi_datatype_get_true_extent(request->u.scatter.pack_src_dtype, &request->u.scatter.pack_src_true_lb, &request->u.scatter.pack_src_true_extent); if ((root == request->u.scatter.my_rank) && (rbuf == MPI_IN_PLACE)) { request->u.scatter.unpack_dst_buf = NULL; request->u.scatter.unpack_dst_count = 0; request->u.scatter.unpack_dst_dtype = MPI_DATATYPE_NULL; } else { request->u.scatter.unpack_dst_buf = rbuf; request->u.scatter.unpack_dst_count = rcount; request->u.scatter.unpack_dst_dtype = rdtype; request->u.scatter.unpack_dst_offset = 0; ompi_datatype_get_extent(request->u.scatter.unpack_dst_dtype, &request->u.scatter.unpack_dst_lb, &request->u.scatter.unpack_dst_extent); ompi_datatype_get_true_extent(request->u.scatter.unpack_dst_dtype, &request->u.scatter.unpack_dst_true_lb, &request->u.scatter.unpack_dst_true_extent); } opal_output_verbose(30, ompi_coll_base_framework.framework_output, "%s:%d:rank(%d): request->u.scatter.unpack_dst_offset(%lu)", __FILE__, __LINE__, request->u.scatter.my_rank, request->u.scatter.unpack_dst_offset); /**********************************/ /* Setup Common Parameters */ /**********************************/ i_am_root = (request->u.scatter.my_rank == request->u.scatter.root_rank); request->u.scatter.coll_count = opal_atomic_add_size_t(&portals4_module->coll_count, 1); ret = setup_scatter_buffers_linear(comm, request, portals4_module); if (MPI_SUCCESS != ret) { line = __LINE__; goto err_hdlr; } ret = setup_scatter_handles(comm, request, portals4_module); if (MPI_SUCCESS != ret) { line = __LINE__; goto err_hdlr; } ret = setup_sync_handles(comm, request, portals4_module); if (MPI_SUCCESS != ret) { line = __LINE__; goto err_hdlr; } /**********************************/ /* do the scatter */ /**********************************/ if (i_am_root) { /* operations on the sync counter */ expected_rtrs = request->u.scatter.size - 1; /* expect RTRs from non-root ranks */ expected_acks = request->u.scatter.size - 1; /* expect Recv-ACKs from non-root ranks */ /* operations on the scatter counter */ expected_puts = 0; expected_chained_rtrs = 1; expected_chained_acks = 1; /* Chain the RTR and Recv-ACK to the Scatter CT */ sync_incr_event.success=1; sync_incr_event.failure=0; ret = PtlTriggeredCTInc(request->u.scatter.scatter_cth, sync_incr_event, request->u.scatter.sync_cth, expected_rtrs); if (PTL_OK != ret) { ret = OMPI_ERROR; line = __LINE__; goto err_hdlr; } ret = PtlTriggeredCTInc(request->u.scatter.scatter_cth, sync_incr_event, request->u.scatter.sync_cth, expected_rtrs + expected_acks); if (PTL_OK != ret) { ret = OMPI_ERROR; line = __LINE__; goto err_hdlr; } /* root, so put packed bytes to other ranks */ for (int32_t i=0; i<request->u.scatter.size; i++) { /* do not put to my scatter_buf. my data gets unpacked into my out buffer in linear_bottom(). */ if (i == request->u.scatter.my_rank) { continue; } ptl_size_t offset = request->u.scatter.packed_size * i; opal_output_verbose(30, ompi_coll_base_framework.framework_output, "%s:%d:rank(%d): offset(%lu)=rank(%d) * packed_size(%ld)", __FILE__, __LINE__, request->u.scatter.my_rank, offset, i, request->u.scatter.packed_size); ret = PtlTriggeredPut(request->u.scatter.scatter_mdh, (ptl_size_t)request->u.scatter.scatter_buf + offset, request->u.scatter.packed_size, PTL_NO_ACK_REQ, ompi_coll_portals4_get_peer(comm, i), mca_coll_portals4_component.pt_idx, request->u.scatter.scatter_match_bits, 0, NULL, 0, request->u.scatter.scatter_cth, expected_chained_rtrs); if (PTL_OK != ret) { ret = OMPI_ERROR; line = __LINE__; goto err_hdlr; } } } else { /* non-root, so do nothing */ /* operations on the sync counter */ expected_rtrs = 0; expected_acks = 0; /* operations on the scatter counter */ expected_puts = 1; /* scatter put from root */ expected_chained_rtrs = 0; expected_chained_acks = 0; } expected_ops = expected_chained_rtrs + expected_puts; /**********************************************/ /* only non-root ranks are PUT to, so only */ /* non-root ranks must PUT a Recv-ACK to root */ /**********************************************/ if (!i_am_root) { ret = PtlTriggeredPut(request->u.scatter.sync_mdh, 0, 0, PTL_NO_ACK_REQ, ompi_coll_portals4_get_peer(comm, request->u.scatter.root_rank), mca_coll_portals4_component.pt_idx, request->u.scatter.sync_match_bits, 0, NULL, 0, request->u.scatter.scatter_cth, expected_ops); if (PTL_OK != ret) { ret = OMPI_ERROR; line = __LINE__; goto err_hdlr; } } expected_ops += expected_chained_acks; if (!request->u.scatter.is_sync) { /******************************************/ /* put to finish pt when all ops complete */ /******************************************/ ret = PtlTriggeredPut(mca_coll_portals4_component.zero_md_h, 0, 0, PTL_NO_ACK_REQ, ompi_coll_portals4_get_peer(comm, request->u.scatter.my_rank), mca_coll_portals4_component.finish_pt_idx, 0, 0, NULL, (uintptr_t) request, request->u.scatter.scatter_cth, expected_ops); if (PTL_OK != ret) { ret = OMPI_ERROR; line = __LINE__; goto err_hdlr; } } /**************************************/ /* all non-root ranks put RTR to root */ /**************************************/ if (!i_am_root) { ret = PtlPut(request->u.scatter.sync_mdh, 0, 0, PTL_NO_ACK_REQ, ompi_coll_portals4_get_peer(comm, request->u.scatter.root_rank), mca_coll_portals4_component.pt_idx, request->u.scatter.sync_match_bits, 0, NULL, 0); if (PTL_OK != ret) { ret = OMPI_ERROR; line = __LINE__; goto err_hdlr; } } if (request->u.scatter.is_sync) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "calling CTWait(expected_ops=%d)\n", expected_ops); /********************************/ /* Wait for all ops to complete */ /********************************/ ret = PtlCTWait(request->u.scatter.scatter_cth, expected_ops, &ct); if (PTL_OK != ret) { ret = OMPI_ERROR; line = __LINE__; goto err_hdlr; } opal_output_verbose(1, ompi_coll_base_framework.framework_output, "completed CTWait(expected_ops=%d)\n", expected_ops); } OPAL_OUTPUT((ompi_coll_base_framework.framework_output, "coll:portals4:scatter_intra_linear_top exit rank %d", request->u.scatter.my_rank)); return OMPI_SUCCESS; err_hdlr: if (NULL != request->u.scatter.scatter_buf) free(request->u.scatter.scatter_buf); opal_output(ompi_coll_base_framework.framework_output, "%s:%4d:%4d\tError occurred ret=%d, rank %2d", __FILE__, __LINE__, line, ret, request->u.scatter.my_rank); return ret; }
static int setup_barrier(uint32_t epoch) { int ret = OMPI_SUCCESS; size_t i; ptl_ct_event_t ct; if (ompi_mtl_portals4.flowctl.i_am_root) { ct.success = ompi_mtl_portals4.flowctl.epoch_counter * ompi_mtl_portals4.flowctl.num_procs; ct.failure = 0; ret = PtlTriggeredCTSet(ompi_mtl_portals4.flowctl.trigger_ct_h, ct, ompi_mtl_portals4.flowctl.fanin_ct_h, epoch * (ompi_mtl_portals4.flowctl.num_children + 1)); if (OPAL_UNLIKELY(PTL_OK != ret)) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: PtlTriggeredCTSet failed: %d\n", __FILE__, __LINE__, ret); goto cleanup; } ret = PtlTriggeredPut(ompi_mtl_portals4.zero_md_h, 0, 0, PTL_NO_ACK_REQ, ompi_mtl_portals4.flowctl.me, ompi_mtl_portals4.flowctl_idx, MTL_PORTALS4_FLOWCTL_FANOUT, 0, NULL, 0, ompi_mtl_portals4.flowctl.fanin_ct_h, epoch * (ompi_mtl_portals4.flowctl.num_children + 1)); if (OPAL_UNLIKELY(PTL_OK != ret)) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: PtlTriggeredPut failed: %d\n", __FILE__, __LINE__, ret); goto cleanup; } } else { ret = PtlTriggeredPut(ompi_mtl_portals4.zero_md_h, 0, 0, PTL_NO_ACK_REQ, ompi_mtl_portals4.flowctl.parent, ompi_mtl_portals4.flowctl_idx, MTL_PORTALS4_FLOWCTL_FANIN, 0, NULL, 0, ompi_mtl_portals4.flowctl.fanin_ct_h, epoch * (ompi_mtl_portals4.flowctl.num_children + 1)); if (OPAL_UNLIKELY(PTL_OK != ret)) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: PtlTriggeredPut failed: %d\n", __FILE__, __LINE__, ret); goto cleanup; } } for (i = 0 ; i < ompi_mtl_portals4.flowctl.num_children ; ++i) { ret = PtlTriggeredPut(ompi_mtl_portals4.zero_md_h, 0, 0, PTL_NO_ACK_REQ, ompi_mtl_portals4.flowctl.children[i], ompi_mtl_portals4.flowctl_idx, MTL_PORTALS4_FLOWCTL_FANOUT, 0, NULL, 0, ompi_mtl_portals4.flowctl.fanout_ct_h, epoch); if (OPAL_UNLIKELY(PTL_OK != ret)) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: PtlTriggeredPut failed: %d\n", __FILE__, __LINE__, ret); return ret; } } cleanup: return ret; }