int
ompi_coll_portals4_iscatter_intra(const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
int root,
struct ompi_communicator_t *comm,
ompi_request_t **ompi_request,
mca_coll_base_module_t *module)
{
int ret, line;
ompi_coll_portals4_request_t *request;
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
"coll:portals4:iscatter_intra enter rank %d", ompi_comm_rank(comm)));
/*
* allocate a portals4 request
*/
OMPI_COLL_PORTALS4_REQUEST_ALLOC(comm, request);
if (NULL == request) {
ret = OMPI_ERR_TEMP_OUT_OF_RESOURCE;
line = __LINE__;
goto err_hdlr;
}
*ompi_request = &request->super;
request->u.scatter.is_sync = 0;
/*
* initiate the scatter
*
* this request is marked asynchronous (is_sync==0), so
* portals4_progress() will handle completion.
*/
ret = ompi_coll_portals4_scatter_intra_linear_top(sbuf, scount, sdtype,
rbuf, rcount, rdtype,
root,
comm,
request,
module);
if (MPI_SUCCESS != ret) {
line = __LINE__;
goto err_hdlr;
}
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
"coll:portals4:iscatter_intra exit rank %d", request->u.scatter.my_rank));
return OMPI_SUCCESS;
err_hdlr:
opal_output(ompi_coll_base_framework.framework_output,
"%s:%4d:%4d\tError occurred ret=%d, rank %2d",
__FILE__, __LINE__, line, ret, request->u.scatter.my_rank);
return ret;
}
int
ompi_coll_portals4_ibarrier_intra(struct ompi_communicator_t *comm,
ompi_request_t **ompi_req,
struct mca_coll_base_module_2_0_0_t *module)
{
mca_coll_portals4_module_t *portals4_module = (mca_coll_portals4_module_t*) module;
int ret, i, dim, hibit, mask, num_msgs;
int size = ompi_comm_size(comm);
int rank = ompi_comm_rank(comm);
ptl_me_t me;
size_t count;
ptl_match_bits_t match_bits;
ompi_coll_portals4_request_t *request;
ptl_handle_md_t md_h;
void *base;
ompi_coll_portals4_get_md(0, &md_h, &base);
OMPI_COLL_PORTALS4_REQUEST_ALLOC(comm, request);
if (NULL == request) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: request alloc failed\n",
__FILE__, __LINE__);
return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
}
*ompi_req = &request->super;
request->type = OMPI_COLL_PORTALS4_TYPE_BARRIER;
count = opal_atomic_add_size_t(&portals4_module->barrier_count, 1);
ret = PtlCTAlloc(mca_coll_portals4_component.ni_h,
&request->ct_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlCTAlloc failed: %d\n",
__FILE__, __LINE__, ret);
return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
}
COLL_PORTALS4_SET_BITS(match_bits, ompi_comm_get_cid(comm),
0, COLL_PORTALS4_BARRIER, count);
/* Build "tree" out of hypercube */
dim = comm->c_cube_dim;
hibit = opal_hibit(rank, dim);
--dim;
/* calculate number of children to receive from */
num_msgs = ompi_coll_portals4_get_nchildren(dim + 1, hibit, rank, size);
/* receive space */
me.start = NULL;
me.length = 0;
me.ct_handle = request->ct_h;
me.min_free = 0;
me.uid = mca_coll_portals4_component.uid;
me.options = PTL_ME_OP_PUT | PTL_ME_EVENT_SUCCESS_DISABLE |
PTL_ME_EVENT_LINK_DISABLE | PTL_ME_EVENT_UNLINK_DISABLE |
PTL_ME_EVENT_CT_COMM | PTL_ME_EVENT_CT_OVERFLOW;
me.match_id.phys.nid = PTL_NID_ANY;
me.match_id.phys.pid = PTL_PID_ANY;
me.match_bits = match_bits;
me.ignore_bits = 0;
ret = PtlMEAppend(mca_coll_portals4_component.ni_h,
mca_coll_portals4_component.pt_idx,
&me,
PTL_PRIORITY_LIST,
NULL,
&request->me_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlMEAppend failed: %d\n",
__FILE__, __LINE__, ret);
return OMPI_ERROR;
}
/* send to parent when children have sent to us */
if (rank > 0) {
int parent = rank & ~(1 << hibit);
ret = PtlTriggeredPut(md_h,
0,
0,
PTL_NO_ACK_REQ,
ompi_coll_portals4_get_peer(comm, parent),
mca_coll_portals4_component.pt_idx,
match_bits,
0,
NULL,
0,
request->ct_h,
num_msgs);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlTriggeredPut failed: %d\n",
__FILE__, __LINE__, ret);
return OMPI_ERROR;
}
/* we'll need to wait for the parent response before the next set of comms */
num_msgs++;
}
/* send to children when parent (or all children if root) has sent to us */
for (i = hibit + 1, mask = 1 << i; i <= dim; ++i, mask <<= 1) {
int peer = rank | mask;
if (peer < size) {
ret = PtlTriggeredPut(md_h,
0,
0,
PTL_NO_ACK_REQ,
ompi_coll_portals4_get_peer(comm, peer),
mca_coll_portals4_component.pt_idx,
match_bits,
0,
NULL,
0,
request->ct_h,
num_msgs);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlTriggeredPut failed: %d\n",
__FILE__, __LINE__, ret);
return OMPI_ERROR;
}
}
}
/* Send a put to self when we've received all our messages... */
ret = PtlTriggeredPut(md_h,
0,
0,
PTL_NO_ACK_REQ,
ompi_coll_portals4_get_peer(comm, rank),
mca_coll_portals4_component.finish_pt_idx,
0,
0,
NULL,
(uintptr_t) request,
request->ct_h,
num_msgs);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: PtlTriggeredPut failed: %d\n",
__FILE__, __LINE__, ret);
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
