int MPID_Rget_accumulate(const void *origin_addr, int origin_count,
MPI_Datatype origin_datatype, void *result_addr, int result_count,
MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp,
int target_count, MPI_Datatype target_datatype, MPI_Op op,
MPIR_Win * win_ptr, MPIR_Request ** request)
{
int mpi_errno = MPI_SUCCESS;
int dt_contig ATTRIBUTE((unused));
MPIR_Datatype*dtp;
MPI_Aint dt_true_lb ATTRIBUTE((unused));
intptr_t data_sz, trg_data_sz;
MPIR_Request *ureq;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_RGET_ACCUMULATE);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_RGET_ACCUMULATE);
/* request-based RMA operations are only valid within a passive epoch */
MPIR_ERR_CHKANDJUMP(win_ptr->states.access_state != MPIDI_RMA_PER_TARGET &&
win_ptr->states.access_state != MPIDI_RMA_LOCK_ALL_CALLED &&
win_ptr->states.access_state != MPIDI_RMA_LOCK_ALL_ISSUED &&
win_ptr->states.access_state != MPIDI_RMA_LOCK_ALL_GRANTED,
mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");
/* Create user request, initially cc=1, ref=1 */
ureq = MPIR_Request_create(MPIR_REQUEST_KIND__RMA);
MPIR_ERR_CHKANDJUMP(ureq == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
/* This request is referenced by user and ch3 by default. */
MPIR_Object_set_ref(ureq, 2);
/* Note that GACC is only a no-op if no data goes in both directions */
MPIDI_Datatype_get_info(origin_count, origin_datatype, dt_contig, data_sz, dtp, dt_true_lb);
MPIDI_Datatype_get_info(origin_count, origin_datatype, dt_contig, trg_data_sz, dtp, dt_true_lb);
/* Enqueue or perform the RMA operation */
if (target_rank != MPI_PROC_NULL && (data_sz != 0 || trg_data_sz != 0)) {
mpi_errno = MPIDI_CH3I_Get_accumulate(origin_addr, origin_count,
origin_datatype, result_addr,
result_count, result_datatype,
target_rank, target_disp,
target_count, target_datatype, op, win_ptr, ureq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
}
else {
mpi_errno = MPID_Request_complete(ureq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
}
*request = ureq;
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_RGET_ACCUMULATE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
void MPIDI_CH3U_Buffer_allocate(
const void * const sbuf, MPI_Aint scount, MPI_Datatype sdt, int * smpi_errno,
void ** rbuf_handle, MPI_Aint rcount, MPI_Datatype rdt, MPIDI_msg_sz_t * rsz,
int * rmpi_errno)
{
int sdt_contig;
int rdt_contig;
MPI_Aint sdt_true_lb, rdt_true_lb;
MPIDI_msg_sz_t sdata_sz;
MPIDI_msg_sz_t rdata_sz;
MPID_Datatype * sdt_ptr;
MPID_Datatype * rdt_ptr;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_BUFFER_ALLOCATE);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_BUFFER_ALLOCATE);
*smpi_errno = MPI_SUCCESS;
*rmpi_errno = MPI_SUCCESS;
MPIDI_Datatype_get_info(scount, sdt, sdt_contig, sdata_sz, sdt_ptr, sdt_true_lb);
MPIDI_Datatype_get_info(rcount, rdt, rdt_contig, rdata_sz, rdt_ptr, rdt_true_lb);
/* --BEGIN ERROR HANDLING-- */
if (sdata_sz > rdata_sz)
{
MPIU_DBG_MSG_FMT(CH3_OTHER,TYPICAL,(MPIU_DBG_FDEST,
"message truncated, sdata_sz=" MPIDI_MSG_SZ_FMT " rdata_sz=" MPIDI_MSG_SZ_FMT,
sdata_sz, rdata_sz));
sdata_sz = rdata_sz;
*rmpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_TRUNCATE, "**truncate", "**truncate %d %d", sdata_sz, rdata_sz );
}
/* --END ERROR HANDLING-- */
if (sdata_sz == 0)
{
*rsz = 0;
goto fn_exit;
}
if (sdt_contig && rdt_contig)
{
*rbuf_handle = (void *)MPIU_Malloc(sdata_sz);
MPIU_Assert(*rbuf_handle);
*rsz = sdata_sz;
}
else
{
/* --BEGIN ERROR HANDLING-- */
MPIU_DBG_MSG(CH3_OTHER,TYPICAL,"Sender and receiver datatypes are not contiguous");
*smpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER, "**zcopybufalloc", "**zcopybufalloc %d %d", scount, rcount);
*rmpi_errno = *smpi_errno;
*rsz = 0;
goto fn_exit;
/* --END ERROR HANDLING-- */
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_BUFFER_ALLOCATE);
}
static int send_sreq_data(MPIDI_VC_t *vc, MPID_Request *sreq, knem_cookie_t *s_cookiep)
{
int mpi_errno = MPI_SUCCESS;
int dt_contig;
MPI_Aint dt_true_lb;
MPIDI_msg_sz_t data_sz;
MPID_Datatype * dt_ptr;
/* MT: this code assumes only one thread can be at this point at a time */
if (knem_fd < 0) {
mpi_errno = open_knem_dev();
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
/* find out contig/noncontig, size, and lb for the datatype */
MPIDI_Datatype_get_info(sreq->dev.user_count, sreq->dev.datatype,
dt_contig, data_sz, dt_ptr, dt_true_lb);
if (dt_contig) {
/* handle the iov creation ourselves */
sreq->dev.iov[0].MPID_IOV_BUF = (char *)sreq->dev.user_buf + dt_true_lb;
sreq->dev.iov[0].MPID_IOV_LEN = data_sz;
sreq->dev.iov_count = 1;
}
else {
/* use the segment routines to handle the iovec creation */
if (sreq->dev.segment_ptr == NULL) {
sreq->dev.iov_count = MPID_IOV_LIMIT;
sreq->dev.iov_offset = 0;
/* segment_ptr may be non-null when this is a continuation of a
many-part message that we couldn't fit in one single flight of
iovs. */
sreq->dev.segment_ptr = MPID_Segment_alloc();
MPIU_ERR_CHKANDJUMP1((sreq->dev.segment_ptr == NULL), mpi_errno,
MPI_ERR_OTHER, "**nomem",
"**nomem %s", "MPID_Segment_alloc");
MPID_Segment_init(sreq->dev.user_buf, sreq->dev.user_count,
sreq->dev.datatype, sreq->dev.segment_ptr, 0);
sreq->dev.segment_first = 0;
sreq->dev.segment_size = data_sz;
/* FIXME we should write our own function that isn't dependent on
the in-request iov array. This will let us use IOVs that are
larger than MPID_IOV_LIMIT. */
mpi_errno = MPIDI_CH3U_Request_load_send_iov(sreq, &sreq->dev.iov[0],
&sreq->dev.iov_count);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
}
mpi_errno = do_dma_send(vc, sreq, sreq->dev.iov_count, sreq->dev.iov, s_cookiep);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_tcp_module_lmt_pre_send (MPIDI_VC_t *vc, MPID_Request *req, MPID_IOV *cookie)
{
int mpi_errno = MPI_SUCCESS;
int len;
MPIDI_msg_sz_t data_sz;
int dt_contig;
MPI_Aint dt_true_lb;
MPID_Datatype * dt_ptr;
MPIDI_CH3I_VC *vc_ch = (MPIDI_CH3I_VC *)vc->channel_private;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_PRE_SEND);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_PRE_SEND);
MPIDI_Datatype_get_info (req->dev.user_count, req->dev.datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
mpi_errno = create_s_cookie (data_sz, &vc_ch->net.tcp.lmt_cookie, &len);
if (mpi_errno) MPIU_ERR_POP (mpi_errno);
cookie->MPID_IOV_BUF = vc_ch->net.tcp.lmt_cookie;
cookie->MPID_IOV_LEN = len;
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_PRE_SEND);
return mpi_errno;
fn_fail:
goto fn_exit;
}
static int handler_recv_dequeue_complete(const ptl_event_t *e)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *const rreq = e->user_ptr;
int is_contig;
MPI_Aint last;
MPI_Aint dt_true_lb;
MPIDI_msg_sz_t data_sz;
MPID_Datatype *dt_ptr ATTRIBUTE((unused));
MPIDI_STATE_DECL(MPID_STATE_HANDLER_RECV_DEQUEUE_COMPLETE);
MPIDI_FUNC_ENTER(MPID_STATE_HANDLER_RECV_DEQUEUE_COMPLETE);
MPIU_Assert(e->type == PTL_EVENT_PUT || e->type == PTL_EVENT_PUT_OVERFLOW);
MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype, is_contig, data_sz, dt_ptr, dt_true_lb);
dequeue_req(e);
if (e->type == PTL_EVENT_PUT_OVERFLOW) {
/* unpack the data from unexpected buffer */
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "is_contig = %d", is_contig);
if (is_contig) {
MPIU_Memcpy((char *)rreq->dev.user_buf + dt_true_lb, e->start, e->mlength);
} else {
last = e->mlength;
MPID_Segment_unpack(rreq->dev.segment_ptr, rreq->dev.segment_first, &last, e->start);
if (last != e->mlength)
MPIR_ERR_SET(rreq->status.MPI_ERROR, MPI_ERR_TYPE, "**dtypemismatch");
}
} else {
/* Data was placed directly into the user buffer, so datatype mismatch
is harder to detect. We use a simple check ensuring the received bytes
are a multiple of a single basic element. Currently, we do not detect
mismatches with datatypes constructed of more than one basic type */
MPI_Datatype dt_basic_type;
MPID_Datatype_get_basic_type(rreq->dev.datatype, dt_basic_type);
if (dt_basic_type != MPI_DATATYPE_NULL && (e->mlength % MPID_Datatype_get_basic_size(dt_basic_type)) != 0)
MPIR_ERR_SET(rreq->status.MPI_ERROR, MPI_ERR_TYPE, "**dtypemismatch");
}
mpi_errno = handler_recv_complete(e);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_HANDLER_RECV_DEQUEUE_COMPLETE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
void
MPIDI_Win_datatype_basic(int count,
MPI_Datatype datatype,
MPIDI_Datatype * dt)
{
MPIDI_Datatype_get_info(dt->count = count,
dt->type = datatype,
dt->contig,
dt->size,
dt->pointer,
dt->true_lb);
TRACE_ERR("DT=%08x DTp=%p count=%d contig=%d size=%zu true_lb=%zu\n",
dt->type, dt->pointer, dt->count, dt->contig, (size_t)dt->size, (size_t)dt->true_lb);
}
/* MSGQUEUE lock is not held */
void
MPIDI_Callback_process_userdefined_dt(pami_context_t context,
const void * sndbuf,
size_t sndlen,
MPID_Request * rreq)
{
unsigned dt_contig, dt_size;
MPID_Datatype *dt_ptr;
MPI_Aint dt_true_lb;
MPIDI_Datatype_get_info(rreq->mpid.userbufcount,
rreq->mpid.datatype,
dt_contig,
dt_size,
dt_ptr,
dt_true_lb);
/* ----------------------------- */
/* Test for truncated message. */
/* ----------------------------- */
if (unlikely(sndlen > dt_size))
{
#if ASSERT_LEVEL > 0
MPIDI_Callback_process_trunc(context, rreq, NULL, sndbuf);
return;
#else
sndlen = dt_size;
#endif
}
/*
* This is to test that the fields don't need to be
* initialized. Remove after this doesn't fail for a while.
*/
if (likely (dt_contig))
{
MPID_assert(rreq->mpid.uebuf == NULL);
MPID_assert(rreq->mpid.uebuflen == 0);
void* rcvbuf = rreq->mpid.userbuf + dt_true_lb;;
memcpy(rcvbuf, sndbuf, sndlen);
MPIDI_Request_complete(rreq);
return;
}
MPIDI_Request_setCA(rreq, MPIDI_CA_UNPACK_UEBUF_AND_COMPLETE);
rreq->mpid.uebuflen = sndlen;
rreq->mpid.uebuf = (void*)sndbuf;
MPIDI_RecvDoneCB(context, rreq, PAMI_SUCCESS);
MPID_Request_release(rreq);
}
/* fills in req->dev.iov{,_offset,_count} based on the datatype info in the
request, creating a segment if necessary */
static int populate_iov_from_req(MPID_Request *req)
{
int mpi_errno = MPI_SUCCESS;
int dt_contig;
MPI_Aint dt_true_lb;
MPIDI_msg_sz_t data_sz;
MPID_Datatype * dt_ptr;
/* find out contig/noncontig, size, and lb for the datatype */
MPIDI_Datatype_get_info(req->dev.user_count, req->dev.datatype,
dt_contig, data_sz, dt_ptr, dt_true_lb);
if (dt_contig) {
/* handle the iov creation ourselves */
req->dev.iov[0].MPL_IOV_BUF = (char *)req->dev.user_buf + dt_true_lb;
req->dev.iov[0].MPL_IOV_LEN = data_sz;
req->dev.iov_count = 1;
}
else {
/* use the segment routines to handle the iovec creation */
MPIU_Assert(req->dev.segment_ptr == NULL);
req->dev.iov_count = MPL_IOV_LIMIT;
req->dev.iov_offset = 0;
/* XXX DJG FIXME where is this segment freed? */
req->dev.segment_ptr = MPID_Segment_alloc();
MPIR_ERR_CHKANDJUMP1((req->dev.segment_ptr == NULL), mpi_errno,
MPI_ERR_OTHER, "**nomem",
"**nomem %s", "MPID_Segment_alloc");
MPID_Segment_init(req->dev.user_buf, req->dev.user_count,
req->dev.datatype, req->dev.segment_ptr, 0);
req->dev.segment_first = 0;
req->dev.segment_size = data_sz;
/* FIXME we should write our own function that isn't dependent on
the in-request iov array. This will let us use IOVs that are
larger than MPL_IOV_LIMIT. */
mpi_errno = MPIDI_CH3U_Request_load_send_iov(req, &req->dev.iov[0],
&req->dev.iov_count);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
}
fn_fail:
return mpi_errno;
}
int MPIDO_Gatherv_simple(const void *sendbuf,
int sendcount,
MPI_Datatype sendtype,
void *recvbuf,
const int *recvcounts,
const int *displs,
MPI_Datatype recvtype,
int root,
MPID_Comm * comm_ptr,
int *mpierrno)
{
#ifndef HAVE_PAMI_IN_PLACE
if (sendbuf == MPI_IN_PLACE)
{
MPID_Abort (NULL, 0, 1, "'MPI_IN_PLACE' requries support for `PAMI_IN_PLACE`");
return -1;
}
#endif
TRACE_ERR("Entering MPIDO_Gatherv_optimized\n");
int snd_contig = 1, rcv_contig = 1;
void *snd_noncontig_buff = NULL, *rcv_noncontig_buff = NULL;
void *sbuf = NULL, *rbuf = NULL;
int *rcounts = NULL;
int *rdispls = NULL;
int send_size = 0;
int recv_size = 0;
int rcvlen = 0;
int totalrecvcount = 0;
pami_type_t rtype = PAMI_TYPE_NULL;
MPID_Segment segment;
MPID_Datatype *data_ptr = NULL;
int send_true_lb, recv_true_lb = 0;
int i, tmp;
volatile unsigned gatherv_active = 1;
const int rank = comm_ptr->rank;
const int size = comm_ptr->local_size;
#if ASSERT_LEVEL==0
/* We can't afford the tracing in ndebug/performance libraries */
const unsigned verbose = 0;
#else
const unsigned verbose = (MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_ALL) && (rank == 0);
#endif
const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid);
int recvok=PAMI_SUCCESS, recvcontinuous=0;
if(sendbuf != MPI_IN_PLACE)
{
MPIDI_Datatype_get_info(sendcount, sendtype, snd_contig,
send_size, data_ptr, send_true_lb);
if(MPIDI_Pamix_collsel_advise != NULL && mpid->collsel_fast_query != NULL)
{
advisor_algorithm_t advisor_algorithms[1];
int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_GATHERV_INT, 64, advisor_algorithms, 1);
if(num_algorithms)
{
if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO)
{
return MPIR_Gatherv(sendbuf, sendcount, sendtype,
recvbuf, recvcounts, displs, recvtype,
root, comm_ptr, mpierrno);
}
else if(advisor_algorithms[0].metadata && advisor_algorithms[0].metadata->check_correct.values.asyncflowctl && !(--(comm_ptr->mpid.num_requests)))
{
comm_ptr->mpid.num_requests = MPIDI_Process.optimized.num_requests;
int tmpmpierrno;
if(unlikely(verbose))
fprintf(stderr,"Query barrier required for %s\n", advisor_algorithms[0].metadata->name);
MPIDO_Barrier(comm_ptr, &tmpmpierrno);
}
}
}
sbuf = (char *)sendbuf + send_true_lb;
if(!snd_contig)
{
snd_noncontig_buff = MPL_malloc(send_size);
sbuf = snd_noncontig_buff;
if(snd_noncontig_buff == NULL)
{
MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1,
"Fatal: Cannot allocate pack buffer");
}
DLOOP_Offset last = send_size;
MPID_Segment_init(sendbuf, sendcount, sendtype, &segment, 0);
MPID_Segment_pack(&segment, 0, &last, snd_noncontig_buff);
}
}
else
{
MPIDI_Datatype_get_info(1, recvtype, rcv_contig,
rcvlen, data_ptr, recv_true_lb);
if(MPIDI_Pamix_collsel_advise != NULL && mpid->collsel_fast_query != NULL)
{
advisor_algorithm_t advisor_algorithms[1];
int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_GATHERV_INT, 64, advisor_algorithms, 1);
if(num_algorithms)
{
if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO)
{
return MPIR_Gatherv(sendbuf, sendcount, sendtype,
recvbuf, recvcounts, displs, recvtype,
root, comm_ptr, mpierrno);
}
else if(advisor_algorithms[0].metadata && advisor_algorithms[0].metadata->check_correct.values.asyncflowctl && !(--(comm_ptr->mpid.num_requests)))
{
comm_ptr->mpid.num_requests = MPIDI_Process.optimized.num_requests;
int tmpmpierrno;
if(unlikely(verbose))
fprintf(stderr,"Query barrier required for %s\n", advisor_algorithms[0].metadata->name);
MPIDO_Barrier(comm_ptr, &tmpmpierrno);
}
}
}
}
pami_xfer_t gatherv;
rbuf = (char *)recvbuf + recv_true_lb;
rcounts = (int*)recvcounts;
rdispls = (int*)displs;
if(rank == root)
{
if((recvok = MPIDI_Datatype_to_pami(recvtype, &rtype, -1, NULL, &tmp)) != MPI_SUCCESS)
{
MPIDI_Datatype_get_info(1, recvtype, rcv_contig,
rcvlen, data_ptr, recv_true_lb);
totalrecvcount = recvcounts[0];
recvcontinuous = displs[0] == 0? 1 : 0 ;
rcounts = (int*)MPL_malloc(size);
rdispls = (int*)MPL_malloc(size);
rdispls[0] = 0;
rcounts[0] = rcvlen * recvcounts[0];
for(i = 1; i < size; i++)
{
rdispls[i]= rcvlen * totalrecvcount;
totalrecvcount += recvcounts[i];
if(displs[i] != (displs[i-1] + recvcounts[i-1]))
recvcontinuous = 0;
rcounts[i] = rcvlen * recvcounts[i];
}
recv_size = rcvlen * totalrecvcount;
rcv_noncontig_buff = MPL_malloc(recv_size);
rbuf = rcv_noncontig_buff;
rtype = PAMI_TYPE_BYTE;
if(rcv_noncontig_buff == NULL)
{
MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1,
"Fatal: Cannot allocate pack buffer");
}
if(sendbuf == MPI_IN_PLACE)
{
size_t extent;
MPID_Datatype_get_extent_macro(recvtype,extent);
MPIR_Localcopy(recvbuf + displs[rank]*extent, recvcounts[rank], recvtype,
rcv_noncontig_buff + rdispls[rank], rcounts[rank],MPI_CHAR);
}
}
if(sendbuf == MPI_IN_PLACE)
{
gatherv.cmd.xfer_gatherv_int.sndbuf = PAMI_IN_PLACE;
}
else
{
gatherv.cmd.xfer_gatherv_int.sndbuf = sbuf;
}
gatherv.cmd.xfer_gatherv_int.stype = PAMI_TYPE_BYTE;/* stype is ignored when sndbuf == PAMI_IN_PLACE */
gatherv.cmd.xfer_gatherv_int.stypecount = send_size;
}
else
{
gatherv.cmd.xfer_gatherv_int.sndbuf = sbuf;
gatherv.cmd.xfer_gatherv_int.stype = PAMI_TYPE_BYTE;
gatherv.cmd.xfer_gatherv_int.stypecount = send_size;
}
gatherv.cb_done = cb_gatherv;
gatherv.cookie = (void *)&gatherv_active;
gatherv.cmd.xfer_gatherv_int.root = MPIDI_Task_to_endpoint(MPID_VCR_GET_LPID(comm_ptr->vcr, root), 0);
gatherv.cmd.xfer_gatherv_int.rcvbuf = rbuf;
gatherv.cmd.xfer_gatherv_int.rtype = rtype;
gatherv.cmd.xfer_gatherv_int.rtypecounts = (int *) rcounts;
gatherv.cmd.xfer_gatherv_int.rdispls = (int *) rdispls;
const pami_metadata_t *my_gatherv_md;
gatherv.algorithm = mpid->coll_algorithm[PAMI_XFER_GATHERV_INT][0][0];
my_gatherv_md = &mpid->coll_metadata[PAMI_XFER_GATHERV_INT][0][0];
MPIDI_Update_last_algorithm(comm_ptr, my_gatherv_md->name);
MPIDI_Post_coll_t gatherv_post;
TRACE_ERR("%s gatherv\n", MPIDI_Process.context_post.active>0?"Posting":"Invoking");
MPIDI_Context_post(MPIDI_Context[0], &gatherv_post.state,
MPIDI_Pami_post_wrapper, (void *)&gatherv);
TRACE_ERR("Gatherv %s\n", MPIDI_Process.context_post.active>0?"posted":"invoked");
TRACE_ERR("Waiting on active %d\n", gatherv_active);
MPID_PROGRESS_WAIT_WHILE(gatherv_active);
if(!rcv_contig || recvok != PAMI_SUCCESS)
{
if(recvcontinuous)
{
MPIR_Localcopy(rcv_noncontig_buff, recv_size, MPI_CHAR,
recvbuf, totalrecvcount, recvtype);
}
else
{
size_t extent;
MPID_Datatype_get_extent_macro(recvtype,extent);
for(i=0; i<size; ++i)
{
char* scbuf = (char*)rcv_noncontig_buff+ rdispls[i];
char* rcbuf = (char*)recvbuf + displs[i]*extent;
MPIR_Localcopy(scbuf, rcounts[i], MPI_CHAR,
rcbuf, recvcounts[i], recvtype);
TRACE_ERR("Pack recv src extent %zu, displ[%zu]=%zu, count[%zu]=%zu buf[%zu]=%u\n",
(size_t)extent, (size_t)i,(size_t)precvdispls[i],(size_t)i,(size_t)precvcounts[i],(size_t)precvdispls[i], *(int*)scbuf);
TRACE_ERR("Pack recv dest extent %zu, displ[%zu]=%zu, count[%zu]=%zu buf[%zu]=%u\n",
(size_t)extent, (size_t)i,(size_t)displs[i],(size_t)i,(size_t)recvcounts[i],(size_t)displs[i], *(int*)rcbuf);
}
}
MPL_free(rcv_noncontig_buff);
if(rank == root)
{
MPL_free(rcounts);
MPL_free(rdispls);
}
}
if(!snd_contig) MPL_free(snd_noncontig_buff);
TRACE_ERR("Leaving MPIDO_Gatherv_optimized\n");
return MPI_SUCCESS;
}
int MPID_nem_newmad_directRecv(MPIDI_VC_t *vc, MPID_Request *rreq)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_NEWMAD_DIRECTRECV);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_NEWMAD_DIRECTRECV);
if (!VC_CH(vc)->is_local)
{
nm_tag_t match_info = 0;
nm_tag_t match_mask = NEM_NMAD_MATCH_FULL_MASK;
MPIR_Rank_t source = rreq->dev.match.parts.rank;
MPIR_Context_id_t context = rreq->dev.match.parts.context_id;
Nmad_Nem_tag_t tag = rreq->dev.match.parts.tag;
int ret;
MPIDI_msg_sz_t data_sz;
int dt_contig;
MPI_Aint dt_true_lb;
MPID_Datatype *dt_ptr;
NEM_NMAD_DIRECT_MATCH(match_info,0,source,context);
if (tag != MPI_ANY_TAG)
{
NEM_NMAD_SET_TAG(match_info,tag);
}
else
{
NEM_NMAD_SET_ANYTAG(match_info);
NEM_NMAD_SET_ANYTAG(match_mask);
}
#ifdef DEBUG
fprintf(stdout,"========> Posting Recv req %p (match is %lx) \n",rreq,match_info);
#endif
MPIDI_Datatype_get_info(rreq->dev.user_count,rreq->dev.datatype, dt_contig, data_sz, dt_ptr,dt_true_lb);
rreq->dev.OnDataAvail = NULL;
if (dt_contig)
{
ret = nm_sr_irecv_with_ref_tagged(mpid_nem_newmad_session,VC_FIELD(vc,p_gate),match_info,match_mask,
(char*)(rreq->dev.user_buf) + dt_true_lb,data_sz,
&(REQ_FIELD(rreq,newmad_req)),(void*)rreq);
REQ_FIELD(rreq,iov) = NULL;
}
else
{
int num_seg = 0;
struct iovec *newmad_iov = (struct iovec *)MPIU_Malloc(NMAD_IOV_MAX_DEPTH*sizeof(struct iovec));
struct iovec *newmad_iov_ptr = &(newmad_iov[0]);
MPID_nem_newmad_process_rdtype(&rreq,dt_ptr,data_sz,&newmad_iov_ptr,&num_seg);
MPIU_Assert(num_seg <= NMAD_IOV_MAX_DEPTH);
#ifdef DEBUG
{
int index;
for(index = 0; index < num_seg ; index++)
{
fprintf(stdout,"======================\n");
fprintf(stdout,"RECV nmad_iov[%i]: [base %p][len %i]\n",index,
newmad_iov[index].iov_base,newmad_iov[index].iov_len);
}
}
#endif
ret = nm_sr_irecv_iov_with_ref_tagged(mpid_nem_newmad_session,VC_FIELD(vc,p_gate),match_info,match_mask,
newmad_iov,num_seg,&(REQ_FIELD(rreq,newmad_req)),(void*)rreq);
REQ_FIELD(rreq,iov) = newmad_iov;
}
}
else
{
/* Fixme : this might not work in the case of multiple netmods */
memset((&(REQ_FIELD(rreq,newmad_req))),0,sizeof(nm_sr_request_t));
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_NEWMAD_DIRECTRECV);
return mpi_errno;
fn_fail: ATTRIBUTE((unused))
goto fn_exit;
}
int MPIDI_CH3U_Request_unpack_uebuf(MPID_Request * rreq)
{
int dt_contig;
MPI_Aint dt_true_lb;
MPIDI_msg_sz_t userbuf_sz;
MPID_Datatype * dt_ptr;
MPIDI_msg_sz_t unpack_sz;
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_REQUEST_UNPACK_UEBUF);
MPIDI_STATE_DECL(MPID_STATE_MEMCPY);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_REQUEST_UNPACK_UEBUF);
MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype,
dt_contig, userbuf_sz, dt_ptr, dt_true_lb);
if (rreq->dev.recv_data_sz <= userbuf_sz)
{
unpack_sz = rreq->dev.recv_data_sz;
}
else
{
/* --BEGIN ERROR HANDLING-- */
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,(MPIU_DBG_FDEST,
"receive buffer overflow; message truncated, msg_sz=" MPIDI_MSG_SZ_FMT
", buf_sz=" MPIDI_MSG_SZ_FMT,
rreq->dev.recv_data_sz, userbuf_sz));
unpack_sz = userbuf_sz;
MPIR_STATUS_SET_COUNT(rreq->status, userbuf_sz);
rreq->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_TRUNCATE,
"**truncate", "**truncate %d %d",
rreq->dev.recv_data_sz, userbuf_sz);
/* --END ERROR HANDLING-- */
}
if (unpack_sz > 0)
{
if (dt_contig)
{
/* TODO - check that amount of data is consistent with datatype.
In other words, if we were to use Segment_unpack()
would last = unpack? If not we should return an error
(unless configured with --enable-fast) */
MPIDI_FUNC_ENTER(MPID_STATE_MEMCPY);
MPIU_Memcpy((char *)rreq->dev.user_buf + dt_true_lb, rreq->dev.tmpbuf,
unpack_sz);
MPIDI_FUNC_EXIT(MPID_STATE_MEMCPY);
}
else
{
MPID_Segment seg;
MPI_Aint last;
MPID_Segment_init(rreq->dev.user_buf, rreq->dev.user_count,
rreq->dev.datatype, &seg, 0);
last = unpack_sz;
MPID_Segment_unpack(&seg, 0, &last, rreq->dev.tmpbuf);
if (last != unpack_sz)
{
/* --BEGIN ERROR HANDLING-- */
/* received data was not entirely consumed by unpack()
because too few bytes remained to fill the next basic
datatype */
MPIR_STATUS_SET_COUNT(rreq->status, last);
rreq->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_TYPE,
"**dtypemismatch", 0);
/* --END ERROR HANDLING-- */
}
}
}
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_REQUEST_UNPACK_UEBUF);
return mpi_errno;
}
static int
MPID_nem_newmad_handle_rreq(MPID_Request *req, nm_tag_t match_info, size_t size)
{
int mpi_errno = MPI_SUCCESS;
int complete = FALSE;
int dt_contig;
MPI_Aint dt_true_lb;
MPIDI_msg_sz_t userbuf_sz;
MPID_Datatype *dt_ptr;
MPIDI_msg_sz_t data_sz;
MPIDI_VC_t *vc = NULL;
#ifdef DEBUG
fprintf(stdout,"========> Completing Recv req %p (match is %lx) \n",req,match_info);
#endif
NEM_NMAD_MATCH_GET_RANK(match_info,req->status.MPI_SOURCE);
NEM_NMAD_MATCH_GET_TAG(match_info,req->status.MPI_TAG);
req->status.count = size;
req->dev.recv_data_sz = size;
MPIDI_Datatype_get_info(req->dev.user_count, req->dev.datatype, dt_contig, userbuf_sz, dt_ptr, dt_true_lb);
if (size <= userbuf_sz) {
data_sz = req->dev.recv_data_sz;
}
else
{
MPIU_DBG_MSG_FMT(CH3_OTHER,VERBOSE,(MPIU_DBG_FDEST,
"receive buffer too small; message truncated, msg_sz="
MPIDI_MSG_SZ_FMT ", userbuf_sz="
MPIDI_MSG_SZ_FMT,
req->dev.recv_data_sz, userbuf_sz));
req->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_TRUNCATE,
"**truncate", "**truncate %d %d %d %d",
req->status.MPI_SOURCE, req->status.MPI_TAG,
req->dev.recv_data_sz, userbuf_sz );
req->status.count = userbuf_sz;
data_sz = userbuf_sz;
}
if ((!dt_contig)&&(req->dev.tmpbuf != NULL))
{
MPIDI_msg_sz_t last;
last = req->dev.recv_data_sz;
MPID_Segment_unpack( req->dev.segment_ptr, 0, &last, req->dev.tmpbuf);
MPIU_Free(req->dev.tmpbuf);
if (last != data_sz) {
req->status.count = (int)last;
if (req->dev.recv_data_sz <= userbuf_sz) {
MPIU_ERR_SETSIMPLE(req->status.MPI_ERROR,MPI_ERR_TYPE,"**dtypemismatch");
}
}
}
if (REQ_FIELD(req,iov) != NULL)
MPIU_Free(REQ_FIELD(req,iov));
MPIDI_Comm_get_vc_set_active(req->comm, req->status.MPI_SOURCE, &vc);
MPIDI_CH3U_Handle_recv_req(vc, req, &complete);
MPIU_Assert(complete == TRUE);
#ifdef DEBUG
fprintf(stdout,"========> Completing Recv req %p done \n",req);
#endif
fn_exit:
return mpi_errno;
fn_fail: ATTRIBUTE((unused))
goto fn_exit;
}
void MPID_nem_newmad_anysource_posted(MPID_Request *rreq)
{
/* This function is called whenever an anyource request has been
posted to the posted receive queue. */
MPIR_Context_id_t context;
Nmad_Nem_tag_t tag;
nm_tag_t match_info = 0;
nm_tag_t match_mask = NEM_NMAD_MATCH_FULL_MASK;
nm_sr_request_t *newmad_req = MPIU_Malloc(sizeof(nm_sr_request_t));
int num_seg = 1;
int ret;
MPIDI_msg_sz_t data_sz;
int dt_contig;
MPI_Aint dt_true_lb;
MPID_Datatype *dt_ptr;
struct iovec *newmad_iov = (struct iovec *)MPIU_Malloc(NMAD_IOV_MAX_DEPTH*sizeof(struct iovec));
tag = rreq->dev.match.parts.tag;
context = rreq->dev.match.parts.context_id;
NEM_NMAD_DIRECT_MATCH(match_info,0,0,context);
if (tag != MPI_ANY_TAG)
{
NEM_NMAD_SET_TAG(match_info,tag);
}
else
{
NEM_NMAD_SET_ANYTAG(match_info);
NEM_NMAD_SET_ANYTAG(match_mask);
}
NEM_NMAD_SET_ANYSRC(match_info);
NEM_NMAD_SET_ANYSRC(match_mask);
#ifdef DEBUG
fprintf(stdout,"========> Any Source : Posting Recv req %p (nmad req is %p) (match is %lx) (mask is %lx) \n",
rreq,newmad_req,match_info,match_mask);
#endif
MPIDI_Datatype_get_info(rreq->dev.user_count,rreq->dev.datatype, dt_contig, data_sz, dt_ptr,dt_true_lb);
rreq->dev.OnDataAvail = NULL;
if (dt_contig)
{
newmad_iov[0].iov_base = (char*)(rreq->dev.user_buf) + dt_true_lb;
newmad_iov[0].iov_len = data_sz;
}
else
{
struct iovec *newmad_iov_ptr = &(newmad_iov[0]);
MPID_nem_newmad_process_rdtype(&rreq,dt_ptr,data_sz,&newmad_iov_ptr,&num_seg);
}
ret = nm_sr_irecv_iov_with_ref_tagged(mpid_nem_newmad_session,NM_ANY_GATE,match_info,match_mask,
newmad_iov,num_seg,newmad_req,(void*)rreq);
REQ_FIELD(rreq,iov) = newmad_iov;
MPID_MEM_NMAD_ADD_REQ_IN_HASH(rreq,newmad_req);
/*
#ifdef DEBUG
fprintf(stdout,"========> Any Source : callback end \n");
#endif
*/
}
static int _mxm_handle_rreq(MPID_Request * req)
{
int complete = FALSE, found = FALSE;
int dt_contig;
MPI_Aint dt_true_lb ATTRIBUTE((unused));
MPIDI_msg_sz_t userbuf_sz;
MPID_Datatype *dt_ptr;
MPIDI_msg_sz_t data_sz;
MPID_nem_mxm_vc_area *vc_area ATTRIBUTE((unused)) = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
void *tmp_buf = NULL;
MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_MSGQ_MUTEX);
found = MPIDI_CH3U_Recvq_DP(req);
MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_MSGQ_MUTEX);
/* an MPI_ANY_SOURCE request may have been previously removed from the
* CH3 queue by an FDP (find and dequeue posted) operation */
if (req->dev.match.parts.rank != MPI_ANY_SOURCE) {
MPIU_Assert(found);
}
MPIDI_Datatype_get_info(req->dev.user_count, req->dev.datatype, dt_contig, userbuf_sz, dt_ptr,
dt_true_lb);
vc_area = VC_BASE(req->ch.vc);
req_area = REQ_BASE(req);
_dbg_mxm_out_buf(req_area->iov_buf[0].ptr,
(req_area->iov_buf[0].length > 16 ? 16 : req_area->iov_buf[0].length));
if (req->dev.recv_data_sz <= userbuf_sz) {
data_sz = req->dev.recv_data_sz;
if (req->status.MPI_ERROR == MPI_ERR_TRUNCATE) {
req->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__,
MPI_ERR_TRUNCATE, "**truncate",
"**truncate %d %d %d %d",
req->status.MPI_SOURCE,
req->status.MPI_TAG, req->dev.recv_data_sz,
userbuf_sz);
}
}
else {
data_sz = userbuf_sz;
MPIR_STATUS_SET_COUNT(req->status, userbuf_sz);
MPIU_DBG_MSG_FMT(CH3_OTHER, VERBOSE, (MPIU_DBG_FDEST,
"receive buffer too small; message truncated, msg_sz="
MPIDI_MSG_SZ_FMT ", userbuf_sz="
MPIDI_MSG_SZ_FMT, req->dev.recv_data_sz, userbuf_sz));
req->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__,
MPI_ERR_TRUNCATE, "**truncate",
"**truncate %d %d %d %d",
req->status.MPI_SOURCE, req->status.MPI_TAG,
req->dev.recv_data_sz, userbuf_sz);
}
if (!dt_contig) {
MPIDI_msg_sz_t last = 0;
if (req->dev.tmpbuf != NULL) {
last = req->dev.recv_data_sz;
MPID_Segment_unpack(req->dev.segment_ptr, 0, &last, req->dev.tmpbuf);
tmp_buf = req->dev.tmpbuf;
}
else {
mxm_req_buffer_t *iov_buf;
MPL_IOV *iov;
int n_iov = 0;
int index;
last = req->dev.recv_data_sz;
n_iov = req_area->iov_count;
iov_buf = req_area->iov_buf;
if (last && n_iov > 0) {
iov = MPIU_Malloc(n_iov * sizeof(*iov));
MPIU_Assert(iov);
for (index = 0; index < n_iov; index++) {
iov[index].MPL_IOV_BUF = iov_buf[index].ptr;
iov[index].MPL_IOV_LEN = iov_buf[index].length;
}
MPID_Segment_unpack_vector(req->dev.segment_ptr, req->dev.segment_first, &last, iov,
&n_iov);
MPIU_Free(iov);
}
if (req_area->iov_count > MXM_MPICH_MAX_IOV) {
tmp_buf = req_area->iov_buf;
req_area->iov_buf = req_area->tmp_buf;
req_area->iov_count = 0;
}
}
if (last != data_sz) {
MPIR_STATUS_SET_COUNT(req->status, last);
if (req->dev.recv_data_sz <= userbuf_sz) {
/* If the data can't be unpacked, the we have a
* mismatch between the datatype and the amount of
* data received. Throw away received data.
*/
MPIR_ERR_SETSIMPLE(req->status.MPI_ERROR, MPI_ERR_TYPE, "**dtypemismatch");
}
}
}
MPIDI_CH3U_Handle_recv_req(req->ch.vc, req, &complete);
MPIU_Assert(complete == TRUE);
if (tmp_buf)
MPIU_Free(tmp_buf);
return complete;
}
int MPID_Send(const void * buf, MPI_Aint count, MPI_Datatype datatype, int rank,
int tag, MPID_Comm * comm, int context_offset,
MPID_Request ** request)
{
MPIDI_msg_sz_t data_sz;
int dt_contig;
MPI_Aint dt_true_lb;
MPID_Datatype * dt_ptr;
MPID_Request * sreq = NULL;
MPIDI_VC_t * vc;
#if defined(MPID_USE_SEQUENCE_NUMBERS)
MPID_Seqnum_t seqnum;
#endif
int eager_threshold = -1;
int mpi_errno = MPI_SUCCESS;
#if defined(FINEGRAIN_MPI)
int destpid=-1, destworldrank=-1;
#endif
MPIDI_STATE_DECL(MPID_STATE_MPID_SEND);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_SEND);
MPIU_DBG_MSG_FMT(CH3_OTHER,VERBOSE,(MPIU_DBG_FDEST,
"rank=%d, tag=%d, context=%d",
rank, tag, comm->context_id + context_offset));
/* Check to make sure the communicator hasn't already been revoked */
if (comm->revoked &&
MPIR_AGREE_TAG != MPIR_TAG_MASK_ERROR_BITS(tag & ~MPIR_Process.tagged_coll_mask) &&
MPIR_SHRINK_TAG != MPIR_TAG_MASK_ERROR_BITS(tag & ~MPIR_Process.tagged_coll_mask)) {
MPIR_ERR_SETANDJUMP(mpi_errno,MPIX_ERR_REVOKED,"**revoked");
}
#if defined(FINEGRAIN_MPI)
MPIDI_Comm_get_pid_worldrank(comm, rank, &destpid, &destworldrank);
if (COMPARE_RANKS(rank,comm,destpid) && comm->comm_kind != MPID_INTERCOMM)
{
mpi_errno = MPIDI_Isend_self(&buf, count, datatype, rank, tag, comm,
context_offset, MPIDI_REQUEST_TYPE_SEND,
&sreq);
if (rank == comm->rank)
{
printf("my_fgrank=%d: %s, self send DEADLOCK\n", my_fgrank, __FUNCTION__);
if (sreq != NULL && sreq->cc != 0) {
MPIR_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,
"**dev|selfsenddeadlock");
}
}
#else
if (rank == comm->rank && comm->comm_kind != MPID_INTERCOMM)
{
mpi_errno = MPIDI_Isend_self(buf, count, datatype, rank, tag, comm,
context_offset, MPIDI_REQUEST_TYPE_SEND,
&sreq);
/* In the single threaded case, sending to yourself will cause
deadlock. Note that in the runtime-thread case, this check
will not be made (long-term FIXME) */
# ifndef MPICH_IS_THREADED
{
if (sreq != NULL && MPID_cc_get(sreq->cc) != 0) {
MPIR_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,
"**dev|selfsenddeadlock");
}
}
# endif
#endif
if (mpi_errno != MPI_SUCCESS) { MPIR_ERR_POP(mpi_errno); }
goto fn_exit;
}
if (rank == MPI_PROC_NULL)
{
goto fn_exit;
}
#if defined(FINEGRAIN_MPI)
MPIDI_Comm_get_vc_set_active_direct(comm, destpid, &vc);
#else
MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
#endif
MPIR_ERR_CHKANDJUMP1(vc->state == MPIDI_VC_STATE_MORIBUND, mpi_errno, MPIX_ERR_PROC_FAILED, "**comm_fail", "**comm_fail %d", rank);
#ifdef ENABLE_COMM_OVERRIDES
if (vc->comm_ops && vc->comm_ops->send)
{
mpi_errno = vc->comm_ops->send( vc, buf, count, datatype, rank, tag, comm, context_offset, &sreq);
goto fn_exit;
}
#endif
MPIDI_Datatype_get_info(count, datatype, dt_contig, data_sz, dt_ptr,
dt_true_lb);
if (data_sz == 0)
{
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_eager_send_t * const eager_pkt = &upkt.eager_send;
MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"sending zero length message");
MPIDI_Pkt_init(eager_pkt, MPIDI_CH3_PKT_EAGER_SEND);
#if defined(FINEGRAIN_MPI)
eager_pkt->match.parts.dest_rank = destworldrank;
#endif
eager_pkt->match.parts.rank = comm->rank;
eager_pkt->match.parts.tag = tag;
eager_pkt->match.parts.context_id = comm->context_id + context_offset;
eager_pkt->sender_req_id = MPI_REQUEST_NULL;
eager_pkt->data_sz = 0;
MPIDI_VC_FAI_send_seqnum(vc, seqnum);
MPIDI_Pkt_set_seqnum(eager_pkt, seqnum);
MPID_THREAD_CS_ENTER(POBJ, vc->pobj_mutex);
mpi_errno = MPIDI_CH3_iStartMsg(vc, eager_pkt, sizeof(*eager_pkt), &sreq);
MPID_THREAD_CS_EXIT(POBJ, vc->pobj_mutex);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
MPIR_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|eagermsg");
}
/* --END ERROR HANDLING-- */
if (sreq != NULL)
{
MPIDI_Request_set_seqnum(sreq, seqnum);
MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_SEND);
/* sreq->comm = comm;
MPIR_Comm_add_ref(comm); -- not necessary for blocking functions */
}
goto fn_exit;
}
MPIDI_CH3_GET_EAGER_THRESHOLD(&eager_threshold, comm, vc);
/* FIXME: flow control: limit number of outstanding eager messages
containing data and need to be buffered by the receiver */
#ifdef USE_EAGER_SHORT
if (dt_contig && data_sz <= MPIDI_EAGER_SHORT_SIZE) {
mpi_errno = MPIDI_CH3_EagerContigShortSend( &sreq,
MPIDI_CH3_PKT_EAGERSHORT_SEND,
(char *)buf + dt_true_lb,
data_sz, rank, tag, comm,
context_offset );
}
else
#endif
if (data_sz + sizeof(MPIDI_CH3_Pkt_eager_send_t) <= eager_threshold)
{
if (dt_contig)
{
mpi_errno = MPIDI_CH3_EagerContigSend( &sreq,
MPIDI_CH3_PKT_EAGER_SEND,
(char *)buf + dt_true_lb,
data_sz, rank, tag, comm,
context_offset );
}
else
{
MPIDI_Request_create_sreq(sreq, mpi_errno, goto fn_exit);
MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_SEND);
mpi_errno = MPIDI_CH3_EagerNoncontigSend( &sreq,
MPIDI_CH3_PKT_EAGER_SEND,
buf, count, datatype,
data_sz, rank, tag,
comm, context_offset );
}
}
else
{
int MPID_nem_ib_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, MPID_IOV s_cookie)
{
int mpi_errno = MPI_SUCCESS;
int dt_contig;
MPIDI_msg_sz_t data_sz;
MPID_Datatype *dt_ptr;
MPI_Aint dt_true_lb;
MPID_nem_ib_vc_area *vc_ib = VC_IB(vc);
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_LMT_START_RECV);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_LMT_START_RECV);
dprintf("lmt_start_recv,enter,%d<-%d,req=%p\n", MPID_nem_ib_myrank, vc->pg_rank, req);
/* obtain dt_true_lb */
/* see MPIDI_Datatype_get_info(in, in, out, out, out, out) (in src/mpid/ch3/include/mpidimpl.h) */
MPIDI_Datatype_get_info(req->dev.user_count, req->dev.datatype, dt_contig, data_sz, dt_ptr,
dt_true_lb);
MPID_nem_ib_lmt_cookie_t *s_cookie_buf = s_cookie.iov_base;
/* stash vc for ib_poll */
req->ch.vc = vc;
void *write_to_buf;
if (dt_contig) {
write_to_buf = (void *) ((char *) req->dev.user_buf + dt_true_lb);
}
else {
//REQ_FIELD(req, lmt_pack_buf) = MPIU_Malloc((size_t)req->ch.lmt_data_sz);
REQ_FIELD(req, lmt_pack_buf) = MPID_nem_ib_stmalloc((size_t) req->ch.lmt_data_sz);
MPIU_ERR_CHKANDJUMP(!REQ_FIELD(req, lmt_pack_buf), mpi_errno, MPI_ERR_OTHER,
"**outofmemory");
write_to_buf = REQ_FIELD(req, lmt_pack_buf);
}
REQ_FIELD(req, buf.to) = write_to_buf;
#ifdef MPID_NEM_IB_LMT_GET_CQE
#else
/* unmark magic */
*((uint8_t *) (write_to_buf + req->ch.lmt_data_sz - sizeof(uint8_t))) = ~s_cookie_buf->tail; /* size in cookie was not set */
#endif
dprintf
("lmt_start_recv,dt_contig=%d,write_to_buf=%p,req->dev.user_buf=%p,REQ_FIELD(req, lmt_pack_buf)=%p,marked-tail=%02x,unmarked-tail=%02x\n",
dt_contig, write_to_buf, req->dev.user_buf, REQ_FIELD(req, lmt_pack_buf),
s_cookie_buf->tail, *((uint8_t *) (write_to_buf + req->ch.lmt_data_sz - sizeof(uint8_t))));
/* stash tail for poll because do_cts in mpid_nem_lmt.c free s_cookie_buf just after this function */
REQ_FIELD(req, lmt_tail) = s_cookie_buf->tail;
dprintf("lmt_start_recv,mem-tail=%p,%02x\n",
write_to_buf + req->ch.lmt_data_sz - sizeof(uint8_t),
*((uint8_t *) (write_to_buf + req->ch.lmt_data_sz - sizeof(uint8_t))));
//dprintf("lmt_start_recv,sendq_empty=%d,ncom=%d,ncqe=%d\n", MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY, MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY);
int last = 1;
long length = req->ch.lmt_data_sz;
if (s_cookie_buf->seg_seq_num != s_cookie_buf->seg_num) {
last = 0;
length = s_cookie_buf->max_msg_sz;
}
REQ_FIELD(req, max_msg_sz) = s_cookie_buf->max_msg_sz; /* store initiator's max_msg_sz */
REQ_FIELD(req, seg_num) = s_cookie_buf->seg_num; /* store number of segments */
/* try to issue RDMA-read command */
int slack = 1; /* slack for control packet bringing sequence number */
if (MPID_nem_ib_sendq_empty(vc_ib->sendq) &&
vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY - slack &&
MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY - slack) {
mpi_errno =
MPID_nem_ib_lmt_start_recv_core(req, s_cookie_buf->addr, s_cookie_buf->rkey, length,
write_to_buf, s_cookie_buf->max_msg_sz, last);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
}
else {
/* enqueue command into send_queue */
dprintf("lmt_start_recv, enqueuing,sendq_empty=%d,ncom=%d,ncqe=%d\n",
MPID_nem_ib_sendq_empty(vc_ib->sendq),
vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY,
MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY);
/* make raddr, (sz is in rreq->ch.lmt_data_sz), rkey, (user_buf is in req->dev.user_buf) survive enqueue, free cookie, dequeue */
REQ_FIELD(req, lmt_raddr) = s_cookie_buf->addr;
REQ_FIELD(req, lmt_rkey) = s_cookie_buf->rkey;
REQ_FIELD(req, lmt_write_to_buf) = write_to_buf;
REQ_FIELD(req, lmt_szsend) = length;
REQ_FIELD(req, last) = last;
MPID_nem_ib_sendq_enqueue(&vc_ib->sendq, req);
}
#if 0 /* moving to packet header */
/* extract embeded RDMA-write-to buffer occupancy information */
dprintf("lmt_start_recv,old lsr_seq_num=%d,s_cookie_buf->seq_num_tail=%d\n",
vc_ib->ibcom->lsr_seq_num_tail, s_cookie_buf->seq_num_tail);
vc_ib->ibcom->lsr_seq_num_tail = s_cookie_buf->seq_num_tail;
//dprintf("lmt_start_recv,new lsr_seq_num=%d\n", vc_ib->ibcom->lsr_seq_num_tail);
#endif
#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE
/* change remote notification policy of RDMA-write-to buf */
//dprintf("lmt_start_recv,reply_seq_num,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, &vc_ib->ibcom->lsr_seq_num_tail);
//dprintf("lmt_start_recv,reply_seq_num,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
#endif
//dprintf("lmt_start_recv,reply_seq_num,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, MPID_nem_ib_ncqe, MPID_nem_ib_diff16(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail));
/* try to send from sendq because at least one RDMA-write-to buffer has been released */
//dprintf("lmt_start_recv,reply_seq_num,send_progress\n");
if (!MPID_nem_ib_sendq_empty(vc_ib->sendq)) {
dprintf("lmt_start_recv,ncom=%d,ncqe=%d,diff=%d\n",
vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY,
MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY,
MPID_nem_ib_diff16(vc_ib->ibcom->sseq_num,
vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG);
}
if (!MPID_nem_ib_sendq_empty(vc_ib->sendq) && MPID_nem_ib_sendq_ready_to_send_head(vc_ib)) {
dprintf("lmt_start_recv,send_progress\n");
fflush(stdout);
MPID_nem_ib_send_progress(vc);
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_LMT_START_RECV);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_Irsend(const void * buf, int count, MPI_Datatype datatype, int rank, int tag, MPID_Comm * comm, int context_offset,
MPID_Request ** request)
{
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_ready_send_t * const ready_pkt = &upkt.ready_send;
MPIDI_msg_sz_t data_sz;
int dt_contig;
MPI_Aint dt_true_lb;
MPID_Datatype * dt_ptr;
MPID_Request * sreq;
MPIDI_VC_t * vc;
#if defined(MPID_USE_SEQUENCE_NUMBERS)
MPID_Seqnum_t seqnum;
#endif
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_IRSEND);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_IRSEND);
MPIU_DBG_MSG_FMT(CH3_OTHER,VERBOSE,(MPIU_DBG_FDEST,
"rank=%d, tag=%d, context=%d",
rank, tag, comm->context_id + context_offset));
/* Check to make sure the communicator hasn't already been revoked */
if (comm->revoked &&
MPIR_AGREE_TAG != MPIR_TAG_MASK_ERROR_BITS(tag & ~MPIR_Process.tagged_coll_mask) &&
MPIR_SHRINK_TAG != MPIR_TAG_MASK_ERROR_BITS(tag & ~MPIR_Process.tagged_coll_mask)) {
MPIR_ERR_SETANDJUMP(mpi_errno,MPIX_ERR_REVOKED,"**revoked");
}
if (rank == comm->rank && comm->comm_kind != MPID_INTERCOMM)
{
mpi_errno = MPIDI_Isend_self(buf, count, datatype, rank, tag, comm, context_offset, MPIDI_REQUEST_TYPE_RSEND, &sreq);
goto fn_exit;
}
if (rank != MPI_PROC_NULL) {
MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
#ifdef ENABLE_COMM_OVERRIDES
/* this needs to come before the sreq is created, since the override
* function is responsible for creating its own request */
if (vc->comm_ops && vc->comm_ops->irsend)
{
mpi_errno = vc->comm_ops->irsend( vc, buf, count, datatype, rank, tag, comm, context_offset, &sreq);
goto fn_exit;
}
#endif
}
MPIDI_Request_create_sreq(sreq, mpi_errno, goto fn_exit);
MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_RSEND);
MPIDI_Request_set_msg_type(sreq, MPIDI_REQUEST_EAGER_MSG);
if (rank == MPI_PROC_NULL)
{
MPIU_Object_set_ref(sreq, 1);
MPID_cc_set(&sreq->cc, 0);
goto fn_exit;
}
MPIDI_Datatype_get_info(count, datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
MPIDI_Pkt_init(ready_pkt, MPIDI_CH3_PKT_READY_SEND);
ready_pkt->match.parts.rank = comm->rank;
ready_pkt->match.parts.tag = tag;
ready_pkt->match.parts.context_id = comm->context_id + context_offset;
ready_pkt->sender_req_id = MPI_REQUEST_NULL;
ready_pkt->data_sz = data_sz;
if (data_sz == 0)
{
MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"sending zero length message");
sreq->dev.OnDataAvail = 0;
MPIDI_VC_FAI_send_seqnum(vc, seqnum);
MPIDI_Pkt_set_seqnum(ready_pkt, seqnum);
MPIDI_Request_set_seqnum(sreq, seqnum);
MPID_THREAD_CS_ENTER(POBJ, vc->pobj_mutex);
mpi_errno = MPIDI_CH3_iSend(vc, sreq, ready_pkt, sizeof(*ready_pkt));
MPID_THREAD_CS_EXIT(POBJ, vc->pobj_mutex);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
MPID_Request_release(sreq);
sreq = NULL;
MPIR_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**ch3|eagermsg");
goto fn_exit;
}
/* --END ERROR HANDLING-- */
goto fn_exit;
}
if (vc->ready_eager_max_msg_sz < 0 || data_sz + sizeof(MPIDI_CH3_Pkt_ready_send_t) <= vc->ready_eager_max_msg_sz) {
if (dt_contig) {
mpi_errno = MPIDI_CH3_EagerContigIsend( &sreq,
MPIDI_CH3_PKT_READY_SEND,
(char*)buf + dt_true_lb,
data_sz, rank, tag,
comm, context_offset );
}
else {
mpi_errno = MPIDI_CH3_EagerNoncontigSend( &sreq,
MPIDI_CH3_PKT_READY_SEND,
buf, count, datatype,
data_sz, rank, tag,
comm, context_offset );
/* If we're not complete, then add a reference to the datatype */
if (sreq && sreq->dev.OnDataAvail) {
sreq->dev.datatype_ptr = dt_ptr;
MPID_Datatype_add_ref(dt_ptr);
}
}
} else {
/* Do rendezvous. This will be sent as a regular send not as
a ready send, so the receiver won't know to send an error
if the receive has not been posted */
MPIDI_Request_set_msg_type( sreq, MPIDI_REQUEST_RNDV_MSG );
mpi_errno = vc->rndvSend_fn( &sreq, buf, count, datatype, dt_contig,
data_sz, dt_true_lb, rank, tag, comm,
context_offset );
if (sreq && dt_ptr != NULL) {
sreq->dev.datatype_ptr = dt_ptr;
MPID_Datatype_add_ref(dt_ptr);
}
}
fn_exit:
*request = sreq;
MPIU_DBG_STMT(CH3_OTHER,VERBOSE,{
if (sreq != NULL)
{
MPIU_DBG_MSG_P(CH3_OTHER,VERBOSE,"request allocated, handle=0x%08x", sreq->handle);
}
}
);
int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype
origin_datatype, int target_rank, MPI_Aint target_disp,
int target_count, MPI_Datatype target_datatype, MPID_Win * win_ptr,
MPID_Request * ureq)
{
int mpi_errno = MPI_SUCCESS;
int dt_contig ATTRIBUTE((unused)), rank;
MPID_Datatype *dtp;
MPI_Aint dt_true_lb ATTRIBUTE((unused));
MPIDI_msg_sz_t data_sz;
MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
int made_progress = 0;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_PUT);
MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_PUT);
MPIR_ERR_CHKANDJUMP(win_ptr->states.access_state == MPIDI_RMA_NONE,
mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");
if (target_rank == MPI_PROC_NULL) {
goto fn_exit;
}
MPIDI_Datatype_get_info(origin_count, origin_datatype, dt_contig, data_sz, dtp, dt_true_lb);
if (data_sz == 0) {
goto fn_exit;
}
rank = win_ptr->comm_ptr->rank;
if (win_ptr->shm_allocated == TRUE && target_rank != rank &&
win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) {
/* check if target is local and shared memory is allocated on window,
* if so, we directly perform this operation on shared memory region. */
/* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on
* the same node. However, in ch3:sock, even if origin and target are on the same node, they do
* not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first,
* which is only set to TRUE when SHM region is allocated in nemesis.
* In future we need to figure out a way to check if origin and target are in the same "SHM comm".
*/
MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc);
MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);
}
/* If the put is a local operation, do it here */
if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED ||
(win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) {
mpi_errno = MPIDI_CH3I_Shm_put_op(origin_addr, origin_count, origin_datatype, target_rank,
target_disp, target_count, target_datatype, win_ptr);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
if (ureq) {
/* Complete user request and release the ch3 ref */
mpi_errno = MPID_Request_complete(ureq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
}
}
else {
MPIDI_RMA_Op_t *op_ptr = NULL;
MPIDI_CH3_Pkt_put_t *put_pkt = NULL;
int use_immed_pkt = FALSE;
int is_origin_contig, is_target_contig;
/* queue it up */
mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &op_ptr);
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
/******************** Setting operation struct areas ***********************/
/* FIXME: For contig and very short operations, use a streamlined op */
op_ptr->origin_addr = (void *) origin_addr;
op_ptr->origin_count = origin_count;
op_ptr->origin_datatype = origin_datatype;
op_ptr->target_rank = target_rank;
/* Remember user request */
op_ptr->ureq = ureq;
/* if source or target datatypes are derived, increment their
* reference counts */
if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) {
MPID_Datatype_get_ptr(origin_datatype, dtp);
MPID_Datatype_add_ref(dtp);
}
if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
MPID_Datatype_get_ptr(target_datatype, dtp);
MPID_Datatype_add_ref(dtp);
}
MPID_Datatype_is_contig(origin_datatype, &is_origin_contig);
MPID_Datatype_is_contig(target_datatype, &is_target_contig);
/* Judge if we can use IMMED data packet */
if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) &&
MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && is_origin_contig && is_target_contig) {
if (data_sz <= MPIDI_RMA_IMMED_BYTES)
use_immed_pkt = TRUE;
}
/* Judge if this operation is an piggyback candidate */
if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) &&
MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
/* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes
* for both origin and target data. We should extend this optimization to derived
* datatypes as well. */
if (data_sz <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE)
op_ptr->piggyback_lock_candidate = 1;
}
/************** Setting packet struct areas in operation ****************/
put_pkt = &(op_ptr->pkt.put);
if (use_immed_pkt) {
MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT_IMMED);
}
else {
MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT);
}
put_pkt->addr = (char *) win_ptr->basic_info_table[target_rank].base_addr +
win_ptr->basic_info_table[target_rank].disp_unit * target_disp;
put_pkt->count = target_count;
put_pkt->datatype = target_datatype;
put_pkt->info.dataloop_size = 0;
put_pkt->target_win_handle = win_ptr->basic_info_table[target_rank].win_handle;
put_pkt->source_win_handle = win_ptr->handle;
put_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
if (use_immed_pkt) {
void *src = (void *) origin_addr, *dest = (void *) (put_pkt->info.data);
mpi_errno = immed_copy(src, dest, data_sz);
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
}
MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, op_ptr);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, target_rank, &made_progress);
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
if (MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD >= 0 &&
MPIDI_CH3I_RMA_Active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) {
while (MPIDI_CH3I_RMA_Active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) {
mpi_errno = wait_progress_engine();
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
}
}
}
fn_exit:
MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_PUT);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
goto fn_exit;
/* --END ERROR HANDLING-- */
}
/**
* \brief MPID buffer copy
*
* Implements non-contiguous buffers correctly.
*
* \param[in] sbuf The address of the input buffer
* \param[in] scount The number of elements in that buffer
* \param[in] sdt The datatype of those elements
* \param[out] smpi_errno Returns errors
* \param[in] rbuf The address of the output buffer
* \param[out] rcount The number of elements in that buffer
* \param[in] rdt The datatype of those elements
* \param[out] rsz The size of the ouput data
* \param[out] rmpi_errno Returns errors
*/
void MPIDI_Buffer_copy(
const void * const sbuf, MPI_Aint scount, MPI_Datatype sdt, int * smpi_errno,
void * const rbuf, MPI_Aint rcount, MPI_Datatype rdt, MPIDI_msg_sz_t * rsz, int * rmpi_errno)
{
int sdt_contig;
int rdt_contig;
MPI_Aint sdt_true_lb, rdt_true_lb;
MPIDI_msg_sz_t sdata_sz;
MPIDI_msg_sz_t rdata_sz;
MPID_Datatype * sdt_ptr;
MPID_Datatype * rdt_ptr;
MPI_Aint sdt_extent;
MPI_Aint rdt_extent;
*smpi_errno = MPI_SUCCESS;
*rmpi_errno = MPI_SUCCESS;
/* printf("bufcopy: src count=%d dt=%d\n", scount, sdt); */
/* printf("bufcopy: dst count=%d dt=%d\n", rcount, rdt); */
MPIDI_Datatype_get_info(scount, sdt, sdt_contig, sdata_sz, sdt_ptr, sdt_true_lb);
MPIDI_Datatype_get_info(rcount, rdt, rdt_contig, rdata_sz, rdt_ptr, rdt_true_lb);
/* --BEGIN ERROR HANDLING-- */
if (sdata_sz > rdata_sz)
{
*rmpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, __FUNCTION__, __LINE__, MPI_ERR_TRUNCATE, "**truncate", "**truncate %d %d", sdata_sz, rdata_sz );
sdata_sz = rdata_sz;
}
/* --END ERROR HANDLING-- */
if (sdata_sz == 0)
{
*rsz = 0;
goto fn_exit;
}
if (sdt_contig && rdt_contig)
{
#if CUDA_AWARE_SUPPORT
if(MPIDI_Process.cuda_aware_support_on && MPIDI_cuda_is_device_buf(rbuf))
{
cudaError_t cudaerr = CudaMemcpy(rbuf + rdt_true_lb, sbuf + sdt_true_lb, sdata_sz, cudaMemcpyHostToDevice);
}
else
#endif
memcpy((char*)rbuf + rdt_true_lb, (const char *)sbuf + sdt_true_lb, sdata_sz);
*rsz = sdata_sz;
}
else if (sdt_contig)
{
#if CUDA_AWARE_SUPPORT
// This will need to be done in two steps:
// 1 - Allocate a temp buffer which is the same size as user buffer and unpack in it.
// 2 - Copy unpacked data into user buffer from temp buffer.
if(MPIDI_Process.cuda_aware_support_on && MPIDI_cuda_is_device_buf(rbuf))
{
MPID_Datatype_get_extent_macro(rdt, rdt_extent);
char *buf = MPL_malloc(rdt_extent * rcount);
memset(buf, 0, rdt_extent * rcount);
MPID_Segment seg;
DLOOP_Offset last;
MPID_Segment_init(buf, rcount, rdt, &seg, 0);
last = sdata_sz;
MPID_Segment_unpack(&seg, 0, &last, (char*)sbuf + sdt_true_lb);
/* --BEGIN ERROR HANDLING-- */
if (last != sdata_sz)
{
*rmpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, __FUNCTION__, __LINE__, MPI_ERR_TYPE, "**dtypemismatch", 0);
}
/* --END ERROR HANDLING-- */
*rsz = last;
cudaError_t cudaerr = CudaMemcpy(rbuf + rdt_true_lb, buf, rdt_extent * rcount, cudaMemcpyHostToDevice);
MPL_free(buf);
goto fn_exit;
}
#endif
MPID_Segment seg;
DLOOP_Offset last;
MPID_Segment_init(rbuf, rcount, rdt, &seg, 0);
last = sdata_sz;
MPID_Segment_unpack(&seg, 0, &last, (char*)sbuf + sdt_true_lb);
/* --BEGIN ERROR HANDLING-- */
if (last != sdata_sz)
{
*rmpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, __FUNCTION__, __LINE__, MPI_ERR_TYPE, "**dtypemismatch", 0);
}
/* --END ERROR HANDLING-- */
*rsz = last;
}
else if (rdt_contig)
{
MPID_Segment seg;
DLOOP_Offset last;
MPID_Segment_init(sbuf, scount, sdt, &seg, 0);
last = sdata_sz;
MPID_Segment_pack(&seg, 0, &last, (char*)rbuf + rdt_true_lb);
/* --BEGIN ERROR HANDLING-- */
if (last != sdata_sz)
{
*rmpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, __FUNCTION__, __LINE__, MPI_ERR_TYPE, "**dtypemismatch", 0);
}
/* --END ERROR HANDLING-- */
*rsz = last;
}
else
{
char * buf;
MPIDI_msg_sz_t buf_off;
MPID_Segment sseg;
MPIDI_msg_sz_t sfirst;
MPID_Segment rseg;
MPIDI_msg_sz_t rfirst;
buf = MPL_malloc(MPIDI_COPY_BUFFER_SZ);
/* --BEGIN ERROR HANDLING-- */
if (buf == NULL)
{
*smpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL, __FUNCTION__, __LINE__, MPI_ERR_OTHER, "**nomem", 0);
*rmpi_errno = *smpi_errno;
*rsz = 0;
goto fn_exit;
}
/* --END ERROR HANDLING-- */
MPID_Segment_init(sbuf, scount, sdt, &sseg, 0);
MPID_Segment_init(rbuf, rcount, rdt, &rseg, 0);
sfirst = 0;
rfirst = 0;
buf_off = 0;
for(;;)
{
DLOOP_Offset last;
char * buf_end;
if (sdata_sz - sfirst > MPIDI_COPY_BUFFER_SZ - buf_off)
{
last = sfirst + (MPIDI_COPY_BUFFER_SZ - buf_off);
}
else
{
last = sdata_sz;
}
MPID_Segment_pack(&sseg, sfirst, &last, buf + buf_off);
/* --BEGIN ERROR HANDLING-- */
MPID_assert(last > sfirst);
/* --END ERROR HANDLING-- */
buf_end = buf + buf_off + (last - sfirst);
sfirst = last;
MPID_Segment_unpack(&rseg, rfirst, &last, buf);
/* --BEGIN ERROR HANDLING-- */
MPID_assert(last > rfirst);
/* --END ERROR HANDLING-- */
rfirst = last;
if (rfirst == sdata_sz)
{
/* successful completion */
break;
}
/* --BEGIN ERROR HANDLING-- */
if (sfirst == sdata_sz)
{
/* datatype mismatch -- remaining bytes could not be unpacked */
*rmpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, __FUNCTION__, __LINE__, MPI_ERR_TYPE, "**dtypemismatch", 0);
break;
}
/* --END ERROR HANDLING-- */
buf_off = sfirst - rfirst;
if (buf_off > 0)
{
memmove(buf, buf_end - buf_off, buf_off);
}
}
*rsz = rfirst;
MPL_free(buf);
}
fn_exit:
return;
}
int
MPIDO_Allgather_simple(const void *sendbuf,
int sendcount,
MPI_Datatype sendtype,
void *recvbuf,
int recvcount,
MPI_Datatype recvtype,
MPID_Comm * comm_ptr,
int *mpierrno)
{
#ifndef HAVE_PAMI_IN_PLACE
if (sendbuf == MPI_IN_PLACE)
{
MPID_Abort (NULL, 0, 1, "'MPI_IN_PLACE' requries support for `PAMI_IN_PLACE`");
return -1;
}
#endif
/* *********************************
* Check the nature of the buffers
* *********************************
*/
const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid);
MPID_Datatype * dt_null = NULL;
void *snd_noncontig_buff = NULL, *rcv_noncontig_buff = NULL;
MPI_Aint send_true_lb = 0;
MPI_Aint recv_true_lb = 0;
int snd_data_contig = 1, rcv_data_contig = 1;
size_t send_size = 0;
size_t recv_size = 0;
MPID_Segment segment;
volatile unsigned allgather_active = 1;
const int rank = comm_ptr->rank;
const int size = comm_ptr->local_size;
#if ASSERT_LEVEL==0
/* We can't afford the tracing in ndebug/performance libraries */
const unsigned verbose = 0;
#else
const unsigned verbose = (MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_ALL) && (rank == 0);
#endif
const pami_metadata_t *my_md;
char *rbuf = NULL, *sbuf = NULL;
if ((sendcount < 1 && sendbuf != MPI_IN_PLACE) || recvcount < 1)
return MPI_SUCCESS;
/* Gather datatype information */
MPIDI_Datatype_get_info(recvcount,
recvtype,
rcv_data_contig,
recv_size,
dt_null,
recv_true_lb);
send_size = recv_size;
if(MPIDI_Pamix_collsel_advise != NULL && mpid->collsel_fast_query != NULL)
{
advisor_algorithm_t advisor_algorithms[1];
int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_ALLGATHER, send_size, advisor_algorithms, 1);
if(num_algorithms)
{
if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO)
{
return MPIR_Allgather(sendbuf, sendcount, sendtype,
recvbuf, recvcount, recvtype,
comm_ptr, mpierrno);
}
else if(advisor_algorithms[0].metadata && advisor_algorithms[0].metadata->check_correct.values.asyncflowctl && !(--(comm_ptr->mpid.num_requests)))
{
comm_ptr->mpid.num_requests = MPIDI_Process.optimized.num_requests;
int tmpmpierrno;
if(unlikely(verbose))
fprintf(stderr,"Query barrier required for %s\n", advisor_algorithms[0].metadata->name);
MPIDO_Barrier(comm_ptr, &tmpmpierrno);
}
}
}
rbuf = (char *)recvbuf+recv_true_lb;
if(!rcv_data_contig)
{
rcv_noncontig_buff = MPL_malloc(recv_size * size);
rbuf = rcv_noncontig_buff;
if(rcv_noncontig_buff == NULL)
{
MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1,
"Fatal: Cannot allocate pack buffer");
}
if(sendbuf == MPI_IN_PLACE)
{
sbuf = PAMI_IN_PLACE;
size_t extent;
MPID_Datatype_get_extent_macro(recvtype,extent);
MPIR_Localcopy(recvbuf + (rank*recvcount*extent), recvcount, recvtype,
rcv_noncontig_buff + (rank*recv_size), recv_size,MPI_CHAR);
}
}
if(sendbuf != MPI_IN_PLACE)
{
MPIDI_Datatype_get_info(sendcount,
sendtype,
snd_data_contig,
send_size,
dt_null,
send_true_lb);
sbuf = (char *)sendbuf+send_true_lb;
if(!snd_data_contig)
{
snd_noncontig_buff = MPL_malloc(send_size);
sbuf = snd_noncontig_buff;
if(snd_noncontig_buff == NULL)
{
MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1,
"Fatal: Cannot allocate pack buffer");
}
DLOOP_Offset last = send_size;
MPID_Segment_init(sendbuf, sendcount, sendtype, &segment, 0);
MPID_Segment_pack(&segment, 0, &last, snd_noncontig_buff);
}
}
else
sbuf = PAMI_IN_PLACE;
TRACE_ERR("Using PAMI-level allgather protocol\n");
pami_xfer_t allgather;
allgather.cb_done = allgather_cb_done;
allgather.cookie = (void *)&allgather_active;
allgather.cmd.xfer_allgather.rcvbuf = rbuf;
allgather.cmd.xfer_allgather.sndbuf = sbuf;
allgather.cmd.xfer_allgather.stype = PAMI_TYPE_BYTE;/* stype is ignored when sndbuf == PAMI_IN_PLACE */
allgather.cmd.xfer_allgather.rtype = PAMI_TYPE_BYTE;
allgather.cmd.xfer_allgather.stypecount = send_size;
allgather.cmd.xfer_allgather.rtypecount = recv_size;
allgather.algorithm = mpid->coll_algorithm[PAMI_XFER_ALLGATHER][0][0];
my_md = &mpid->coll_metadata[PAMI_XFER_ALLGATHER][0][0];
TRACE_ERR("Calling PAMI_Collective with allgather structure\n");
MPIDI_Post_coll_t allgather_post;
MPIDI_Context_post(MPIDI_Context[0], &allgather_post.state, MPIDI_Pami_post_wrapper, (void *)&allgather);
TRACE_ERR("Allgather %s\n", MPIDI_Process.context_post.active>0?"posted":"invoked");
MPIDI_Update_last_algorithm(comm_ptr, my_md->name);
MPID_PROGRESS_WAIT_WHILE(allgather_active);
if(!rcv_data_contig)
{
MPIR_Localcopy(rcv_noncontig_buff, recv_size * size, MPI_CHAR,
recvbuf, recvcount, recvtype);
MPL_free(rcv_noncontig_buff);
}
if(!snd_data_contig) MPL_free(snd_noncontig_buff);
TRACE_ERR("Allgather done\n");
return MPI_SUCCESS;
}
int
MPIDO_Allgather(const void *sendbuf,
int sendcount,
MPI_Datatype sendtype,
void *recvbuf,
int recvcount,
MPI_Datatype recvtype,
MPID_Comm * comm_ptr,
int *mpierrno)
{
#ifndef HAVE_PAMI_IN_PLACE
if (sendbuf == MPI_IN_PLACE)
{
MPID_Abort (NULL, 0, 1, "'MPI_IN_PLACE' requries support for `PAMI_IN_PLACE`");
return -1;
}
#endif
/* *********************************
* Check the nature of the buffers
* *********************************
*/
const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid);
int config[6], i;
MPID_Datatype * dt_null = NULL;
MPI_Aint send_true_lb = 0;
MPI_Aint recv_true_lb = 0;
int comm_size = comm_ptr->local_size;
size_t send_bytes = 0;
size_t recv_bytes = 0;
volatile unsigned allred_active = 1;
volatile unsigned allgather_active = 1;
pami_xfer_t allred;
const int rank = comm_ptr->rank;
int queryreq = 0;
#if ASSERT_LEVEL==0
/* We can't afford the tracing in ndebug/performance libraries */
const unsigned verbose = 0;
#else
const unsigned verbose = (MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_ALL) && (rank == 0);
#endif
const int selected_type = mpid->user_selected_type[PAMI_XFER_ALLGATHER];
for (i=0;i<6;i++) config[i] = 1;
const pami_metadata_t *my_md = (pami_metadata_t *)NULL;
allred.cb_done = allred_cb_done;
allred.cookie = (void *)&allred_active;
/* Pick an algorithm that is guaranteed to work for the pre-allreduce */
/* TODO: This needs selection for fast(er|est) allreduce protocol */
allred.algorithm = mpid->coll_algorithm[PAMI_XFER_ALLREDUCE][0][0];
allred.cmd.xfer_allreduce.sndbuf = (void *)config;
allred.cmd.xfer_allreduce.stype = PAMI_TYPE_SIGNED_INT;
allred.cmd.xfer_allreduce.rcvbuf = (void *)config;
allred.cmd.xfer_allreduce.rtype = PAMI_TYPE_SIGNED_INT;
allred.cmd.xfer_allreduce.stypecount = 6;
allred.cmd.xfer_allreduce.rtypecount = 6;
allred.cmd.xfer_allreduce.op = PAMI_DATA_BAND;
char use_tree_reduce, use_alltoall, use_bcast, use_pami, use_opt;
char *rbuf = NULL, *sbuf = NULL;
const char * const allgathers = mpid->allgathers;
use_alltoall = allgathers[2];
use_tree_reduce = allgathers[0];
use_bcast = allgathers[1];
use_pami = (selected_type == MPID_COLL_USE_MPICH) ? 0 : 1;
use_opt = use_alltoall || use_tree_reduce || use_bcast || use_pami;
TRACE_ERR("flags before: b: %d a: %d t: %d p: %d\n", use_bcast, use_alltoall, use_tree_reduce, use_pami);
if(!use_opt)
{
if(unlikely(verbose))
fprintf(stderr,"Using MPICH allgather algorithm\n");
TRACE_ERR("No options set/available; using MPICH for allgather\n");
MPIDI_Update_last_algorithm(comm_ptr, "ALLGATHER_MPICH");
#if CUDA_AWARE_SUPPORT
if(MPIDI_Process.cuda_aware_support_on)
{
MPI_Aint sdt_extent,rdt_extent;
MPID_Datatype_get_extent_macro(sendtype, sdt_extent);
MPID_Datatype_get_extent_macro(recvtype, rdt_extent);
char *scbuf = NULL;
char *rcbuf = NULL;
int is_send_dev_buf = MPIDI_cuda_is_device_buf(sendbuf);
int is_recv_dev_buf = MPIDI_cuda_is_device_buf(recvbuf);
if(is_send_dev_buf)
{
scbuf = MPL_malloc(sdt_extent * sendcount);
cudaError_t cudaerr = CudaMemcpy(scbuf, sendbuf, sdt_extent * sendcount, cudaMemcpyDeviceToHost);
if (cudaSuccess != cudaerr)
fprintf(stderr, "cudaMemcpy failed: %s\n", CudaGetErrorString(cudaerr));
}
else
scbuf = sendbuf;
if(is_recv_dev_buf)
{
rcbuf = MPL_malloc(rdt_extent * recvcount);
if(sendbuf == MPI_IN_PLACE)
{
cudaError_t cudaerr = CudaMemcpy(rcbuf, recvbuf, rdt_extent * recvcount, cudaMemcpyDeviceToHost);
if (cudaSuccess != cudaerr)
fprintf(stderr, "cudaMemcpy failed: %s\n", CudaGetErrorString(cudaerr));
}
else
memset(rcbuf, 0, rdt_extent * recvcount);
}
else
rcbuf = recvbuf;
int cuda_res = MPIR_Allgather(scbuf, sendcount, sendtype, rcbuf, recvcount, recvtype, comm_ptr, mpierrno);
if(is_send_dev_buf)MPL_free(scbuf);
if(is_recv_dev_buf)
{
cudaError_t cudaerr = CudaMemcpy(recvbuf, rcbuf, rdt_extent * recvcount, cudaMemcpyHostToDevice);
if (cudaSuccess != cudaerr)
fprintf(stderr, "cudaMemcpy failed: %s\n", CudaGetErrorString(cudaerr));
MPL_free(rcbuf);
}
return cuda_res;
}
else
#endif
return MPIR_Allgather(sendbuf, sendcount, sendtype,
recvbuf, recvcount, recvtype,
comm_ptr, mpierrno);
}
if ((sendcount < 1 && sendbuf != MPI_IN_PLACE) || recvcount < 1)
return MPI_SUCCESS;
/* Gather datatype information */
MPIDI_Datatype_get_info(recvcount,
recvtype,
config[MPID_RECV_CONTIG],
recv_bytes,
dt_null,
recv_true_lb);
send_bytes = recv_bytes;
rbuf = (char *)recvbuf+recv_true_lb;
sbuf = PAMI_IN_PLACE;
if(sendbuf != MPI_IN_PLACE)
{
MPIDI_Datatype_get_info(sendcount,
sendtype,
config[MPID_SEND_CONTIG],
send_bytes,
dt_null,
send_true_lb);
sbuf = (char *)sendbuf+send_true_lb;
}
else
if(unlikely(verbose))
fprintf(stderr,"allgather MPI_IN_PLACE buffering\n");
/* verify everyone's datatype contiguity */
/* Check buffer alignment now, since we're pre-allreducing anyway */
/* Only do this if one of the glue protocols is likely to be used */
if(use_alltoall || use_tree_reduce || use_bcast)
{
config[MPID_ALIGNEDBUFFER] =
!((long)sendbuf & 0x0F) && !((long)recvbuf & 0x0F);
/* #warning need to determine best allreduce for short messages */
if(mpid->preallreduces[MPID_ALLGATHER_PREALLREDUCE])
{
TRACE_ERR("Preallreducing in allgather\n");
MPIDI_Post_coll_t allred_post;
MPIDI_Context_post(MPIDI_Context[0], &allred_post.state,
MPIDI_Pami_post_wrapper, (void *)&allred);
MPID_PROGRESS_WAIT_WHILE(allred_active);
}
use_alltoall = allgathers[2] &&
config[MPID_RECV_CONTIG] && config[MPID_SEND_CONTIG];;
/* Note: some of the glue protocols use recv_bytes*comm_size rather than
* recv_bytes so we use that for comparison here, plus we pass that in
* to those protocols. */
use_tree_reduce = allgathers[0] &&
config[MPID_RECV_CONTIG] && config[MPID_SEND_CONTIG] &&
config[MPID_RECV_CONTINUOUS] && (recv_bytes*comm_size%sizeof(unsigned)) == 0;
use_bcast = allgathers[1];
TRACE_ERR("flags after: b: %d a: %d t: %d p: %d\n", use_bcast, use_alltoall, use_tree_reduce, use_pami);
}
if(use_pami)
{
TRACE_ERR("Using PAMI-level allgather protocol\n");
pami_xfer_t allgather;
allgather.cb_done = allgather_cb_done;
allgather.cookie = (void *)&allgather_active;
allgather.cmd.xfer_allgather.rcvbuf = rbuf;
allgather.cmd.xfer_allgather.sndbuf = sbuf;
allgather.cmd.xfer_allgather.stype = PAMI_TYPE_BYTE;
allgather.cmd.xfer_allgather.rtype = PAMI_TYPE_BYTE;
allgather.cmd.xfer_allgather.stypecount = send_bytes;
allgather.cmd.xfer_allgather.rtypecount = recv_bytes;
if(selected_type == MPID_COLL_OPTIMIZED)
{
if((mpid->cutoff_size[PAMI_XFER_ALLGATHER][0] == 0) ||
(mpid->cutoff_size[PAMI_XFER_ALLGATHER][0] > 0 && mpid->cutoff_size[PAMI_XFER_ALLGATHER][0] >= send_bytes))
{
allgather.algorithm = mpid->opt_protocol[PAMI_XFER_ALLGATHER][0];
my_md = &mpid->opt_protocol_md[PAMI_XFER_ALLGATHER][0];
queryreq = mpid->must_query[PAMI_XFER_ALLGATHER][0];
}
else
{
return MPIR_Allgather(sendbuf, sendcount, sendtype,
recvbuf, recvcount, recvtype,
comm_ptr, mpierrno);
}
}
else
{
allgather.algorithm = mpid->user_selected[PAMI_XFER_ALLGATHER];
my_md = &mpid->user_metadata[PAMI_XFER_ALLGATHER];
queryreq = selected_type;
}
if(unlikely( queryreq == MPID_COLL_ALWAYS_QUERY ||
queryreq == MPID_COLL_CHECK_FN_REQUIRED))
{
metadata_result_t result = {0};
TRACE_ERR("Querying allgather protocol %s, type was: %d\n",
my_md->name,
selected_type);
if(my_md->check_fn == NULL)
{
/* process metadata bits */
if((!my_md->check_correct.values.inplace) && (sendbuf == MPI_IN_PLACE))
result.check.unspecified = 1;
if(my_md->check_correct.values.rangeminmax)
{
if((my_md->range_lo <= recv_bytes) &&
(my_md->range_hi >= recv_bytes))
; /* ok, algorithm selected */
else
{
result.check.range = 1;
if(unlikely(verbose))
{
fprintf(stderr,"message size (%zu) outside range (%zu<->%zu) for %s.\n",
recv_bytes,
my_md->range_lo,
my_md->range_hi,
my_md->name);
}
}
}
}
else /* calling the check fn is sufficient */
result = my_md->check_fn(&allgather);
TRACE_ERR("bitmask: %#X\n", result.bitmask);
result.check.nonlocal = 0; /* #warning REMOVE THIS WHEN IMPLEMENTED */
if(result.bitmask)
{
if(unlikely(verbose))
fprintf(stderr,"Query failed for %s. Using MPICH allgather\n",
my_md->name);
MPIDI_Update_last_algorithm(comm_ptr, "ALLGATHER_MPICH");
return MPIR_Allgather(sendbuf, sendcount, sendtype,
recvbuf, recvcount, recvtype,
comm_ptr, mpierrno);
}
if(my_md->check_correct.values.asyncflowctl && !(--(comm_ptr->mpid.num_requests)))
{
comm_ptr->mpid.num_requests = MPIDI_Process.optimized.num_requests;
int tmpmpierrno;
if(unlikely(verbose))
fprintf(stderr,"Query barrier required for %s\n", my_md->name);
MPIDO_Barrier(comm_ptr, &tmpmpierrno);
}
}
if(unlikely(verbose))
{
unsigned long long int threadID;
MPL_thread_id_t tid;
MPL_thread_self(&tid);
threadID = (unsigned long long int)tid;
fprintf(stderr,"<%llx> Using protocol %s for allgather on %u\n",
threadID,
my_md->name,
(unsigned) comm_ptr->context_id);
}
TRACE_ERR("Calling PAMI_Collective with allgather structure\n");
MPIDI_Post_coll_t allgather_post;
MPIDI_Context_post(MPIDI_Context[0], &allgather_post.state, MPIDI_Pami_post_wrapper, (void *)&allgather);
MPIDI_Update_last_algorithm(comm_ptr, my_md->name);
MPID_PROGRESS_WAIT_WHILE(allgather_active);
TRACE_ERR("Allgather done\n");
return PAMI_SUCCESS;
}
if(use_tree_reduce)
{
if(unlikely(verbose))
fprintf(stderr,"Using protocol GLUE_ALLREDUCE for allgather\n");
TRACE_ERR("Using allgather via allreduce\n");
MPIDI_Update_last_algorithm(comm_ptr, "ALLGATHER_OPT_ALLREDUCE");
return MPIDO_Allgather_allreduce(sendbuf, sendcount, sendtype,
recvbuf, recvcount, recvtype,
send_true_lb, recv_true_lb, send_bytes, recv_bytes*comm_size, comm_ptr, mpierrno);
}
if(use_alltoall)
{
if(unlikely(verbose))
fprintf(stderr,"Using protocol GLUE_BCAST for allgather\n");
TRACE_ERR("Using allgather via alltoall\n");
MPIDI_Update_last_algorithm(comm_ptr, "ALLGATHER_OPT_ALLTOALL");
return MPIDO_Allgather_alltoall(sendbuf, sendcount, sendtype,
recvbuf, recvcount, recvtype,
send_true_lb, recv_true_lb, send_bytes, recv_bytes*comm_size, comm_ptr, mpierrno);
}
if(use_bcast)
{
if(unlikely(verbose))
fprintf(stderr,"Using protocol GLUE_ALLTOALL for allgather\n");
TRACE_ERR("Using allgather via bcast\n");
MPIDI_Update_last_algorithm(comm_ptr, "ALLGATHER_OPT_BCAST");
return MPIDO_Allgather_bcast(sendbuf, sendcount, sendtype,
recvbuf, recvcount, recvtype,
send_true_lb, recv_true_lb, send_bytes, recv_bytes*comm_size, comm_ptr, mpierrno);
}
/* Nothing used yet; dump to MPICH */
if(unlikely(verbose))
fprintf(stderr,"Using MPICH allgather algorithm\n");
TRACE_ERR("Using allgather via mpich\n");
MPIDI_Update_last_algorithm(comm_ptr, "ALLGATHER_MPICH");
return MPIR_Allgather(sendbuf, sendcount, sendtype,
recvbuf, recvcount, recvtype,
comm_ptr, mpierrno);
}
int MPID_nem_ib_lmt_switch_send(struct MPIDI_VC *vc, struct MPID_Request *req)
{
int mpi_errno = MPI_SUCCESS;
int dt_contig;
MPIDI_msg_sz_t data_sz;
MPID_Datatype *dt_ptr;
MPI_Aint dt_true_lb;
MPID_IOV r_cookie = req->ch.lmt_tmp_cookie;
MPID_nem_ib_lmt_cookie_t *r_cookie_buf = r_cookie.iov_base;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_LMT_SWITCH_SEND);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_LMT_SWITCH_SEND);
MPIDI_Datatype_get_info(req->dev.user_count, req->dev.datatype, dt_contig, data_sz, dt_ptr,
dt_true_lb);
void *write_from_buf;
if (dt_contig) {
write_from_buf = req->dev.user_buf;
}
else {
/* see MPIDI_CH3_EagerNoncontigSend (in ch3u_eager.c) */
req->dev.segment_ptr = MPID_Segment_alloc();
MPIU_ERR_CHKANDJUMP((req->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER,
"**outofmemory");
MPID_Segment_init(req->dev.user_buf, req->dev.user_count, req->dev.datatype,
req->dev.segment_ptr, 0);
req->dev.segment_first = 0;
req->dev.segment_size = data_sz;
MPIDI_msg_sz_t last;
last = req->dev.segment_size; /* segment_size is byte offset */
MPIU_Assert(last > 0);
REQ_FIELD(req, lmt_pack_buf) = MPIU_Malloc(data_sz);
MPIU_ERR_CHKANDJUMP(!REQ_FIELD(req, lmt_pack_buf), mpi_errno, MPI_ERR_OTHER,
"**outofmemory");
MPID_Segment_pack(req->dev.segment_ptr, req->dev.segment_first, &last,
(char *) (REQ_FIELD(req, lmt_pack_buf)));
MPIU_Assert(last == req->dev.segment_size);
write_from_buf = REQ_FIELD(req, lmt_pack_buf);
}
//assert(dt_true_lb == 0);
uint8_t *tailp =
(uint8_t *) ((uint8_t *) write_from_buf /*+ dt_true_lb */ + data_sz - sizeof(uint8_t));
#if 0
*is_end_flag_same = (r_cookie_buf->tail == *tailp) ? 1 : 0;
#else
REQ_FIELD(req, lmt_receiver_tail) = r_cookie_buf->tail;
REQ_FIELD(req, lmt_sender_tail) = *tailp;
dprintf("lmt_switch_send,tail on sender=%02x,tail onreceiver=%02x,req=%p\n", *tailp,
r_cookie_buf->tail, req);
#ifdef MPID_NEM_IB_DEBUG_LMT
uint8_t *tail_wordp = (uint8_t *) ((uint8_t *) write_from_buf + data_sz - sizeof(uint32_t) * 2);
#endif
dprintf("lmt_switch_send,tail on sender=%d\n", *tail_wordp);
fflush(stdout);
#endif
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_LMT_SWITCH_SEND);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDO_Gatherv(const void *sendbuf,
int sendcount,
MPI_Datatype sendtype,
void *recvbuf,
const int *recvcounts,
const int *displs,
MPI_Datatype recvtype,
int root,
MPID_Comm * comm_ptr,
int *mpierrno)
{
#ifndef HAVE_PAMI_IN_PLACE
if (sendbuf == MPI_IN_PLACE)
{
MPID_Abort (NULL, 0, 1, "'MPI_IN_PLACE' requries support for `PAMI_IN_PLACE`");
return -1;
}
#endif
TRACE_ERR("Entering MPIDO_Gatherv\n");
int i;
int contig ATTRIBUTE((unused)), rsize ATTRIBUTE((unused)), ssize ATTRIBUTE((unused));
int pamidt = 1;
MPID_Datatype *dt_ptr = NULL;
MPI_Aint send_true_lb, recv_true_lb;
char *sbuf, *rbuf;
pami_type_t stype, rtype;
int tmp;
volatile unsigned gatherv_active = 1;
const int rank = comm_ptr->rank;
const int size = comm_ptr->local_size;
#if ASSERT_LEVEL==0
/* We can't afford the tracing in ndebug/performance libraries */
const unsigned verbose = 0;
#else
const unsigned verbose = (MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_ALL) && (rank == 0);
#endif
const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid);
const int selected_type = mpid->user_selected_type[PAMI_XFER_GATHERV_INT];
/* Check for native PAMI types and MPI_IN_PLACE on sendbuf */
/* MPI_IN_PLACE is a nonlocal decision. We will need a preallreduce if we ever have
* multiple "good" gathervs that work on different counts for example */
if((sendbuf != MPI_IN_PLACE) && (MPIDI_Datatype_to_pami(sendtype, &stype, -1, NULL, &tmp) != MPI_SUCCESS))
pamidt = 0;
if(MPIDI_Datatype_to_pami(recvtype, &rtype, -1, NULL, &tmp) != MPI_SUCCESS)
pamidt = 0;
if(pamidt == 0 || selected_type == MPID_COLL_USE_MPICH)
{
if(unlikely(verbose))
fprintf(stderr,"Using MPICH gatherv algorithm\n");
TRACE_ERR("GATHERV using MPICH\n");
MPIDI_Update_last_algorithm(comm_ptr, "GATHERV_MPICH");
#if CUDA_AWARE_SUPPORT
if(MPIDI_Process.cuda_aware_support_on)
{
MPI_Aint sdt_extent,rdt_extent;
MPID_Datatype_get_extent_macro(sendtype, sdt_extent);
MPID_Datatype_get_extent_macro(recvtype, rdt_extent);
char *scbuf = NULL;
char *rcbuf = NULL;
int is_send_dev_buf = MPIDI_cuda_is_device_buf(sendbuf);
int is_recv_dev_buf = (rank == root) ? MPIDI_cuda_is_device_buf(recvbuf) : 0;
if(is_send_dev_buf)
{
scbuf = MPL_malloc(sdt_extent * sendcount);
cudaError_t cudaerr = CudaMemcpy(scbuf, sendbuf, sdt_extent * sendcount, cudaMemcpyDeviceToHost);
if (cudaSuccess != cudaerr)
fprintf(stderr, "cudaMemcpy failed: %s\n", CudaGetErrorString(cudaerr));
}
else
scbuf = sendbuf;
size_t rtotal_buf;
if(is_recv_dev_buf)
{
//Since displs can be non-continous, we need to calculate max buffer size
int highest_displs = displs[size - 1];
int highest_recvcount = recvcounts[size - 1];
for(i = 0; i < size; i++)
{
if(displs[i]+recvcounts[i] > highest_displs+highest_recvcount)
{
highest_displs = displs[i];
highest_recvcount = recvcounts[i];
}
}
rtotal_buf = (highest_displs+highest_recvcount)*rdt_extent;
rcbuf = MPL_malloc(rtotal_buf);
if(sendbuf == MPI_IN_PLACE)
{
cudaError_t cudaerr = CudaMemcpy(rcbuf, recvbuf, rtotal_buf, cudaMemcpyDeviceToHost);
if (cudaSuccess != cudaerr)
fprintf(stderr, "cudaMemcpy failed: %s\n", CudaGetErrorString(cudaerr));
}
else
memset(rcbuf, 0, rtotal_buf);
}
else
rcbuf = recvbuf;
int cuda_res = MPIR_Gatherv(scbuf, sendcount, sendtype, rcbuf, recvcounts, displs, recvtype, root, comm_ptr, mpierrno);
if(is_send_dev_buf)MPL_free(scbuf);
if(is_recv_dev_buf)
{
cudaError_t cudaerr = CudaMemcpy(recvbuf, rcbuf, rtotal_buf, cudaMemcpyHostToDevice);
if (cudaSuccess != cudaerr)
fprintf(stderr, "cudaMemcpy failed: %s\n", CudaGetErrorString(cudaerr));
MPL_free(rcbuf);
}
return cuda_res;
}
else
#endif
return MPIR_Gatherv(sendbuf, sendcount, sendtype,
recvbuf, recvcounts, displs, recvtype,
root, comm_ptr, mpierrno);
}
MPIDI_Datatype_get_info(1, recvtype, contig, rsize, dt_ptr, recv_true_lb);
rbuf = (char *)recvbuf + recv_true_lb;
sbuf = (void *) sendbuf;
pami_xfer_t gatherv;
gatherv.cb_done = cb_gatherv;
gatherv.cookie = (void *)&gatherv_active;
gatherv.cmd.xfer_gatherv_int.root = MPIDI_Task_to_endpoint(MPID_VCR_GET_LPID(comm_ptr->vcr, root), 0);
gatherv.cmd.xfer_gatherv_int.rcvbuf = rbuf;
gatherv.cmd.xfer_gatherv_int.rtype = rtype;
gatherv.cmd.xfer_gatherv_int.rtypecounts = (int *) recvcounts;
gatherv.cmd.xfer_gatherv_int.rdispls = (int *) displs;
gatherv.cmd.xfer_gatherv_int.sndbuf = NULL;
gatherv.cmd.xfer_gatherv_int.stype = stype;
gatherv.cmd.xfer_gatherv_int.stypecount = sendcount;
if(rank == root)
{
if(sendbuf == MPI_IN_PLACE)
{
if(unlikely(verbose))
fprintf(stderr,"gatherv MPI_IN_PLACE buffering\n");
sbuf = PAMI_IN_PLACE;
gatherv.cmd.xfer_gatherv_int.stype = rtype;
gatherv.cmd.xfer_gatherv_int.stypecount = recvcounts[rank];
}
else
{
MPIDI_Datatype_get_info(1, sendtype, contig, ssize, dt_ptr, send_true_lb);
sbuf = (char *)sbuf + send_true_lb;
}
}
gatherv.cmd.xfer_gatherv_int.sndbuf = sbuf;
pami_algorithm_t my_gatherv;
const pami_metadata_t *my_md = (pami_metadata_t *)NULL;
int queryreq = 0;
if(selected_type == MPID_COLL_OPTIMIZED)
{
TRACE_ERR("Optimized gatherv %s was selected\n",
mpid->opt_protocol_md[PAMI_XFER_GATHERV_INT][0].name);
my_gatherv = mpid->opt_protocol[PAMI_XFER_GATHERV_INT][0];
my_md = &mpid->opt_protocol_md[PAMI_XFER_GATHERV_INT][0];
queryreq = mpid->must_query[PAMI_XFER_GATHERV_INT][0];
}
else
{
TRACE_ERR("Optimized gatherv %s was set by user\n",
mpid->user_metadata[PAMI_XFER_GATHERV_INT].name);
my_gatherv = mpid->user_selected[PAMI_XFER_GATHERV_INT];
my_md = &mpid->user_metadata[PAMI_XFER_GATHERV_INT];
queryreq = selected_type;
}
gatherv.algorithm = my_gatherv;
if(unlikely(queryreq == MPID_COLL_ALWAYS_QUERY ||
queryreq == MPID_COLL_CHECK_FN_REQUIRED))
{
metadata_result_t result = {0};
TRACE_ERR("querying gatherv protocol %s, type was %d\n",
my_md->name, queryreq);
if(my_md->check_fn == NULL)
{
/* process metadata bits */
if((!my_md->check_correct.values.inplace) && (sendbuf == MPI_IN_PLACE))
result.check.unspecified = 1;
/* Can't check ranges like this. Non-local. Comment out for now.
if(my_md->check_correct.values.rangeminmax)
{
MPI_Aint data_true_lb;
MPID_Datatype *data_ptr;
int data_size, data_contig;
MPIDI_Datatype_get_info(sendcount, sendtype, data_contig, data_size, data_ptr, data_true_lb);
if((my_md->range_lo <= data_size) &&
(my_md->range_hi >= data_size))
;
else
{
result.check.range = 1;
if(unlikely(verbose))
{
fprintf(stderr,"message size (%u) outside range (%zu<->%zu) for %s.\n",
data_size,
my_md->range_lo,
my_md->range_hi,
my_md->name);
}
}
}
*/
}
else /* calling the check fn is sufficient */
result = my_md->check_fn(&gatherv);
TRACE_ERR("bitmask: %#X\n", result.bitmask);
result.check.nonlocal = 0; /* #warning REMOVE THIS WHEN IMPLEMENTED */
if(result.bitmask)
{
if(unlikely(verbose))
fprintf(stderr,"Query failed for %s. Using MPICH gatherv.\n", my_md->name);
MPIDI_Update_last_algorithm(comm_ptr, "GATHERV_MPICH");
return MPIR_Gatherv(sendbuf, sendcount, sendtype,
recvbuf, recvcounts, displs, recvtype,
root, comm_ptr, mpierrno);
}
if(my_md->check_correct.values.asyncflowctl && !(--(comm_ptr->mpid.num_requests)))
{
comm_ptr->mpid.num_requests = MPIDI_Process.optimized.num_requests;
int tmpmpierrno;
if(unlikely(verbose))
fprintf(stderr,"Query barrier required for %s\n", my_md->name);
MPIDO_Barrier(comm_ptr, &tmpmpierrno);
}
}
MPIDI_Update_last_algorithm(comm_ptr, my_md->name);
if(unlikely(verbose))
{
unsigned long long int threadID;
MPL_thread_id_t tid;
MPL_thread_self(&tid);
threadID = (unsigned long long int)tid;
fprintf(stderr,"<%llx> Using protocol %s for gatherv on %u\n",
threadID,
my_md->name,
(unsigned) comm_ptr->context_id);
}
MPIDI_Post_coll_t gatherv_post;
MPIDI_Context_post(MPIDI_Context[0], &gatherv_post.state,
MPIDI_Pami_post_wrapper, (void *)&gatherv);
TRACE_ERR("Waiting on active %d\n", gatherv_active);
MPID_PROGRESS_WAIT_WHILE(gatherv_active);
TRACE_ERR("Leaving MPIDO_Gatherv\n");
return 0;
}
int MPIDO_Scatterv_simple(const void *sendbuf,
const int *sendcounts,
const int *displs,
MPI_Datatype sendtype,
void *recvbuf,
int recvcount,
MPI_Datatype recvtype,
int root,
MPID_Comm *comm_ptr,
int *mpierrno)
{
#ifndef HAVE_PAMI_IN_PLACE
if (sendbuf == MPI_IN_PLACE)
{
MPID_Abort (NULL, 0, 1, "'MPI_IN_PLACE' requries support for `PAMI_IN_PLACE`");
return -1;
}
#endif
int snd_contig = 1;
int rcv_contig = 1;
int send_size = 0, recv_size = 0;
int ssize = 0;
MPID_Datatype *dt_ptr = NULL;
MPI_Aint send_true_lb=0, recv_true_lb=0;
void *snd_noncontig_buff = NULL, *rcv_noncontig_buff = NULL;
void *sbuf = NULL, *rbuf = NULL;
int *sdispls = NULL, *scounts = NULL;
int sndcount = 0;
MPID_Segment segment;
int tmp, i;
pami_type_t stype = PAMI_TYPE_NULL;
const int rank = comm_ptr->rank;
const int size = comm_ptr->local_size;
const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid);
if (rank == root && sendtype != MPI_DATATYPE_NULL && sendcounts[0] >= 0)
{
MPIDI_Datatype_get_info(1, sendtype, snd_contig, ssize, dt_ptr, send_true_lb);
if(MPIDI_Pamix_collsel_advise != NULL && mpid->collsel_fast_query != NULL)
{
advisor_algorithm_t advisor_algorithms[1];
int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_SCATTERV_INT, 64, advisor_algorithms, 1);
if(num_algorithms)
{
if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO)
{
return MPIR_Scatterv(sendbuf, sendcounts, displs, sendtype,
recvbuf, recvcount, recvtype,
root, comm_ptr, mpierrno);
}
else if(advisor_algorithms[0].metadata && advisor_algorithms[0].metadata->check_correct.values.asyncflowctl && !(--(comm_ptr->mpid.num_requests)))
{
comm_ptr->mpid.num_requests = MPIDI_Process.optimized.num_requests;
int tmpmpierrno;
MPIDO_Barrier(comm_ptr, &tmpmpierrno);
}
}
}
}
if (recvtype != MPI_DATATYPE_NULL && recvcount >= 0)
{
MPIDI_Datatype_get_info(recvcount, recvtype, rcv_contig,
recv_size, dt_ptr, recv_true_lb);
if(MPIDI_Pamix_collsel_advise != NULL && mpid->collsel_fast_query != NULL)
{
advisor_algorithm_t advisor_algorithms[1];
int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_SCATTERV_INT, 64, advisor_algorithms, 1);
if(num_algorithms)
{
if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO)
{
return MPIR_Scatterv(sendbuf, sendcounts, displs, sendtype,
recvbuf, recvcount, recvtype,
root, comm_ptr, mpierrno);
}
else if(advisor_algorithms[0].metadata && advisor_algorithms[0].metadata->check_correct.values.asyncflowctl && !(--(comm_ptr->mpid.num_requests)))
{
comm_ptr->mpid.num_requests = MPIDI_Process.optimized.num_requests;
int tmpmpierrno;
MPIDO_Barrier(comm_ptr, &tmpmpierrno);
}
}
}
}
pami_xfer_t scatterv;
const pami_metadata_t *my_scatterv_md;
volatile unsigned scatterv_active = 1;
sbuf = (char *)sendbuf + send_true_lb;
rbuf = (char *)recvbuf + recv_true_lb;
scounts = (int*)sendcounts;
sdispls = (int*)displs;
if(rank == root)
{
if(MPIDI_Datatype_to_pami(sendtype, &stype, -1, NULL, &tmp) != MPI_SUCCESS)
{
if (!snd_contig)
{
scounts = (int*)MPIU_Malloc(size);
sdispls = (int*)MPIU_Malloc(size);
for(i = 0; i < size; i++)
{
scounts[i] = ssize * sendcounts[i];
sdispls[i] = ssize * displs[i];
send_size += scounts[i];
sndcount += sendcounts[i];
}
snd_noncontig_buff = MPIU_Malloc(send_size);
sbuf = snd_noncontig_buff;
stype = PAMI_TYPE_BYTE;
if(snd_noncontig_buff == NULL)
{
MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1,
"Fatal: Cannot allocate pack buffer");
}
DLOOP_Offset last = send_size;
MPID_Segment_init(sendbuf, sndcount, sendtype, &segment, 0);
MPID_Segment_pack(&segment, 0, &last, snd_noncontig_buff);
}
}
if(recvbuf == MPI_IN_PLACE)
{
rbuf = PAMI_IN_PLACE;
}
}
if(recvbuf != MPI_IN_PLACE)
{
if (!rcv_contig)
{
rcv_noncontig_buff = MPIU_Malloc(recv_size);
rbuf = rcv_noncontig_buff;
if(rcv_noncontig_buff == NULL)
{
MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1,
"Fatal: Cannot allocate pack buffer");
}
}
}
scatterv.cb_done = cb_scatterv;
scatterv.cookie = (void *)&scatterv_active;
scatterv.cmd.xfer_scatterv_int.root = MPIDI_Task_to_endpoint(MPID_VCR_GET_LPID(comm_ptr->vcr, root), 0);
scatterv.algorithm = mpid->coll_algorithm[PAMI_XFER_SCATTERV_INT][0][0];
my_scatterv_md = &mpid->coll_metadata[PAMI_XFER_SCATTERV_INT][0][0];
scatterv.cmd.xfer_scatterv_int.rcvbuf = rbuf;
scatterv.cmd.xfer_scatterv_int.sndbuf = sbuf;
scatterv.cmd.xfer_scatterv_int.stype = stype;
scatterv.cmd.xfer_scatterv_int.rtype = PAMI_TYPE_BYTE;/* rtype is ignored when rcvbuf == PAMI_IN_PLACE */
scatterv.cmd.xfer_scatterv_int.stypecounts = (int *) scounts;
scatterv.cmd.xfer_scatterv_int.rtypecount = recv_size;
scatterv.cmd.xfer_scatterv_int.sdispls = (int *) sdispls;
MPIDI_Update_last_algorithm(comm_ptr, my_scatterv_md->name);
MPIDI_Post_coll_t scatterv_post;
TRACE_ERR("%s scatterv\n", MPIDI_Process.context_post.active>0?"Posting":"Invoking");
MPIDI_Context_post(MPIDI_Context[0], &scatterv_post.state,
MPIDI_Pami_post_wrapper, (void *)&scatterv);
TRACE_ERR("Waiting on active %d\n", scatterv_active);
MPID_PROGRESS_WAIT_WHILE(scatterv_active);
if(!rcv_contig)
{
MPIR_Localcopy(rcv_noncontig_buff, recv_size, MPI_CHAR,
recvbuf, recvcount, recvtype);
MPIU_Free(rcv_noncontig_buff);
}
if(!snd_contig)
{
MPIU_Free(snd_noncontig_buff);
MPIU_Free(scounts);
MPIU_Free(sdispls);
}
TRACE_ERR("Leaving MPIDO_Scatterv_optimized\n");
return MPI_SUCCESS;
}
int MPID_Isend(const void * buf, int count, MPI_Datatype datatype, int rank,
int tag, MPID_Comm * comm, int context_offset,
MPID_Request ** request)
{
MPIDI_msg_sz_t data_sz;
int dt_contig;
MPI_Aint dt_true_lb;
MPID_Datatype * dt_ptr;
MPID_Request * sreq;
MPIDI_VC_t * vc=0;
#if defined(MPID_USE_SEQUENCE_NUMBERS)
MPID_Seqnum_t seqnum;
#endif
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_ISEND);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_ISEND);
MPIU_DBG_MSG_FMT(CH3_OTHER,VERBOSE,(MPIU_DBG_FDEST,
"rank=%d, tag=%d, context=%d",
rank, tag, comm->context_id + context_offset));
if (rank == comm->rank && comm->comm_kind != MPID_INTERCOMM)
{
#if defined (_OSU_PSM_)
goto skip_self_send; /* psm will internally do self-send, no special
handling is needed here */
#endif /* _OSU_PSM_ */
mpi_errno = MPIDI_Isend_self(buf, count, datatype, rank, tag, comm,
context_offset, MPIDI_REQUEST_TYPE_SEND, &sreq);
goto fn_exit;
}
#if defined (_OSU_PSM_)
skip_self_send:
#endif
if (rank != MPI_PROC_NULL) {
MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
#ifdef ENABLE_COMM_OVERRIDES
/* this needs to come before the sreq is created, since the override
* function is responsible for creating its own request */
if (vc->comm_ops && vc->comm_ops->isend)
{
mpi_errno = vc->comm_ops->isend( vc, buf, count, datatype, rank, tag, comm, context_offset, &sreq);
goto fn_exit;
}
#endif
}
MPIDI_Request_create_sreq(sreq, mpi_errno, goto fn_exit);
MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_SEND);
if (rank == MPI_PROC_NULL)
{
MPIU_Object_set_ref(sreq, 1);
MPID_cc_set(&sreq->cc, 0);
goto fn_exit;
}
MPIDI_Datatype_get_info(count, datatype, dt_contig, data_sz, dt_ptr,
dt_true_lb);
if (data_sz == 0)
{
#if defined (_OSU_PSM_)
goto eager_send;
#endif /* _OSU_PSM_ */
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_eager_send_t * const eager_pkt = &upkt.eager_send;
MPIDI_Request_set_msg_type(sreq, MPIDI_REQUEST_EAGER_MSG);
sreq->dev.OnDataAvail = 0;
MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"sending zero length message");
MPIDI_Pkt_init(eager_pkt, MPIDI_CH3_PKT_EAGER_SEND);
eager_pkt->match.parts.rank = comm->rank;
eager_pkt->match.parts.tag = tag;
eager_pkt->match.parts.context_id = comm->context_id + context_offset;
eager_pkt->sender_req_id = sreq->handle;
eager_pkt->data_sz = 0;
MPIDI_VC_FAI_send_seqnum(vc, seqnum);
MPIDI_Pkt_set_seqnum(eager_pkt, seqnum);
MPIDI_Request_set_seqnum(sreq, seqnum);
MPIU_THREAD_CS_ENTER(CH3COMM,vc);
mpi_errno = MPIU_CALL(MPIDI_CH3,iSend(vc, sreq, eager_pkt,
sizeof(*eager_pkt)));
MPIU_THREAD_CS_EXIT(CH3COMM,vc);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
MPIU_Object_set_ref(sreq, 0);
MPIDI_CH3_Request_destroy(sreq);
sreq = NULL;
MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**ch3|eagermsg");
goto fn_exit;
}
/* --END ERROR HANDLING-- */
goto fn_exit;
}
#if defined (_OSU_PSM_)
if(HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN) {
sreq->dev.datatype_ptr = dt_ptr;
MPID_Datatype_add_ref(dt_ptr);
sreq->psm_flags |= PSM_NEED_DTYPE_RELEASE;
}
if(vc->force_eager)
goto eager_send;
#endif /* _OSU_PSM_ */
#if defined(_OSU_MVAPICH_)
int i;
for (i = 0 ; i < rdma_num_extra_polls; i++)
{
if (rdma_global_ext_sendq_size > 1)
MPID_Progress_test();
}
#endif
/* FIXME: flow control: limit number of outstanding eager messsages
containing data and need to be buffered by the receiver */
#if defined(_OSU_MVAPICH_)
if (data_sz + sizeof(MPIDI_CH3_Pkt_eager_send_t) <= vc->eager_max_msg_sz
&& !vc->force_rndv)
#else /* defined(_OSU_MVAPICH_) */
if (data_sz + sizeof(MPIDI_CH3_Pkt_eager_send_t) <= vc->eager_max_msg_sz)
#endif /* defined(_OSU_MVAPICH_) */
{
#if defined (_OSU_PSM_)
eager_send:
#endif /* _OSU_PSM */
if (dt_contig)
{
mpi_errno = MPIDI_CH3_EagerContigIsend( &sreq,
MPIDI_CH3_PKT_EAGER_SEND,
(char*)buf + dt_true_lb,
data_sz, rank, tag,
comm, context_offset );
}
else
{
#if defined (_OSU_PSM_)
sreq->psm_flags |= PSM_NON_BLOCKING_SEND;
#endif
mpi_errno = MPIDI_CH3_EagerNoncontigSend( &sreq,
MPIDI_CH3_PKT_EAGER_SEND,
buf, count, datatype,
data_sz, rank, tag,
comm, context_offset );
#if defined (_OSU_PSM_)
goto fn_exit;
#endif
/* If we're not complete, then add a reference to the datatype */
if (sreq && sreq->dev.OnDataAvail) {
sreq->dev.datatype_ptr = dt_ptr;
MPID_Datatype_add_ref(dt_ptr);
}
}
}
else
{
/* Note that the sreq was created above */
MPIDI_Request_set_msg_type( sreq, MPIDI_REQUEST_RNDV_MSG );
mpi_errno = vc->rndvSend_fn( &sreq, buf, count, datatype, dt_contig,
data_sz, dt_true_lb, rank, tag, comm,
context_offset );
/* FIXME: fill temporary IOV or pack temporary buffer after send to
hide some latency. This requires synchronization
because the CTS packet could arrive and be processed before the
above iStartmsg completes (depending on the progress
engine, threads, etc.). */
#if defined(_OSU_MVAPICH_)
/* rndv transfers need to process CTS packet to initiate the actual RDMA transfer */
MPID_Progress_test();
#endif /* defined(_OSU_MVAPICH_) */
if (sreq && dt_ptr != NULL)
{
sreq->dev.datatype_ptr = dt_ptr;
MPID_Datatype_add_ref(dt_ptr);
}
}
fn_exit:
*request = sreq;
#if defined(_OSU_MVAPICH_)
for (i = 0 ; i < rdma_num_extra_polls; i++)
{
if (rdma_global_ext_sendq_size > 1)
MPID_Progress_test();
}
#endif
MPIU_DBG_STMT(CH3_OTHER,VERBOSE,
{
if (sreq != NULL)
{
MPIU_DBG_MSG_P(CH3_OTHER,VERBOSE,"request allocated, handle=0x%08x", sreq->handle);
}
}
);
int MPID_nem_mxm_recv(MPIDI_VC_t * vc, MPID_Request * rreq)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_msg_sz_t data_sz;
int dt_contig;
MPI_Aint dt_true_lb;
MPID_Datatype *dt_ptr;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_RECV);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_RECV);
MPIU_Assert(rreq);
MPIU_Assert(((rreq->dev.match.parts.rank == MPI_ANY_SOURCE) && (vc == NULL)) ||
(vc && !vc->ch.is_local));
MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype, dt_contig, data_sz,
dt_ptr, dt_true_lb);
{
MPIU_Context_id_t context_id = rreq->dev.match.parts.context_id;
int tag = rreq->dev.match.parts.tag;
MPID_nem_mxm_vc_area *vc_area = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
mxm_mq_h *mq_h_v = (mxm_mq_h *) rreq->comm->dev.ch.netmod_priv;
rreq->dev.OnDataAvail = NULL;
rreq->dev.tmpbuf = NULL;
rreq->ch.vc = vc;
rreq->ch.noncontig = FALSE;
_dbg_mxm_output(5,
"Recv ========> Getting USER msg for req %p (context %d from %d tag %d size %d) \n",
rreq, context_id, rreq->dev.match.parts.rank, tag, data_sz);
vc_area = VC_BASE(vc);
req_area = REQ_BASE(rreq);
req_area->ctx = rreq;
req_area->iov_buf = req_area->tmp_buf;
req_area->iov_count = 0;
req_area->iov_buf[0].ptr = NULL;
req_area->iov_buf[0].length = 0;
if (dt_contig) {
req_area->iov_count = 1;
req_area->iov_buf[0].ptr = (char *) (rreq->dev.user_buf) + dt_true_lb;
req_area->iov_buf[0].length = data_sz;
}
else {
rreq->ch.noncontig = TRUE;
mpi_errno = _mxm_process_rdtype(&rreq, rreq->dev.datatype, dt_ptr, data_sz,
rreq->dev.user_buf, rreq->dev.user_count,
&req_area->iov_buf, &req_area->iov_count);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
}
mpi_errno = _mxm_irecv((vc_area ? vc_area->mxm_ep : NULL), req_area,
tag,
(rreq->comm ? mq_h_v[0] : mxm_obj->mxm_mq), _mxm_tag_mpi2mxm(tag,
context_id));
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
}
if (vc)
_dbg_mxm_out_req(rreq);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_RECV);
return mpi_errno;
fn_fail:ATTRIBUTE((unused))
goto fn_exit;
}
int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count,
MPI_Datatype origin_datatype, void *result_addr, int result_count,
MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp,
int target_count, MPI_Datatype target_datatype, MPI_Op op,
MPID_Win * win_ptr, MPID_Request * ureq)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_msg_sz_t orig_data_sz, target_data_sz;
int rank;
int dt_contig ATTRIBUTE((unused));
MPI_Aint dt_true_lb ATTRIBUTE((unused));
MPID_Datatype *dtp;
MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
int made_progress = 0;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_GET_ACCUMULATE);
MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_GET_ACCUMULATE);
MPIR_ERR_CHKANDJUMP(win_ptr->states.access_state == MPIDI_RMA_NONE,
mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");
if (target_rank == MPI_PROC_NULL) {
goto fn_exit;
}
MPIDI_Datatype_get_info(target_count, target_datatype, dt_contig, target_data_sz, dtp,
dt_true_lb);
if (target_data_sz == 0) {
goto fn_exit;
}
rank = win_ptr->comm_ptr->rank;
if (win_ptr->shm_allocated == TRUE && target_rank != rank &&
win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) {
/* check if target is local and shared memory is allocated on window,
* if so, we directly perform this operation on shared memory region. */
/* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on
* the same node. However, in ch3:sock, even if origin and target are on the same node, they do
* not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first,
* which is only set to TRUE when SHM region is allocated in nemesis.
* In future we need to figure out a way to check if origin and target are in the same "SHM comm".
*/
MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc);
MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);
}
/* Do =! rank first (most likely branch?) */
if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED ||
(win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) {
mpi_errno = MPIDI_CH3I_Shm_get_acc_op(origin_addr, origin_count, origin_datatype,
result_addr, result_count, result_datatype,
target_rank, target_disp, target_count,
target_datatype, op, win_ptr);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
if (ureq) {
/* Complete user request and release the ch3 ref */
mpi_errno = MPID_Request_complete(ureq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
}
}
else {
MPIDI_RMA_Op_t *op_ptr = NULL;
MPIDI_CH3_Pkt_get_accum_t *get_accum_pkt;
MPI_Aint origin_type_size;
MPI_Aint target_type_size;
int use_immed_pkt = FALSE, i;
int is_origin_contig, is_target_contig, is_result_contig;
MPI_Aint stream_elem_count, stream_unit_count;
MPI_Aint predefined_dtp_size, predefined_dtp_count, predefined_dtp_extent;
MPID_Datatype *origin_dtp = NULL, *target_dtp = NULL, *result_dtp = NULL;
int is_empty_origin = FALSE;
/* Judge if origin buffer is empty */
if (op == MPI_NO_OP)
is_empty_origin = TRUE;
/* Append the operation to the window's RMA ops queue */
mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &op_ptr);
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
/* TODO: Can we use the MPIDI_RMA_ACC_CONTIG optimization? */
MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
/******************** Setting operation struct areas ***********************/
op_ptr->origin_addr = (void *) origin_addr;
op_ptr->origin_count = origin_count;
op_ptr->origin_datatype = origin_datatype;
op_ptr->result_addr = result_addr;
op_ptr->result_count = result_count;
op_ptr->result_datatype = result_datatype;
op_ptr->target_rank = target_rank;
/* Remember user request */
op_ptr->ureq = ureq;
/* if source or target datatypes are derived, increment their
* reference counts */
if (is_empty_origin == FALSE && !MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) {
MPID_Datatype_get_ptr(origin_datatype, origin_dtp);
}
if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) {
MPID_Datatype_get_ptr(result_datatype, result_dtp);
}
if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
MPID_Datatype_get_ptr(target_datatype, target_dtp);
}
if (is_empty_origin == FALSE) {
MPID_Datatype_get_size_macro(origin_datatype, origin_type_size);
MPIU_Assign_trunc(orig_data_sz, origin_count * origin_type_size, MPIDI_msg_sz_t);
}
else {
/* If origin buffer is empty, set origin data size to 0 */
orig_data_sz = 0;
}
MPID_Datatype_get_size_macro(target_datatype, target_type_size);
/* Get size and count for predefined datatype elements */
if (MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
predefined_dtp_size = target_type_size;
predefined_dtp_count = target_count;
MPID_Datatype_get_extent_macro(target_datatype, predefined_dtp_extent);
}
else {
MPIU_Assert(target_dtp->basic_type != MPI_DATATYPE_NULL);
MPID_Datatype_get_size_macro(target_dtp->basic_type, predefined_dtp_size);
predefined_dtp_count = target_data_sz / predefined_dtp_size;
MPID_Datatype_get_extent_macro(target_dtp->basic_type, predefined_dtp_extent);
}
MPIU_Assert(predefined_dtp_count > 0 && predefined_dtp_size > 0 &&
predefined_dtp_extent > 0);
/* Calculate number of predefined elements in each stream unit, and
* total number of stream units. */
stream_elem_count = MPIDI_CH3U_Acc_stream_size / predefined_dtp_extent;
stream_unit_count = (predefined_dtp_count - 1) / stream_elem_count + 1;
MPIU_Assert(stream_elem_count > 0 && stream_unit_count > 0);
for (i = 0; i < stream_unit_count; i++) {
if (origin_dtp != NULL) {
MPID_Datatype_add_ref(origin_dtp);
}
if (target_dtp != NULL) {
MPID_Datatype_add_ref(target_dtp);
}
if (result_dtp != NULL) {
MPID_Datatype_add_ref(result_dtp);
}
}
if (is_empty_origin == FALSE) {
MPID_Datatype_is_contig(origin_datatype, &is_origin_contig);
}
else {
/* If origin buffer is empty, mark origin data as contig data */
is_origin_contig = 1;
}
MPID_Datatype_is_contig(target_datatype, &is_target_contig);
MPID_Datatype_is_contig(result_datatype, &is_result_contig);
/* Judge if we can use IMMED data packet */
if ((is_empty_origin == TRUE || MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) &&
MPIR_DATATYPE_IS_PREDEFINED(result_datatype) &&
MPIR_DATATYPE_IS_PREDEFINED(target_datatype) &&
is_origin_contig && is_target_contig && is_result_contig) {
if (target_data_sz <= MPIDI_RMA_IMMED_BYTES)
use_immed_pkt = TRUE;
}
/* Judge if this operation is a piggyback candidate */
if ((is_empty_origin == TRUE || MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) &&
MPIR_DATATYPE_IS_PREDEFINED(result_datatype) &&
MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
/* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes
* for origin, target and result data. We should extend this optimization to derived
* datatypes as well. */
if (orig_data_sz <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE)
op_ptr->piggyback_lock_candidate = 1;
}
/************** Setting packet struct areas in operation ****************/
get_accum_pkt = &(op_ptr->pkt.get_accum);
if (use_immed_pkt) {
MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM_IMMED);
}
else {
MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM);
}
get_accum_pkt->addr = (char *) win_ptr->basic_info_table[target_rank].base_addr +
win_ptr->basic_info_table[target_rank].disp_unit * target_disp;
get_accum_pkt->count = target_count;
get_accum_pkt->datatype = target_datatype;
get_accum_pkt->info.dataloop_size = 0;
get_accum_pkt->op = op;
get_accum_pkt->target_win_handle = win_ptr->basic_info_table[target_rank].win_handle;
get_accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
if (use_immed_pkt) {
void *src = (void *) origin_addr, *dest = (void *) (get_accum_pkt->info.data);
mpi_errno = immed_copy(src, dest, orig_data_sz);
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
}
MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, op_ptr);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, target_rank, &made_progress);
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
if (MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD >= 0 &&
MPIDI_CH3I_RMA_Active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) {
while (MPIDI_CH3I_RMA_Active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) {
mpi_errno = wait_progress_engine();
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
}
}
}
fn_exit:
MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_GET_ACCUMULATE);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
goto fn_exit;
/* --END ERROR HANDLING-- */
}
int MPIDO_Scatter(void *sendbuf,
int sendcount,
MPI_Datatype sendtype,
void *recvbuf,
int recvcount,
MPI_Datatype recvtype,
int root,
MPID_Comm * comm)
{
MPIDO_Embedded_Info_Set * properties = &(comm->dcmf.properties);
MPID_Datatype * data_ptr;
MPI_Aint true_lb = 0;
char *sbuf = sendbuf, *rbuf = recvbuf;
int contig, nbytes = 0, rc = 0;
int rank = comm->rank;
int success = 1;
if (rank == root)
{
if (recvtype != MPI_DATATYPE_NULL && recvcount >= 0)
{
MPIDI_Datatype_get_info(sendcount, sendtype, contig,
nbytes, data_ptr, true_lb);
if (!contig) success = 0;
}
else
success = 0;
if (success)
{
if (recvtype != MPI_DATATYPE_NULL && recvcount >= 0)
{
MPIDI_Datatype_get_info(recvcount, recvtype, contig,
nbytes, data_ptr, true_lb);
if (!contig) success = 0;
}
else success = 0;
}
}
else
{
if (sendtype != MPI_DATATYPE_NULL && sendcount >= 0)
{
MPIDI_Datatype_get_info(recvcount, recvtype, contig,
nbytes, data_ptr, true_lb);
if (!contig) success = 0;
}
else
success = 0;
}
if (MPIDO_INFO_ISSET(properties, MPIDO_USE_MPICH_SCATTER) ||
MPIDO_INFO_ISSET(properties, MPIDO_IRREG_COMM) ||
(!MPIDO_INFO_ISSET(properties, MPIDO_USE_TREE_BCAST) && nbytes <= 64))
{
comm->dcmf.last_algorithm = MPIDO_USE_MPICH_SCATTER;
return MPIR_Scatter_intra(sendbuf, sendcount, sendtype,
recvbuf, recvcount, recvtype,
root, comm);
}
/* set the internal control flow to disable internal star tuning */
STAR_info.internal_control_flow = 1;
MPIDO_Allreduce(MPI_IN_PLACE, &success, 1, MPI_INT, MPI_BAND, comm);
/* reset flag */
STAR_info.internal_control_flow = 0;
if (!success)
return MPIR_Scatter_intra(sendbuf, sendcount, sendtype,
recvbuf, recvcount, recvtype,
root, comm);
MPIDI_VerifyBuffer(sendbuf, sbuf, true_lb);
MPIDI_VerifyBuffer(recvbuf, rbuf, true_lb);
if (!STAR_info.enabled || STAR_info.internal_control_flow ||
STAR_info.scatter_algorithms == 1)
{
if (MPIDO_INFO_ISSET(properties, MPIDO_USE_BCAST_SCATTER))
{
comm->dcmf.last_algorithm = MPIDO_USE_BCAST_SCATTER;
return MPIDO_Scatter_bcast(sbuf, sendcount, sendtype,
rbuf, recvcount, recvtype,
root, comm);
}
}
else
{
int id;
unsigned char same_callsite = 1;
void ** tb_ptr = (void **) MPIU_Malloc(sizeof(void *) *
STAR_info.traceback_levels);
/* set the internal control flow to disable internal star tuning */
STAR_info.internal_control_flow = 1;
/* get backtrace info for caller to this func, use that as callsite_id */
backtrace(tb_ptr, STAR_info.traceback_levels);
id = (int) tb_ptr[STAR_info.traceback_levels - 1];
/* find out if all participants agree on the callsite id */
if (STAR_info.agree_on_callsite)
{
int tmp[2], result[2];
tmp[0] = id;
tmp[1] = ~id;
MPIDO_Allreduce(tmp, result, 2, MPI_UNSIGNED_LONG, MPI_MAX, comm);
if (result[0] != (~result[1]))
same_callsite = 0;
}
if (same_callsite)
{
STAR_Callsite collective_site;
/* create a signature callsite info for this particular call site */
collective_site.call_type = SCATTER_CALL;
collective_site.comm = comm;
collective_site.bytes = nbytes;
collective_site.op_type_support = MPIDO_SUPPORT_NOT_NEEDED;
collective_site.id = id;
rc = STAR_Scatter(sbuf, sendcount, sendtype,
rbuf, recvcount, recvtype,
root, &collective_site,
STAR_scatter_repository,
STAR_info.scatter_algorithms);
}
if (rc == STAR_FAILURE || !same_callsite)
rc = MPIR_Scatter_intra(sendbuf, sendcount, sendtype,
recvbuf, recvcount, recvtype,
root, comm);
/* unset the internal control flow */
STAR_info.internal_control_flow = 0;
MPIU_Free(tb_ptr);
}
return rc;
}
int MPID_nem_mxm_issend(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatype datatype,
int rank, int tag, MPID_Comm * comm, int context_offset,
MPID_Request ** sreq_ptr)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *sreq = NULL;
MPID_Datatype *dt_ptr;
int dt_contig;
MPIDI_msg_sz_t data_sz;
MPI_Aint dt_true_lb;
MPID_nem_mxm_vc_area *vc_area = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_ISSEND);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_ISSEND);
MPIDI_Datatype_get_info(count, datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
/* create a request */
MPIDI_Request_create_sreq(sreq, mpi_errno, goto fn_exit);
MPIU_Assert(sreq != NULL);
MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_SEND);
MPIDI_VC_FAI_send_seqnum(vc, seqnum);
MPIDI_Request_set_seqnum(sreq, seqnum);
if (HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN) {
MPID_Datatype_get_ptr(datatype, sreq->dev.datatype_ptr);
MPID_Datatype_add_ref(sreq->dev.datatype_ptr);
}
sreq->partner_request = NULL;
sreq->dev.OnDataAvail = NULL;
sreq->dev.tmpbuf = NULL;
sreq->ch.vc = vc;
sreq->ch.noncontig = FALSE;
_dbg_mxm_output(5,
"isSend ========> Sending USER msg for req %p (context %d to %d tag %d size %d) \n",
sreq, comm->context_id + context_offset, rank, tag, data_sz);
vc_area = VC_BASE(vc);
req_area = REQ_BASE(sreq);
req_area-> ctx = sreq;
req_area->iov_buf = req_area->tmp_buf;
req_area->iov_count = 0;
req_area->iov_buf[0].ptr = NULL;
req_area->iov_buf[0].length = 0;
if (data_sz) {
if (dt_contig) {
req_area->iov_count = 1;
req_area->iov_buf[0].ptr = (char *) (buf) + dt_true_lb;
req_area->iov_buf[0].length = data_sz;
}
else {
MPIDI_msg_sz_t last;
MPI_Aint packsize = 0;
sreq->ch.noncontig = TRUE;
sreq->dev.segment_ptr = MPID_Segment_alloc();
MPIU_ERR_CHKANDJUMP1((sreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER,
"**nomem", "**nomem %s", "MPID_Segment_alloc");
MPIR_Pack_size_impl(count, datatype, &packsize);
last = data_sz;
if (packsize > 0) {
sreq->dev.tmpbuf = MPIU_Malloc((size_t) packsize);
MPIU_Assert(sreq->dev.tmpbuf);
MPID_Segment_init(buf, count, datatype, sreq->dev.segment_ptr, 0);
MPID_Segment_pack(sreq->dev.segment_ptr, 0, &last, sreq->dev.tmpbuf);
req_area->iov_count = 1;
req_area->iov_buf[0].ptr = sreq->dev.tmpbuf;
req_area->iov_buf[0].length = last;
}
}
}
vc_area->pending_sends += 1;
mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND_SYNC,
(mxm_mq_h) comm->dev.ch.netmod_priv, comm->rank, tag, _mxm_tag_mpi2mxm(tag,
comm->context_id
+
context_offset),
0);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
_dbg_mxm_out_req(sreq);
fn_exit:
*sreq_ptr = sreq;
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_ISSEND);
return mpi_errno;
fn_fail:
goto fn_exit;
}
/**
* \brief The standard callback for a new short message
* \note Because this is a short message, the data is already received
* \param[in] clientdata Unused
* \param[in] msginfo The 16-byte msginfo struct
* \param[in] count The number of msginfo quads (1)
* \param[in] senderrank The sender's rank
* \param[in] sndlen The length of the incoming data
* \param[in] sndbuf Where the data is stored
*/
void MPIDI_BG2S_RecvShortCB(void * clientdata,
const DCQuad * msgquad,
unsigned count,
size_t senderrank,
const char * sndbuf,
size_t sndlen)
{
const MPIDI_DCMF_MsgInfo *msginfo = (const MPIDI_DCMF_MsgInfo *)msgquad;
MPID_Request * rreq = NULL;
int found;
int rcvlen = sndlen;
/* Handle cancel requests */
if (msginfo->msginfo.type == MPIDI_DCMF_REQUEST_TYPE_CANCEL_REQUEST)
{
MPIDI_DCMF_procCancelReq(msginfo, senderrank);
return;
}
/* -------------------------- */
/* match request */
/* -------------------------- */
MPIDI_Message_match match;
match.rank = msginfo->msginfo.MPIrank;
match.tag = msginfo->msginfo.MPItag;
match.context_id = msginfo->msginfo.MPIctxt;
rreq = MPIDI_Recvq_FDP_or_AEU(match.rank, match.tag, match.context_id, &found);
if (rreq == NULL)
{
/* ------------------------------------------------- */
/* we have failed to match the request. */
/* allocate and initialize a request object instead. */
/* ------------------------------------------------- */
int mpi_errno = MPIR_Err_create_code(MPI_SUCCESS,
MPIR_ERR_FATAL,
"mpid_recv",
__LINE__,
MPI_ERR_OTHER,
"**nomem", 0);
rreq->status.MPI_ERROR = mpi_errno;
rreq->status.count = 0;
MPID_Abort(NULL, mpi_errno, -1, "Cannot allocate message");
}
/* -------------------------------------- */
/* Signal that the recv has been started. */
/* -------------------------------------- */
MPID_Progress_signal ();
/* ------------------------ */
/* copy in information */
/* ------------------------ */
rreq->status.MPI_SOURCE = match.rank;
rreq->status.MPI_TAG = match.tag;
MPID_Request_setPeerRank(rreq,senderrank);
MPID_Request_setPeerRequest(rreq,msginfo->msginfo.req);
MPID_Request_setSync(rreq, msginfo->msginfo.isSync);
MPID_Request_setRzv(rreq, 0);
/*
*
* Whitespace to sync lines of code with mpidi_callback.c
*
*/
/* ----------------------------------------- */
/* figure out target buffer for request data */
/* ----------------------------------------- */
MPID_Request_setCA(rreq, MPIDI_DCMF_CA_COMPLETE);
rreq->status.count = rcvlen;
if (found)
{
/* --------------------------- */
/* request was already posted. */
/* if synchronized, post ack. */
/* --------------------------- */
if (msginfo->msginfo.isSync)
MPIDI_DCMF_postSyncAck(rreq);
/* -------------------------------------- */
/* calculate message length for reception */
/* calculate receive message "count" */
/* -------------------------------------- */
unsigned dt_contig, dt_size;
MPID_Datatype *dt_ptr;
MPI_Aint dt_true_lb;
MPIDI_Datatype_get_info (rreq->dcmf.userbufcount,
rreq->dcmf.datatype,
dt_contig,
dt_size,
dt_ptr,
dt_true_lb);
/* -------------------------------------- */
/* test for truncated message. */
/* -------------------------------------- */
if (rcvlen > dt_size)
{
rcvlen = dt_size;
rreq->status.MPI_ERROR = MPI_ERR_TRUNCATE;
rreq->status.count = rcvlen;
}
/* -------------------------------------- */
/* if buffer is contiguous ... */
/* -------------------------------------- */
if (dt_contig)
{
char *rcvbuf;
rreq->dcmf.uebuf = NULL;
rreq->dcmf.uebuflen = 0;
rcvbuf = (char *)rreq->dcmf.userbuf + dt_true_lb;
memcpy(rcvbuf, sndbuf, rcvlen);
MPIDI_DCMF_RecvDoneCB(rreq, NULL);
return;
}
/* --------------------------------------------- */
/* buffer is non-contiguous. we need to specify */
/* the send buffer as temporary and unpack. */
/* --------------------------------------------- */
else
{
MPID_Request_setCA(rreq, MPIDI_DCMF_CA_UNPACK_UEBUF_AND_COMPLETE_NOFREE);
rreq->dcmf.uebuflen = rcvlen ;
rreq->dcmf.uebuf = (char *) sndbuf ;
MPIDI_DCMF_RecvDoneCB(rreq, NULL);
return;
}
}
/* ------------------------------------------------------------- */
/* Request was not posted. We must allocate enough space to hold */
/* the message temporarily and copy the data into the temporary */
/* buffer. The temporary buffer will be unpacked later. */
/* ------------------------------------------------------------- */
rreq->dcmf.uebuflen = rcvlen ;
if ((rreq->dcmf.uebuf = MPIU_Malloc (rcvlen)) == NULL)
{
/* ------------------------------------ */
/* creation of temporary buffer failed. */
/* we are in trouble and must bail out. */
/* ------------------------------------ */
int mpi_errno = MPIR_Err_create_code(MPI_SUCCESS,
MPIR_ERR_FATAL,
"mpid_recv",
__LINE__,
MPI_ERR_OTHER,
"**nomem", 0);
rreq->status.MPI_ERROR = mpi_errno;
rreq->status.count = 0;
MPID_Abort(NULL, mpi_errno, -1, "Cannot allocate unexpected buffer");
}
/* ------------------------------------------------ */
/* Copy the data into the unexpected buffer. */
/* ------------------------------------------------ */
memcpy(rreq->dcmf.uebuf, sndbuf, rreq->dcmf.uebuflen);
MPIDI_DCMF_RecvDoneCB(rreq, NULL);
return;
}
