int MPIDO_Gatherv_simple(const void *sendbuf,
int sendcount,
MPI_Datatype sendtype,
void *recvbuf,
const int *recvcounts,
const int *displs,
MPI_Datatype recvtype,
int root,
MPID_Comm * comm_ptr,
int *mpierrno)
{
#ifndef HAVE_PAMI_IN_PLACE
if (sendbuf == MPI_IN_PLACE)
{
MPID_Abort (NULL, 0, 1, "'MPI_IN_PLACE' requries support for `PAMI_IN_PLACE`");
return -1;
}
#endif
TRACE_ERR("Entering MPIDO_Gatherv_optimized\n");
int snd_contig = 1, rcv_contig = 1;
void *snd_noncontig_buff = NULL, *rcv_noncontig_buff = NULL;
void *sbuf = NULL, *rbuf = NULL;
int *rcounts = NULL;
int *rdispls = NULL;
int send_size = 0;
int recv_size = 0;
int rcvlen = 0;
int totalrecvcount = 0;
pami_type_t rtype = PAMI_TYPE_NULL;
MPID_Segment segment;
MPID_Datatype *data_ptr = NULL;
int send_true_lb, recv_true_lb = 0;
int i, tmp;
volatile unsigned gatherv_active = 1;
const int rank = comm_ptr->rank;
const int size = comm_ptr->local_size;
#if ASSERT_LEVEL==0
/* We can't afford the tracing in ndebug/performance libraries */
const unsigned verbose = 0;
#else
const unsigned verbose = (MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_ALL) && (rank == 0);
#endif
const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid);
int recvok=PAMI_SUCCESS, recvcontinuous=0;
if(sendbuf != MPI_IN_PLACE)
{
MPIDI_Datatype_get_info(sendcount, sendtype, snd_contig,
send_size, data_ptr, send_true_lb);
if(MPIDI_Pamix_collsel_advise != NULL && mpid->collsel_fast_query != NULL)
{
advisor_algorithm_t advisor_algorithms[1];
int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_GATHERV_INT, 64, advisor_algorithms, 1);
if(num_algorithms)
{
if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO)
{
return MPIR_Gatherv(sendbuf, sendcount, sendtype,
recvbuf, recvcounts, displs, recvtype,
root, comm_ptr, mpierrno);
}
else if(advisor_algorithms[0].metadata && advisor_algorithms[0].metadata->check_correct.values.asyncflowctl && !(--(comm_ptr->mpid.num_requests)))
{
comm_ptr->mpid.num_requests = MPIDI_Process.optimized.num_requests;
int tmpmpierrno;
if(unlikely(verbose))
fprintf(stderr,"Query barrier required for %s\n", advisor_algorithms[0].metadata->name);
MPIDO_Barrier(comm_ptr, &tmpmpierrno);
}
}
}
sbuf = (char *)sendbuf + send_true_lb;
if(!snd_contig)
{
snd_noncontig_buff = MPL_malloc(send_size);
sbuf = snd_noncontig_buff;
if(snd_noncontig_buff == NULL)
{
MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1,
"Fatal: Cannot allocate pack buffer");
}
DLOOP_Offset last = send_size;
MPID_Segment_init(sendbuf, sendcount, sendtype, &segment, 0);
MPID_Segment_pack(&segment, 0, &last, snd_noncontig_buff);
}
}
else
{
MPIDI_Datatype_get_info(1, recvtype, rcv_contig,
rcvlen, data_ptr, recv_true_lb);
if(MPIDI_Pamix_collsel_advise != NULL && mpid->collsel_fast_query != NULL)
{
advisor_algorithm_t advisor_algorithms[1];
int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_GATHERV_INT, 64, advisor_algorithms, 1);
if(num_algorithms)
{
if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO)
{
return MPIR_Gatherv(sendbuf, sendcount, sendtype,
recvbuf, recvcounts, displs, recvtype,
root, comm_ptr, mpierrno);
}
else if(advisor_algorithms[0].metadata && advisor_algorithms[0].metadata->check_correct.values.asyncflowctl && !(--(comm_ptr->mpid.num_requests)))
{
comm_ptr->mpid.num_requests = MPIDI_Process.optimized.num_requests;
int tmpmpierrno;
if(unlikely(verbose))
fprintf(stderr,"Query barrier required for %s\n", advisor_algorithms[0].metadata->name);
MPIDO_Barrier(comm_ptr, &tmpmpierrno);
}
}
}
}
pami_xfer_t gatherv;
rbuf = (char *)recvbuf + recv_true_lb;
rcounts = (int*)recvcounts;
rdispls = (int*)displs;
if(rank == root)
{
if((recvok = MPIDI_Datatype_to_pami(recvtype, &rtype, -1, NULL, &tmp)) != MPI_SUCCESS)
{
MPIDI_Datatype_get_info(1, recvtype, rcv_contig,
rcvlen, data_ptr, recv_true_lb);
totalrecvcount = recvcounts[0];
recvcontinuous = displs[0] == 0? 1 : 0 ;
rcounts = (int*)MPL_malloc(size);
rdispls = (int*)MPL_malloc(size);
rdispls[0] = 0;
rcounts[0] = rcvlen * recvcounts[0];
for(i = 1; i < size; i++)
{
rdispls[i]= rcvlen * totalrecvcount;
totalrecvcount += recvcounts[i];
if(displs[i] != (displs[i-1] + recvcounts[i-1]))
recvcontinuous = 0;
rcounts[i] = rcvlen * recvcounts[i];
}
recv_size = rcvlen * totalrecvcount;
rcv_noncontig_buff = MPL_malloc(recv_size);
rbuf = rcv_noncontig_buff;
rtype = PAMI_TYPE_BYTE;
if(rcv_noncontig_buff == NULL)
{
MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1,
"Fatal: Cannot allocate pack buffer");
}
if(sendbuf == MPI_IN_PLACE)
{
size_t extent;
MPID_Datatype_get_extent_macro(recvtype,extent);
MPIR_Localcopy(recvbuf + displs[rank]*extent, recvcounts[rank], recvtype,
rcv_noncontig_buff + rdispls[rank], rcounts[rank],MPI_CHAR);
}
}
if(sendbuf == MPI_IN_PLACE)
{
gatherv.cmd.xfer_gatherv_int.sndbuf = PAMI_IN_PLACE;
}
else
{
gatherv.cmd.xfer_gatherv_int.sndbuf = sbuf;
}
gatherv.cmd.xfer_gatherv_int.stype = PAMI_TYPE_BYTE;/* stype is ignored when sndbuf == PAMI_IN_PLACE */
gatherv.cmd.xfer_gatherv_int.stypecount = send_size;
}
else
{
gatherv.cmd.xfer_gatherv_int.sndbuf = sbuf;
gatherv.cmd.xfer_gatherv_int.stype = PAMI_TYPE_BYTE;
gatherv.cmd.xfer_gatherv_int.stypecount = send_size;
}
gatherv.cb_done = cb_gatherv;
gatherv.cookie = (void *)&gatherv_active;
gatherv.cmd.xfer_gatherv_int.root = MPIDI_Task_to_endpoint(MPID_VCR_GET_LPID(comm_ptr->vcr, root), 0);
gatherv.cmd.xfer_gatherv_int.rcvbuf = rbuf;
gatherv.cmd.xfer_gatherv_int.rtype = rtype;
gatherv.cmd.xfer_gatherv_int.rtypecounts = (int *) rcounts;
gatherv.cmd.xfer_gatherv_int.rdispls = (int *) rdispls;
const pami_metadata_t *my_gatherv_md;
gatherv.algorithm = mpid->coll_algorithm[PAMI_XFER_GATHERV_INT][0][0];
my_gatherv_md = &mpid->coll_metadata[PAMI_XFER_GATHERV_INT][0][0];
MPIDI_Update_last_algorithm(comm_ptr, my_gatherv_md->name);
MPIDI_Post_coll_t gatherv_post;
TRACE_ERR("%s gatherv\n", MPIDI_Process.context_post.active>0?"Posting":"Invoking");
MPIDI_Context_post(MPIDI_Context[0], &gatherv_post.state,
MPIDI_Pami_post_wrapper, (void *)&gatherv);
TRACE_ERR("Gatherv %s\n", MPIDI_Process.context_post.active>0?"posted":"invoked");
TRACE_ERR("Waiting on active %d\n", gatherv_active);
MPID_PROGRESS_WAIT_WHILE(gatherv_active);
if(!rcv_contig || recvok != PAMI_SUCCESS)
{
if(recvcontinuous)
{
MPIR_Localcopy(rcv_noncontig_buff, recv_size, MPI_CHAR,
recvbuf, totalrecvcount, recvtype);
}
else
{
size_t extent;
MPID_Datatype_get_extent_macro(recvtype,extent);
for(i=0; i<size; ++i)
{
char* scbuf = (char*)rcv_noncontig_buff+ rdispls[i];
char* rcbuf = (char*)recvbuf + displs[i]*extent;
MPIR_Localcopy(scbuf, rcounts[i], MPI_CHAR,
rcbuf, recvcounts[i], recvtype);
TRACE_ERR("Pack recv src extent %zu, displ[%zu]=%zu, count[%zu]=%zu buf[%zu]=%u\n",
(size_t)extent, (size_t)i,(size_t)precvdispls[i],(size_t)i,(size_t)precvcounts[i],(size_t)precvdispls[i], *(int*)scbuf);
TRACE_ERR("Pack recv dest extent %zu, displ[%zu]=%zu, count[%zu]=%zu buf[%zu]=%u\n",
(size_t)extent, (size_t)i,(size_t)displs[i],(size_t)i,(size_t)recvcounts[i],(size_t)displs[i], *(int*)rcbuf);
}
}
MPL_free(rcv_noncontig_buff);
if(rank == root)
{
MPL_free(rcounts);
MPL_free(rdispls);
}
}
if(!snd_contig) MPL_free(snd_noncontig_buff);
TRACE_ERR("Leaving MPIDO_Gatherv_optimized\n");
return MPI_SUCCESS;
}
int MPIDO_Scatterv_simple(const void *sendbuf,
const int *sendcounts,
const int *displs,
MPI_Datatype sendtype,
void *recvbuf,
int recvcount,
MPI_Datatype recvtype,
int root,
MPID_Comm *comm_ptr,
int *mpierrno)
{
#ifndef HAVE_PAMI_IN_PLACE
if (sendbuf == MPI_IN_PLACE)
{
MPID_Abort (NULL, 0, 1, "'MPI_IN_PLACE' requries support for `PAMI_IN_PLACE`");
return -1;
}
#endif
int snd_contig = 1;
int rcv_contig = 1;
int send_size = 0, recv_size = 0;
int ssize = 0;
MPID_Datatype *dt_ptr = NULL;
MPI_Aint send_true_lb=0, recv_true_lb=0;
void *snd_noncontig_buff = NULL, *rcv_noncontig_buff = NULL;
void *sbuf = NULL, *rbuf = NULL;
int *sdispls = NULL, *scounts = NULL;
int sndcount = 0;
MPID_Segment segment;
int tmp, i;
pami_type_t stype = PAMI_TYPE_NULL;
const int rank = comm_ptr->rank;
const int size = comm_ptr->local_size;
const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid);
if (rank == root && sendtype != MPI_DATATYPE_NULL && sendcounts[0] >= 0)
{
MPIDI_Datatype_get_info(1, sendtype, snd_contig, ssize, dt_ptr, send_true_lb);
if(MPIDI_Pamix_collsel_advise != NULL && mpid->collsel_fast_query != NULL)
{
advisor_algorithm_t advisor_algorithms[1];
int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_SCATTERV_INT, 64, advisor_algorithms, 1);
if(num_algorithms)
{
if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO)
{
return MPIR_Scatterv(sendbuf, sendcounts, displs, sendtype,
recvbuf, recvcount, recvtype,
root, comm_ptr, mpierrno);
}
else if(advisor_algorithms[0].metadata && advisor_algorithms[0].metadata->check_correct.values.asyncflowctl && !(--(comm_ptr->mpid.num_requests)))
{
comm_ptr->mpid.num_requests = MPIDI_Process.optimized.num_requests;
int tmpmpierrno;
MPIDO_Barrier(comm_ptr, &tmpmpierrno);
}
}
}
}
if (recvtype != MPI_DATATYPE_NULL && recvcount >= 0)
{
MPIDI_Datatype_get_info(recvcount, recvtype, rcv_contig,
recv_size, dt_ptr, recv_true_lb);
if(MPIDI_Pamix_collsel_advise != NULL && mpid->collsel_fast_query != NULL)
{
advisor_algorithm_t advisor_algorithms[1];
int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_SCATTERV_INT, 64, advisor_algorithms, 1);
if(num_algorithms)
{
if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO)
{
return MPIR_Scatterv(sendbuf, sendcounts, displs, sendtype,
recvbuf, recvcount, recvtype,
root, comm_ptr, mpierrno);
}
else if(advisor_algorithms[0].metadata && advisor_algorithms[0].metadata->check_correct.values.asyncflowctl && !(--(comm_ptr->mpid.num_requests)))
{
comm_ptr->mpid.num_requests = MPIDI_Process.optimized.num_requests;
int tmpmpierrno;
MPIDO_Barrier(comm_ptr, &tmpmpierrno);
}
}
}
}
pami_xfer_t scatterv;
const pami_metadata_t *my_scatterv_md;
volatile unsigned scatterv_active = 1;
sbuf = (char *)sendbuf + send_true_lb;
rbuf = (char *)recvbuf + recv_true_lb;
scounts = (int*)sendcounts;
sdispls = (int*)displs;
if(rank == root)
{
if(MPIDI_Datatype_to_pami(sendtype, &stype, -1, NULL, &tmp) != MPI_SUCCESS)
{
if (!snd_contig)
{
scounts = (int*)MPIU_Malloc(size);
sdispls = (int*)MPIU_Malloc(size);
for(i = 0; i < size; i++)
{
scounts[i] = ssize * sendcounts[i];
sdispls[i] = ssize * displs[i];
send_size += scounts[i];
sndcount += sendcounts[i];
}
snd_noncontig_buff = MPIU_Malloc(send_size);
sbuf = snd_noncontig_buff;
stype = PAMI_TYPE_BYTE;
if(snd_noncontig_buff == NULL)
{
MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1,
"Fatal: Cannot allocate pack buffer");
}
DLOOP_Offset last = send_size;
MPID_Segment_init(sendbuf, sndcount, sendtype, &segment, 0);
MPID_Segment_pack(&segment, 0, &last, snd_noncontig_buff);
}
}
if(recvbuf == MPI_IN_PLACE)
{
rbuf = PAMI_IN_PLACE;
}
}
if(recvbuf != MPI_IN_PLACE)
{
if (!rcv_contig)
{
rcv_noncontig_buff = MPIU_Malloc(recv_size);
rbuf = rcv_noncontig_buff;
if(rcv_noncontig_buff == NULL)
{
MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1,
"Fatal: Cannot allocate pack buffer");
}
}
}
scatterv.cb_done = cb_scatterv;
scatterv.cookie = (void *)&scatterv_active;
scatterv.cmd.xfer_scatterv_int.root = MPIDI_Task_to_endpoint(MPID_VCR_GET_LPID(comm_ptr->vcr, root), 0);
scatterv.algorithm = mpid->coll_algorithm[PAMI_XFER_SCATTERV_INT][0][0];
my_scatterv_md = &mpid->coll_metadata[PAMI_XFER_SCATTERV_INT][0][0];
scatterv.cmd.xfer_scatterv_int.rcvbuf = rbuf;
scatterv.cmd.xfer_scatterv_int.sndbuf = sbuf;
scatterv.cmd.xfer_scatterv_int.stype = stype;
scatterv.cmd.xfer_scatterv_int.rtype = PAMI_TYPE_BYTE;/* rtype is ignored when rcvbuf == PAMI_IN_PLACE */
scatterv.cmd.xfer_scatterv_int.stypecounts = (int *) scounts;
scatterv.cmd.xfer_scatterv_int.rtypecount = recv_size;
scatterv.cmd.xfer_scatterv_int.sdispls = (int *) sdispls;
MPIDI_Update_last_algorithm(comm_ptr, my_scatterv_md->name);
MPIDI_Post_coll_t scatterv_post;
TRACE_ERR("%s scatterv\n", MPIDI_Process.context_post.active>0?"Posting":"Invoking");
MPIDI_Context_post(MPIDI_Context[0], &scatterv_post.state,
MPIDI_Pami_post_wrapper, (void *)&scatterv);
TRACE_ERR("Waiting on active %d\n", scatterv_active);
MPID_PROGRESS_WAIT_WHILE(scatterv_active);
if(!rcv_contig)
{
MPIR_Localcopy(rcv_noncontig_buff, recv_size, MPI_CHAR,
recvbuf, recvcount, recvtype);
MPIU_Free(rcv_noncontig_buff);
}
if(!snd_contig)
{
MPIU_Free(snd_noncontig_buff);
MPIU_Free(scounts);
MPIU_Free(sdispls);
}
TRACE_ERR("Leaving MPIDO_Scatterv_optimized\n");
return MPI_SUCCESS;
}
int MPIDO_Bcast_simple(void *buffer,
int count,
MPI_Datatype datatype,
int root,
MPID_Comm *comm_ptr,
int *mpierrno)
{
TRACE_ERR("Entering MPIDO_Bcast_optimized\n");
int data_contig;
void *data_buffer = NULL,
*noncontig_buff = NULL;
volatile unsigned active = 1;
MPI_Aint data_true_lb = 0;
MPID_Datatype *data_ptr;
MPID_Segment segment;
MPIDI_Post_coll_t bcast_post;
const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid);
const int rank = comm_ptr->rank;
/* Must calculate data_size based on count=1 in case it's total size is > integer */
int data_size_one;
MPIDI_Datatype_get_info(1, datatype,
data_contig, data_size_one, data_ptr, data_true_lb);
if(MPIDI_Pamix_collsel_advise != NULL && mpid->collsel_fast_query != NULL)
{
advisor_algorithm_t advisor_algorithms[1];
int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_BROADCAST, data_size_one * count, advisor_algorithms, 1);
if(num_algorithms)
{
if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO)
{
return MPIR_Bcast_intra(buffer, count, datatype, root, comm_ptr, mpierrno);
}
}
}
const int data_size = data_size_one*(size_t)count;
data_buffer = (char *)buffer + data_true_lb;
if(!data_contig)
{
noncontig_buff = MPIU_Malloc(data_size);
data_buffer = noncontig_buff;
if(noncontig_buff == NULL)
{
MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1,
"Fatal: Cannot allocate pack buffer");
}
if(rank == root)
{
DLOOP_Offset last = data_size;
MPID_Segment_init(buffer, count, datatype, &segment, 0);
MPID_Segment_pack(&segment, 0, &last, noncontig_buff);
}
}
pami_xfer_t bcast;
const pami_metadata_t *my_bcast_md;
int queryreq = 0;
bcast.cb_done = cb_bcast;
bcast.cookie = (void *)&active;
bcast.cmd.xfer_broadcast.root = MPIDI_Task_to_endpoint(MPID_VCR_GET_LPID(comm_ptr->vcr, root), 0);
bcast.algorithm = mpid->coll_algorithm[PAMI_XFER_BROADCAST][0][0];
bcast.cmd.xfer_broadcast.buf = data_buffer;
bcast.cmd.xfer_broadcast.type = PAMI_TYPE_BYTE;
/* Needs to be sizeof(type)*count since we are using bytes as * the generic type */
bcast.cmd.xfer_broadcast.typecount = data_size;
my_bcast_md = &mpid->coll_metadata[PAMI_XFER_BROADCAST][0][0];
MPIDI_Context_post(MPIDI_Context[0], &bcast_post.state, MPIDI_Pami_post_wrapper, (void *)&bcast);
MPIDI_Update_last_algorithm(comm_ptr, my_bcast_md->name);
MPID_PROGRESS_WAIT_WHILE(active);
TRACE_ERR("bcast done\n");
if(!data_contig)
{
if(rank != root)
MPIR_Localcopy(noncontig_buff, data_size, MPI_CHAR,
buffer, count, datatype);
MPIU_Free(noncontig_buff);
}
TRACE_ERR("Exiting MPIDO_Bcast_optimized\n");
return 0;
}
int
MPIDO_Allgather_simple(const void *sendbuf,
int sendcount,
MPI_Datatype sendtype,
void *recvbuf,
int recvcount,
MPI_Datatype recvtype,
MPID_Comm * comm_ptr,
int *mpierrno)
{
#ifndef HAVE_PAMI_IN_PLACE
if (sendbuf == MPI_IN_PLACE)
{
MPID_Abort (NULL, 0, 1, "'MPI_IN_PLACE' requries support for `PAMI_IN_PLACE`");
return -1;
}
#endif
/* *********************************
* Check the nature of the buffers
* *********************************
*/
const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid);
MPID_Datatype * dt_null = NULL;
void *snd_noncontig_buff = NULL, *rcv_noncontig_buff = NULL;
MPI_Aint send_true_lb = 0;
MPI_Aint recv_true_lb = 0;
int snd_data_contig = 1, rcv_data_contig = 1;
size_t send_size = 0;
size_t recv_size = 0;
MPID_Segment segment;
volatile unsigned allgather_active = 1;
const int rank = comm_ptr->rank;
const int size = comm_ptr->local_size;
#if ASSERT_LEVEL==0
/* We can't afford the tracing in ndebug/performance libraries */
const unsigned verbose = 0;
#else
const unsigned verbose = (MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_ALL) && (rank == 0);
#endif
const pami_metadata_t *my_md;
char *rbuf = NULL, *sbuf = NULL;
if ((sendcount < 1 && sendbuf != MPI_IN_PLACE) || recvcount < 1)
return MPI_SUCCESS;
/* Gather datatype information */
MPIDI_Datatype_get_info(recvcount,
recvtype,
rcv_data_contig,
recv_size,
dt_null,
recv_true_lb);
send_size = recv_size;
if(MPIDI_Pamix_collsel_advise != NULL && mpid->collsel_fast_query != NULL)
{
advisor_algorithm_t advisor_algorithms[1];
int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_ALLGATHER, send_size, advisor_algorithms, 1);
if(num_algorithms)
{
if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO)
{
return MPIR_Allgather(sendbuf, sendcount, sendtype,
recvbuf, recvcount, recvtype,
comm_ptr, mpierrno);
}
else if(advisor_algorithms[0].metadata && advisor_algorithms[0].metadata->check_correct.values.asyncflowctl && !(--(comm_ptr->mpid.num_requests)))
{
comm_ptr->mpid.num_requests = MPIDI_Process.optimized.num_requests;
int tmpmpierrno;
if(unlikely(verbose))
fprintf(stderr,"Query barrier required for %s\n", advisor_algorithms[0].metadata->name);
MPIDO_Barrier(comm_ptr, &tmpmpierrno);
}
}
}
rbuf = (char *)recvbuf+recv_true_lb;
if(!rcv_data_contig)
{
rcv_noncontig_buff = MPL_malloc(recv_size * size);
rbuf = rcv_noncontig_buff;
if(rcv_noncontig_buff == NULL)
{
MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1,
"Fatal: Cannot allocate pack buffer");
}
if(sendbuf == MPI_IN_PLACE)
{
sbuf = PAMI_IN_PLACE;
size_t extent;
MPID_Datatype_get_extent_macro(recvtype,extent);
MPIR_Localcopy(recvbuf + (rank*recvcount*extent), recvcount, recvtype,
rcv_noncontig_buff + (rank*recv_size), recv_size,MPI_CHAR);
}
}
if(sendbuf != MPI_IN_PLACE)
{
MPIDI_Datatype_get_info(sendcount,
sendtype,
snd_data_contig,
send_size,
dt_null,
send_true_lb);
sbuf = (char *)sendbuf+send_true_lb;
if(!snd_data_contig)
{
snd_noncontig_buff = MPL_malloc(send_size);
sbuf = snd_noncontig_buff;
if(snd_noncontig_buff == NULL)
{
MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1,
"Fatal: Cannot allocate pack buffer");
}
DLOOP_Offset last = send_size;
MPID_Segment_init(sendbuf, sendcount, sendtype, &segment, 0);
MPID_Segment_pack(&segment, 0, &last, snd_noncontig_buff);
}
}
else
sbuf = PAMI_IN_PLACE;
TRACE_ERR("Using PAMI-level allgather protocol\n");
pami_xfer_t allgather;
allgather.cb_done = allgather_cb_done;
allgather.cookie = (void *)&allgather_active;
allgather.cmd.xfer_allgather.rcvbuf = rbuf;
allgather.cmd.xfer_allgather.sndbuf = sbuf;
allgather.cmd.xfer_allgather.stype = PAMI_TYPE_BYTE;/* stype is ignored when sndbuf == PAMI_IN_PLACE */
allgather.cmd.xfer_allgather.rtype = PAMI_TYPE_BYTE;
allgather.cmd.xfer_allgather.stypecount = send_size;
allgather.cmd.xfer_allgather.rtypecount = recv_size;
allgather.algorithm = mpid->coll_algorithm[PAMI_XFER_ALLGATHER][0][0];
my_md = &mpid->coll_metadata[PAMI_XFER_ALLGATHER][0][0];
TRACE_ERR("Calling PAMI_Collective with allgather structure\n");
MPIDI_Post_coll_t allgather_post;
MPIDI_Context_post(MPIDI_Context[0], &allgather_post.state, MPIDI_Pami_post_wrapper, (void *)&allgather);
TRACE_ERR("Allgather %s\n", MPIDI_Process.context_post.active>0?"posted":"invoked");
MPIDI_Update_last_algorithm(comm_ptr, my_md->name);
MPID_PROGRESS_WAIT_WHILE(allgather_active);
if(!rcv_data_contig)
{
MPIR_Localcopy(rcv_noncontig_buff, recv_size * size, MPI_CHAR,
recvbuf, recvcount, recvtype);
MPL_free(rcv_noncontig_buff);
}
if(!snd_data_contig) MPL_free(snd_noncontig_buff);
TRACE_ERR("Allgather done\n");
return MPI_SUCCESS;
}
int
MPIDO_Allgatherv_simple(const void *sendbuf,
int sendcount,
MPI_Datatype sendtype,
void *recvbuf,
const int *recvcounts,
const int *displs,
MPI_Datatype recvtype,
MPID_Comm * comm_ptr,
int *mpierrno)
{
#ifndef HAVE_PAMI_IN_PLACE
if (sendbuf == MPI_IN_PLACE)
{
MPID_Abort (NULL, 0, 1, "'MPI_IN_PLACE' requries support for `PAMI_IN_PLACE`");
return -1;
}
#endif
TRACE_ERR("Entering MPIDO_Allgatherv_optimized\n");
/* function pointer to be used to point to approperiate algorithm */
/* Check the nature of the buffers */
MPID_Datatype *dt_null = NULL;
MPI_Aint send_true_lb = 0;
MPI_Aint recv_true_lb = 0;
size_t send_size = 0;
size_t recv_size = 0;
size_t rcvtypelen = 0;
int snd_data_contig = 0, rcv_data_contig = 0;
void *snd_noncontig_buff = NULL, *rcv_noncontig_buff = NULL;
int scount=sendcount;
char *sbuf, *rbuf;
pami_type_t stype = NULL, rtype;
const int rank = comm_ptr->rank;
const int size = comm_ptr->local_size;
const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid);
#if ASSERT_LEVEL==0
/* We can't afford the tracing in ndebug/performance libraries */
const unsigned verbose = 0;
#else
const unsigned verbose = (MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_ALL) && (rank == 0);
#endif
int recvcontinuous=0;
size_t totalrecvcount=0;
int *lrecvdispls = NULL; /* possible local displs calculated for noncontinous */
int *lrecvcounts = NULL;/* possible local counts calculated for noncontinous */
const int *precvdispls = displs; /* pointer to displs to use as pami parmi */
const int *precvcounts = recvcounts; /* pointer to counts to use as pami parmi */
int inplace = sendbuf == MPI_IN_PLACE? 1 : 0;
volatile unsigned allgatherv_active = 1;
int recvok=PAMI_SUCCESS, sendok=PAMI_SUCCESS;
int tmp;
const pami_metadata_t *my_md;
MPIDI_Datatype_get_info(1,
recvtype,
rcv_data_contig,
rcvtypelen,
dt_null,
recv_true_lb);
if(MPIDI_Pamix_collsel_advise != NULL && mpid->collsel_fast_query != NULL)
{
advisor_algorithm_t advisor_algorithms[1];
int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_ALLGATHERV_INT, rcvtypelen * recvcounts[0], advisor_algorithms, 1);
if(num_algorithms)
{
if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO)
{
return MPIR_Allgatherv(sendbuf, sendcount, sendtype,
recvbuf, recvcounts, displs, recvtype,
comm_ptr, mpierrno);
}
else if(advisor_algorithms[0].metadata && advisor_algorithms[0].metadata->check_correct.values.asyncflowctl && !(--(comm_ptr->mpid.num_requests)))
{
comm_ptr->mpid.num_requests = MPIDI_Process.optimized.num_requests;
int tmpmpierrno;
if(unlikely(verbose))
fprintf(stderr,"Query barrier required for %s\n", advisor_algorithms[0].metadata->name);
MPIDO_Barrier(comm_ptr, &tmpmpierrno);
}
}
}
if(!inplace)
{
sendok = MPIDI_Datatype_to_pami(sendtype, &stype, -1, NULL, &tmp);
MPIDI_Datatype_get_info(sendcount, sendtype, snd_data_contig, send_size, dt_null, send_true_lb);
sbuf = (char *)sendbuf + send_true_lb;
if(!snd_data_contig || (sendok != PAMI_SUCCESS))
{
stype = PAMI_TYPE_UNSIGNED_CHAR;
scount = send_size;
if(!snd_data_contig)
{
snd_noncontig_buff = MPIU_Malloc(send_size);
sbuf = snd_noncontig_buff;
if(snd_noncontig_buff == NULL)
{
MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1,
"Fatal: Cannot allocate pack buffer");
}
MPIR_Localcopy(sendbuf, sendcount, sendtype,
snd_noncontig_buff, send_size,MPI_CHAR);
}
}
}
else
sbuf = PAMI_IN_PLACE;
recvok = MPIDI_Datatype_to_pami(recvtype, &rtype, -1, NULL, &tmp);
rbuf = (char *)recvbuf+recv_true_lb;
if(!rcv_data_contig || (recvok != PAMI_SUCCESS))
{
rtype = PAMI_TYPE_UNSIGNED_CHAR;
totalrecvcount = recvcounts[0];
recvcontinuous = displs[0] == 0? 1 : 0 ;
int i;
precvdispls = lrecvdispls = MPIU_Malloc(size*sizeof(int));
precvcounts = lrecvcounts = MPIU_Malloc(size*sizeof(int));
lrecvdispls[0]= 0;
lrecvcounts[0]= rcvtypelen * recvcounts[0];
for(i=1; i<size; ++i)
{
lrecvdispls[i]= rcvtypelen * totalrecvcount;
totalrecvcount += recvcounts[i];
if(displs[i] != (displs[i-1] + recvcounts[i-1]))
recvcontinuous = 0;
lrecvcounts[i]= rcvtypelen * recvcounts[i];
}
recv_size = rcvtypelen * totalrecvcount;
TRACE_ERR("Pack receive rcv_contig %zu, recvok %zd, totalrecvcount %zu, recvcontinuous %zu, rcvtypelen %zu, recv_size %zu\n",
(size_t)rcv_data_contig, (size_t)recvok, (size_t)totalrecvcount, (size_t)recvcontinuous,(size_t)rcvtypelen, (size_t)recv_size);
rcv_noncontig_buff = MPIU_Malloc(recv_size);
rbuf = rcv_noncontig_buff;
if(rcv_noncontig_buff == NULL)
{
MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1,
"Fatal: Cannot allocate pack buffer");
}
if(inplace)
{
size_t extent;
MPID_Datatype_get_extent_macro(recvtype,extent);
MPIR_Localcopy(recvbuf + displs[rank]*extent, recvcounts[rank], recvtype,
rcv_noncontig_buff + precvdispls[rank], precvcounts[rank],MPI_CHAR);
scount = precvcounts[rank];
stype = PAMI_TYPE_UNSIGNED_CHAR;
sbuf = PAMI_IN_PLACE;
}
}
pami_xfer_t allgatherv;
allgatherv.cb_done = allgatherv_cb_done;
allgatherv.cookie = (void *)&allgatherv_active;
allgatherv.cmd.xfer_allgatherv_int.sndbuf = sbuf;
allgatherv.cmd.xfer_allgatherv_int.rcvbuf = rbuf;
allgatherv.cmd.xfer_allgatherv_int.stype = stype;/* stype is ignored when sndbuf == PAMI_IN_PLACE */
allgatherv.cmd.xfer_allgatherv_int.rtype = rtype;
allgatherv.cmd.xfer_allgatherv_int.stypecount = scount;
allgatherv.cmd.xfer_allgatherv_int.rtypecounts = (int *) precvcounts;
allgatherv.cmd.xfer_allgatherv_int.rdispls = (int *) precvdispls;
allgatherv.algorithm = mpid->coll_algorithm[PAMI_XFER_ALLGATHERV_INT][0][0];
my_md = &mpid->coll_metadata[PAMI_XFER_ALLGATHERV_INT][0][0];
TRACE_ERR("Calling allgatherv via %s()\n", MPIDI_Process.context_post.active>0?"PAMI_Collective":"PAMI_Context_post");
MPIDI_Post_coll_t allgatherv_post;
MPIDI_Context_post(MPIDI_Context[0], &allgatherv_post.state,
MPIDI_Pami_post_wrapper, (void *)&allgatherv);
MPIDI_Update_last_algorithm(comm_ptr, my_md->name);
TRACE_ERR("Rank %d waiting on active %d\n", rank, allgatherv_active);
MPID_PROGRESS_WAIT_WHILE(allgatherv_active);
if(!rcv_data_contig || (recvok != PAMI_SUCCESS))
{
if(recvcontinuous)
{
MPIR_Localcopy(rcv_noncontig_buff, recv_size,MPI_CHAR,
recvbuf, totalrecvcount, recvtype);
}
else
{
size_t extent;
int i;
MPID_Datatype_get_extent_macro(recvtype,extent);
for(i=0; i<size; ++i)
{
char* scbuf = (char*)rcv_noncontig_buff+ precvdispls[i];
char* rcbuf = (char*)recvbuf + displs[i]*extent;
MPIR_Localcopy(scbuf, precvcounts[i], MPI_CHAR,
rcbuf, recvcounts[i], recvtype);
TRACE_ERR("Pack recv src extent %zu, displ[%zu]=%zu, count[%zu]=%zu buf[%zu]=%u\n",
(size_t)extent, (size_t)i,(size_t)precvdispls[i],(size_t)i,(size_t)precvcounts[i],(size_t)precvdispls[i], *(int*)scbuf);
TRACE_ERR("Pack recv dest extent %zu, displ[%zu]=%zu, count[%zu]=%zu buf[%zu]=%u\n",
(size_t)extent, (size_t)i,(size_t)displs[i],(size_t)i,(size_t)recvcounts[i],(size_t)displs[i], *(int*)rcbuf);
}
}
MPIU_Free(rcv_noncontig_buff);
}
if(!snd_data_contig) MPIU_Free(snd_noncontig_buff);
if(lrecvdispls) MPIU_Free(lrecvdispls);
if(lrecvcounts) MPIU_Free(lrecvcounts);
return MPI_SUCCESS;
}
