/* simple chained MPI_Ibcast */
static inline int bcast_sched_chain(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count, MPI_Datatype datatype, int fragsize, size_t size) {
int res, vrank, rpeer, speer, numfrag, fragcount, thiscount;
MPI_Aint ext;
char *buf;
RANK2VRANK(rank, vrank, root);
VRANK2RANK(rpeer, vrank-1, root);
VRANK2RANK(speer, vrank+1, root);
res = ompi_datatype_type_extent(datatype, &ext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
if (count == 0) {
return OMPI_SUCCESS;
}
numfrag = count * size/fragsize;
if ((count * size) % fragsize != 0) {
numfrag++;
}
fragcount = count/numfrag;
for (int fragnum = 0 ; fragnum < numfrag ; ++fragnum) {
buf = (char *) buffer + fragnum * fragcount * ext;
thiscount = fragcount;
if (fragnum == numfrag-1) {
/* last fragment may not be full */
thiscount = count - fragcount * fragnum;
}
/* root does not receive */
if (vrank != 0) {
res = NBC_Sched_recv (buf, false, thiscount, datatype, rpeer, schedule, true);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
}
/* last rank does not send */
if (vrank != p-1) {
res = NBC_Sched_send (buf, false, thiscount, datatype, speer, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
/* this barrier here seems awaward but isn't!!!! */
if (vrank == 0) {
res = NBC_Sched_barrier (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
}
}
}
return OMPI_SUCCESS;
}
static inline int bcast_sched_binomial(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count, MPI_Datatype datatype) {
int maxr, vrank, peer, r, res;
maxr = (int)ceil((log((double)p)/LOG2));
RANK2VRANK(rank, vrank, root);
/* receive from the right hosts */
if(vrank != 0) {
for(r=0; r<maxr; r++) {
if((vrank >= (1<<r)) && (vrank < (1<<(r+1)))) {
VRANK2RANK(peer, vrank-(1<<r), root);
res = NBC_Sched_recv(buffer, false, count, datatype, peer, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
}
}
res = NBC_Sched_barrier(schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; }
}
/* now send to the right hosts */
for(r=0; r<maxr; r++) {
if(((vrank + (1<<r) < p) && (vrank < (1<<r))) || (vrank == 0)) {
VRANK2RANK(peer, vrank+(1<<r), root);
res = NBC_Sched_send(buffer, false, count, datatype, peer, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
}
}
return NBC_OK;
}
/* simple chained MPI_Ibcast */
static inline int bcast_sched_chain(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count, MPI_Datatype datatype, int fragsize, int size) {
int res, vrank, rpeer, speer, numfrag, fragnum, fragcount, thiscount;
MPI_Aint ext;
char *buf;
RANK2VRANK(rank, vrank, root);
VRANK2RANK(rpeer, vrank-1, root);
VRANK2RANK(speer, vrank+1, root);
res = MPI_Type_extent(datatype, &ext);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
if(count == 0) return NBC_OK;
numfrag = count*size/fragsize;
if((count*size)%fragsize != 0) numfrag++;
fragcount = count/numfrag;
/*if(!rank) printf("numfrag: %i, count: %i, size: %i, fragcount: %i\n", numfrag, count, size, fragcount);*/
for(fragnum = 0; fragnum < numfrag; fragnum++) {
buf = (char*)buffer+fragnum*fragcount*ext;
thiscount = fragcount;
if(fragnum == numfrag-1) {
/* last fragment may not be full */
thiscount = count-fragcount*fragnum;
}
/* root does not receive */
if(vrank != 0) {
res = NBC_Sched_recv(buf, false, thiscount, datatype, rpeer, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
res = NBC_Sched_barrier(schedule);
}
/* last rank does not send */
if(vrank != p-1) {
res = NBC_Sched_send(buf, false, thiscount, datatype, speer, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
/* this barrier here seems awaward but isn't!!!! */
if(vrank == 0) res = NBC_Sched_barrier(schedule);
}
}
return NBC_OK;
}
static inline int bcast_sched_binomial(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count, MPI_Datatype datatype) {
int maxr, vrank, peer, res;
maxr = (int)ceil((log((double)p)/LOG2));
RANK2VRANK(rank, vrank, root);
/* receive from the right hosts */
if (vrank != 0) {
for (int r = 0 ; r < maxr ; ++r) {
if ((vrank >= (1 << r)) && (vrank < (1 << (r + 1)))) {
VRANK2RANK(peer, vrank - (1 << r), root);
res = NBC_Sched_recv (buffer, false, count, datatype, peer, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
}
}
res = NBC_Sched_barrier (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
}
/* now send to the right hosts */
for (int r = 0 ; r < maxr ; ++r) {
if (((vrank + (1 << r) < p) && (vrank < (1 << r))) || (vrank == 0)) {
VRANK2RANK(peer, vrank + (1 << r), root);
res = NBC_Sched_send (buffer, false, count, datatype, peer, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
}
}
return OMPI_SUCCESS;
}