unsigned long
comm_send_to_rank(void* buf, int len, int dst_rank, void* _request)
{
MPI_Request* request = (MPI_Request*)_request;
if(dst_rank < 0 || dst_rank >= comm_size()){
printf("ERROR: Invalid dst rank(%d)\n", dst_rank);
comm_exit(1);
}
int sendtag = 99;
MPI_Isend(buf, len, MPI_BYTE, dst_rank, sendtag, MPI_COMM_WORLD, request);
return (unsigned long)request;
}
void recursive_inertial_bisection(unsigned* n, point** o, rcopy** idx)
{
int osize;
int orank;
int size;
mpi* oldcomm;
osize = comm_size();
orank = comm_rank();
for (size = osize; size != 1; size /= 2) {
ASSERT(size % 2 == 0);
oldcomm = enter_groups(orank / size, orank % size);
inertial_bisection(n, o, idx);
exit_groups(oldcomm);
}
}
unsigned long
comm_recv_from_rank(void* buf, int len, int src_rank, void* _request)
{
MPI_Request* request = (MPI_Request*)_request;
if(src_rank < 0 || src_rank >= comm_size()){
printf("ERROR: Invalid src rank(%d)\n", src_rank);
comm_exit(1);
}
int recvtag = 99;
MPI_Irecv(buf, len, MPI_BYTE, src_rank, recvtag, MPI_COMM_WORLD, request);
return (unsigned long)request;
}
unsigned long
comm_send_to_rank(void* buf, int len, int dst_rank)
{
MPI_Request* request = (MPI_Request*)malloc(sizeof(MPI_Request));
if (request == NULL){
printf("ERROR: malloc failed for mpi request\n");
comm_exit(1);
}
if(dst_rank < 0 || dst_rank >= comm_size()){
printf("ERROR: Invalid dst rank(%d)\n", dst_rank);
comm_exit(1);
}
int sendtag = 99;
MPI_Isend(buf, len, MPI_BYTE, dst_rank, sendtag, MPI_COMM_WORLD, request);
return (unsigned long)request;
}
unsigned long
comm_recv_from_rank(void* buf, int len, int src_rank)
{
MPI_Request* request = (MPI_Request*)malloc(sizeof(MPI_Request));
if (request == NULL){
printf("ERROR: malloc failed for mpi request\n");
comm_exit(1);
}
if(src_rank < 0 || src_rank >= comm_size()){
printf("ERROR: Invalid src rank(%d)\n", src_rank);
comm_exit(1);
}
int recvtag = 99;
MPI_Irecv(buf, len, MPI_BYTE, src_rank, recvtag, MPI_COMM_WORLD, request);
return (unsigned long)request;
}
static void partition(unsigned* n, point** o, rcopy** rc, plane mp)
{
unsigned pn;
unsigned nn;
point* po;
point* no;
rcopy* prc;
rcopy* nrc;
unsigned lin;
unsigned long tin;
unsigned lout;
unsigned long tout;
unsigned long in_i;
unsigned long out_i;
int rank_is_in;
unsigned ranks_in;
unsigned ranks_out;
unsigned long quo;
unsigned long rem;
unsigned i;
unsigned long dest_i;
int dest_rank;
unsigned rank, size;
rank = (unsigned) comm_rank();
size = (unsigned) comm_size();
pn = *n;
po = *o;
prc = *rc;
lin = count_local_in(pn, po, mp);
lout = pn - lin;
tin = mpi_add_ulong(comm_mpi(), lin);
tout = mpi_add_ulong(comm_mpi(), lout);
ranks_in = size / 2;
rank_is_in = (rank < ranks_in);
ranks_out = size - ranks_in;
if (rank_is_in) {
quo = tin / ranks_in;
rem = tin % ranks_in;
nn = (unsigned) quo;
if (rank == ranks_in - 1)
nn += (unsigned) rem;
} else {
quo = tout / ranks_out;
rem = tout % ranks_out;
nn = (unsigned) quo;
if (rank - ranks_in == ranks_out - 1)
nn += (unsigned) rem;
}
no = my_malloc(sizeof(point) * nn);
nrc = my_malloc(sizeof(rcopy) * nn);
in_i = mpi_exscan_ulong(comm_mpi(), lin);
out_i = mpi_exscan_ulong(comm_mpi(), lout);
for (i = 0; i < pn; ++i) {
if (plane_has(mp, po[i])) {
dest_i = in_i++;
dest_rank = (int) (dest_i / quo);
dest_rank = MIN(dest_rank, (int)ranks_in - 1);
ASSERT(dest_rank < (int)ranks_in);
} else {
dest_i = out_i++;
dest_rank = (int) (dest_i / quo);
dest_rank += ranks_in;
dest_rank = MIN(dest_rank, comm_size() - 1);
ASSERT((dest_rank - (int)ranks_in) < (int)ranks_out);
}
COMM_PACK(dest_i, dest_rank);
COMM_PACK(po[i], dest_rank);
COMM_PACK(prc[i], dest_rank);
}
comm_exch();
while (comm_recv()) {
COMM_UNPACK(dest_i);
if (rank_is_in)
i = (unsigned) (dest_i - (rank * quo));
else
i = (unsigned) (dest_i - ((rank - ranks_in) * quo));
COMM_UNPACK(no[i]);
ASSERT(plane_has(mp, no[i]) == rank_is_in);
COMM_UNPACK(nrc[i]);
}
my_free(po);
my_free(prc);
*n = nn;
*o = no;
*rc = nrc;
}