int MPIX_Irsend_x(BIGMPI_CONST void *buf, MPI_Count count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request)
{
int rc = MPI_SUCCESS;
if (likely (count <= bigmpi_int_max )) {
rc = MPI_Irsend(buf, (int)count, datatype, dest, tag, comm, request);
} else {
MPI_Datatype newtype;
BigMPI_Type_contiguous(0,count, datatype, &newtype);
MPI_Type_commit(&newtype);
rc = MPI_Irsend(buf, 1, newtype, dest, tag, comm, request);
MPI_Type_free(&newtype);
}
return rc;
}
int main(int argc, char* argv[]) {
int myRank, nProcs;
MPI_Request req1, req2;
MPI_Status status1, status2;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
MPI_Comm_size(MPI_COMM_WORLD, &nProcs);
int right = (myRank + 1) % nProcs;
int left = myRank - 1;
if (left < 0) left = nProcs - 1;
int buf1[10], buf2[10];
for (int i = 0; i < 10; i++) {
buf2[i] = myRank * nProcs + 1;
}
MPI_Irecv(buf1, 10, MPI_INT, left, 123, MPI_COMM_WORLD, &req1);
MPI_Barrier(MPI_COMM_WORLD);
MPI_Irsend(buf2, 10, MPI_INT, right, 123, MPI_COMM_WORLD, &req2);
MPI_Wait(&req1, &status1);
MPI_Wait(&req2, &status2);
fprintf(stdout, "rank %d, send & recv finished\n", myRank);
MPI_Finalize();
return 0;
}
static int
kmr_alltoall_naive(KMR *mr, void *sbuf, void *rbuf, int cnt)
{
MPI_Comm comm = mr->comm;
int nprocs = mr->nprocs;
int rank = mr->rank;
int tag = KMR_TAG_ATOA;
MPI_Request *rqs = kmr_malloc(sizeof(MPI_Request) * (size_t)(nprocs * 2));
int cc;
char *r = rbuf;
for (int i = 0; i < nprocs; i++) {
cc = MPI_Irecv(&r[i * cnt], cnt, MPI_BYTE,
i, tag, comm, &rqs[i]);
assert(cc == MPI_SUCCESS);
}
char *s = sbuf;
int peer;
peer = (rank % nprocs);
for (int i = 0; i < nprocs; i++) {
peer++;
if (peer >= nprocs) {
peer -= nprocs;
}
cc = MPI_Irsend(&s[peer * cnt], cnt, MPI_BYTE,
peer, tag, comm, &rqs[nprocs + peer]);
assert(cc == MPI_SUCCESS);
}
cc = MPI_Waitall((2 * nprocs), rqs, MPI_STATUSES_IGNORE);
assert(cc == MPI_SUCCESS);
kmr_free(rqs, (sizeof(MPI_Request) * (size_t)(nprocs * 2)));
return MPI_SUCCESS;
}
void TreeCommunicatorLevel::send_sample(const struct geopm_sample_message_s &sample)
{
MPI_Request request;
// Don't check return code or hold onto request, drop message if receiver not ready
(void) MPI_Irsend(const_cast<struct geopm_sample_message_s*>(&sample), 1, m_sample_mpi_type, 0, GEOPM_SAMPLE_TAG, m_comm, &request);
}
void mpi_irsend_(void *buf, int* count, int* datatype, int* dst,
int* tag, int* comm, int* request, int* ierr) {
MPI_Request req;
*ierr = MPI_Irsend(buf, *count, get_datatype(*datatype), *dst, *tag,
get_comm(*comm), &req);
if(*ierr == MPI_SUCCESS) {
*request = new_request(req);
}
}
HYPRE_Int
hypre_MPI_Irsend( void *buf,
HYPRE_Int count,
hypre_MPI_Datatype datatype,
HYPRE_Int dest,
HYPRE_Int tag,
hypre_MPI_Comm comm,
hypre_MPI_Request *request )
{
return (HYPRE_Int) MPI_Irsend(buf, (hypre_int)count, datatype,
(hypre_int)dest, (hypre_int)tag, comm, request);
}
/**
* vsg_packed_msg_irsend:
* @pm: a #VsgPackedMsg.
* @dst: the destination task id.
* @tag: an integer message tag.
* @request: the corresponding request object
*
* Sends stored message to the specified destination with the specified tag in
* a non blocking mode. @request is provided for output.
*/
void vsg_packed_msg_irsend (VsgPackedMsg *pm, gint dst, gint tag,
MPI_Request *request)
{
gint ierr;
_trace_write_msg_send (pm, "irsend", dst, tag);
ierr = MPI_Irsend (pm->buffer, pm->position, MPI_PACKED, dst, tag,
pm->communicator, request);
if (ierr != MPI_SUCCESS) vsg_mpi_error_output (ierr);
}
void TreeCommunicatorLevel::send_policy(const std::vector<struct geopm_policy_message_s> &policy, size_t length)
{
size_t dest;
MPI_Request request;
if (m_rank != 0) {
throw Exception("called send_policy() from rank not at root of level", GEOPM_ERROR_CTL_COMM, __FILE__, __LINE__);
}
dest = 0;
for (auto policy_it = policy.begin(); dest != length; ++policy_it, ++dest) {
// Don't check return code or hold onto request, drop message if receiver not ready
(void) MPI_Irsend(const_cast<struct geopm_policy_message_s*>(&(*policy_it)), 1, m_policy_mpi_type, dest, GEOPM_POLICY_TAG, m_comm, &request);
}
}
void mpi_irsend_f(char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *request, MPI_Fint *ierr)
{
MPI_Datatype c_type = MPI_Type_f2c(*datatype);
MPI_Request c_req;
MPI_Comm c_comm;
c_comm = MPI_Comm_f2c (*comm);
*ierr = OMPI_INT_2_FINT(MPI_Irsend(OMPI_F2C_BOTTOM(buf), OMPI_FINT_2_INT(*count),
c_type, OMPI_FINT_2_INT(*dest),
OMPI_FINT_2_INT(*tag), c_comm,
&c_req));
if (MPI_SUCCESS == OMPI_FINT_2_INT(*ierr)) {
*request = MPI_Request_c2f(c_req);
}
}
int main( int argc, char *argv[] )
{
int errs = 0;
int rank, size, /* source, */ dest;
MPI_Comm comm;
MPI_Status status;
MPI_Request req;
static int bufsizes[4] = { 1, 100, 10000, 1000000 };
char *buf;
#ifdef TEST_IRSEND
int veryPicky = 0; /* Set to 1 to test "quality of implementation" in
a tricky part of cancel */
#endif
int cs, flag, n;
MTest_Init( &argc, &argv );
comm = MPI_COMM_WORLD;
MPI_Comm_rank( comm, &rank );
MPI_Comm_size( comm, &size );
/* source = 0; */
dest = size - 1;
MTestPrintfMsg( 1, "Starting scancel test\n" );
for (cs=0; cs<4; cs++) {
if (rank == 0) {
n = bufsizes[cs];
buf = (char *)malloc( n );
if (!buf) {
fprintf( stderr, "Unable to allocate %d bytes\n", n );
MPI_Abort( MPI_COMM_WORLD, 1 );
exit(1);
}
MTestPrintfMsg( 1, "(%d) About to create isend and cancel\n",cs );
MPI_Isend( buf, n, MPI_CHAR, dest, cs+n+1, comm, &req );
MPI_Cancel( &req );
MPI_Wait( &req, &status );
MTestPrintfMsg( 1, "Completed wait on isend\n" );
MPI_Test_cancelled( &status, &flag );
if (!flag) {
errs ++;
printf( "Failed to cancel an Isend request\n" );
fflush(stdout);
}
else
{
n = 0;
}
/* Send the size, zero for successfully cancelled */
MPI_Send( &n, 1, MPI_INT, dest, 123, comm );
/* Send the tag so the message can be received */
n = cs+n+1;
MPI_Send( &n, 1, MPI_INT, dest, 123, comm );
free( buf );
}
else if (rank == dest)
{
int nn, tag;
char *btemp;
MPI_Recv( &nn, 1, MPI_INT, 0, 123, comm, &status );
MPI_Recv( &tag, 1, MPI_INT, 0, 123, comm, &status );
if (nn > 0)
{
/* If the message was not cancelled, receive it here */
btemp = (char*)malloc( nn );
if (!btemp)
{
fprintf( stderr, "Unable to allocate %d bytes\n", nn );
MPI_Abort( MPI_COMM_WORLD, 1 );
exit(1);
}
MPI_Recv( btemp, nn, MPI_CHAR, 0, tag, comm, &status );
free(btemp);
}
}
MPI_Barrier( comm );
if (rank == 0) {
char *bsendbuf;
int bsendbufsize;
int bf, bs;
n = bufsizes[cs];
buf = (char *)malloc( n );
if (!buf) {
fprintf( stderr, "Unable to allocate %d bytes\n", n );
MPI_Abort( MPI_COMM_WORLD, 1 );
exit(1);
}
bsendbufsize = n + MPI_BSEND_OVERHEAD;
bsendbuf = (char *)malloc( bsendbufsize );
if (!bsendbuf) {
fprintf( stderr, "Unable to allocate %d bytes for bsend\n", n );
MPI_Abort( MPI_COMM_WORLD, 1 );
exit(1);
}
MPI_Buffer_attach( bsendbuf, bsendbufsize );
MTestPrintfMsg( 1, "About to create and cancel ibsend\n" );
MPI_Ibsend( buf, n, MPI_CHAR, dest, cs+n+2, comm, &req );
MPI_Cancel( &req );
MPI_Wait( &req, &status );
MPI_Test_cancelled( &status, &flag );
if (!flag) {
errs ++;
printf( "Failed to cancel an Ibsend request\n" );
fflush(stdout);
}
else
{
n = 0;
}
/* Send the size, zero for successfully cancelled */
MPI_Send( &n, 1, MPI_INT, dest, 123, comm );
/* Send the tag so the message can be received */
n = cs+n+2;
MPI_Send( &n, 1, MPI_INT, dest, 123, comm );
free( buf );
MPI_Buffer_detach( &bf, &bs );
free( bsendbuf );
}
else if (rank == dest)
{
int nn, tag;
char *btemp;
MPI_Recv( &nn, 1, MPI_INT, 0, 123, comm, &status );
MPI_Recv( &tag, 1, MPI_INT, 0, 123, comm, &status );
if (nn > 0)
{
/* If the message was not cancelled, receive it here */
btemp = (char*)malloc( nn );
if (!btemp)
{
fprintf( stderr, "Unable to allocate %d bytes\n", nn);
MPI_Abort( MPI_COMM_WORLD, 1 );
exit(1);
}
MPI_Recv( btemp, nn, MPI_CHAR, 0, tag, comm, &status );
free(btemp);
}
}
MPI_Barrier( comm );
/* Because this test is erroneous, we do not perform it unless
TEST_IRSEND is defined. */
#ifdef TEST_IRSEND
/* We avoid ready send to self because an implementation
is free to detect the error in delivering a message to
itself without a pending receive; we could also check
for an error return from the MPI_Irsend */
if (rank == 0 && dest != rank) {
n = bufsizes[cs];
buf = (char *)malloc( n );
if (!buf) {
fprintf( stderr, "Unable to allocate %d bytes\n", n );
MPI_Abort( MPI_COMM_WORLD, 1 );
exit(1);
}
MTestPrintfMsg( 1, "About to create and cancel irsend\n" );
MPI_Irsend( buf, n, MPI_CHAR, dest, cs+n+3, comm, &req );
MPI_Cancel( &req );
MPI_Wait( &req, &status );
MPI_Test_cancelled( &status, &flag );
/* This can be pretty ugly. The standard is clear (Section 3.8)
that either a sent message is received or the
sent message is successfully cancelled. Since this message
can never be received, the cancel must complete
successfully.
However, since there is no matching receive, this
program is erroneous. In this case, we can't really
flag this as an error */
if (!flag && veryPicky) {
errs ++;
printf( "Failed to cancel an Irsend request\n" );
fflush(stdout);
}
if (flag)
{
n = 0;
}
/* Send the size, zero for successfully cancelled */
MPI_Send( &n, 1, MPI_INT, dest, 123, comm );
/* Send the tag so the message can be received */
n = cs+n+3;
MPI_Send( &n, 1, MPI_INT, dest, 123, comm );
free( buf );
}
else if (rank == dest)
{
int n, tag;
char *btemp;
MPI_Recv( &n, 1, MPI_INT, 0, 123, comm, &status );
MPI_Recv( &tag, 1, MPI_INT, 0, 123, comm, &status );
if (n > 0)
{
/* If the message was not cancelled, receive it here */
btemp = (char*)malloc( n );
if (!btemp)
{
fprintf( stderr, "Unable to allocate %d bytes\n", n);
MPI_Abort( MPI_COMM_WORLD, 1 );
exit(1);
}
MPI_Recv( btemp, n, MPI_CHAR, 0, tag, comm, &status );
free(btemp);
}
}
MPI_Barrier( comm );
#endif
if (rank == 0) {
n = bufsizes[cs];
buf = (char *)malloc( n );
if (!buf) {
fprintf( stderr, "Unable to allocate %d bytes\n", n );
MPI_Abort( MPI_COMM_WORLD, 1 );
exit(1);
}
MTestPrintfMsg( 1, "About to create and cancel issend\n" );
MPI_Issend( buf, n, MPI_CHAR, dest, cs+n+4, comm, &req );
MPI_Cancel( &req );
MPI_Wait( &req, &status );
MPI_Test_cancelled( &status, &flag );
if (!flag) {
errs ++;
printf( "Failed to cancel an Issend request\n" );
fflush(stdout);
}
else
{
n = 0;
}
/* Send the size, zero for successfully cancelled */
MPI_Send( &n, 1, MPI_INT, dest, 123, comm );
/* Send the tag so the message can be received */
n = cs+n+4;
MPI_Send( &n, 1, MPI_INT, dest, 123, comm );
free( buf );
}
else if (rank == dest)
{
int nn, tag;
char *btemp;
MPI_Recv( &nn, 1, MPI_INT, 0, 123, comm, &status );
MPI_Recv( &tag, 1, MPI_INT, 0, 123, comm, &status );
if (nn > 0)
{
/* If the message was not cancelled, receive it here */
btemp = (char*)malloc( nn );
if (!btemp)
{
fprintf( stderr, "Unable to allocate %d bytes\n", nn);
MPI_Abort( MPI_COMM_WORLD, 1 );
exit(1);
}
MPI_Recv( btemp, nn, MPI_CHAR, 0, tag, comm, &status );
free(btemp);
}
}
MPI_Barrier( comm );
}
MTest_Finalize( errs );
MPI_Finalize();
return 0;
}
int
main (int argc, char **argv)
{
int nprocs = -1;
int rank = -1;
MPI_Comm comm = MPI_COMM_WORLD;
char processor_name[128];
int namelen = 128;
int bbuf[(BUF_SIZE + MPI_BSEND_OVERHEAD) * 2 * NUM_BSEND_TYPES];
int buf[BUF_SIZE * 2 * NUM_SEND_TYPES];
int i, j, k, at_size, send_t_number, index, outcount, total, flag;
int num_errors, error_count, indices[2 * NUM_SEND_TYPES];
MPI_Request aReq[2 * NUM_SEND_TYPES];
MPI_Status aStatus[2 * NUM_SEND_TYPES];
/* init */
MPI_Init (&argc, &argv);
MPI_Comm_size (comm, &nprocs);
MPI_Comm_rank (comm, &rank);
MPI_Get_processor_name (processor_name, &namelen);
printf ("(%d) is alive on %s\n", rank, processor_name);
fflush (stdout);
MPI_Buffer_attach (bbuf, sizeof(int) *
(BUF_SIZE + MPI_BSEND_OVERHEAD) * 2 * NUM_BSEND_TYPES);
if (rank == 0) {
/* set up persistent sends... */
send_t_number = NUM_SEND_TYPES - NUM_PERSISTENT_SEND_TYPES;
MPI_Send_init (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Send_init (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
send_t_number++;
MPI_Bsend_init (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Bsend_init (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
send_t_number++;
MPI_Rsend_init (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Rsend_init (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
send_t_number++;
MPI_Ssend_init (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Ssend_init (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
}
for (k = 0; k < (NUM_COMPLETION_MECHANISMS * 2); k++) {
if (rank == 0) {
/* initialize all of the send buffers */
for (j = 0; j < NUM_SEND_TYPES; j++) {
for (i = 0; i < BUF_SIZE; i++) {
buf[2 * j * BUF_SIZE + i] = i;
buf[((2 * j + 1) * BUF_SIZE) + i] = BUF_SIZE - 1 - i;
}
}
}
else if (rank == 1) {
/* zero out all of the receive buffers */
bzero (buf, sizeof(int) * BUF_SIZE * 2 * NUM_SEND_TYPES);
}
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
/* set up transient sends... */
send_t_number = 0;
MPI_Isend (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Isend (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
send_t_number++;
MPI_Ibsend (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Ibsend (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
send_t_number++;
/* Barrier to ensure receives are posted for rsends... */
MPI_Barrier(MPI_COMM_WORLD);
MPI_Irsend (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Irsend (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
send_t_number++;
MPI_Issend (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Issend (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
/* just to be paranoid */
send_t_number++;
assert (send_t_number == NUM_SEND_TYPES - NUM_PERSISTENT_SEND_TYPES);
/* start the persistent sends... */
if (k % 2) {
MPI_Startall (NUM_PERSISTENT_SEND_TYPES * 2, &aReq[2 * send_t_number]);
}
else {
for (j = 0; j < NUM_PERSISTENT_SEND_TYPES * 2; j++) {
MPI_Start (&aReq[2 * send_t_number + j]);
}
}
/* NOTE: Changing the send buffer of a Bsend is NOT an error... */
for (j = 0; j < NUM_SEND_TYPES; j++) {
/* muck the buffers */
buf[j * 2 * BUF_SIZE + (BUF_SIZE >> 1)] = BUF_SIZE;
}
printf ("USER MSG: 6 change send buffer errors in iteration #%d:\n", k);
/* complete the sends */
switch (k/2) {
case 0:
/* use MPI_Wait */
for (j = 0; j < NUM_SEND_TYPES * 2; j++) {
MPI_Wait (&aReq[j], &aStatus[j]);
}
break;
case 1:
/* use MPI_Waitall */
MPI_Waitall (NUM_SEND_TYPES * 2, aReq, aStatus);
break;
case 2:
/* use MPI_Waitany */
for (j = 0; j < NUM_SEND_TYPES * 2; j++) {
MPI_Waitany (NUM_SEND_TYPES * 2, aReq, &index, aStatus);
}
break;
case 3:
/* use MPI_Waitsome */
total = 0;
while (total < NUM_SEND_TYPES * 2) {
MPI_Waitsome (NUM_SEND_TYPES * 2, aReq, &outcount, indices, aStatus);
total += outcount;
}
break;
case 4:
/* use MPI_Test */
for (j = 0; j < NUM_SEND_TYPES * 2; j++) {
flag = 0;
while (!flag) {
MPI_Test (&aReq[j], &flag, &aStatus[j]);
}
}
break;
case 5:
/* use MPI_Testall */
flag = 0;
while (!flag) {
MPI_Testall (NUM_SEND_TYPES * 2, aReq, &flag, aStatus);
}
break;
case 6:
/* use MPI_Testany */
for (j = 0; j < NUM_SEND_TYPES * 2; j++) {
flag = 0;
while (!flag) {
MPI_Testany (NUM_SEND_TYPES * 2, aReq, &index, &flag, aStatus);
}
}
break;
case 7:
/* use MPI_Testsome */
total = 0;
while (total < NUM_SEND_TYPES * 2) {
outcount = 0;
while (!outcount) {
MPI_Testsome (NUM_SEND_TYPES * 2, aReq,
&outcount, indices, aStatus);
}
total += outcount;
}
break;
default:
assert (0);
break;
}
}
else if (rank == 1) {
static void test_pair (void)
{
int prev, next, count, tag, index, i, outcount, indices[2];
int rank, size, flag, ierr, reqcount;
double send_buf[TEST_SIZE], recv_buf[TEST_SIZE];
double buffered_send_buf[TEST_SIZE * 2 + MPI_BSEND_OVERHEAD]; /* factor of two is based on guessing - only dynamic allocation would be safe */
void *buffer;
MPI_Status statuses[2];
MPI_Status status;
MPI_Request requests[2];
MPI_Comm dupcom, intercom;
#ifdef V_T
struct _VT_FuncFrameHandle {
char *name;
int func;
int frame;
};
typedef struct _VT_FuncFrameHandle VT_FuncFrameHandle_t;
VT_FuncFrameHandle_t normal_sends,
buffered_sends,
buffered_persistent_sends,
ready_sends,
sync_sends,
nblock_sends,
nblock_rsends,
nblock_ssends,
pers_sends,
pers_rsends,
pers_ssends,
sendrecv,
sendrecv_repl,
intercomm;
int classid;
VT_classdef( "Application:test_pair", &classid );
#define VT_REGION_DEF( _name, _nameframe, _class ) \
(_nameframe).name=_name; \
VT_funcdef( (_nameframe).name, _class, &((_nameframe).func) );
#define VT_BEGIN_REGION( _nameframe ) \
LOCDEF(); \
VT_begin( (_nameframe).func )
#define VT_END_REGION( _nameframe ) \
LOCDEF(); VT_end( (_nameframe).func )
#else
#define VT_REGION_DEF( _name, _nameframe, _class )
#define VT_BEGIN_REGION( _nameframe )
#define VT_END_REGION( _nameframe )
#endif
ierr = MPI_Comm_rank(MPI_COMM_WORLD, &rank);
ierr = MPI_Comm_size(MPI_COMM_WORLD, &size);
if ( size < 2 ) {
if ( rank == 0 ) {
printf("Program needs to be run on at least 2 processes.\n");
}
ierr = MPI_Abort( MPI_COMM_WORLD, 66 );
}
ierr = MPI_Comm_dup(MPI_COMM_WORLD, &dupcom);
if ( rank >= 2 ) {
/* printf( "%d Calling finalize.\n", rank ); */
ierr = MPI_Finalize( );
exit(0);
}
next = rank + 1;
if (next >= 2)
next = 0;
prev = rank - 1;
if (prev < 0)
prev = 1;
VT_REGION_DEF( "Normal_Sends", normal_sends, classid );
VT_REGION_DEF( "Buffered_Sends", buffered_sends, classid );
VT_REGION_DEF( "Buffered_Persistent_Sends", buffered_persistent_sends, classid );
VT_REGION_DEF( "Ready_Sends", ready_sends, classid );
VT_REGION_DEF( "Sync_Sends", sync_sends, classid );
VT_REGION_DEF( "nblock_Sends", nblock_sends, classid );
VT_REGION_DEF( "nblock_RSends", nblock_rsends, classid );
VT_REGION_DEF( "nblock_SSends", nblock_ssends, classid );
VT_REGION_DEF( "Pers_Sends", pers_sends, classid );
VT_REGION_DEF( "Pers_RSends", pers_rsends, classid );
VT_REGION_DEF( "Pers_SSends", pers_ssends, classid );
VT_REGION_DEF( "SendRecv", sendrecv, classid );
VT_REGION_DEF( "SendRevc_Repl", sendrecv_repl, classid );
VT_REGION_DEF( "InterComm", intercomm, classid );
/*
* Normal sends
*/
VT_BEGIN_REGION( normal_sends );
if (rank == 0)
printf ("Send\n");
tag = 0x100;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
LOCDEF();
MPI_Send(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, &status);
msg_check(recv_buf, prev, tag, count, &status, TEST_SIZE, "send and recv");
}
else {
LOCDEF();
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,"send and recv");
init_test_data(recv_buf,TEST_SIZE,1);
MPI_Send(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( normal_sends );
/*
* Buffered sends
*/
VT_BEGIN_REGION( buffered_sends );
if (rank == 0)
printf ("Buffered Send\n");
tag = 138;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
LOCDEF();
MPI_Buffer_attach(buffered_send_buf, sizeof(buffered_send_buf));
MPI_Bsend(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
MPI_Buffer_detach(&buffer, &size);
if(buffer != buffered_send_buf || size != sizeof(buffered_send_buf)) {
printf ("[%d] Unexpected buffer returned by MPI_Buffer_detach(): %p/%d != %p/%d\n", rank, buffer, size, buffered_send_buf, (int)sizeof(buffered_send_buf));
MPI_Abort(MPI_COMM_WORLD, 201);
}
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, &status);
msg_check(recv_buf, prev, tag, count, &status, TEST_SIZE, "send and recv");
}
else {
LOCDEF();
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,"send and recv");
init_test_data(recv_buf,TEST_SIZE,1);
MPI_Send(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( buffered_sends );
/*
* Buffered sends
*/
VT_BEGIN_REGION( buffered_persistent_sends );
if (rank == 0)
printf ("Buffered Persistent Send\n");
tag = 238;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
LOCDEF();
MPI_Buffer_attach(buffered_send_buf, sizeof(buffered_send_buf));
MPI_Bsend_init(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD, requests);
MPI_Start(requests);
MPI_Wait(requests, statuses);
MPI_Request_free(requests);
MPI_Buffer_detach(&buffer, &size);
if(buffer != buffered_send_buf || size != sizeof(buffered_send_buf)) {
printf ("[%d] Unexpected buffer returned by MPI_Buffer_detach(): %p/%d != %p/%d\n", rank, buffer, size, buffered_send_buf, (int)sizeof(buffered_send_buf));
MPI_Abort(MPI_COMM_WORLD, 201);
}
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, &status);
msg_check(recv_buf, prev, tag, count, &status, TEST_SIZE, "send and recv");
}
else {
LOCDEF();
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,"send and recv");
init_test_data(recv_buf,TEST_SIZE,1);
MPI_Send(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( buffered_persistent_sends );
/*
* Ready sends. Note that we must insure that the receive is posted
* before the rsend; this requires using Irecv.
*/
VT_BEGIN_REGION( ready_sends );
if (rank == 0)
printf ("Rsend\n");
tag = 1456;
count = TEST_SIZE / 3;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
MPI_Recv(MPI_BOTTOM, 0, MPI_INT, next, tag, MPI_COMM_WORLD, &status);
MPI_Rsend(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
MPI_Probe(MPI_ANY_SOURCE, tag, MPI_COMM_WORLD, &status);
if (status.MPI_SOURCE != prev)
printf ("Incorrect src, expected %d, got %d\n",prev, status.MPI_SOURCE);
if (status.MPI_TAG != tag)
printf ("Incorrect tag, expected %d, got %d\n",tag, status.MPI_TAG);
MPI_Get_count(&status, MPI_DOUBLE, &i);
if (i != count)
printf ("Incorrect count, expected %d, got %d\n",count,i);
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"rsend and recv");
}
else {
MPI_Irecv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, requests);
MPI_Send( MPI_BOTTOM, 0, MPI_INT, next, tag, MPI_COMM_WORLD);
MPI_Wait(requests, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"rsend and recv");
init_test_data(recv_buf,TEST_SIZE,1);
MPI_Send(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( ready_sends );
/*
* Synchronous sends
*/
VT_BEGIN_REGION( sync_sends );
if (rank == 0)
printf ("Ssend\n");
tag = 1789;
count = TEST_SIZE / 3;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
MPI_Iprobe(MPI_ANY_SOURCE, tag, MPI_COMM_WORLD, &flag, &status);
if (flag)
printf ("Iprobe succeeded! source %d, tag %d\n",status.MPI_SOURCE,
status.MPI_TAG);
MPI_Ssend(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
while (!flag)
MPI_Iprobe(MPI_ANY_SOURCE, tag, MPI_COMM_WORLD, &flag, &status);
if (status.MPI_SOURCE != prev)
printf ("Incorrect src, expected %d, got %d\n",prev, status.MPI_SOURCE);
if (status.MPI_TAG != tag)
printf ("Incorrect tag, expected %d, got %d\n",tag, status.MPI_TAG);
MPI_Get_count(&status, MPI_DOUBLE, &i);
if (i != count)
printf ("Incorrect count, expected %d, got %d\n",count,i);
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE, "ssend and recv");
}
else {
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE, "ssend and recv"); init_test_data(recv_buf,TEST_SIZE,1);
MPI_Ssend(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( sync_sends );
/*
* Nonblocking normal sends
*/
VT_BEGIN_REGION( nblock_sends );
if (rank == 0)
printf ("Isend\n");
tag = 2123;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
MPI_Irecv(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, requests);
init_test_data(send_buf,TEST_SIZE,0);
MPI_Isend(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD,
(requests+1));
MPI_Waitall(2, requests, statuses);
rq_check( requests, 2, "isend and irecv" );
msg_check(recv_buf,prev,tag,count,statuses, TEST_SIZE,"isend and irecv");
}
else {
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check(recv_buf,prev,tag,count,&status, TEST_SIZE,"isend and irecv"); init_test_data(recv_buf,TEST_SIZE,1);
MPI_Isend(recv_buf, count, MPI_DOUBLE, next, tag,MPI_COMM_WORLD,
(requests));
MPI_Wait((requests), &status);
rq_check(requests, 1, "isend (and recv)");
}
VT_END_REGION( nblock_sends );
/*
* Nonblocking ready sends
*/
VT_BEGIN_REGION( nblock_rsends );
if (rank == 0)
printf ("Irsend\n");
tag = 2456;
count = TEST_SIZE / 3;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
MPI_Irecv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, requests);
init_test_data(send_buf,TEST_SIZE,0);
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, next, 0,
MPI_BOTTOM, 0, MPI_INT, next, 0,
dupcom, &status);
MPI_Irsend(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD,
(requests+1));
reqcount = 0;
while (reqcount != 2) {
MPI_Waitany( 2, requests, &index, statuses);
if( index == 0 ) {
memcpy( &status, statuses, sizeof(status) );
}
reqcount++;
}
rq_check( requests, 1, "irsend and irecv");
msg_check(recv_buf,prev,tag,count,&status, TEST_SIZE,"irsend and irecv");
}
else {
MPI_Irecv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, requests);
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, next, 0,
MPI_BOTTOM, 0, MPI_INT, next, 0,
dupcom, &status);
flag = 0;
while (!flag)
MPI_Test(requests, &flag, &status);
rq_check( requests, 1, "irsend and irecv (test)");
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"irsend and irecv"); init_test_data(recv_buf,TEST_SIZE,1);
MPI_Irsend(recv_buf, count, MPI_DOUBLE, next, tag,
MPI_COMM_WORLD, requests);
MPI_Waitall(1, requests, statuses);
rq_check( requests, 1, "irsend and irecv");
}
VT_END_REGION( nblock_rsends );
/*
* Nonblocking synchronous sends
*/
VT_BEGIN_REGION( nblock_ssends );
if (rank == 0)
printf ("Issend\n");
tag = 2789;
count = TEST_SIZE / 3;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
MPI_Irecv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, requests );
init_test_data(send_buf,TEST_SIZE,0);
MPI_Issend(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD,
(requests+1));
flag = 0;
while (!flag)
MPI_Testall(2, requests, &flag, statuses);
rq_check( requests, 2, "issend and irecv (testall)");
msg_check( recv_buf, prev, tag, count, statuses, TEST_SIZE,
"issend and recv");
}
else {
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"issend and recv"); init_test_data(recv_buf,TEST_SIZE,1);
MPI_Issend(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD,requests);
flag = 0;
while (!flag)
MPI_Testany(1, requests, &index, &flag, statuses);
rq_check( requests, 1, "issend and recv (testany)");
}
VT_END_REGION( nblock_ssends );
/*
* Persistent normal sends
*/
VT_BEGIN_REGION( pers_sends );
if (rank == 0)
printf ("Send_init\n");
tag = 3123;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
MPI_Send_init(send_buf, count, MPI_DOUBLE, next, tag,
MPI_COMM_WORLD, requests);
MPI_Recv_init(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, (requests+1));
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
MPI_Startall(2, requests);
MPI_Waitall(2, requests, statuses);
msg_check( recv_buf, prev, tag, count, (statuses+1),
TEST_SIZE, "persistent send/recv");
}
else {
MPI_Start((requests+1));
MPI_Wait((requests+1), &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"persistent send/recv");
init_test_data(send_buf,TEST_SIZE,1);
MPI_Start(requests);
MPI_Wait(requests, &status);
}
MPI_Request_free(requests);
MPI_Request_free((requests+1));
VT_END_REGION( pers_sends );
/*
* Persistent ready sends
*/
VT_BEGIN_REGION( pers_rsends );
if (rank == 0)
printf ("Rsend_init\n");
tag = 3456;
count = TEST_SIZE / 3;
clear_test_data(recv_buf,TEST_SIZE);
MPI_Rsend_init(send_buf, count, MPI_DOUBLE, next, tag,
MPI_COMM_WORLD, requests);
MPI_Recv_init(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, (requests+1));
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0); MPI_Barrier( MPI_COMM_WORLD );
MPI_Startall(2, requests);
reqcount = 0;
while (reqcount != 2) {
MPI_Waitsome(2, requests, &outcount, indices, statuses);
for (i=0; i<outcount; i++) {
if (indices[i] == 1) {
msg_check( recv_buf, prev, tag, count, (statuses+i),
TEST_SIZE, "waitsome");
}
reqcount++;
}
}
}
else {
MPI_Start((requests+1)); MPI_Barrier( MPI_COMM_WORLD );
flag = 0;
while (!flag)
MPI_Test((requests+1), &flag, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE, "test");
init_test_data(send_buf,TEST_SIZE,1);
MPI_Start(requests);
MPI_Wait(requests, &status);
}
MPI_Request_free(requests);
MPI_Request_free((requests+1));
VT_END_REGION( pers_rsends );
/*
* Persistent synchronous sends
*/
VT_BEGIN_REGION( pers_ssends );
if (rank == 0)
printf ("Ssend_init\n");
tag = 3789;
count = TEST_SIZE / 3;
clear_test_data(recv_buf,TEST_SIZE);
MPI_Ssend_init(send_buf, count, MPI_DOUBLE, next, tag,
MPI_COMM_WORLD, (requests+1));
MPI_Recv_init(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, requests);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
MPI_Startall(2, requests);
reqcount = 0;
while (reqcount != 2) {
MPI_Testsome(2, requests, &outcount, indices, statuses);
for (i=0; i<outcount; i++) {
if (indices[i] == 0) {
msg_check( recv_buf, prev, tag, count, (statuses+i),
TEST_SIZE, "testsome");
}
reqcount++;
}
}
}
else {
MPI_Start(requests);
flag = 0;
while (!flag)
MPI_Testany(1, requests, &index, &flag, statuses);
msg_check( recv_buf, prev, tag, count, statuses, TEST_SIZE, "testany" );
init_test_data(send_buf,TEST_SIZE,1);
MPI_Start((requests+1));
MPI_Wait((requests+1), &status);
}
MPI_Request_free(requests);
MPI_Request_free((requests+1));
VT_END_REGION( pers_ssends );
/*
* Send/receive.
*/
VT_BEGIN_REGION( sendrecv );
if (rank == 0)
printf ("Sendrecv\n");
tag = 4123;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
MPI_Sendrecv(send_buf, count, MPI_DOUBLE, next, tag,
recv_buf, count, MPI_DOUBLE, prev, tag,
MPI_COMM_WORLD, &status );
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"sendrecv");
}
else {
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"recv/send"); init_test_data(recv_buf,TEST_SIZE,1);
MPI_Send(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( sendrecv );
#ifdef V_T
VT_flush();
#endif
/*
* Send/receive replace.
*/
VT_BEGIN_REGION( sendrecv_repl );
if (rank == 0)
printf ("Sendrecv_replace\n");
tag = 4456;
count = TEST_SIZE / 3;
if (rank == 0) {
init_test_data(recv_buf, TEST_SIZE,0);
for (i=count; i< TEST_SIZE; i++)
recv_buf[i] = 0.0;
MPI_Sendrecv_replace(recv_buf, count, MPI_DOUBLE,
next, tag, prev, tag, MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"sendrecvreplace");
}
else {
clear_test_data(recv_buf,TEST_SIZE);
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"recv/send for replace"); init_test_data(recv_buf,TEST_SIZE,1);
MPI_Send(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( sendrecv_repl );
/*
* Send/Receive via inter-communicator
*/
VT_BEGIN_REGION( intercomm );
MPI_Intercomm_create(MPI_COMM_SELF, 0, MPI_COMM_WORLD, next, 1, &intercom);
if (rank == 0)
printf ("Send via inter-communicator\n");
tag = 4018;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
LOCDEF();
MPI_Send(send_buf, count, MPI_DOUBLE, 0, tag, intercom);
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, intercom, &status);
msg_check(recv_buf, 0, tag, count, &status, TEST_SIZE, "send and recv via inter-communicator");
}
else if (rank == 1) {
LOCDEF();
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
intercom, &status);
msg_check( recv_buf, 0, tag, count, &status, TEST_SIZE,"send and recv via inter-communicator");
init_test_data(recv_buf,TEST_SIZE,0);
MPI_Send(recv_buf, count, MPI_DOUBLE, 0, tag, intercom);
}
VT_END_REGION( normal_sends );
MPI_Comm_free(&intercom);
MPI_Comm_free(&dupcom);
}
int main(int argc, char* argv[])
{
int requested = MPI_THREAD_FUNNELED;
int provided = MPI_THREAD_SINGLE;
MPI_Init_thread(&argc, &argv, requested, &provided);
if (provided<requested)
exit(provided);
int world_rank, world_size;
MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
MPI_Comm_size(MPI_COMM_WORLD, &world_size);
if (world_rank==0)
{
printf("hello world from rank %d of %d \n", world_rank, world_size );
fflush(stdout);
}
/************************************************************************/
for (int n=1; n<=67108864; n*=2)
{
size_t bytes = n * sizeof(int);
int * rbuf = (int *) safemalloc(bytes);
for (int i=0; i<n; i++)
rbuf[i] = -1;
int * sbuf = (int *) safemalloc(bytes);
for (int i=0; i<n; i++)
sbuf[i] = world_rank;
int send_to = (world_rank>0 ? world_rank-1 : world_size-1);
int recv_from = (world_rank<(world_size-1) ? world_rank+1 : 0);
MPI_Request req[2];
MPI_Irecv( rbuf, n, MPI_INT, recv_from, 0, MPI_COMM_WORLD, &req[0] );
MPI_Barrier(MPI_COMM_WORLD);
uint64_t t0 = GetTimeBase();
MPI_Irsend( sbuf, n, MPI_INT, send_to, 0, MPI_COMM_WORLD, &req[1] );
MPI_Waitall( 2, req, MPI_STATUSES_IGNORE );
uint64_t t1 = GetTimeBase();
uint64_t dt = t1-t0;
MPI_Barrier(MPI_COMM_WORLD);
printf("%d: MPI_Irsend/Irecv/Waitall of %ld bytes achieves %lf MB/s \n", world_rank, bytes, 1.6e9*1e-6*(double)bytes/(double)dt );
fflush(stdout);
int errors = 0;
for (int i=0; i<n; i++)
if (rbuf[i] != recv_from)
errors++;
if (errors>0)
for (int i=0; i<n; i++)
if (rbuf[i] != recv_from)
printf("%d: rbuf[%d] = %d (%d) \n", world_rank, i, rbuf[i], recv_from);
else
printf("%d: no errors :-) \n", world_rank);
fflush(stdout);
if (errors>0)
exit(13);
MPI_Barrier(MPI_COMM_WORLD);
free(sbuf);
free(rbuf);
}
/************************************************************************/
MPI_Finalize();
if (world_rank==0)
printf("%d: end of test \n", world_rank );
fflush(stdout);
return 0;
}
void declareBindings (void)
{
/* === Point-to-point === */
void* buf;
int count;
MPI_Datatype datatype;
int dest;
int tag;
MPI_Comm comm;
MPI_Send (buf, count, datatype, dest, tag, comm); // L12
int source;
MPI_Status status;
MPI_Recv (buf, count, datatype, source, tag, comm, &status); // L15
MPI_Get_count (&status, datatype, &count);
MPI_Bsend (buf, count, datatype, dest, tag, comm);
MPI_Ssend (buf, count, datatype, dest, tag, comm);
MPI_Rsend (buf, count, datatype, dest, tag, comm);
void* buffer;
int size;
MPI_Buffer_attach (buffer, size); // L22
MPI_Buffer_detach (buffer, &size);
MPI_Request request;
MPI_Isend (buf, count, datatype, dest, tag, comm, &request); // L25
MPI_Ibsend (buf, count, datatype, dest, tag, comm, &request);
MPI_Issend (buf, count, datatype, dest, tag, comm, &request);
MPI_Irsend (buf, count, datatype, dest, tag, comm, &request);
MPI_Irecv (buf, count, datatype, source, tag, comm, &request);
MPI_Wait (&request, &status);
int flag;
MPI_Test (&request, &flag, &status); // L32
MPI_Request_free (&request);
MPI_Request* array_of_requests;
int index;
MPI_Waitany (count, array_of_requests, &index, &status); // L36
MPI_Testany (count, array_of_requests, &index, &flag, &status);
MPI_Status* array_of_statuses;
MPI_Waitall (count, array_of_requests, array_of_statuses); // L39
MPI_Testall (count, array_of_requests, &flag, array_of_statuses);
int incount;
int outcount;
int* array_of_indices;
MPI_Waitsome (incount, array_of_requests, &outcount, array_of_indices,
array_of_statuses); // L44--45
MPI_Testsome (incount, array_of_requests, &outcount, array_of_indices,
array_of_statuses); // L46--47
MPI_Iprobe (source, tag, comm, &flag, &status); // L48
MPI_Probe (source, tag, comm, &status);
MPI_Cancel (&request);
MPI_Test_cancelled (&status, &flag);
MPI_Send_init (buf, count, datatype, dest, tag, comm, &request);
MPI_Bsend_init (buf, count, datatype, dest, tag, comm, &request);
MPI_Ssend_init (buf, count, datatype, dest, tag, comm, &request);
MPI_Rsend_init (buf, count, datatype, dest, tag, comm, &request);
MPI_Recv_init (buf, count, datatype, source, tag, comm, &request);
MPI_Start (&request);
MPI_Startall (count, array_of_requests);
void* sendbuf;
int sendcount;
MPI_Datatype sendtype;
int sendtag;
void* recvbuf;
int recvcount;
MPI_Datatype recvtype;
MPI_Datatype recvtag;
MPI_Sendrecv (sendbuf, sendcount, sendtype, dest, sendtag,
recvbuf, recvcount, recvtype, source, recvtag,
comm, &status); // L67--69
MPI_Sendrecv_replace (buf, count, datatype, dest, sendtag, source, recvtag,
comm, &status); // L70--71
MPI_Datatype oldtype;
MPI_Datatype newtype;
MPI_Type_contiguous (count, oldtype, &newtype); // L74
int blocklength;
{
int stride;
MPI_Type_vector (count, blocklength, stride, oldtype, &newtype); // L78
}
{
MPI_Aint stride;
MPI_Type_hvector (count, blocklength, stride, oldtype, &newtype); // L82
}
int* array_of_blocklengths;
{
int* array_of_displacements;
MPI_Type_indexed (count, array_of_blocklengths, array_of_displacements,
oldtype, &newtype); // L87--88
}
{
MPI_Aint* array_of_displacements;
MPI_Type_hindexed (count, array_of_blocklengths, array_of_displacements,
oldtype, &newtype); // L92--93
MPI_Datatype* array_of_types;
MPI_Type_struct (count, array_of_blocklengths, array_of_displacements,
array_of_types, &newtype); // L95--96
}
void* location;
MPI_Aint address;
MPI_Address (location, &address); // L100
MPI_Aint extent;
MPI_Type_extent (datatype, &extent); // L102
MPI_Type_size (datatype, &size);
MPI_Aint displacement;
MPI_Type_lb (datatype, &displacement); // L105
MPI_Type_ub (datatype, &displacement);
MPI_Type_commit (&datatype);
MPI_Type_free (&datatype);
MPI_Get_elements (&status, datatype, &count);
void* inbuf;
void* outbuf;
int outsize;
int position;
MPI_Pack (inbuf, incount, datatype, outbuf, outsize, &position, comm); // L114
int insize;
MPI_Unpack (inbuf, insize, &position, outbuf, outcount, datatype,
comm); // L116--117
MPI_Pack_size (incount, datatype, comm, &size);
/* === Collectives === */
MPI_Barrier (comm); // L121
int root;
MPI_Bcast (buffer, count, datatype, root, comm); // L123
MPI_Gather (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
root, comm); // L124--125
int* recvcounts;
int* displs;
MPI_Gatherv (sendbuf, sendcount, sendtype,
recvbuf, recvcounts, displs, recvtype,
root, comm); // L128--130
MPI_Scatter (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
root, comm); // L131--132
int* sendcounts;
MPI_Scatterv (sendbuf, sendcounts, displs, sendtype,
recvbuf, recvcount, recvtype, root, comm); // L134--135
MPI_Allgather (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
comm); // L136--137
MPI_Allgatherv (sendbuf, sendcount, sendtype,
recvbuf, recvcounts, displs, recvtype,
comm); // L138--140
MPI_Alltoall (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
comm); // L141--142
int* sdispls;
int* rdispls;
MPI_Alltoallv (sendbuf, sendcounts, sdispls, sendtype,
recvbuf, recvcounts, rdispls, recvtype,
comm); // L145--147
MPI_Op op;
MPI_Reduce (sendbuf, recvbuf, count, datatype, op, root, comm); // L149
#if 0
MPI_User_function function;
int commute;
MPI_Op_create (function, commute, &op); // L153
#endif
MPI_Op_free (&op); // L155
MPI_Allreduce (sendbuf, recvbuf, count, datatype, op, comm);
MPI_Reduce_scatter (sendbuf, recvbuf, recvcounts, datatype, op, comm);
MPI_Scan (sendbuf, recvbuf, count, datatype, op, comm);
/* === Groups, contexts, and communicators === */
MPI_Group group;
MPI_Group_size (group, &size); // L162
int rank;
MPI_Group_rank (group, &rank); // L164
MPI_Group group1;
int n;
int* ranks1;
MPI_Group group2;
int* ranks2;
MPI_Group_translate_ranks (group1, n, ranks1, group2, ranks2); // L170
int result;
MPI_Group_compare (group1, group2, &result); // L172
MPI_Group newgroup;
MPI_Group_union (group1, group2, &newgroup); // L174
MPI_Group_intersection (group1, group2, &newgroup);
MPI_Group_difference (group1, group2, &newgroup);
int* ranks;
MPI_Group_incl (group, n, ranks, &newgroup); // L178
MPI_Group_excl (group, n, ranks, &newgroup);
extern int ranges[][3];
MPI_Group_range_incl (group, n, ranges, &newgroup); // L181
MPI_Group_range_excl (group, n, ranges, &newgroup);
MPI_Group_free (&group);
MPI_Comm_size (comm, &size);
MPI_Comm_rank (comm, &rank);
MPI_Comm comm1;
MPI_Comm comm2;
MPI_Comm_compare (comm1, comm2, &result);
MPI_Comm newcomm;
MPI_Comm_dup (comm, &newcomm);
MPI_Comm_create (comm, group, &newcomm);
int color;
int key;
MPI_Comm_split (comm, color, key, &newcomm); // L194
MPI_Comm_free (&comm);
MPI_Comm_test_inter (comm, &flag);
MPI_Comm_remote_size (comm, &size);
MPI_Comm_remote_group (comm, &group);
MPI_Comm local_comm;
int local_leader;
MPI_Comm peer_comm;
int remote_leader;
MPI_Comm newintercomm;
MPI_Intercomm_create (local_comm, local_leader, peer_comm, remote_leader, tag,
&newintercomm); // L204--205
MPI_Comm intercomm;
MPI_Comm newintracomm;
int high;
MPI_Intercomm_merge (intercomm, high, &newintracomm); // L209
int keyval;
#if 0
MPI_Copy_function copy_fn;
MPI_Delete_function delete_fn;
void* extra_state;
MPI_Keyval_create (copy_fn, delete_fn, &keyval, extra_state); // L215
#endif
MPI_Keyval_free (&keyval); // L217
void* attribute_val;
MPI_Attr_put (comm, keyval, attribute_val); // L219
MPI_Attr_get (comm, keyval, attribute_val, &flag);
MPI_Attr_delete (comm, keyval);
/* === Environmental inquiry === */
char* name;
int resultlen;
MPI_Get_processor_name (name, &resultlen); // L226
MPI_Errhandler errhandler;
#if 0
MPI_Handler_function function;
MPI_Errhandler_create (function, &errhandler); // L230
#endif
MPI_Errhandler_set (comm, errhandler); // L232
MPI_Errhandler_get (comm, &errhandler);
MPI_Errhandler_free (&errhandler);
int errorcode;
char* string;
MPI_Error_string (errorcode, string, &resultlen); // L237
int errorclass;
MPI_Error_class (errorcode, &errorclass); // L239
MPI_Wtime ();
MPI_Wtick ();
int argc;
char** argv;
MPI_Init (&argc, &argv); // L244
MPI_Finalize ();
MPI_Initialized (&flag);
MPI_Abort (comm, errorcode);
}
int main( int argc, char **argv )
{
MPI_Datatype *types;
void **inbufs, **outbufs;
char **names;
int *counts, *bytesize, ntype;
MPI_Comm comms[20];
int ncomm = 20, rank, np, partner, tag;
int i, j, k, err, toterr, world_rank, errloc;
MPI_Status status, statuses[2];
int flag, index;
char *obuf;
MPI_Request requests[2];
MPI_Init( &argc, &argv );
AllocateForData( &types, &inbufs, &outbufs, &counts, &bytesize,
&names, &ntype );
GenerateData( types, inbufs, outbufs, counts, bytesize, names, &ntype );
MPI_Comm_rank( MPI_COMM_WORLD, &world_rank );
MakeComms( comms, 20, &ncomm, 0 );
/* Test over a wide range of datatypes and communicators */
err = 0;
for (i=0; i<ncomm; i++) {
MPI_Comm_rank( comms[i], &rank );
MPI_Comm_size( comms[i], &np );
if (np < 2) continue;
tag = i;
for (j=0; j<ntype; j++) {
if (world_rank == 0){
/* SI make size of outputindependent of number of processes */
if (i<2) fprintf( stdout, "Testing type %s\n",names[j] );
}
/* This test does an irsend between both partners, with
a sendrecv after the irecv used to guarentee that the
irsend has a matching receive
*/
if (rank == 0) {
partner = np - 1;
#if 0
MPIR_PrintDatatypePack( stdout, counts[j], types[j], 0, 0 );
#endif
obuf = outbufs[j];
for (k=0; k<bytesize[j]; k++)
obuf[k] = 0;
MPI_Irecv(outbufs[j], counts[j], types[j], partner, tag,
comms[i], &requests[0] );
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, partner, ncomm+i,
MPI_BOTTOM, 0, MPI_INT, partner, ncomm+i,
comms[i], &status );
MPI_Irsend( inbufs[j], counts[j], types[j], partner, tag,
comms[i], &requests[1] );
do {
MPI_Waitany( 2, requests, &index, &status );
} while (index != 0);
/* Always the possiblity that the Irsend is still waiting */
MPI_Waitall( 2, requests, statuses );
if ((errloc = CheckData( inbufs[j], outbufs[j], bytesize[j] ))) {
char *p1, *p2;
fprintf( stderr,
"Error in data with type %s (type %d on %d) at byte %d\n",
names[j], j, world_rank, errloc - 1 );
p1 = (char *)inbufs[j];
p2 = (char *)outbufs[j];
fprintf( stderr,
"Got %x expected %x\n", p1[errloc-1], p2[errloc-1] );
err++;
#if 0
MPIR_PrintDatatypeUnpack( stderr, counts[j], types[j],
0, 0 );
#endif
}
}
else if (rank == np - 1) {
partner = 0;
obuf = outbufs[j];
for (k=0; k<bytesize[j]; k++)
obuf[k] = 0;
MPI_Irecv(outbufs[j], counts[j], types[j], partner, tag,
comms[i], &requests[0] );
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, partner, ncomm+i,
MPI_BOTTOM, 0, MPI_INT, partner, ncomm+i,
comms[i], &status );
/* Wait for irecv to complete */
do {
MPI_Test( &requests[0], &flag, &status );
} while (!flag);
if ((errloc = CheckData( inbufs[j], outbufs[j], bytesize[j] ))) {
char *p1, *p2;
fprintf( stderr,
"Error in data with type %s (type %d on %d) at byte %d\n",
names[j], j, world_rank, errloc - 1 );
p1 = (char *)inbufs[j];
p2 = (char *)outbufs[j];
fprintf( stderr,
"Got %x expected %x\n", p1[errloc-1], p2[errloc-1] );
err++;
#if 0
MPIR_PrintDatatypeUnpack( stderr, counts[j], types[j],
0, 0 );
#endif
}
MPI_Irsend( inbufs[j], counts[j], types[j], partner, tag,
comms[i], &requests[1] );
MPI_Waitall(1, &requests[1], &status );
}
}
}
if (err > 0) {
fprintf( stderr, "%d errors on %d\n", err, rank );
}
MPI_Allreduce( &err, &toterr, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
if (world_rank == 0) {
if (toterr == 0) {
printf( " No Errors\n" );
}
else {
printf (" Found %d errors\n", toterr );
}
}
FreeDatatypes( types, inbufs, outbufs, counts, bytesize, names, ntype );
FreeComms( comms, ncomm );
MPI_Finalize();
return err;
}
int buflen;
va_start(ap, unknown);
buf = unknown;
if (_numargs() == NUMPARAMS+1) {
buflen = va_arg(ap, int) /8; /* This is in bits. */
}
count = va_arg (ap, int *);
datatype = va_arg(ap, MPI_Datatype*);
dest = va_arg(ap, int *);
tag = va_arg(ap, int *);
comm = va_arg(ap, MPI_Comm *);
request = va_arg(ap, MPI_Request *);
__ierr = va_arg(ap, int *);
*__ierr = MPI_Irsend(MPIR_F_PTR(buf),*count,*datatype,*dest,*tag,*comm,
&lrequest);
*(int*)request = MPIR_FromPointer(lrequest);
}
#else
void mpi_irsend_( buf, count, datatype, dest, tag, comm, request, __ierr )
void *buf;
int*count;
MPI_Datatype *datatype;
int*dest;
int*tag;
MPI_Comm *comm;
MPI_Request *request;
int *__ierr;
{
