void simulate_generation(){
MPI_Startall(vertical_req_index, vertical_reqs);
int i, loc = n+1;
for(i = 0; i < m * n; i++){
if(i % n == 0){
loc += 2;
}
/*next_generation_grid[loc] = (current_generation_grid[loc] % 1000)
+ ((current_generation + 1) * 1000);*/
next_generation_grid[loc] = evolve(loc++);
//eventually, need to exclude a 2 layer border
}
MPI_Waitall(vertical_req_index, vertical_reqs, MPI_STATUSES_IGNORE);
MPI_Startall(horizontal_req_index, horizontal_reqs);
MPI_Waitall(horizontal_req_index, horizontal_reqs, MPI_STATUSES_IGNORE);
for(i = 0; i <(m+2); i++){
current_generation_grid[i * (n+2)] = recv_right_buffer[i];
current_generation_grid[(n+1) + i * (n+2)] = recv_left_buffer[i];
}
//now have neighbor dependencies, can calculate remaining cells
MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);
MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);
MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);
MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);
MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);
if(world_rank == 0) printf("ROUND %d:\n", zi);
for(i=0; i<world_size; i++){
if(world_rank == i){
int j;
for(j=0; j < (m+2) * (n+2); j++){
if(j%(n+2) == 0) printf("\n");
printf("%4d ", current_generation_grid[j]);
}
printf("\n\n");
}
MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);
MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);
MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);
MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);
MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);
MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);MPI_Barrier(grid_comm);
}
memcpy(current_generation_grid, next_generation_grid, (m+2) * (n+2) * sizeof(int));
}
void _XMP_reflect_async_cardinal(_XMP_array_t *a, int async_id)
{
_XMP_async_comm_t *async = _XMP_get_current_async();
MPI_Request *reqs = &async->reqs[async->nreqs];
int nreqs = 0;
_XMP_TSTART(t0);
for (int i = 0; i < a->dim; i++){
_XMP_array_info_t *ai = &(a->info[i]);
if (ai->shadow_type == _XMP_N_SHADOW_NONE){
continue;
}
else if (ai->shadow_type == _XMP_N_SHADOW_NORMAL){
_XMP_reflect_sched_t *reflect = ai->reflect_sched;
if (_xmp_lwidth[i] || _xmp_uwidth[i]){
_XMP_ASSERT(reflect);
if (reflect->is_periodic == -1 /* not set yet */ ||
_xmp_lwidth[i] != reflect->lo_width ||
_xmp_uwidth[i] != reflect->hi_width ||
_xmp_is_periodic[i] != reflect->is_periodic){
reflect->lo_width = _xmp_lwidth[i];
reflect->hi_width = _xmp_uwidth[i];
reflect->is_periodic = _xmp_is_periodic[i];
_XMP_reflect_normal_sched_dim(a, i, _xmp_lwidth[i], _xmp_uwidth[i], _xmp_is_periodic[i]);
}
if (async->nreqs + nreqs + 4 > _XMP_MAX_ASYNC_REQS){
_XMP_fatal("too many arrays in an asynchronous reflect");
}
memcpy(&reqs[nreqs], reflect->req, 4 * sizeof(MPI_Request));
nreqs += 4;
_XMP_TSTART(t0);
if (reflect->req[0] != MPI_REQUEST_NULL) // if req[0] isn't null, any others shouldn't be null.
MPI_Startall(4, reflect->req);
_XMP_TEND2(xmptiming_.t_comm, xmptiming_.tdim_comm[i], t0);
}
}
else { /* _XMP_N_SHADOW_FULL */
_XMP_reflect_shadow_FULL(a->array_addr_p, a, i);
}
}
_XMP_TEND(xmptiming_.t_sched, t0);
async->nreqs += nreqs;
}
JNIEXPORT void JNICALL Java_mpi_Prequest_startAll(
JNIEnv *env, jclass clazz, jlongArray prequests)
{
int count = (*env)->GetArrayLength(env, prequests);
jlong* jReq;
MPI_Request *cReq;
ompi_java_getPtrArray(env, prequests, &jReq, (void***)&cReq);
int rc = MPI_Startall(count, cReq);
ompi_java_exceptionCheck(env, rc);
ompi_java_releasePtrArray(env, prequests, jReq, (void**)cReq);
}
void mpi_startall_(int* count, int* requests, int* ierr) {
MPI_Request* reqs;
int i;
reqs = xbt_new(MPI_Request, *count);
for(i = 0; i < *count; i++) {
reqs[i] = find_request(requests[i]);
}
*ierr = MPI_Startall(*count, reqs);
free(reqs);
}
/*
* This example causes the IBM SP2 MPI version to generate the message
* ERROR: 0032-158 Persistent request already active (2) in MPI_Startall, task 0
* in the SECOND set of MPI_Startall (after the MPI_Request_free).
*/
int main( int argc, char **argv )
{
MPI_Request r[4];
MPI_Status statuses[4];
double sbuf1[10], sbuf2[10];
double rbuf1[10], rbuf2[10];
int size, rank, up_nbr, down_nbr, i;
MPI_Init( &argc, &argv );
MPI_Comm_size( MPI_COMM_WORLD, &size );
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
up_nbr = (rank + 1) % size;
down_nbr = (size + rank - 1) % size;
MPI_Recv_init( rbuf1, 10, MPI_DOUBLE, down_nbr, 0, MPI_COMM_WORLD, &r[0] );
MPI_Recv_init( rbuf2, 10, MPI_DOUBLE, up_nbr, 1, MPI_COMM_WORLD, &r[1] );
MPI_Send_init( sbuf1, 10, MPI_DOUBLE, up_nbr, 0, MPI_COMM_WORLD, &r[2] );
MPI_Send_init( sbuf2, 10, MPI_DOUBLE, down_nbr, 1, MPI_COMM_WORLD, &r[3] );
MPI_Startall( 4, r );
MPI_Waitall( 4, r, statuses );
for (i=0; i<4; i++) {
MPI_Request_free( &r[i] );
}
MPI_Recv_init( rbuf1, 10, MPI_DOUBLE, down_nbr, 0, MPI_COMM_WORLD, &r[0] );
MPI_Recv_init( rbuf2, 10, MPI_DOUBLE, up_nbr, 1, MPI_COMM_WORLD, &r[1] );
MPI_Send_init( sbuf1, 10, MPI_DOUBLE, up_nbr, 0, MPI_COMM_WORLD, &r[2] );
MPI_Send_init( sbuf2, 10, MPI_DOUBLE, down_nbr, 1, MPI_COMM_WORLD, &r[3] );
MPI_Startall( 4, r );
MPI_Waitall( 4, r, statuses );
for (i=0; i<4; i++) {
MPI_Request_free( &r[i] );
}
if (rank == 0) printf( "No errors\n" );
MPI_Finalize();
return 0;
}
QMP_status_t
QMP_start_mpi (QMP_msghandle_t mh)
{
int err = QMP_SUCCESS;
if(mh->type==MH_multiple) {
MPI_Startall(mh->num, mh->request_array);
} else {
MPI_Start(&mh->request);
}
return err;
}
static void _XMP_reflect_start(_XMP_array_t *a, int dummy)
{
int packSkipDim = 0;
if (_XMPF_running && !_XMPC_running){ /* for XMP/F */
packSkipDim = a->dim - 1;
} else if (!_XMPF_running && _XMPC_running){ /* for XMP/C */
packSkipDim = 0;
} else {
_XMP_fatal("cannot determin the base language.");
}
TLOG_LOG(TLOG_EVENT_3_IN);
for (int i = 0; i < a->dim; i++){
_XMP_array_info_t *ai = &(a->info[i]);
if(! ai->is_shadow_comm_member) continue;
_XMP_reflect_sched_t *reflect = ai->reflect_acc_sched;
if (ai->shadow_type == _XMP_N_SHADOW_NORMAL){
if(packVector && (i != packSkipDim)){
gpu_pack_vector2(reflect, a->type_size);
}
TLOG_LOG(TLOG_EVENT_9);
if(useHostBuffer){
gpu_update_host(reflect);
}
}
}
TLOG_LOG(TLOG_EVENT_3_OUT);
TLOG_LOG(TLOG_EVENT_4_IN);
for (int i = 0; i < a->dim; i++){
_XMP_array_info_t *ai = &(a->info[i]);
if(! ai->is_shadow_comm_member) continue;
_XMP_reflect_sched_t *reflect = ai->reflect_acc_sched;
int lo_width = reflect->lo_width;
int hi_width = reflect->hi_width;
if (!lo_width && !hi_width) continue;
if (ai->shadow_type == _XMP_N_SHADOW_NORMAL){
if((packVector && i != packSkipDim) || useHostBuffer){
gpu_pack_wait(reflect);
TLOG_LOG(TLOG_EVENT_2);
}
MPI_Startall(4, reflect->req);
TLOG_LOG(TLOG_EVENT_1);
}
}
TLOG_LOG(TLOG_EVENT_4_OUT);
}
void
sendp2()
{
static MPI_Request req[4];
static int flag = 0;
if(! flag){
#pragma acc data present(sendp2_lo_recvbuf[0:imax*kmax], sendp2_lo_sendbuf[0:imax*kmax], sendp2_hi_sendbuf[0:imax*kmax], sendp2_hi_recvbuf[0:imax*kmax])
#pragma acc host_data use_device(sendp2_lo_sendbuf, sendp2_lo_recvbuf, sendp2_hi_sendbuf, sendp2_hi_recvbuf)
{
MPI_Recv_init(sendp2_hi_recvbuf,
1,
ikvec,
npy[1],
1,
mpi_comm_cart,
req+2);
MPI_Recv_init(sendp2_lo_recvbuf,
1,
ikvec,
npy[0],
2,
mpi_comm_cart,
req+0);
MPI_Send_init(sendp2_hi_sendbuf,
1,
ikvec,
npy[0],
1,
mpi_comm_cart,
req+3);
MPI_Send_init(sendp2_lo_sendbuf,
1,
ikvec,
npy[1],
2,
mpi_comm_cart,
req+1);
flag = 1;
}
}
sendp2_pack();
MPI_Startall(4, req);
MPI_Waitall(4,
req,
MPI_STATUSES_IGNORE);
sendp2_unpack();
}
void
sendp1()
{
static MPI_Request req[4];
static int flag = 0;
if(! flag){
#pragma acc data present(p)
#pragma acc host_data use_device(p)
{
MPI_Recv_init(&p[imax-1][0][0],
1,
jkvec,
npx[1],
1,
mpi_comm_cart,
req+2);
MPI_Recv_init(&p[0][0][0],
1,
jkvec,
npx[0],
2,
mpi_comm_cart,
req+0);
MPI_Send_init(&p[1][0][0],
1,
jkvec,
npx[0],
1,
mpi_comm_cart,
req+3);
MPI_Send_init(&p[imax-2][0][0],
1,
jkvec,
npx[1],
2,
mpi_comm_cart,
req+1);
flag = 1;
}
}
MPI_Startall(4, req);
MPI_Waitall(4,
req,
MPI_STATUSES_IGNORE);
}
/* 3. */
void xchange_halffield() {
# ifdef MPI
MPI_Status status[16];
# ifdef PARALLELT
int reqcount = 4;
# elif defined PARALLELXT
int reqcount = 8;
# elif defined PARALLELXYT
int reqcount = 12;
# elif defined PARALLELXYZT
int x0=0, x1=0, x2=0, ix=0;
int reqcount = 16;
# endif
# if (defined XLC && defined BGL)
__alignx(16, HalfSpinor);
# endif
MPI_Startall(reqcount, prequests);
MPI_Waitall(reqcount, prequests, status);
# endif /* MPI */
return;
}
void mpi_startall_f(MPI_Fint *count, MPI_Fint *array_of_requests,
MPI_Fint *ierr)
{
MPI_Request *c_req;
int i;
c_req = malloc(*count * sizeof(MPI_Request));
if (NULL == c_req) {
*ierr = OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_NO_MEM,
FUNC_NAME);
return;
}
for(i = 0; i < *count; i++ ) {
c_req[i] = MPI_Request_f2c(array_of_requests[i]);
}
*ierr = OMPI_INT_2_FINT(MPI_Startall(OMPI_FINT_2_INT(*count), c_req));
for( i = 0; i < *count; i++ ) {
array_of_requests[i] = MPI_Request_c2f(c_req[i]);
}
free(c_req);
}
static void test_pair (void)
{
int prev, next, count, tag, index, i, outcount, indices[2];
int rank, size, flag, ierr, reqcount;
double send_buf[TEST_SIZE], recv_buf[TEST_SIZE];
double buffered_send_buf[TEST_SIZE * 2 + MPI_BSEND_OVERHEAD]; /* factor of two is based on guessing - only dynamic allocation would be safe */
void *buffer;
MPI_Status statuses[2];
MPI_Status status;
MPI_Request requests[2];
MPI_Comm dupcom, intercom;
#ifdef V_T
struct _VT_FuncFrameHandle {
char *name;
int func;
int frame;
};
typedef struct _VT_FuncFrameHandle VT_FuncFrameHandle_t;
VT_FuncFrameHandle_t normal_sends,
buffered_sends,
buffered_persistent_sends,
ready_sends,
sync_sends,
nblock_sends,
nblock_rsends,
nblock_ssends,
pers_sends,
pers_rsends,
pers_ssends,
sendrecv,
sendrecv_repl,
intercomm;
int classid;
VT_classdef( "Application:test_pair", &classid );
#define VT_REGION_DEF( _name, _nameframe, _class ) \
(_nameframe).name=_name; \
VT_funcdef( (_nameframe).name, _class, &((_nameframe).func) );
#define VT_BEGIN_REGION( _nameframe ) \
LOCDEF(); \
VT_begin( (_nameframe).func )
#define VT_END_REGION( _nameframe ) \
LOCDEF(); VT_end( (_nameframe).func )
#else
#define VT_REGION_DEF( _name, _nameframe, _class )
#define VT_BEGIN_REGION( _nameframe )
#define VT_END_REGION( _nameframe )
#endif
ierr = MPI_Comm_rank(MPI_COMM_WORLD, &rank);
ierr = MPI_Comm_size(MPI_COMM_WORLD, &size);
if ( size < 2 ) {
if ( rank == 0 ) {
printf("Program needs to be run on at least 2 processes.\n");
}
ierr = MPI_Abort( MPI_COMM_WORLD, 66 );
}
ierr = MPI_Comm_dup(MPI_COMM_WORLD, &dupcom);
if ( rank >= 2 ) {
/* printf( "%d Calling finalize.\n", rank ); */
ierr = MPI_Finalize( );
exit(0);
}
next = rank + 1;
if (next >= 2)
next = 0;
prev = rank - 1;
if (prev < 0)
prev = 1;
VT_REGION_DEF( "Normal_Sends", normal_sends, classid );
VT_REGION_DEF( "Buffered_Sends", buffered_sends, classid );
VT_REGION_DEF( "Buffered_Persistent_Sends", buffered_persistent_sends, classid );
VT_REGION_DEF( "Ready_Sends", ready_sends, classid );
VT_REGION_DEF( "Sync_Sends", sync_sends, classid );
VT_REGION_DEF( "nblock_Sends", nblock_sends, classid );
VT_REGION_DEF( "nblock_RSends", nblock_rsends, classid );
VT_REGION_DEF( "nblock_SSends", nblock_ssends, classid );
VT_REGION_DEF( "Pers_Sends", pers_sends, classid );
VT_REGION_DEF( "Pers_RSends", pers_rsends, classid );
VT_REGION_DEF( "Pers_SSends", pers_ssends, classid );
VT_REGION_DEF( "SendRecv", sendrecv, classid );
VT_REGION_DEF( "SendRevc_Repl", sendrecv_repl, classid );
VT_REGION_DEF( "InterComm", intercomm, classid );
/*
* Normal sends
*/
VT_BEGIN_REGION( normal_sends );
if (rank == 0)
printf ("Send\n");
tag = 0x100;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
LOCDEF();
MPI_Send(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, &status);
msg_check(recv_buf, prev, tag, count, &status, TEST_SIZE, "send and recv");
}
else {
LOCDEF();
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,"send and recv");
init_test_data(recv_buf,TEST_SIZE,1);
MPI_Send(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( normal_sends );
/*
* Buffered sends
*/
VT_BEGIN_REGION( buffered_sends );
if (rank == 0)
printf ("Buffered Send\n");
tag = 138;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
LOCDEF();
MPI_Buffer_attach(buffered_send_buf, sizeof(buffered_send_buf));
MPI_Bsend(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
MPI_Buffer_detach(&buffer, &size);
if(buffer != buffered_send_buf || size != sizeof(buffered_send_buf)) {
printf ("[%d] Unexpected buffer returned by MPI_Buffer_detach(): %p/%d != %p/%d\n", rank, buffer, size, buffered_send_buf, (int)sizeof(buffered_send_buf));
MPI_Abort(MPI_COMM_WORLD, 201);
}
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, &status);
msg_check(recv_buf, prev, tag, count, &status, TEST_SIZE, "send and recv");
}
else {
LOCDEF();
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,"send and recv");
init_test_data(recv_buf,TEST_SIZE,1);
MPI_Send(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( buffered_sends );
/*
* Buffered sends
*/
VT_BEGIN_REGION( buffered_persistent_sends );
if (rank == 0)
printf ("Buffered Persistent Send\n");
tag = 238;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
LOCDEF();
MPI_Buffer_attach(buffered_send_buf, sizeof(buffered_send_buf));
MPI_Bsend_init(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD, requests);
MPI_Start(requests);
MPI_Wait(requests, statuses);
MPI_Request_free(requests);
MPI_Buffer_detach(&buffer, &size);
if(buffer != buffered_send_buf || size != sizeof(buffered_send_buf)) {
printf ("[%d] Unexpected buffer returned by MPI_Buffer_detach(): %p/%d != %p/%d\n", rank, buffer, size, buffered_send_buf, (int)sizeof(buffered_send_buf));
MPI_Abort(MPI_COMM_WORLD, 201);
}
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, &status);
msg_check(recv_buf, prev, tag, count, &status, TEST_SIZE, "send and recv");
}
else {
LOCDEF();
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,"send and recv");
init_test_data(recv_buf,TEST_SIZE,1);
MPI_Send(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( buffered_persistent_sends );
/*
* Ready sends. Note that we must insure that the receive is posted
* before the rsend; this requires using Irecv.
*/
VT_BEGIN_REGION( ready_sends );
if (rank == 0)
printf ("Rsend\n");
tag = 1456;
count = TEST_SIZE / 3;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
MPI_Recv(MPI_BOTTOM, 0, MPI_INT, next, tag, MPI_COMM_WORLD, &status);
MPI_Rsend(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
MPI_Probe(MPI_ANY_SOURCE, tag, MPI_COMM_WORLD, &status);
if (status.MPI_SOURCE != prev)
printf ("Incorrect src, expected %d, got %d\n",prev, status.MPI_SOURCE);
if (status.MPI_TAG != tag)
printf ("Incorrect tag, expected %d, got %d\n",tag, status.MPI_TAG);
MPI_Get_count(&status, MPI_DOUBLE, &i);
if (i != count)
printf ("Incorrect count, expected %d, got %d\n",count,i);
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"rsend and recv");
}
else {
MPI_Irecv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, requests);
MPI_Send( MPI_BOTTOM, 0, MPI_INT, next, tag, MPI_COMM_WORLD);
MPI_Wait(requests, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"rsend and recv");
init_test_data(recv_buf,TEST_SIZE,1);
MPI_Send(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( ready_sends );
/*
* Synchronous sends
*/
VT_BEGIN_REGION( sync_sends );
if (rank == 0)
printf ("Ssend\n");
tag = 1789;
count = TEST_SIZE / 3;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
MPI_Iprobe(MPI_ANY_SOURCE, tag, MPI_COMM_WORLD, &flag, &status);
if (flag)
printf ("Iprobe succeeded! source %d, tag %d\n",status.MPI_SOURCE,
status.MPI_TAG);
MPI_Ssend(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
while (!flag)
MPI_Iprobe(MPI_ANY_SOURCE, tag, MPI_COMM_WORLD, &flag, &status);
if (status.MPI_SOURCE != prev)
printf ("Incorrect src, expected %d, got %d\n",prev, status.MPI_SOURCE);
if (status.MPI_TAG != tag)
printf ("Incorrect tag, expected %d, got %d\n",tag, status.MPI_TAG);
MPI_Get_count(&status, MPI_DOUBLE, &i);
if (i != count)
printf ("Incorrect count, expected %d, got %d\n",count,i);
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE, "ssend and recv");
}
else {
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE, "ssend and recv"); init_test_data(recv_buf,TEST_SIZE,1);
MPI_Ssend(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( sync_sends );
/*
* Nonblocking normal sends
*/
VT_BEGIN_REGION( nblock_sends );
if (rank == 0)
printf ("Isend\n");
tag = 2123;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
MPI_Irecv(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, requests);
init_test_data(send_buf,TEST_SIZE,0);
MPI_Isend(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD,
(requests+1));
MPI_Waitall(2, requests, statuses);
rq_check( requests, 2, "isend and irecv" );
msg_check(recv_buf,prev,tag,count,statuses, TEST_SIZE,"isend and irecv");
}
else {
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check(recv_buf,prev,tag,count,&status, TEST_SIZE,"isend and irecv"); init_test_data(recv_buf,TEST_SIZE,1);
MPI_Isend(recv_buf, count, MPI_DOUBLE, next, tag,MPI_COMM_WORLD,
(requests));
MPI_Wait((requests), &status);
rq_check(requests, 1, "isend (and recv)");
}
VT_END_REGION( nblock_sends );
/*
* Nonblocking ready sends
*/
VT_BEGIN_REGION( nblock_rsends );
if (rank == 0)
printf ("Irsend\n");
tag = 2456;
count = TEST_SIZE / 3;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
MPI_Irecv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, requests);
init_test_data(send_buf,TEST_SIZE,0);
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, next, 0,
MPI_BOTTOM, 0, MPI_INT, next, 0,
dupcom, &status);
MPI_Irsend(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD,
(requests+1));
reqcount = 0;
while (reqcount != 2) {
MPI_Waitany( 2, requests, &index, statuses);
if( index == 0 ) {
memcpy( &status, statuses, sizeof(status) );
}
reqcount++;
}
rq_check( requests, 1, "irsend and irecv");
msg_check(recv_buf,prev,tag,count,&status, TEST_SIZE,"irsend and irecv");
}
else {
MPI_Irecv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, requests);
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, next, 0,
MPI_BOTTOM, 0, MPI_INT, next, 0,
dupcom, &status);
flag = 0;
while (!flag)
MPI_Test(requests, &flag, &status);
rq_check( requests, 1, "irsend and irecv (test)");
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"irsend and irecv"); init_test_data(recv_buf,TEST_SIZE,1);
MPI_Irsend(recv_buf, count, MPI_DOUBLE, next, tag,
MPI_COMM_WORLD, requests);
MPI_Waitall(1, requests, statuses);
rq_check( requests, 1, "irsend and irecv");
}
VT_END_REGION( nblock_rsends );
/*
* Nonblocking synchronous sends
*/
VT_BEGIN_REGION( nblock_ssends );
if (rank == 0)
printf ("Issend\n");
tag = 2789;
count = TEST_SIZE / 3;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
MPI_Irecv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, requests );
init_test_data(send_buf,TEST_SIZE,0);
MPI_Issend(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD,
(requests+1));
flag = 0;
while (!flag)
MPI_Testall(2, requests, &flag, statuses);
rq_check( requests, 2, "issend and irecv (testall)");
msg_check( recv_buf, prev, tag, count, statuses, TEST_SIZE,
"issend and recv");
}
else {
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"issend and recv"); init_test_data(recv_buf,TEST_SIZE,1);
MPI_Issend(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD,requests);
flag = 0;
while (!flag)
MPI_Testany(1, requests, &index, &flag, statuses);
rq_check( requests, 1, "issend and recv (testany)");
}
VT_END_REGION( nblock_ssends );
/*
* Persistent normal sends
*/
VT_BEGIN_REGION( pers_sends );
if (rank == 0)
printf ("Send_init\n");
tag = 3123;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
MPI_Send_init(send_buf, count, MPI_DOUBLE, next, tag,
MPI_COMM_WORLD, requests);
MPI_Recv_init(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, (requests+1));
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
MPI_Startall(2, requests);
MPI_Waitall(2, requests, statuses);
msg_check( recv_buf, prev, tag, count, (statuses+1),
TEST_SIZE, "persistent send/recv");
}
else {
MPI_Start((requests+1));
MPI_Wait((requests+1), &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"persistent send/recv");
init_test_data(send_buf,TEST_SIZE,1);
MPI_Start(requests);
MPI_Wait(requests, &status);
}
MPI_Request_free(requests);
MPI_Request_free((requests+1));
VT_END_REGION( pers_sends );
/*
* Persistent ready sends
*/
VT_BEGIN_REGION( pers_rsends );
if (rank == 0)
printf ("Rsend_init\n");
tag = 3456;
count = TEST_SIZE / 3;
clear_test_data(recv_buf,TEST_SIZE);
MPI_Rsend_init(send_buf, count, MPI_DOUBLE, next, tag,
MPI_COMM_WORLD, requests);
MPI_Recv_init(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, (requests+1));
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0); MPI_Barrier( MPI_COMM_WORLD );
MPI_Startall(2, requests);
reqcount = 0;
while (reqcount != 2) {
MPI_Waitsome(2, requests, &outcount, indices, statuses);
for (i=0; i<outcount; i++) {
if (indices[i] == 1) {
msg_check( recv_buf, prev, tag, count, (statuses+i),
TEST_SIZE, "waitsome");
}
reqcount++;
}
}
}
else {
MPI_Start((requests+1)); MPI_Barrier( MPI_COMM_WORLD );
flag = 0;
while (!flag)
MPI_Test((requests+1), &flag, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE, "test");
init_test_data(send_buf,TEST_SIZE,1);
MPI_Start(requests);
MPI_Wait(requests, &status);
}
MPI_Request_free(requests);
MPI_Request_free((requests+1));
VT_END_REGION( pers_rsends );
/*
* Persistent synchronous sends
*/
VT_BEGIN_REGION( pers_ssends );
if (rank == 0)
printf ("Ssend_init\n");
tag = 3789;
count = TEST_SIZE / 3;
clear_test_data(recv_buf,TEST_SIZE);
MPI_Ssend_init(send_buf, count, MPI_DOUBLE, next, tag,
MPI_COMM_WORLD, (requests+1));
MPI_Recv_init(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, requests);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
MPI_Startall(2, requests);
reqcount = 0;
while (reqcount != 2) {
MPI_Testsome(2, requests, &outcount, indices, statuses);
for (i=0; i<outcount; i++) {
if (indices[i] == 0) {
msg_check( recv_buf, prev, tag, count, (statuses+i),
TEST_SIZE, "testsome");
}
reqcount++;
}
}
}
else {
MPI_Start(requests);
flag = 0;
while (!flag)
MPI_Testany(1, requests, &index, &flag, statuses);
msg_check( recv_buf, prev, tag, count, statuses, TEST_SIZE, "testany" );
init_test_data(send_buf,TEST_SIZE,1);
MPI_Start((requests+1));
MPI_Wait((requests+1), &status);
}
MPI_Request_free(requests);
MPI_Request_free((requests+1));
VT_END_REGION( pers_ssends );
/*
* Send/receive.
*/
VT_BEGIN_REGION( sendrecv );
if (rank == 0)
printf ("Sendrecv\n");
tag = 4123;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
MPI_Sendrecv(send_buf, count, MPI_DOUBLE, next, tag,
recv_buf, count, MPI_DOUBLE, prev, tag,
MPI_COMM_WORLD, &status );
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"sendrecv");
}
else {
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"recv/send"); init_test_data(recv_buf,TEST_SIZE,1);
MPI_Send(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( sendrecv );
#ifdef V_T
VT_flush();
#endif
/*
* Send/receive replace.
*/
VT_BEGIN_REGION( sendrecv_repl );
if (rank == 0)
printf ("Sendrecv_replace\n");
tag = 4456;
count = TEST_SIZE / 3;
if (rank == 0) {
init_test_data(recv_buf, TEST_SIZE,0);
for (i=count; i< TEST_SIZE; i++)
recv_buf[i] = 0.0;
MPI_Sendrecv_replace(recv_buf, count, MPI_DOUBLE,
next, tag, prev, tag, MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"sendrecvreplace");
}
else {
clear_test_data(recv_buf,TEST_SIZE);
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"recv/send for replace"); init_test_data(recv_buf,TEST_SIZE,1);
MPI_Send(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( sendrecv_repl );
/*
* Send/Receive via inter-communicator
*/
VT_BEGIN_REGION( intercomm );
MPI_Intercomm_create(MPI_COMM_SELF, 0, MPI_COMM_WORLD, next, 1, &intercom);
if (rank == 0)
printf ("Send via inter-communicator\n");
tag = 4018;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
LOCDEF();
MPI_Send(send_buf, count, MPI_DOUBLE, 0, tag, intercom);
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, intercom, &status);
msg_check(recv_buf, 0, tag, count, &status, TEST_SIZE, "send and recv via inter-communicator");
}
else if (rank == 1) {
LOCDEF();
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
intercom, &status);
msg_check( recv_buf, 0, tag, count, &status, TEST_SIZE,"send and recv via inter-communicator");
init_test_data(recv_buf,TEST_SIZE,0);
MPI_Send(recv_buf, count, MPI_DOUBLE, 0, tag, intercom);
}
VT_END_REGION( normal_sends );
MPI_Comm_free(&intercom);
MPI_Comm_free(&dupcom);
}
int
main (int argc, char **argv)
{
int nprocs = -1;
int rank = -1;
MPI_Comm comm = MPI_COMM_WORLD;
char processor_name[128];
int namelen = 128;
int bbuf[(BUF_SIZE + MPI_BSEND_OVERHEAD) * 2 * NUM_BSEND_TYPES];
int buf[BUF_SIZE * 2 * NUM_SEND_TYPES];
int i, j, k, at_size, send_t_number, index, outcount, total, flag;
int num_errors, error_count, indices[2 * NUM_SEND_TYPES];
MPI_Request aReq[2 * NUM_SEND_TYPES];
MPI_Status aStatus[2 * NUM_SEND_TYPES];
/* init */
MPI_Init (&argc, &argv);
MPI_Comm_size (comm, &nprocs);
MPI_Comm_rank (comm, &rank);
MPI_Get_processor_name (processor_name, &namelen);
printf ("(%d) is alive on %s\n", rank, processor_name);
fflush (stdout);
MPI_Buffer_attach (bbuf, sizeof(int) *
(BUF_SIZE + MPI_BSEND_OVERHEAD) * 2 * NUM_BSEND_TYPES);
if (rank == 0) {
/* set up persistent sends... */
send_t_number = NUM_SEND_TYPES - NUM_PERSISTENT_SEND_TYPES;
MPI_Send_init (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Send_init (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
send_t_number++;
MPI_Bsend_init (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Bsend_init (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
send_t_number++;
MPI_Rsend_init (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Rsend_init (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
send_t_number++;
MPI_Ssend_init (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Ssend_init (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
}
for (k = 0; k < (NUM_COMPLETION_MECHANISMS * 2); k++) {
if (rank == 0) {
/* initialize all of the send buffers */
for (j = 0; j < NUM_SEND_TYPES; j++) {
for (i = 0; i < BUF_SIZE; i++) {
buf[2 * j * BUF_SIZE + i] = i;
buf[((2 * j + 1) * BUF_SIZE) + i] = BUF_SIZE - 1 - i;
}
}
}
else if (rank == 1) {
/* zero out all of the receive buffers */
bzero (buf, sizeof(int) * BUF_SIZE * 2 * NUM_SEND_TYPES);
}
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
/* set up transient sends... */
send_t_number = 0;
MPI_Isend (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Isend (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
send_t_number++;
MPI_Ibsend (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Ibsend (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
send_t_number++;
/* Barrier to ensure receives are posted for rsends... */
MPI_Barrier(MPI_COMM_WORLD);
MPI_Irsend (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Irsend (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
send_t_number++;
MPI_Issend (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Issend (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
/* just to be paranoid */
send_t_number++;
assert (send_t_number == NUM_SEND_TYPES - NUM_PERSISTENT_SEND_TYPES);
/* start the persistent sends... */
if (k % 2) {
MPI_Startall (NUM_PERSISTENT_SEND_TYPES * 2, &aReq[2 * send_t_number]);
}
else {
for (j = 0; j < NUM_PERSISTENT_SEND_TYPES * 2; j++) {
MPI_Start (&aReq[2 * send_t_number + j]);
}
}
/* NOTE: Changing the send buffer of a Bsend is NOT an error... */
for (j = 0; j < NUM_SEND_TYPES; j++) {
/* muck the buffers */
buf[j * 2 * BUF_SIZE + (BUF_SIZE >> 1)] = BUF_SIZE;
}
printf ("USER MSG: 6 change send buffer errors in iteration #%d:\n", k);
/* complete the sends */
switch (k/2) {
case 0:
/* use MPI_Wait */
for (j = 0; j < NUM_SEND_TYPES * 2; j++) {
MPI_Wait (&aReq[j], &aStatus[j]);
}
break;
case 1:
/* use MPI_Waitall */
MPI_Waitall (NUM_SEND_TYPES * 2, aReq, aStatus);
break;
case 2:
/* use MPI_Waitany */
for (j = 0; j < NUM_SEND_TYPES * 2; j++) {
MPI_Waitany (NUM_SEND_TYPES * 2, aReq, &index, aStatus);
}
break;
case 3:
/* use MPI_Waitsome */
total = 0;
while (total < NUM_SEND_TYPES * 2) {
MPI_Waitsome (NUM_SEND_TYPES * 2, aReq, &outcount, indices, aStatus);
total += outcount;
}
break;
case 4:
/* use MPI_Test */
for (j = 0; j < NUM_SEND_TYPES * 2; j++) {
flag = 0;
while (!flag) {
MPI_Test (&aReq[j], &flag, &aStatus[j]);
}
}
break;
case 5:
/* use MPI_Testall */
flag = 0;
while (!flag) {
MPI_Testall (NUM_SEND_TYPES * 2, aReq, &flag, aStatus);
}
break;
case 6:
/* use MPI_Testany */
for (j = 0; j < NUM_SEND_TYPES * 2; j++) {
flag = 0;
while (!flag) {
MPI_Testany (NUM_SEND_TYPES * 2, aReq, &index, &flag, aStatus);
}
}
break;
case 7:
/* use MPI_Testsome */
total = 0;
while (total < NUM_SEND_TYPES * 2) {
outcount = 0;
while (!outcount) {
MPI_Testsome (NUM_SEND_TYPES * 2, aReq,
&outcount, indices, aStatus);
}
total += outcount;
}
break;
default:
assert (0);
break;
}
}
else if (rank == 1) {
int
main (int argc, char **argv)
{
int nprocs = -1;
int rank = -1;
int comm = MPI_COMM_WORLD;
char processor_name[128];
int namelen = 128;
int buf[BUF_SIZE * 2];
int i, j, k, index, outcount, flag;
int indices[2];
MPI_Request aReq[2];
MPI_Status aStatus[2];
/* init */
MPI_Init (&argc, &argv);
MPI_Comm_size (comm, &nprocs);
MPI_Comm_rank (comm, &rank);
MPI_Get_processor_name (processor_name, &namelen);
printf ("(%d) is alive on %s\n", rank, processor_name);
fflush (stdout);
if (rank == 0) {
/* set up persistent sends... */
MPI_Send_init (&buf[0], BUF_SIZE, MPI_INT, 1, 0, comm, &aReq[0]);
MPI_Send_init (&buf[BUF_SIZE], BUF_SIZE, MPI_INT, 1, 1, comm, &aReq[1]);
/* initialize the send buffers */
for (i = 0; i < BUF_SIZE; i++) {
buf[i] = i;
buf[BUF_SIZE + i] = BUF_SIZE - 1 - i;
}
}
for (k = 0; k < 4; k++) {
if (rank == 1) {
/* zero out the receive buffers */
bzero (buf, sizeof(int) * BUF_SIZE * 2);
}
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
/* start the persistent sends... */
if (k % 2) {
MPI_Startall (2, &aReq[0]);
}
else {
for (j = 0; j < 2; j++) {
MPI_Start (&aReq[j]);
}
}
/* complete the sends */
if (k < 2) {
/* use MPI_Waitany */
for (j = 0; j < 2; j++)
MPI_Waitany (2, aReq, &index, aStatus);
}
else {
/* use MPI_Waitsome */
j = 0;
while (j < 2) {
MPI_Waitsome (2, aReq, &outcount, indices, aStatus);
j += outcount;
}
}
}
else if (rank == 1) {
/* set up receives for all of the sends */
for (j = 0; j < 2; j++) {
MPI_Irecv (&buf[j * BUF_SIZE], BUF_SIZE,
MPI_INT, 0, j, comm, &aReq[j]);
}
/* complete all of the receives... */
MPI_Waitall (2, aReq, aStatus);
}
}
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
/* free the persistent requests */
for (i = 0 ; i < 2; i++) {
MPI_Request_free (&aReq[i]);
}
}
MPI_Finalize ();
printf ("(%d) Finished normally\n", rank);
}
void _XMP_reflect_async_ordinal(_XMP_array_t *a, int async_id){
int n = a->dim;
_XMP_async_reflect_t *async_reflect;
_Bool reusable_sched = false;
if (!a->async_reflect){
int max_nreqs = (pow(3, n) - 1) * 2;
async_reflect = (_XMP_async_reflect_t *)_XMP_alloc(sizeof(_XMP_async_reflect_t));
async_reflect->datatype = (MPI_Datatype *)_XMP_alloc(sizeof(MPI_Datatype) * max_nreqs);
async_reflect->reqs = (MPI_Request *)_XMP_alloc(sizeof(MPI_Request) * max_nreqs);
for (int i = 0; i < max_nreqs; i++){
async_reflect->datatype[i] = MPI_DATATYPE_NULL;
async_reflect->reqs[i] = MPI_REQUEST_NULL;
}
async_reflect->nreqs = 0;
a->async_reflect = async_reflect;
}
else {
reusable_sched = true;
async_reflect = a->async_reflect;
for (int i = 0; i < n; i++){
if (async_reflect->lwidth[i] != _xmp_lwidth[i] ||
async_reflect->uwidth[i] != _xmp_uwidth[i] ||
async_reflect->is_periodic[i] != _xmp_is_periodic[i]){
reusable_sched = false;
break;
}
}
}
if (!reusable_sched){
int lb[_XMP_N_MAX_DIM] = { 0 };
int ub[_XMP_N_MAX_DIM] = { 0 };
for (int i = 0; i < n; i++){
async_reflect->lwidth[i] = _xmp_lwidth[i];
async_reflect->uwidth[i] = _xmp_uwidth[i];
async_reflect->is_periodic[i] = _xmp_is_periodic[i];
if (_xmp_lwidth[i] > 0) lb[i] = -1;
if (_xmp_uwidth[i] > 0) ub[i] = 1;
}
for (int i = 0; i < async_reflect->nreqs; i++){
if (async_reflect->datatype[i] != MPI_DATATYPE_NULL)
MPI_Type_free(&async_reflect->datatype[i]);
if (async_reflect->reqs[i] != MPI_REQUEST_NULL)
MPI_Request_free(&async_reflect->reqs[i]);
}
async_reflect->nreqs = 0;
int ishadow[_XMP_N_MAX_DIM];
for (ishadow[0] = lb[0]; ishadow[0] <= ub[0]; ishadow[0]++){
for (ishadow[1] = lb[1]; ishadow[1] <= ub[1]; ishadow[1]++){
for (ishadow[2] = lb[2]; ishadow[2] <= ub[2]; ishadow[2]++){
for (ishadow[3] = lb[3]; ishadow[3] <= ub[3]; ishadow[3]++){
for (ishadow[4] = lb[4]; ishadow[4] <= ub[4]; ishadow[4]++){
for (ishadow[5] = lb[5]; ishadow[5] <= ub[5]; ishadow[5]++){
for (ishadow[6] = lb[6]; ishadow[6] <= ub[6]; ishadow[6]++){
// When ishadow > 0, upper shadow is to be updated, and vice versa.
int nnzero = 0;
for (int i = 0; i < n; i++){
if (ishadow[i] != 0) nnzero++;
}
if (nnzero == 0) continue;
_XMP_reflect_sched_dir(a, ishadow, _xmp_lwidth, _xmp_uwidth, _xmp_is_periodic);
}}}}}}}
}
_XMP_async_comm_t *async = _XMP_get_current_async();
MPI_Request *reqs = &async->reqs[async->nreqs];
// copy to async
if (async->nreqs + async_reflect->nreqs > _XMP_MAX_ASYNC_REQS){
_XMP_fatal("too many arrays in an asynchronous reflect");
}
memcpy(reqs, async_reflect->reqs, async_reflect->nreqs * sizeof(MPI_Request));
async->nreqs += async_reflect->nreqs;
_XMP_TSTART(t0);
MPI_Startall(async_reflect->nreqs, reqs);
_XMP_TEND(xmptiming_.t_start, t0);
}
HYPRE_Int
hypre_MPI_Startall( HYPRE_Int count,
hypre_MPI_Request *array_of_requests )
{
return (HYPRE_Int) MPI_Startall((hypre_int)count, array_of_requests);
}
void _XMP_reflect__(_XMP_array_t *a)
{
int is_ordinal = 1;
//_XMP_RETURN_IF_SINGLE;
if (!a->is_allocated){
_xmp_set_reflect_flag = 0;
return;
}
if (!_xmp_set_reflect_flag){
for (int i = 0; i < a->dim; i++){
_XMP_array_info_t *ai = &(a->info[i]);
_xmp_lwidth[i] = ai->shadow_size_lo;
_xmp_uwidth[i] = ai->shadow_size_hi;
_xmp_is_periodic[i] = 0;
}
}
_XMP_TSTART(t0);
for (int i = 0; i < a->dim; i++){
_XMP_array_info_t *ai = &(a->info[i]);
if (ai->shadow_type == _XMP_N_SHADOW_NONE){
continue;
}
else if (ai->shadow_type == _XMP_N_SHADOW_NORMAL){
_XMP_reflect_sched_t *reflect = ai->reflect_sched;
if (_xmp_lwidth[i] || _xmp_uwidth[i]){
_XMP_ASSERT(reflect);
/* if (!reflect->reflect_is_initialized || */
/* _xmp_lwidth[i] != reflect->lo_width || */
/* _xmp_uwidth[i] != reflect->hi_width || */
/* _xmp_is_periodic[i] != reflect->is_periodic){ */
/* reflect->lo_width = _xmp_lwidth[i]; */
/* reflect->hi_width = _xmp_uwidth[i]; */
/* reflect->is_periodic = _xmp_is_periodic[i]; */
/* if (_xmp_reflect_pack_flag){ */
/* _XMP_reflect_pcopy_sched_dim(a, i, _xmp_lwidth[i], _xmp_uwidth[i], _xmp_is_periodic[i], 0); */
/* } */
/* else { */
/* _XMP_reflect_normal_sched_dim(a, i, _xmp_lwidth[i], _xmp_uwidth[i], _xmp_is_periodic[i]); */
/* } */
/* reflect->reflect_is_initialized = 1; */
/* } */
if (!reflect->reflect_is_initialized ||
_xmp_lwidth[i] != reflect->lo_width ||
_xmp_uwidth[i] != reflect->hi_width ||
_xmp_is_periodic[i] != reflect->is_periodic){
if (_xmp_reflect_pack_flag){
_XMP_reflect_pcopy_sched_dim(a, i, _xmp_lwidth[i], _xmp_uwidth[i], _xmp_is_periodic[i], _XMP_COMM_REFLECT);
}
else {
_XMP_reflect_normal_sched_dim(a, i, _xmp_lwidth[i], _xmp_uwidth[i], _xmp_is_periodic[i]);
}
reflect->reflect_is_initialized = 1;
reflect->lo_width = _xmp_lwidth[i];
reflect->hi_width = _xmp_uwidth[i];
reflect->is_periodic = _xmp_is_periodic[i];
}
if (_xmp_reflect_pack_flag && reflect->req[0] != MPI_REQUEST_NULL){
_XMP_TSTART(t0);
_XMP_reflect_pack_dim(a, i, _xmp_lwidth, _xmp_uwidth, _xmp_is_periodic, _XMP_COMM_REFLECT);
_XMP_TEND(xmptiming_.t_copy, t0);
}
_XMP_TSTART(t0);
if (reflect->req[0] != MPI_REQUEST_NULL) // if req[0] isn't null, any others shouldn't be null.
MPI_Startall(4, reflect->req);
_XMP_TEND2(xmptiming_.t_comm, xmptiming_.tdim_comm[i], t0);
if (is_ordinal){
_XMP_TSTART(t0);
MPI_Waitall(4, reflect->req, MPI_STATUSES_IGNORE);
_XMP_TEND2(xmptiming_.t_wait, xmptiming_.tdim_wait[i], t0);
if (_xmp_reflect_pack_flag && reflect->req[0] != MPI_REQUEST_NULL){
_XMP_TSTART(t0);
_XMP_reflect_unpack_dim(a, i, _xmp_lwidth, _xmp_uwidth, _xmp_is_periodic);
_XMP_TEND(xmptiming_.t_copy, t0);
}
}
}
}
else { /* _XMP_N_SHADOW_FULL */
_XMP_reflect_shadow_FULL(a->array_addr_p, a, i);
}
}
_XMP_TEND(xmptiming_.t_sched, t0);
// t0 = MPI_Wtime();
if (!is_ordinal)
_XMP_reflect_wait(a, _xmp_lwidth, _xmp_uwidth, _xmp_is_periodic);
// t_wait = t_wait + (MPI_Wtime() - t0);
_xmp_set_reflect_flag = 0;
for (int i = 0; i < a->dim; i++){
_xmp_lwidth[i] = 0;
_xmp_uwidth[i] = 0;
_xmp_is_periodic[i] = 0;
}
}
/**
The \Ref{amps_IExchangePackage} initiates the communication of the
invoices found in the {\bf package} structure that is passed in. Once a
\Ref{amps_IExchangePackage} is issued it is illegal to access the
variables that are being communicated. An \Ref{amps_IExchangePackage}
is always followed by an \Ref{amps_Wait} on the {\bf handle} that is
returned.
{\large Example:}
\begin{verbatim}
// Initialize exchange of boundary points
handle = amps_IExchangePackage(package);
// Compute on the "interior points"
// Wait for the exchange to complete
amps_Wait(handle);
\end{verbatim}
{\large Notes:}
This routine can be optimized on some architectures so if your
communication can be formulated using it there might be
some performance advantages.
@memo Initiate package communication
@param package the collection of invoices to communicate
@return Handle for the asynchronous communication
*/
amps_Handle amps_IExchangePackage(amps_Package package)
{
int i;
int num;
num = package -> num_send + package -> num_recv;
/*-------------------------------------------------------------------
* Check if we need to allocate the MPI types and requests
*------------------------------------------------------------------*/
if(!package -> commited)
{
package -> commited = TRUE;
/*--------------------------------------------------------------------
* Allocate the arrays need for MPI
*--------------------------------------------------------------------*/
if(num)
{
package -> recv_requests = (MPI_Request *)calloc((size_t)(num),
sizeof(MPI_Request));
package -> status = (MPI_Status *)calloc((size_t)(num),
sizeof(MPI_Status));
package -> send_requests = package -> recv_requests +
package -> num_recv;
}
/*--------------------------------------------------------------------
* Set up the receive types and requests
*--------------------------------------------------------------------*/
if( package -> num_recv)
{
for(i = 0; i < package -> num_recv; i++)
{
amps_create_mpi_type(oas3Comm, package -> recv_invoices[i]);
MPI_Type_commit(&(package -> recv_invoices[i] -> mpi_type));
// Temporaries needed by insure++
MPI_Datatype type = package -> recv_invoices[i] -> mpi_type;
MPI_Request *request_ptr = &(package -> recv_requests[i]);
MPI_Recv_init(MPI_BOTTOM, 1,
type,
package -> src[i], 0, oas3Comm,
request_ptr);
}
}
/*--------------------------------------------------------------------
* Set up the send types and requests
*--------------------------------------------------------------------*/
if(package -> num_send)
{
for(i = 0; i < package -> num_send; i++)
{
amps_create_mpi_type(oas3Comm,
package -> send_invoices[i]);
MPI_Type_commit(&(package -> send_invoices[i] -> mpi_type));
// Temporaries needed by insure++
MPI_Datatype type = package -> send_invoices[i] -> mpi_type;
MPI_Request* request_ptr = &(package -> send_requests[i]);
MPI_Ssend_init(MPI_BOTTOM, 1,
type,
package -> dest[i], 0, oas3Comm,
request_ptr);
}
}
}
if(num)
{
/*--------------------------------------------------------------------
* post send and receives
*--------------------------------------------------------------------*/
MPI_Startall(num, package -> recv_requests);
}
return( amps_NewHandle(NULL, 0, NULL, package));
}
int
main (int argc, char **argv)
{
int nprocs = -1;
int rank = -1;
char processor_name[128];
int namelen = 128;
int buf0[buf_size];
int buf1[buf_size];
MPI_Request aReq[2];
MPI_Status aStatus[2];
MPI_Status status;
/* init */
MPI_Init (&argc, &argv);
MPI_Comm_size (MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank (MPI_COMM_WORLD, &rank);
MPI_Get_processor_name (processor_name, &namelen);
printf ("(%d) is alive on %s\n", rank, processor_name);
fflush (stdout);
MPI_Barrier (MPI_COMM_WORLD);
if (nprocs < 2) {
printf ("not enough tasks\n");
}
else {
if (rank == 0) {
memset (buf0, 0, buf_size);
MPI_Send_init (buf0, buf_size, MPI_INT, 1, 0, MPI_COMM_WORLD, &aReq[0]);
MPI_Recv_init (buf1, buf_size, MPI_INT, 1, 0, MPI_COMM_WORLD, &aReq[1]);
MPI_Start (&aReq[0]);
MPI_Start (&aReq[1]);
MPI_Waitall (2, aReq, aStatus);
memset (buf0, 1, buf_size);
MPI_Startall (2, aReq);
MPI_Waitall (2, aReq, aStatus);
}
else if (rank == 1) {
memset (buf1, 1, buf_size);
MPI_Recv_init (buf0, buf_size, MPI_INT, 0, 0, MPI_COMM_WORLD, &aReq[0]);
MPI_Send_init (buf1, buf_size, MPI_INT, 0, 0, MPI_COMM_WORLD, &aReq[1]);
MPI_Start (&aReq[0]);
MPI_Start (&aReq[1]);
MPI_Waitall (2, aReq, aStatus);
memset (buf1, 0, buf_size);
MPI_Startall (2, aReq);
MPI_Waitall (2, aReq, aStatus);
}
}
MPI_Barrier (MPI_COMM_WORLD);
MPI_Request_free (&aReq[0]);
MPI_Request_free (&aReq[1]);
MPI_Finalize ();
printf ("(%d) Finished normally\n", rank);
}
void declareBindings (void)
{
/* === Point-to-point === */
void* buf;
int count;
MPI_Datatype datatype;
int dest;
int tag;
MPI_Comm comm;
MPI_Send (buf, count, datatype, dest, tag, comm); // L12
int source;
MPI_Status status;
MPI_Recv (buf, count, datatype, source, tag, comm, &status); // L15
MPI_Get_count (&status, datatype, &count);
MPI_Bsend (buf, count, datatype, dest, tag, comm);
MPI_Ssend (buf, count, datatype, dest, tag, comm);
MPI_Rsend (buf, count, datatype, dest, tag, comm);
void* buffer;
int size;
MPI_Buffer_attach (buffer, size); // L22
MPI_Buffer_detach (buffer, &size);
MPI_Request request;
MPI_Isend (buf, count, datatype, dest, tag, comm, &request); // L25
MPI_Ibsend (buf, count, datatype, dest, tag, comm, &request);
MPI_Issend (buf, count, datatype, dest, tag, comm, &request);
MPI_Irsend (buf, count, datatype, dest, tag, comm, &request);
MPI_Irecv (buf, count, datatype, source, tag, comm, &request);
MPI_Wait (&request, &status);
int flag;
MPI_Test (&request, &flag, &status); // L32
MPI_Request_free (&request);
MPI_Request* array_of_requests;
int index;
MPI_Waitany (count, array_of_requests, &index, &status); // L36
MPI_Testany (count, array_of_requests, &index, &flag, &status);
MPI_Status* array_of_statuses;
MPI_Waitall (count, array_of_requests, array_of_statuses); // L39
MPI_Testall (count, array_of_requests, &flag, array_of_statuses);
int incount;
int outcount;
int* array_of_indices;
MPI_Waitsome (incount, array_of_requests, &outcount, array_of_indices,
array_of_statuses); // L44--45
MPI_Testsome (incount, array_of_requests, &outcount, array_of_indices,
array_of_statuses); // L46--47
MPI_Iprobe (source, tag, comm, &flag, &status); // L48
MPI_Probe (source, tag, comm, &status);
MPI_Cancel (&request);
MPI_Test_cancelled (&status, &flag);
MPI_Send_init (buf, count, datatype, dest, tag, comm, &request);
MPI_Bsend_init (buf, count, datatype, dest, tag, comm, &request);
MPI_Ssend_init (buf, count, datatype, dest, tag, comm, &request);
MPI_Rsend_init (buf, count, datatype, dest, tag, comm, &request);
MPI_Recv_init (buf, count, datatype, source, tag, comm, &request);
MPI_Start (&request);
MPI_Startall (count, array_of_requests);
void* sendbuf;
int sendcount;
MPI_Datatype sendtype;
int sendtag;
void* recvbuf;
int recvcount;
MPI_Datatype recvtype;
MPI_Datatype recvtag;
MPI_Sendrecv (sendbuf, sendcount, sendtype, dest, sendtag,
recvbuf, recvcount, recvtype, source, recvtag,
comm, &status); // L67--69
MPI_Sendrecv_replace (buf, count, datatype, dest, sendtag, source, recvtag,
comm, &status); // L70--71
MPI_Datatype oldtype;
MPI_Datatype newtype;
MPI_Type_contiguous (count, oldtype, &newtype); // L74
int blocklength;
{
int stride;
MPI_Type_vector (count, blocklength, stride, oldtype, &newtype); // L78
}
{
MPI_Aint stride;
MPI_Type_hvector (count, blocklength, stride, oldtype, &newtype); // L82
}
int* array_of_blocklengths;
{
int* array_of_displacements;
MPI_Type_indexed (count, array_of_blocklengths, array_of_displacements,
oldtype, &newtype); // L87--88
}
{
MPI_Aint* array_of_displacements;
MPI_Type_hindexed (count, array_of_blocklengths, array_of_displacements,
oldtype, &newtype); // L92--93
MPI_Datatype* array_of_types;
MPI_Type_struct (count, array_of_blocklengths, array_of_displacements,
array_of_types, &newtype); // L95--96
}
void* location;
MPI_Aint address;
MPI_Address (location, &address); // L100
MPI_Aint extent;
MPI_Type_extent (datatype, &extent); // L102
MPI_Type_size (datatype, &size);
MPI_Aint displacement;
MPI_Type_lb (datatype, &displacement); // L105
MPI_Type_ub (datatype, &displacement);
MPI_Type_commit (&datatype);
MPI_Type_free (&datatype);
MPI_Get_elements (&status, datatype, &count);
void* inbuf;
void* outbuf;
int outsize;
int position;
MPI_Pack (inbuf, incount, datatype, outbuf, outsize, &position, comm); // L114
int insize;
MPI_Unpack (inbuf, insize, &position, outbuf, outcount, datatype,
comm); // L116--117
MPI_Pack_size (incount, datatype, comm, &size);
/* === Collectives === */
MPI_Barrier (comm); // L121
int root;
MPI_Bcast (buffer, count, datatype, root, comm); // L123
MPI_Gather (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
root, comm); // L124--125
int* recvcounts;
int* displs;
MPI_Gatherv (sendbuf, sendcount, sendtype,
recvbuf, recvcounts, displs, recvtype,
root, comm); // L128--130
MPI_Scatter (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
root, comm); // L131--132
int* sendcounts;
MPI_Scatterv (sendbuf, sendcounts, displs, sendtype,
recvbuf, recvcount, recvtype, root, comm); // L134--135
MPI_Allgather (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
comm); // L136--137
MPI_Allgatherv (sendbuf, sendcount, sendtype,
recvbuf, recvcounts, displs, recvtype,
comm); // L138--140
MPI_Alltoall (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
comm); // L141--142
int* sdispls;
int* rdispls;
MPI_Alltoallv (sendbuf, sendcounts, sdispls, sendtype,
recvbuf, recvcounts, rdispls, recvtype,
comm); // L145--147
MPI_Op op;
MPI_Reduce (sendbuf, recvbuf, count, datatype, op, root, comm); // L149
#if 0
MPI_User_function function;
int commute;
MPI_Op_create (function, commute, &op); // L153
#endif
MPI_Op_free (&op); // L155
MPI_Allreduce (sendbuf, recvbuf, count, datatype, op, comm);
MPI_Reduce_scatter (sendbuf, recvbuf, recvcounts, datatype, op, comm);
MPI_Scan (sendbuf, recvbuf, count, datatype, op, comm);
/* === Groups, contexts, and communicators === */
MPI_Group group;
MPI_Group_size (group, &size); // L162
int rank;
MPI_Group_rank (group, &rank); // L164
MPI_Group group1;
int n;
int* ranks1;
MPI_Group group2;
int* ranks2;
MPI_Group_translate_ranks (group1, n, ranks1, group2, ranks2); // L170
int result;
MPI_Group_compare (group1, group2, &result); // L172
MPI_Group newgroup;
MPI_Group_union (group1, group2, &newgroup); // L174
MPI_Group_intersection (group1, group2, &newgroup);
MPI_Group_difference (group1, group2, &newgroup);
int* ranks;
MPI_Group_incl (group, n, ranks, &newgroup); // L178
MPI_Group_excl (group, n, ranks, &newgroup);
extern int ranges[][3];
MPI_Group_range_incl (group, n, ranges, &newgroup); // L181
MPI_Group_range_excl (group, n, ranges, &newgroup);
MPI_Group_free (&group);
MPI_Comm_size (comm, &size);
MPI_Comm_rank (comm, &rank);
MPI_Comm comm1;
MPI_Comm comm2;
MPI_Comm_compare (comm1, comm2, &result);
MPI_Comm newcomm;
MPI_Comm_dup (comm, &newcomm);
MPI_Comm_create (comm, group, &newcomm);
int color;
int key;
MPI_Comm_split (comm, color, key, &newcomm); // L194
MPI_Comm_free (&comm);
MPI_Comm_test_inter (comm, &flag);
MPI_Comm_remote_size (comm, &size);
MPI_Comm_remote_group (comm, &group);
MPI_Comm local_comm;
int local_leader;
MPI_Comm peer_comm;
int remote_leader;
MPI_Comm newintercomm;
MPI_Intercomm_create (local_comm, local_leader, peer_comm, remote_leader, tag,
&newintercomm); // L204--205
MPI_Comm intercomm;
MPI_Comm newintracomm;
int high;
MPI_Intercomm_merge (intercomm, high, &newintracomm); // L209
int keyval;
#if 0
MPI_Copy_function copy_fn;
MPI_Delete_function delete_fn;
void* extra_state;
MPI_Keyval_create (copy_fn, delete_fn, &keyval, extra_state); // L215
#endif
MPI_Keyval_free (&keyval); // L217
void* attribute_val;
MPI_Attr_put (comm, keyval, attribute_val); // L219
MPI_Attr_get (comm, keyval, attribute_val, &flag);
MPI_Attr_delete (comm, keyval);
/* === Environmental inquiry === */
char* name;
int resultlen;
MPI_Get_processor_name (name, &resultlen); // L226
MPI_Errhandler errhandler;
#if 0
MPI_Handler_function function;
MPI_Errhandler_create (function, &errhandler); // L230
#endif
MPI_Errhandler_set (comm, errhandler); // L232
MPI_Errhandler_get (comm, &errhandler);
MPI_Errhandler_free (&errhandler);
int errorcode;
char* string;
MPI_Error_string (errorcode, string, &resultlen); // L237
int errorclass;
MPI_Error_class (errorcode, &errorclass); // L239
MPI_Wtime ();
MPI_Wtick ();
int argc;
char** argv;
MPI_Init (&argc, &argv); // L244
MPI_Finalize ();
MPI_Initialized (&flag);
MPI_Abort (comm, errorcode);
}
