int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_rank_t numranks, myrank, n;
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_segment_create(0, _2GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
gaspi_atomic_value_t val;
gaspi_atomic_value_t compare = 0;
if(myrank == 0)
for(n = 0; n < numranks; n++)
ASSERT(gaspi_atomic_compare_swap(0, 0, n, compare, (gaspi_atomic_value_t) 42, &val, GASPI_BLOCK));
gaspi_pointer_t _vptr;
ASSERT (gaspi_segment_ptr(0, &_vptr));
//sync
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
gaspi_atomic_value_t *end_val;
end_val = (gaspi_atomic_value_t *)_vptr;
assert(*end_val == 42);
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
static const int VLEN = 1 << 2;
SUCCESS_OR_DIE( gaspi_proc_init(GASPI_BLOCK) );
gaspi_rank_t iProc, nProc;
SUCCESS_OR_DIE( gaspi_proc_rank(&iProc) );
SUCCESS_OR_DIE( gaspi_proc_num(&nProc) );
gaspi_segment_id_t const segment_id = 0;
gaspi_size_t const segment_size = VLEN * sizeof (double);
SUCCESS_OR_DIE
( gaspi_segment_create
( segment_id, segment_size
, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED
)
);
gaspi_pointer_t array;
SUCCESS_OR_DIE( gaspi_segment_ptr (segment_id, &array) );
for (int j = 0; j < VLEN; ++j)
{
( (double *)array )[j]= (double)( iProc * VLEN + j );
printf("rank %d elem %d: %f \n", iProc,j,( (double *)array )[j] );
}
SUCCESS_OR_DIE( gaspi_proc_term(GASPI_BLOCK) );
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
unsigned long i;
gaspi_pointer_t _vptr;
gaspi_rank_t num_ranks, myrank;
gaspi_number_t qmax ;
gaspi_number_t queueSize;
gaspi_rank_t left_rank, right_rank;
const unsigned long N = (1 << 13);
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT (gaspi_proc_num(&num_ranks));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT(gaspi_segment_create(0,
_2MB,
GASPI_GROUP_ALL,
GASPI_BLOCK,
GASPI_MEM_INITIALIZED));
ASSERT(gaspi_segment_ptr(0, &_vptr));
ASSERT (gaspi_queue_size_max(&qmax));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
left_rank = (myrank + num_ranks - 1 ) % num_ranks;
right_rank = (myrank + num_ranks + 1) % num_ranks;
ASSERT( gaspi_write(0, //seg
0, //local off
left_rank, //rank
0, //seg rem
0, //remote off
1, //size 32KB
0, //queue
GASPI_BLOCK));
ASSERT( gaspi_write(0, //seg
0, //local off
right_rank, //rank
0, //seg rem
0, //remote off
1, //size 32KB
0, //queue
GASPI_BLOCK));
ASSERT (gaspi_wait(0, GASPI_BLOCK));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, 5000));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
printf("Rank %d: Finish\n", myrank);
fflush(stdout);
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_rank_t P, myrank;
ASSERT (gaspi_proc_num(&P));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_segment_create(0, _2MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
int * int_GlbMem;
gaspi_pointer_t _vptr;
ASSERT(gaspi_segment_ptr(0, &_vptr));
int_GlbMem = ( int *) _vptr;
gaspi_size_t msgSize;
gaspi_size_t i;
for(msgSize = 4; msgSize < 8192; msgSize *=2)
if(myrank == 0)
{
for(i = 0; i < msgSize / sizeof(int); i++)
int_GlbMem[i] = (int) msgSize;
gaspi_rank_t n;
for(n = 1; n < P; n++)
ASSERT(gaspi_passive_send(0, 0, n, msgSize, GASPI_BLOCK));
}
else
{
gaspi_rank_t sender;
ASSERT(gaspi_passive_receive(0, 0, &sender, msgSize, GASPI_BLOCK));
for(i = 0; i < msgSize / sizeof(int); i++)
assert( int_GlbMem[i] == (int) msgSize );
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
const int ntimes = 1000;
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_rank_t numranks, myrank, n,m;
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_segment_create(0, _2MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
gaspi_atomic_value_t val;
for(m = 0; m < ntimes; m++)
for(n = 0; n < numranks; n++)
{
gaspi_return_t ret;
do
{
ret = gaspi_atomic_fetch_add(0, 0, n, 1, &val, GASPI_BLOCK);
}
while(ret == GASPI_TIMEOUT);
}
gaspi_pointer_t _vptr;
ASSERT (gaspi_segment_ptr(0, &_vptr));
//sync to make sure everyone did it
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
gaspi_atomic_value_t *end_val;
end_val = (gaspi_atomic_value_t *)_vptr;
assert(*end_val == (numranks * ntimes));
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
mctpInitUser(2);
int tid = mctpRegisterThread();
gaspi_rank_t P, myrank;
ASSERT (gaspi_proc_num(&P));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_segment_create(0, _4GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
int * int_GlbMem;
gaspi_pointer_t _vptr;
ASSERT(gaspi_segment_ptr(0, &_vptr));
int_GlbMem = ( int *) _vptr;
mctpStartSingleThread(recvThread, int_GlbMem);
int_GlbMem[0] = 11223344;
//send to myself (need trick to avoid blocking)
gaspi_return_t ret = GASPI_ERROR;
do
{
ret = gaspi_passive_send(0, 0, myrank, sizeof(int), GASPI_TEST);
assert (ret != GASPI_ERROR);
sleep(1);
}
while (ret != GASPI_SUCCESS);
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_rank_t P, myrank;
ASSERT (gaspi_proc_num(&P));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_segment_create(0, _4GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
int * int_GlbMem;
gaspi_pointer_t _vptr;
ASSERT(gaspi_segment_ptr(0, &_vptr));
int_GlbMem = ( int *) _vptr;
const gaspi_size_t msgSize = sizeof(int);
if(myrank == 0)
{
int_GlbMem[0] = 11223344;
gaspi_rank_t n;
for(n = 1; n < P; n++)
ASSERT(gaspi_passive_send(0, 0, n, msgSize, GASPI_BLOCK));
}
else
{
gaspi_rank_t sender;
ASSERT(gaspi_passive_receive(0, 0, &sender, msgSize, GASPI_BLOCK));
gaspi_printf("Received msg from %d\n", sender);
assert( int_GlbMem[0] == 11223344 );
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT(gaspi_threads_init_user(2));
int tid;
ASSERT (gaspi_threads_register(&tid));
gaspi_rank_t P, myrank;
ASSERT (gaspi_proc_num(&P));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_segment_create(0, _2MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
int * int_GlbMem;
gaspi_pointer_t _vptr;
ASSERT(gaspi_segment_ptr(0, &_vptr));
int_GlbMem = ( int *) _vptr;
ASSERT(gaspi_threads_run(recvThread, int_GlbMem));
int_GlbMem[0] = 11223344;
ASSERT(gaspi_passive_send(0, 0, myrank, sizeof(int), GASPI_BLOCK));
gaspi_threads_sync();
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main (int argc, char *argv[])
{
gaspi_configuration_t config = { 0 };
//argc, argv, "", (1UL << 30)
gaspi_proc_init (config, GASPI_BLOCK); // 1 GiB DMA enabled memory per node
gaspi_rank_t iProc, nProc;
gaspi_proc_rank (&iProc);
gaspi_proc_num (&nProc);
void* temp_ptr;
gaspi_segment_ptr(GPI_SEGMENT, &temp_ptr);
int *mem = (int *) temp_ptr; // begin of DMA enabled memory
int *src = mem; // offset 0
int *dst = mem + nProc; // offset nProc * sizeof(int)
for (gaspi_rank_t p = 0; p < nProc; ++p)
{
src[p] = iProc * nProc + p;
const unsigned long locOff = p * sizeof (int);
const unsigned long remOff = (nProc + iProc) * sizeof (int);
gaspi_write(GPI_SEGMENT, locOff, p, GPI_SEGMENT, remOff, sizeof (int), 0, GASPI_BLOCK);
}
gaspi_wait (0, GASPI_BLOCK);
gaspi_barrier (GASPI_GROUP_ALL, GASPI_BLOCK);
dump (src, iProc, nProc, "src");
dump (dst, iProc, nProc, "dst");
gaspi_proc_term (GASPI_BLOCK);
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
int k = 0;
int ret = 0;
unsigned long j;
const gaspi_size_t size = 4096;
const gaspi_size_t memSize = _4GB;
gaspi_offset_t offset_write = 0;
gaspi_offset_t offset_read = _2GB;
gaspi_offset_t offset_check = 3221225472;
gaspi_number_t qmax ;
gaspi_number_t queueSize;
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT (gaspi_queue_size_max(&qmax));
ASSERT (gaspi_segment_create(0, memSize, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
gaspi_pointer_t _vptr;
ASSERT (gaspi_segment_ptr(0, &_vptr));
/* get memory area pointer */
float *mptr_f = (float *) _vptr;
char *mptr_c = (char *) _vptr;
gaspi_rank_t myrank, highestnode;
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_proc_num(&highestnode));
while(k <= RUNS)
{
//generate random
srand((unsigned)time(0));
srand48((unsigned) time(0));
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
//clean
for(j = 0; j < memSize; j++)
mptr_c[j]= 0;
/* fill randoms up to 1GB */
for(j = 0; j < (GB / sizeof(float)); j++)
{
mptr_f[j]= drand48() + (myrank * 1.0);
}
#ifdef DEBUG
gaspi_printf("random value in pos 0 %f\n", mptr_f[0]);
#endif
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
gaspi_printf("\n....Running iteration %d of %d...\n",k, RUNS);
const unsigned long packets = (GB / size);
for(j = 0; j < packets; j++)
{
ASSERT(gaspi_queue_size(0, &queueSize));
if (queueSize > qmax - 24)
{
ASSERT(gaspi_wait(0, GASPI_BLOCK));
}
ASSERT (gaspi_write(0, offset_write, (myrank + 1) % highestnode,
0, offset_read, size,
0, GASPI_BLOCK));
offset_write += size;
offset_read += size;
}
offset_write=0;
offset_read = _2GB;
#ifdef DEBUG
gaspi_printf("%d bytes written!\n", packets * size);
#endif
/* notify remote that data is written */
ASSERT (gaspi_notify( 0, (myrank + 1) % highestnode, 0, 1, 0, GASPI_BLOCK));
gaspi_notification_id_t recv_id;
ASSERT(gaspi_notify_waitsome(0, 0, 1, &recv_id, GASPI_BLOCK));
assert(recv_id == 0);
gaspi_notification_t notification_val;
ASSERT( gaspi_notify_reset(0, recv_id, ¬ification_val));
/* notify remote that data has arrived */
ASSERT (gaspi_notify( 0, (myrank + highestnode - 1) % highestnode, 1, 1, 0, GASPI_BLOCK));
gaspi_notification_id_t ack_id;
ASSERT(gaspi_notify_waitsome(0, 1, 1, &ack_id, GASPI_BLOCK));
assert(ack_id == 1);
ASSERT( gaspi_notify_reset(0, ack_id, ¬ification_val));
/* check if data was written successfully */
ASSERT (gaspi_read(0, offset_check, (myrank + 1) % highestnode,
0, offset_read, GB / 2,
0, GASPI_BLOCK));
ASSERT (gaspi_read(0, offset_check + (GB / 2), (myrank + 1) % highestnode,
0, offset_read + (GB / 2), GB / 2,
0, GASPI_BLOCK));
ASSERT (gaspi_wait(0, GASPI_BLOCK));
#ifdef DEBUG
gaspi_printf("Values %f %f %f \n", mptr_f[0], mptr_f[offset_read / sizeof(float)], mptr_f[offset_check / sizeof(float)]);
#endif
j = 0;
while(j < GB / sizeof(float) )
{
if(mptr_f[j] != mptr_f[offset_check / sizeof(float) + j]){
gaspi_printf("value incorrect %f-%f at %d \n",
mptr_f[j],
mptr_f[offset_check / sizeof(float) + j],
j);
ret = -1;
goto out;
}
j++;
}
#ifdef DEBUG
gaspi_printf("Check!\n");
#endif
k++;
}
out:
gaspi_printf("Waiting to finish...\n");
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return ret;
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_rank_t numranks, myrank;
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
int rankSend = (myrank + 1) % numranks;
ASSERT(gaspi_segment_create(0, _1MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
gaspi_size_t segSize;
ASSERT( gaspi_segment_size(0, myrank, &segSize));
unsigned char * pGlbMem;
gaspi_pointer_t _vptr;
ASSERT(gaspi_segment_ptr(0, &_vptr));
pGlbMem = ( unsigned char *) _vptr;
gaspi_number_t queueSize, qmax ;
ASSERT (gaspi_queue_size_max(&qmax));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
const unsigned long localOff = 0;
const unsigned long remOff = 0;
/* write_notify */
do
{
ASSERT(gaspi_write_notify(0, localOff, rankSend,
0, remOff, 1,
(gaspi_notification_id_t) myrank, 1,
1, GASPI_BLOCK));
gaspi_queue_size(1, &queueSize);
}
while(queueSize < qmax);
EXPECT_FAIL_WITH(gaspi_write_notify(0, localOff, rankSend,
0, remOff, 1,
(gaspi_notification_id_t) myrank, 1,
1, GASPI_BLOCK),
GASPI_QUEUE_FULL);
ASSERT (gaspi_wait(1, GASPI_BLOCK));
/* write */
do
{
ASSERT(gaspi_write(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK));
gaspi_queue_size(1, &queueSize);
}
while(queueSize < qmax);
EXPECT_FAIL_WITH(gaspi_write(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK),
GASPI_QUEUE_FULL);
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT(gaspi_write(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK));
/* write + write_notify */
do
{
ASSERT(gaspi_write(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK));
gaspi_queue_size(1, &queueSize);
}
while(queueSize < qmax - 1);
EXPECT_FAIL_WITH(gaspi_write_notify(0, localOff, rankSend,
0, remOff, 1,
(gaspi_notification_id_t) myrank, 1,
1, GASPI_BLOCK),
GASPI_QUEUE_FULL);
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT(gaspi_write_notify(0, localOff, rankSend,
0, remOff, 1,
(gaspi_notification_id_t) myrank, 1,
1, GASPI_BLOCK));
/* read */
do
{
ASSERT(gaspi_read(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK));
gaspi_queue_size(1, &queueSize);
}
while(queueSize < qmax);
EXPECT_FAIL_WITH(gaspi_read(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK),
GASPI_QUEUE_FULL);
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT(gaspi_read(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK));
/* write_list_notify */
{
const gaspi_number_t nListElems = 255;
gaspi_number_t n;
gaspi_segment_id_t localSegs[nListElems];
gaspi_offset_t localOffs[nListElems];
const gaspi_rank_t rank2send = (myrank + 1) % numranks;
gaspi_segment_id_t remSegs[nListElems];
gaspi_offset_t remOffs[nListElems];
gaspi_size_t sizes[nListElems];
const unsigned int bytes = sizeof(int);
gaspi_offset_t initLocOff = 0;
gaspi_offset_t initRemOff = (bytes * nListElems + 64);
for(n = 0; n < nListElems; n++)
{
sizes[n] = bytes;
localSegs[n] = 0;
localOffs[n] = initLocOff;
initLocOff += bytes;
remSegs[n] = 0;
remOffs[n] = initRemOff;
initRemOff += bytes;
}
do
{
ASSERT( gaspi_write_list_notify( nListElems,
localSegs, localOffs, rank2send,
remSegs, remOffs, sizes,
0, myrank, 1,
0, GASPI_BLOCK));
gaspi_queue_size(0, &queueSize);
}
while(queueSize < qmax);
EXPECT_FAIL_WITH( gaspi_write_list_notify( nListElems,
localSegs, localOffs, rank2send,
remSegs, remOffs, sizes,
0, myrank, 1,
0, GASPI_BLOCK),
GASPI_QUEUE_FULL);
ASSERT (gaspi_wait(0, GASPI_BLOCK));
ASSERT( gaspi_write_list_notify( nListElems,
localSegs, localOffs, rank2send,
remSegs, remOffs, sizes,
0, myrank, 1,
0, GASPI_BLOCK));
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
const unsigned long N = (1 << 13);
gaspi_rank_t P, myrank;
ASSERT (gaspi_proc_num(&P));
ASSERT (gaspi_proc_rank(&myrank));
gaspi_printf("P = %d N = %lu\n", P, N);
gaspi_printf("Seg size: %lu MB\n", MAX (_4GB, 2 * ((N/P) * N * 2 * sizeof (double)))/1024/1024);
if(gaspi_segment_create(0, _1GB,
GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED) != GASPI_SUCCESS){
gaspi_printf("Failed to create segment\n");
return -1;
}
gaspi_pointer_t _vptr;
if(gaspi_segment_ptr(0, &_vptr) != GASPI_SUCCESS)
printf("gaspi_segment_ptr failed\n");
gaspi_number_t qmax ;
ASSERT (gaspi_queue_size_max(&qmax));
gaspi_printf("Queue max: %lu\n", qmax);
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
int i;
gaspi_number_t queueSize;
int rankSend = (myrank + 1) % P;
gaspi_printf("rank to: %d\n", rankSend);
for (i = 0; i < 2 * N; i ++)
{
gaspi_queue_size(1, &queueSize);
if (queueSize > qmax - 24)
{
gaspi_return_t ret;
do
{
ret = gaspi_wait(1, GASPI_TEST);
assert (ret != GASPI_ERROR);
}
while(ret != GASPI_SUCCESS);
gaspi_queue_size(1, &queueSize);
assert(queueSize == 0);
}
ASSERT (gaspi_write(0, 4, rankSend, 0, 6, 32768, 1, GASPI_TEST));
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main (int argc, char *argv[])
{
gaspi_proc_init(GASPI_BLOCK);
gaspi_rank_t myRank;
gaspi_rank_t nProc;
gaspi_proc_rank(&myRank);
gaspi_proc_num(&nProc);
if(nProc < 2)
goto end;
gaspi_number_t queue_size;
gaspi_number_t queue_max;
gaspi_queue_size_max(&queue_max);
if (myRank == 0)
gaspi_printf("Queue max is %d\n", queue_max);
gaspi_printf("Rank %i of %i started.\n", myRank, nProc);
const gaspi_segment_id_t segment_id = 0;
const gaspi_size_t nrReads = NR_OF_READS;
gaspi_group_commit(GASPI_GROUP_ALL,GASPI_BLOCK);
gaspi_segment_create(segment_id, nrReads * (RAWREADLENGTH) * sizeof(gaspi_char),GASPI_GROUP_ALL,GASPI_BLOCK,GASPI_ALLOC_DEFAULT);
gaspi_pointer_t _vptr; //pointer to the segment
if(gaspi_segment_ptr(segment_id, &_vptr) != GASPI_SUCCESS)
printf("gaspi_segment_ptr failed\n");
gaspi_char * shared_ptr = (gaspi_char *) _vptr;
// initialize and print segment
initReads(shared_ptr, nrReads, READLENGTH, myRank);
gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK);
//push the reads from the master to the slaves
gaspi_size_t r = 0;
int rawReadSize = RAWREADLENGTH * sizeof(gaspi_char);
int nrWorkers = nProc - 1;
int toRank;
gaspi_notification_id_t notif_id;
if (myRank == 0) {
for (r = 0; r < nrReads; r++) {
gaspi_queue_size(0, &queue_size);
if(queue_size > queue_max - 1)
gaspi_wait(0, GASPI_BLOCK); //wait for queue to become free again... (note: max is 1024)
toRank = (r % nrWorkers) + 1;
// notif_id = r + 1;
notif_id = ((r / nrWorkers) + 1);
if ( gaspi_write_notify( segment_id, // from segment
r*rawReadSize, // from offset
toRank, // to-rank
segment_id, // to segment
// ((int)(r/nrWorkers))*rawReadSize, // to-offset
r * rawReadSize,
rawReadSize, // size
notif_id, // notification id
r+1, // notification value (> 0!)
(gaspi_queue_id_t) 0, // notification queue
GASPI_BLOCK) == GASPI_SUCCESS) // block until written
gaspi_printf("Sending read %d from %d to rank %d with id %d\n", r, myRank, toRank, notif_id);
if (toRank == 2)
print_read(shared_ptr, r, READLENGTH, myRank);
}
}
//ranks receive reads from the master rank
if (myRank != 0) {
gaspi_notification_id_t fid;
gaspi_notification_t notification_value;
int nrOfReceives = (int)(nrReads / (nProc-1));
if (myRank <= nrReads % nrWorkers)
nrOfReceives++;
gaspi_printf("Rank %d -- listening for %d events...\n", myRank, nrOfReceives);
int complete = 0;
while (complete < nrOfReceives) {
if(gaspi_notify_waitsome( segment_id, // segment
1, // id of first notification to wait for
// nrReads,
nrOfReceives, // id of last notification to wait for (alternative)
&fid, // identifier (output parameter with the identifier of a received notification (?))
GASPI_TEST // immediately return (GASPI_TEST)
) == GASPI_SUCCESS) {
if(gaspi_notify_reset( segment_id, // segment
fid, // notification identifier
¬ification_value // notification value
) == GASPI_SUCCESS) {
complete++ ;
gaspi_printf("Rank %d -- got notification: read %d received (%d completed)\n", myRank, notification_value-1, complete);
if (myRank == 2)
print_read(shared_ptr, notification_value-1, READLENGTH, myRank);
}
}
}
}
// all values received ! print !
gaspi_barrier(GASPI_GROUP_ALL,GASPI_BLOCK);
gaspi_printf("Printing reads\n");
print_char_array_segment(shared_ptr, nrReads, READLENGTH, myRank);
// print_read(shared_ptr, 0, READLENGTH, myRank);
gaspi_barrier(GASPI_GROUP_ALL,GASPI_BLOCK);
gaspi_printf("Rank %d done\n", myRank);
//block and exit
end:
gaspi_barrier(GASPI_GROUP_ALL,GASPI_BLOCK);
gaspi_proc_term(GASPI_BLOCK);
return EXIT_SUCCESS;
}
int main(int argc, char* argv[])
{
int i, j;
gaspi_number_t gsize;
int comm_state = WORKING;
int num_failures = 0;
int timesteps = 0;
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_proc_num(&numprocs));
read_params(argc, argv, ×teps, &numprocs_idle);
numprocs_working = numprocs - numprocs_idle;
numprocs_working_and_idle = numprocs_working + numprocs_idle;
gaspi_rank_t *comm_main_ranks = malloc( numprocs_idle * sizeof(gaspi_rank_t));
init_array_2(comm_main_ranks, numprocs_working);
/* contains info of all processes:
which are working(0), broken(1) and idle(2).
keeps updated all the time(iterations) */
int * status_processes = (int *) malloc(numprocs * sizeof(int));
init_array_3(status_processes, numprocs, WORKING);
for(i = numprocs-1, j=0; j < numprocs_idle;--i,++j)
{
status_processes[i] = IDLE; // putting last processes to IDLE
}
// ===== GASPI group creation =====
if(status_processes[myrank]==WORKING)
{
ASSERT(gaspi_group_create(&COMM_MAIN));
gaspi_number_t i;
for(i=0; i<numprocs; i++)
{
if(status_processes[i]==WORKING)
{
ASSERT(gaspi_group_add(COMM_MAIN, i));
ASSERT(gaspi_group_size(COMM_MAIN, &gsize));
}
}
ASSERT(gaspi_group_ranks (COMM_MAIN, comm_main_ranks));
ASSERT(gaspi_group_commit (COMM_MAIN, GASPI_BLOCK));
}
/* ====== Init a SYNC FLAGS Segment ====== */
/* used to communicate the WORKING, BROKEN, or FINISHED_WORK status between the working and idle processes. */
gaspi_size_t SYNC_global_mem_size;
SYNC_global_mem_size = numprocs * sizeof(int);
gaspi_pointer_t gm_ptr_sync=NULL;
ASSERT(init_segment (gm_seg_sync_flags_id, SYNC_global_mem_size));
ASSERT(gaspi_segment_ptr (gm_seg_sync_flags_id, &gm_ptr_sync));
int * sync_flags = (int *) gm_ptr_sync;
init_array_3(sync_flags, numprocs, WORKING);
/* ====== Init a health check write FLAGS Segment ====== */
/* This array is used to send the gaspi_write message write before health_chk routine,
which will then update the gaspi internal health vector */
gaspi_size_t health_chk_global_mem_size;
health_chk_global_mem_size = numprocs*sizeof(int);
gaspi_pointer_t gm_ptr_health_chk=NULL;
ASSERT(init_segment (gm_seg_health_chk_array_id, health_chk_global_mem_size));
ASSERT(gaspi_segment_ptr (gm_seg_health_chk_array_id, &gm_ptr_health_chk));
gaspi_state_vector_t health_vec = (gaspi_state_vector_t) malloc(numprocs);
ASSERT(gaspi_state_vec_get(health_vec));
gaspi_rank_t * avoid_list= (gaspi_rank_t *) malloc(numprocs * sizeof(gaspi_rank_t));
for(i = 0;i < numprocs; ++i)
avoid_list[i] = (gaspi_rank_t) 0;
gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK);
/* ===== TIME-STEP LOOP ===== */
if(status_processes[myrank]==IDLE)
{
/* IDLE processes remain in this loop */
while(1)
{
gaspi_printf("%d.", myrank);
if(sync_flags[0] == WORKING)
{
/* NO FAILURE REPORTED */
usleep(1000000);
}
if(sync_flags[0] == BROKEN)
{
/* FAILURE REPORTED */
gaspi_printf("myrank: %d Broken reported\n", myrank);
comm_state=BROKEN;
break;
}
if(sync_flags[0] == WORKFINISHED)
{
/* WORKFINISHED REPORTED */
gaspi_printf("myrank: %d WorkFinished reported\n", myrank);
comm_state = WORKFINISHED;
break;
}
}
}
int time_step;
for(time_step=1; time_step <= timesteps && comm_state!=WORKFINISHED; time_step++)
{
gaspi_printf("== time_step: %d ==\n", time_step);
if(comm_state==WORKING && status_processes[myrank]==WORKING)
{
gaspi_barrier(COMM_MAIN, GASPI_TIMEOUT_TIME);
sleep(1); // NOTE: this is the work section.
if(time_step == 5 && myrank== 1)
{
exit (-1);
}
}
if(time_step<timesteps )
{
send_global_msg_to_check_state(health_vec, avoid_list);
num_failures = check_comm_health(status_processes, health_vec);
gaspi_printf("%d NUM_FAILURES at timestep %d = %d\n", myrank, time_step, num_failures);
if( num_failures != 0 )
{
rescue_process = numprocs_working;
if(myrank==0)
{
// message the IDLE process
sync_flags[0]=BROKEN;
for(i = 0 ; i < num_failures ; ++i)
{
/* TODO: multiple failures at the same time. */
gaspi_printf("messaging rescue_process: %d\n", rescue_process);
ASSERT(gaspi_write(gm_seg_sync_flags_id, 0, rescue_process, gm_seg_sync_flags_id, 0, sizeof(int), 0, GASPI_BLOCK));
rescue_process++;
}
}
if(myrank==0 || myrank==rescue_process)
gaspi_printf("%d REPAIRING COMM_MAIN FLAG 1\n", myrank);
update_status_processes_array(status_processes, health_vec);
numprocs_working_and_idle = refresh_numprocs_working_and_idle(status_processes);
if(myrank != rescue_process)
{
ASSERT(gaspi_group_delete(COMM_MAIN));
ASSERT(recover());
}
ASSERT(gaspi_group_create(&COMM_MAIN_NEW));
for(i = 0; i < numprocs; i++)
{
if(status_processes[i]==WORKING)
{
ASSERT(gaspi_group_add(COMM_MAIN_NEW, i));
ASSERT(gaspi_group_size(COMM_MAIN_NEW, &gsize));
if(gsize == numprocs_working)
break;
}
}
gaspi_printf("%d: COMM_MAIN_NEW size is: %hi\n", myrank, gsize);
ASSERT(gaspi_group_commit (COMM_MAIN_NEW, GASPI_BLOCK));
init_array_2(comm_main_ranks, numprocs_working);
ASSERT(gaspi_group_ranks (COMM_MAIN_NEW, comm_main_ranks));
gaspi_printf("printing group_ranks_main: \n");
gaspi_printf_array(comm_main_ranks, numprocs_working);
comm_state = WORKING;
gaspi_printf("%d REPAIRING COMM_MAIN_NEW FLAG 2\n", myrank);
if(status_processes[myrank] == WORKING)
{
ASSERT(gaspi_barrier(COMM_MAIN_NEW, GASPI_BLOCK));
ASSERT(gaspi_barrier(COMM_MAIN_NEW, GASPI_BLOCK));
}
/* set things to work again */
COMM_MAIN = COMM_MAIN_NEW;
time_step = 5;
}
}
}
if(myrank == 0)
{
gaspi_printf("finished successfully\n");
}
gaspi_proc_term(10000);
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
int i, iter;
int ret = 0;
gaspi_rank_t myrank, numranks;
gaspi_size_t mem_size = 0UL, j;
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_proc_num(&numranks));
if( numranks < 2 )
{
return EXIT_SUCCESS;
}
mem_size = 2 * SLOT_SIZE * (numranks - 1);
if(myrank == 0)
{
printf("Mem size: %lu (%.2f MB)\nProcs: %u Max Slot size %lu Iterations %d\n",
mem_size,
mem_size * 1.0f / 1024/ 1024,
numranks,
(gaspi_size_t) SLOT_SIZE,
MAX_ITERATIONS);
#ifdef WITH_SYNC
printf("Using notifications only\n");
#endif
}
ASSERT (gaspi_segment_create(0, mem_size, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
gaspi_pointer_t _vptr;
ASSERT (gaspi_segment_ptr(0, &_vptr));
float *mptr = (float *) _vptr;
//generate random
srand((unsigned)time(0));
srand48((unsigned) time(0));
gaspi_size_t cur_slot_size = SLOT_SIZE;
for(cur_slot_size = SLOT_SIZE; cur_slot_size >= sizeof(float); cur_slot_size/=2)
{
if(myrank == 0)
printf("===== Slot Size %lu ====\n", cur_slot_size);
for(iter = 0; iter < MAX_ITERATIONS; iter++)
{
if(myrank == 0)
printf("iteration %3d... ", iter);
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
/* fill slots with randoms */
for(j = 0; j < (mem_size / sizeof(float) / 2); j++)
{
mptr[j]= drand48() + (myrank*1.0);
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
gaspi_offset_t offset_in = 0, offset_out = mem_size / 2;
/* rank 0 write to all others */
if(myrank == 0)
{
for (i = 1; i < numranks; i++)
{
offset_in = (i - 1) * cur_slot_size;
#ifdef WITH_SYNC
ASSERT (gaspi_write_notify(0, offset_in, i,
0, 0, cur_slot_size,
0, 1,
0, GASPI_BLOCK));
#else
ASSERT (gaspi_write(0, offset_in, i,
0, 0, cur_slot_size,
0, GASPI_BLOCK));
#endif
}
ASSERT(gaspi_wait(0, GASPI_BLOCK));
}
#ifdef WITH_SYNC
else
{
gaspi_notification_id_t id;
gaspi_notification_t val;
ASSERT(gaspi_notify_waitsome(0, 0, 1, &id, GASPI_BLOCK));
ASSERT(gaspi_notify_reset(0, id, &val));
assert(val == 1);
}
#endif
#ifndef WITH_SYNC
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
#endif
/* other ranks all write back to 0 */
if(myrank != 0)
{
offset_in = 0;
offset_out = (mem_size / 2) + (cur_slot_size * (myrank - 1));
#ifdef WITH_SYNC
ASSERT (gaspi_write_notify(0, offset_in, 0,
0, offset_out, cur_slot_size,
myrank, 1,
0, GASPI_BLOCK));
#else
ASSERT (gaspi_write(0, offset_in, i,
0, offset_out, cur_slot_size,
0, GASPI_BLOCK));
#endif
ASSERT(gaspi_wait(0, GASPI_BLOCK));
}
#ifdef WITH_SYNC
else
{
gaspi_notification_id_t id;
gaspi_notification_t val;
int notification_counter = 0;
do
{
ASSERT(gaspi_notify_waitsome(0, 1, numranks - 1, &id, GASPI_BLOCK));
ASSERT(gaspi_notify_reset(0, id, &val));
assert(val == 1);
notification_counter++;
}
while( notification_counter < numranks - 1);
}
#endif
#ifndef WITH_SYNC
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
#endif
if(myrank == 0)
{
/* check correctness */
float *in = (float *) _vptr;
float *out = (float *) ((char *) _vptr + mem_size / 2);
const gaspi_size_t total_elems = (cur_slot_size * (numranks - 1) / sizeof(float));
for(j = 0; j < total_elems; j++)
{
if(in[j] != out[j])
{
printf("Different values at pos %lu: %f %f (iterations %d)\n", j, in[j], out[j], iter);
ret = -1;
goto end;
}
}
printf("All fine!\n");
}
}
}
end:
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return ret;
}
int
main(int argc, char *argv[])
{
const int num_elems = 1024;
TSUITE_INIT( argc, argv );
ASSERT( gaspi_proc_init(GASPI_BLOCK) );
gaspi_rank_t rank, nprocs;
ASSERT( gaspi_proc_num(&nprocs) );
ASSERT( gaspi_proc_rank(&rank) );
const gaspi_rank_t left = (rank + nprocs - 1 ) % nprocs;
const gaspi_rank_t right = (rank + nprocs + 1) % nprocs;
/* Create and fill buffer */
int * const buf = (int *) malloc(num_elems * sizeof(int));
assert( buf != NULL);
int i;
for (i = 0; i < num_elems; i++)
{
buf[i] = rank;
}
ASSERT( gaspi_segment_use( 0, buf, num_elems * sizeof(int),
GASPI_GROUP_ALL, GASPI_BLOCK,
0) );
ASSERT( gaspi_segment_create( 1, num_elems * sizeof(int),
GASPI_GROUP_ALL, GASPI_BLOCK,
GASPI_MEM_INITIALIZED) );
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
/* write data to neighbour ( from seg 0 to seg 1) */
ASSERT( gaspi_write_notify( 0, 0, right,
1, 0, num_elems * sizeof(int),
0, 1,
0, GASPI_BLOCK) );
gaspi_notification_id_t id;
ASSERT( gaspi_notify_waitsome( 1, 0, 1, &id, GASPI_BLOCK ) );
ASSERT( gaspi_wait( 0, GASPI_BLOCK ) );
/* Check data as segment */
gaspi_pointer_t seg1_ptr;
ASSERT( gaspi_segment_ptr( 1, &seg1_ptr ) );
int * recv_buf = (int *) seg1_ptr;
for (i = 0; i < num_elems; i++)
{
assert(recv_buf[i] == left);
}
ASSERT( gaspi_segment_delete(0));
ASSERT( gaspi_segment_delete(1));
/* Check data in buffer */
for (i = 0; i < num_elems; i++)
{
assert(buf[i] == rank);
}
ASSERT( gaspi_barrier( GASPI_GROUP_ALL, GASPI_BLOCK ) );
ASSERT( gaspi_proc_term( GASPI_BLOCK ) );
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_rank_t numranks, myrank;
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_segment_create(0, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
ASSERT (gaspi_segment_create(1, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
gaspi_number_t queueSize, qmax;
ASSERT (gaspi_queue_size_max(&qmax));
gaspi_pointer_t segPtr, segPtr1;
ASSERT( gaspi_segment_ptr(0, &segPtr));
int* segInt = (int *) segPtr;
ASSERT( gaspi_segment_ptr(1, &segPtr1));
int* segInt1 = (int *) segPtr1;
gaspi_size_t segSize;
ASSERT( gaspi_segment_size(0, myrank, &segSize));
int i;
for(i = 0; i < segSize / sizeof(int); i++)
{
segInt[i] = myrank;
segInt1[i] = -1;
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
gaspi_size_t commSize ;
gaspi_offset_t localOff= 0;
const gaspi_offset_t remOff = 0;
for(commSize = sizeof(int); commSize < _8MB; commSize *= 2 )
{
gaspi_rank_t rankSend;
for(rankSend = 0; rankSend < numranks; rankSend++)
{
if(rankSend == myrank)
continue;
gaspi_queue_size(1, &queueSize);
if (queueSize > qmax - 24)
{
ASSERT (gaspi_wait(1, GASPI_BLOCK));
}
ASSERT (gaspi_read(1, localOff, rankSend, 0, remOff, commSize, 1, GASPI_BLOCK));
localOff+= commSize;
}
ASSERT (gaspi_wait(1, GASPI_BLOCK));
const int elems_per_rank = commSize / sizeof(int);
int c, pos = 0;
int* seg_read = (int *) segPtr1;
for(rankSend = 0; rankSend < numranks; rankSend++)
{
if(rankSend == myrank)
continue;
for(c = 0; c < elems_per_rank; c++)
{
assert (seg_read[pos] == rankSend);
pos++;
}
}
localOff = 0;
}
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int
main (int argc, char *argv[])
{
int i, j, t;
gaspi_rank_t myrank;
char *ptr0;
//on numa architectures you have to map this process to the numa node where nic is installed
if (start_bench (2) != 0)
{
printf ("Initialization failed\n");
exit (-1);
}
// BENCH //
gaspi_proc_rank (&myrank);
if (gaspi_segment_ptr (0, (void **) &ptr0) != GASPI_SUCCESS)
{
printf ("gaspi_segment_ptr failed !\n");
exit (-1);
}
gaspi_float cpu_freq;
gaspi_cpu_frequency(&cpu_freq);
if (myrank < 2)
{
if(myrank == 0)
{
printf("-----------------------------------\n");
printf ("%12s\t%5s\n", "Bytes", "Lat(usecs)");
printf("-----------------------------------\n");
}
int bytes = 2;
volatile char *postBuf = (volatile char *) ptr0;
for (i = 1; i < 24; i++)
{
volatile char *pollBuf = (volatile char *) (ptr0 + ( 2 * bytes -1 ));
int rcnt = 0;
int cnt = 0;
gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK);
for (j = 0; j < ITERATIONS; j++)
{
if (rcnt < ITERATIONS && !(cnt < 1 && myrank == 1))
{
rcnt++;
while (*pollBuf != (char) rcnt)
{
#ifdef MIC
_mm_delay_32(32);
#else
_mm_pause();
#endif
}
}
stamp[j] = get_mcycles ();
postBuf[bytes - 1] = (char) ++cnt;
gaspi_write (0, 0, myrank ^ 0x1,
0, bytes, bytes,
0, GASPI_BLOCK);
gaspi_wait (0, GASPI_BLOCK);
}
for (t = 0; t < (ITERATIONS - 1); t++)
delta[t] = stamp[t + 1] - stamp[t];
qsort (delta, (ITERATIONS - 1), sizeof *delta, mcycles_compare);
const double div = 1.0 / cpu_freq;
const double ts = (double) delta[ITERATIONS / 2] * div * 0.5;
if(myrank == 0)
printf ("%12d\t%4.2f\n", bytes, ts);
bytes <<= 1;
}
}
end_bench ();
return 0;
}
int
main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT( gaspi_proc_init(GASPI_BLOCK) );
gaspi_rank_t numranks, myrank;
ASSERT( gaspi_proc_num(&numranks) );
ASSERT( gaspi_proc_rank(&myrank) );
ASSERT( gaspi_segment_create(0, _8MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED) );
ASSERT( gaspi_segment_create(1, _8MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED) );
//prepare memory segment
gaspi_pointer_t _vptr, _vptr1;
ASSERT (gaspi_segment_ptr(0, &_vptr));
ASSERT (gaspi_segment_ptr(1, &_vptr1));
int *mem = (int *) _vptr;
int *mem_read = (int *) _vptr1;
unsigned long i;
const unsigned long maxInts = _8MB / sizeof(int);
for(i = 0; i < maxInts; i++)
{
mem[i] = (int) myrank;
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
//construct list of n elems
gaspi_number_t queue_size = 0;
const gaspi_number_t nListElems = 255;
gaspi_number_t n;
gaspi_segment_id_t localSegs[nListElems];
gaspi_offset_t localOffs[nListElems];
const gaspi_rank_t rank2read = (myrank + 1) % numranks;
gaspi_segment_id_t remSegs[nListElems];
gaspi_offset_t remOffs[nListElems];
gaspi_size_t sizes[nListElems];
const unsigned int bytes = sizeof(int);
gaspi_offset_t initLocOff = 0;
gaspi_offset_t initRemOff = 0;
for(n = 0; n < nListElems; n++)
{
sizes[n] = bytes;
localSegs[n] = 1;
localOffs[n] = initLocOff;
initLocOff += bytes;
remSegs[n] = 0;
remOffs[n] = initRemOff;
initRemOff += bytes;
}
ASSERT( gaspi_read_list_notify( nListElems,
localSegs, localOffs, rank2read,
remSegs, remOffs, sizes,
1, myrank,
0, GASPI_BLOCK));
gaspi_notification_id_t id;
ASSERT (gaspi_notify_waitsome(1, myrank, 1, &id, GASPI_BLOCK));
gaspi_notification_t notification_val;
ASSERT( gaspi_notify_reset(1, id, ¬ification_val));
assert(notification_val == 1);
//check
gaspi_number_t l;
for(l = 0; l < nListElems; l++)
{
assert(mem_read[l] == (int) rank2read);
}
ASSERT (gaspi_wait(0, GASPI_BLOCK));
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main (int argc, char *argv[])
{
SUCCESS_OR_DIE (gaspi_proc_init (GASPI_BLOCK));
gaspi_rank_t iProc, nProc;
SUCCESS_OR_DIE (gaspi_proc_rank (&iProc));
SUCCESS_OR_DIE (gaspi_proc_num (&nProc));
// number of threads
const int NTHREADS = 2;
// number of buffers
const int NWAY = 2;
// allocate segment for array for local vector, left halo and right halo
gaspi_segment_id_t const segment_id = 0;
SUCCESS_OR_DIE ( gaspi_segment_create
( segment_id, NWAY * (NTHREADS + 2) * 2 * VLEN * sizeof (double)
, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED));
gaspi_pointer_t array;
SUCCESS_OR_DIE ( gaspi_segment_ptr ( segment_id, &array) );
// initial buffer id
int buffer_id = 0;
// set notification values
gaspi_notification_id_t left_data_available[NWAY];
gaspi_notification_id_t right_data_available[NWAY];
for (gaspi_notification_id_t id = 0; id < NWAY; ++id)
{
left_data_available[id] = id;
right_data_available[id] = NWAY + id;
}
// set queue id
gaspi_queue_id_t queue_id = 0;
// initialize slice data structures
slice *ssl = (slice *) malloc (NTHREADS * sizeof (slice));
ASSERT (ssl);
init_slices (ssl, NTHREADS);
// initialize data
data_init (NTHREADS,iProc, buffer_id, array);
const int right_halo = NTHREADS+1;
const int left_halo = 0;
// issue initial write to left ngb
wait_for_queue_max_half (&queue_id);
SUCCESS_OR_DIE ( gaspi_write_notify
( segment_id, array_OFFSET_left (buffer_id, left_halo + 1, 0), LEFT(iProc, nProc)
, segment_id, array_OFFSET_left (buffer_id, right_halo, 0), VLEN * sizeof (double)
, right_data_available[buffer_id], 1, queue_id, GASPI_BLOCK));
// issue initial write to right ngb
wait_for_queue_max_half (&queue_id);
SUCCESS_OR_DIE ( gaspi_write_notify
( segment_id, array_OFFSET_right (buffer_id, right_halo - 1, 0), RIGHT(iProc, nProc)
, segment_id, array_OFFSET_right (buffer_id, left_halo, 0), VLEN * sizeof (double)
, left_data_available[buffer_id], 1, queue_id, GASPI_BLOCK));
// set total number of iterations per slice
const int num = nProc * NTHREADS * NITER;
omp_set_num_threads (NTHREADS);
double time = -now();
#pragma omp parallel default (none) firstprivate (buffer_id, queue_id) \
shared (array, left_data_available, right_data_available, ssl, stderr)
{
slice* sl;
while ((sl = get_slice_and_lock (ssl, NTHREADS, num)))
{
handle_slice ( sl, array, left_data_available, right_data_available
, segment_id, queue_id, NWAY, NTHREADS, num);
/*
TODO
====
- Which functionality do we need in 'handle_slice' ?
(asynchronous dataflow for 1-D halo-exchange)
- Discuss.
- Bonus question:
Can we be at different iteration stages for left and right halo ?
if yes: Why ?
*/
omp_unset_lock (&sl->lock);
}
#pragma omp barrier
}
time += now();
data_verify (NTHREADS, iProc, (NITER * nProc * NTHREADS) % NWAY, array);
printf ("# gaspi %s nProc %d vlen %i niter %d nthreads %i nway %i time %g\n"
, argv[0], nProc, VLEN, NITER, NTHREADS, NWAY, time
);
gaspi_proc_term (GASPI_BLOCK);
return EXIT_SUCCESS;
}
int
main(int argc,
char *argv[])
{
static const int VLEN = 1 << 2;
SUCCESS_OR_DIE(gaspi_proc_init (GASPI_BLOCK));
gaspi_rank_t iProc, nProc;
SUCCESS_OR_DIE(gaspi_proc_rank (&iProc));
SUCCESS_OR_DIE(gaspi_proc_num (&nProc));
gaspi_segment_id_t const segment_id = 0;
gaspi_size_t const segment_size = 2 * VLEN * sizeof(double);
SUCCESS_OR_DIE(gaspi_segment_create
(segment_id,
segment_size,
GASPI_GROUP_ALL,
GASPI_BLOCK,
GASPI_MEM_UNINITIALIZED
));
gaspi_pointer_t array;
SUCCESS_OR_DIE(gaspi_segment_ptr (segment_id,
&array));
double * src_array = (double *) (array);
double * rcv_array = src_array + VLEN;
for (int j = 0; j < VLEN; ++j)
{
src_array[j] = (double) (iProc * VLEN + j);
}
gaspi_notification_id_t data_available = 0;
gaspi_queue_id_t queue_id = 0;
gaspi_offset_t loc_off = 0;
gaspi_offset_t rem_off = VLEN * sizeof(double);
/*
TODO
====
- check queue.
- gaspi_write_notify 1-dim array of size VLEN to right neighbour
- wait for notification from left neighbour
*/
for (int j = 0; j < VLEN; ++j)
{
printf ("rank %d rcv elem %d: %f \n",
iProc,
j,
rcv_array[j]);
}
/*
TODO
====
- why do we have to wait for the queue here ?
*/
SUCCESS_OR_DIE(gaspi_wait (queue_id,
GASPI_BLOCK));
SUCCESS_OR_DIE(gaspi_proc_term (GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_rank_t numranks, myrank;
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_segment_create(0, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
//prepare memory segment
gaspi_pointer_t _vptr;
ASSERT (gaspi_segment_ptr(0, &_vptr));
int *mem = (int *) _vptr;
unsigned long i;
const unsigned long maxInts = _128MB / sizeof(int);
for(i = 0; i < maxInts; i++)
{
mem[i] = (int) myrank;
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
//construct list of n elems
gaspi_number_t queue_size = 0;
const gaspi_number_t nListElems = 255;
gaspi_number_t n;
gaspi_segment_id_t localSegs[nListElems];
gaspi_offset_t localOffs[nListElems];
const gaspi_rank_t rank2read = (myrank + 1) % numranks;
gaspi_segment_id_t remSegs[nListElems];
gaspi_offset_t remOffs[nListElems];
gaspi_size_t sizes[nListElems];
const unsigned int bytes = sizeof(int);
gaspi_offset_t initLocOff = 0;
gaspi_offset_t initRemOff = (bytes * nListElems + 64);
for(n = 0; n < nListElems; n++)
{
sizes[n] = bytes;
localSegs[n] = 0;
localOffs[n] = initLocOff;
initLocOff += bytes;
remSegs[n] = 0;
remOffs[n] = initRemOff;
initRemOff += bytes;
}
ASSERT (gaspi_read_list(nListElems, localSegs,localOffs, rank2read, remSegs, remOffs, sizes, 0, GASPI_BLOCK));
ASSERT (gaspi_queue_size(0, &queue_size));
assert (queue_size == nListElems);
ASSERT (gaspi_wait(0, GASPI_BLOCK));
//sync
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
//check
gaspi_number_t l;
gaspi_offset_t off2check = 0;
char * chPtr = (char *) _vptr;
mem = (int *) (chPtr + off2check);
for(l = 0; l < nListElems; l++)
{
assert(mem[l] == (int) rank2read);
}
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_rank_t numranks, myrank;
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
int rankSend = (myrank + 1) % numranks;
gaspi_printf("Seg size: %lu MB\n", _2GB / 1024 / 1024);
ASSERT(gaspi_segment_create(0, _2GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
gaspi_size_t segSize;
ASSERT( gaspi_segment_size(0, myrank, &segSize));
unsigned char * pGlbMem;
gaspi_pointer_t _vptr;
ASSERT(gaspi_segment_ptr(0, &_vptr));
pGlbMem = ( unsigned char *) _vptr;
gaspi_number_t qmax ;
ASSERT (gaspi_queue_size_max(&qmax));
unsigned long i;
unsigned long size = 1800;
for(i = 0; i < size / sizeof(unsigned char); i++)
pGlbMem[i] = myrank;
gaspi_printf("Queue max: %lu\n", qmax);
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
unsigned long localOff = 0;
unsigned long remOff = size;
ASSERT(gaspi_write_notify(0, localOff, rankSend,
0, remOff, size,
(gaspi_notification_id_t) myrank, 1,
1, GASPI_BLOCK));
gaspi_rank_t rankGet = (myrank + numranks - 1) % numranks;
gaspi_notification_t got_val;
gaspi_notification_id_t got;
ASSERT(gaspi_notify_waitsome(0, (gaspi_notification_id_t) rankGet, 1, &got, GASPI_BLOCK));
ASSERT(gaspi_notify_reset(0, got, &got_val));
ASSERT (gaspi_wait(1, GASPI_BLOCK));
/* check */
for(i = size; i < 2 * size / sizeof(unsigned char); i++)
assert(pGlbMem[i] == rankGet);
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
int j,i,k=0;
int ret=0;
const gaspi_size_t size=4096;//4k
const gaspi_size_t memSize = 4294967296; //4GB
gaspi_offset_t offset_write=0, offset_read = memSize / 2, offset_check = 3221225472 ;
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT (gaspi_segment_create(0, memSize, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
gaspi_pointer_t _vptr;
ASSERT (gaspi_segment_ptr(0, &_vptr));
/* get memory area pointer */
#ifdef FLOAT
float *mptr = (float *) _vptr;
#else
int *mptr = (int *) _vptr;
#endif
gaspi_rank_t myrank, highestnode;
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_proc_num(&highestnode));
while(k <= RUNS)
{
//generate random
srand((unsigned)time(0));
#ifdef FLOAT
srand48((unsigned) time(0));
#endif
//clean
for(j = 0; j < (memSize / 4); j++)
mptr[j]= 0;
//fill randoms up to 1GB
for(j = 0; j < (memSize / 16); j++)
{
#ifdef FLOAT
mptr[j]= drand48() + (myrank*1.0);
#else
mptr[j]= rand() + myrank;
#endif
}
#ifdef DEBUG
#ifdef FLOAT
gaspi_printf("random value in pos 0 %f\n", mptr[0]);
#else
gaspi_printf("random value in pos 0 %d\n", mptr[0]);
#endif
#endif //DEBUG
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
gaspi_printf("\n....Running iteration %d of %d...\n",k, RUNS);
for(i = 0; i < ITERATIONS; i++)
{
for(j = 0; j < ITERATIONS; j++)
{
ASSERT (gaspi_write(0, offset_write, (myrank + 1) % highestnode,
0, offset_read, size, 0, GASPI_BLOCK));
offset_write += size;
offset_read += size;
}
ASSERT (gaspi_wait(0, GASPI_BLOCK));
}
#ifdef DEBUG
gaspi_printf("%d bytes written!\n", ITERATIONS * ITERATIONS * size);
#endif
//check if data was written successfully
ASSERT (gaspi_read(0, offset_check, (myrank + 1) % highestnode,
0, memSize/2, GB, 0, GASPI_BLOCK));
ASSERT (gaspi_wait(0, GASPI_BLOCK));
#ifdef DEBUG
gaspi_printf("%d bytes read!\n",GB);
#endif
j=0;
#ifdef DEBUG
#ifdef FLOAT
gaspi_printf("Values %f %f %f \n", mptr[0], mptr[memSize/8], mptr[offset_check/4]);
#else
gaspi_printf("Values %d %d %d \n", mptr[0], mptr[memSize/8], mptr[offset_check/4]);
#endif
#endif//DEBUG
while(j < GB / 4 )
{
if(mptr[j] != mptr[offset_check / 4 + j]){
#ifdef FLOAT
gaspi_printf("value incorrect %f-%f at %d \n",mptr[j],mptr[offset_check / 4],j);
#else
gaspi_printf("value incorrect %d-%d at %d \n",mptr[j],mptr[offset_check / 4],j);
#endif
ret = -1;
goto out;
}
j++;
}
offset_write=0;
offset_read = memSize / 2;
#ifdef DEBUG
gaspi_printf("Check!\n");
#endif
k++;
}
out:
gaspi_printf("Waiting to finish...\n");
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return ret;
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_notification_id_t n=0;
gaspi_rank_t rank, nprocs, i;
const gaspi_segment_id_t seg_id = 0;
gaspi_offset_t offset;
gaspi_number_t queue_size;
gaspi_number_t queue_max;
ASSERT (gaspi_queue_size_max(&queue_max));
ASSERT(gaspi_proc_num(&nprocs));
ASSERT (gaspi_proc_rank(&rank));
ASSERT (gaspi_segment_create(seg_id, nprocs * sizeof(int), GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED));
offset = rank * sizeof(int);
//set memory
gaspi_pointer_t _vptr;
ASSERT (gaspi_segment_ptr(0, &_vptr));
int *mem = (int *) _vptr;
for(i = 0; i < nprocs; i++)
{
mem[i] = (int) rank;
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
//go
for(i = 0; i < nprocs; i++)
{
if (i == rank)
continue;
ASSERT (gaspi_queue_size(0, &queue_size));
if(queue_size > queue_max - 1)
ASSERT (gaspi_wait(0, GASPI_BLOCK));
ASSERT (gaspi_write_notify( seg_id, offset, i,
seg_id, offset, sizeof(int),
(gaspi_notification_id_t) rank, 1,
0, GASPI_BLOCK));
}
do
{
gaspi_notification_id_t id;
ASSERT (gaspi_notify_waitsome(seg_id, 0, (gaspi_notification_id_t) nprocs , &id, GASPI_BLOCK));
gaspi_notification_t notification_val;
ASSERT( gaspi_notify_reset(seg_id, id, ¬ification_val));
assert(notification_val == 1);
n++;
}
while(n < (nprocs - 1));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main (int argc, char *argv[])
{
SUCCESS_OR_DIE (gaspi_proc_init (GASPI_BLOCK));
gaspi_rank_t iProc, nProc;
SUCCESS_OR_DIE (gaspi_proc_rank (&iProc));
SUCCESS_OR_DIE (gaspi_proc_num (&nProc));
// number of threads
const int NTHREADS = 2;
// number of buffers
const int NWAY = 2;
gaspi_segment_id_t const segment_id = 0;
// allocate segment for array for local vector, left halo and right halo
SUCCESS_OR_DIE ( gaspi_segment_create
( segment_id, NWAY * (NTHREADS + 2) * 2 * VLEN * sizeof (double)
, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED));
gaspi_pointer_t array;
SUCCESS_OR_DIE ( gaspi_segment_ptr ( segment_id, &array) );
// initial buffer id
int buffer_id = 0;
// set notification values
gaspi_notification_id_t left_data_available[NWAY];
gaspi_notification_id_t right_data_available[NWAY];
for (gaspi_notification_id_t id = 0; id < NWAY; ++id)
{
left_data_available[id] = id;
right_data_available[id] = NWAY + id;
}
// set queue id
gaspi_queue_id_t queue_id = 0;
// initialize data
data_init (NTHREADS, iProc, buffer_id, array);
omp_set_num_threads (NTHREADS);
double time = -now();
#pragma omp parallel default (shared) firstprivate (buffer_id)
{
const int tid = omp_get_thread_num();
for (int k = 0; k < NITER; ++k)
{
for ( int i = 0; i < nProc * NTHREADS; ++i )
{
const int left_halo = 0;
const int slice_id = tid + 1;
const int right_halo = NTHREADS+1;
if (tid == 0)
{
// issue write
wait_for_queue_max_half (&queue_id);
SUCCESS_OR_DIE ( gaspi_write_notify
( segment_id, array_OFFSET_left (buffer_id, left_halo + 1, 0), LEFT(iProc, nProc)
, segment_id, array_OFFSET_left (buffer_id, right_halo, 0), VLEN * sizeof (double)
, right_data_available[buffer_id], 1 + i, queue_id, GASPI_BLOCK));
// issue write
wait_for_queue_max_half (&queue_id);
SUCCESS_OR_DIE ( gaspi_write_notify
( segment_id, array_OFFSET_right (buffer_id, right_halo - 1, 0), RIGHT(iProc, nProc)
, segment_id, array_OFFSET_right (buffer_id, left_halo, 0), VLEN * sizeof (double)
, left_data_available[buffer_id], 1 + i, queue_id, GASPI_BLOCK));
// wait for data notification
wait_or_die (segment_id, right_data_available[buffer_id], 1 + i);
// wait for data notification
wait_or_die (segment_id, left_data_available[buffer_id], 1 + i);
}
#pragma omp barrier
// compute data, read from id "buffer_id", write to id "1 - buffer_id"
data_compute ( NTHREADS, array, 1 - buffer_id, buffer_id, slice_id);
#pragma omp barrier
// alternate the buffer
buffer_id = 1 - buffer_id;
}
}
}
time += now();
data_verify (NTHREADS, iProc, (NITER * nProc * NTHREADS) % NWAY, array);
printf ("# gaspi %s nProc %d vlen %i niter %d nthreads %i nway %i time %g\n"
, argv[0], nProc, VLEN, NITER, NTHREADS, NWAY, time
);
gaspi_proc_term (GASPI_BLOCK);
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_rank_t numranks, myrank;
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
//2 segments
ASSERT (gaspi_segment_create(0, _2GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
ASSERT (gaspi_segment_create(1, _2GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
//construct list of n elems
gaspi_number_t i, n, max;
gaspi_number_t queue_size = 0;
ASSERT( gaspi_rw_list_elem_max(&max));
for(n = 1; n < max; n++)
{
gaspi_number_t nListElems = n;
gaspi_segment_id_t localSegs[n];
gaspi_offset_t localOffs[n];
const gaspi_rank_t rank2send = (myrank + 1) % numranks;
gaspi_segment_id_t remSegs[n];
gaspi_offset_t remOffs[n];
gaspi_size_t sizes[n];
unsigned char flip = 0;
unsigned int bytes = 1;
gaspi_offset_t off = 0;
gaspi_size_t total_size = 0;
for(i = 0; i < nListElems; i++)
{
sizes[i] = bytes;
localSegs[i] = flip;
localOffs[i] = off;
remSegs[i] = flip;
remOffs[i] = off;
off+=sizes[i];
total_size+= bytes;
flip ^= 0x1;
bytes += 2;
}
ASSERT(gaspi_write_list(nListElems, localSegs, localOffs, rank2send, remSegs, remOffs, sizes, 0, GASPI_BLOCK));
ASSERT(gaspi_queue_size(0, &queue_size));
assert(queue_size == nListElems);
ASSERT(gaspi_wait(0, GASPI_BLOCK));
}
gaspi_pointer_t _vptr;
ASSERT (gaspi_segment_ptr(0, &_vptr));
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int
main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_rank_t rank, nprocs, i;
const gaspi_segment_id_t seg_id = 0;
const gaspi_offset_t offset = 0;
const gaspi_size_t transfer_size = 8192;
gaspi_number_t queue_size;
gaspi_number_t queue_max;
ASSERT (gaspi_queue_size_max(&queue_max));
ASSERT(gaspi_proc_num(&nprocs));
ASSERT (gaspi_proc_rank(&rank));
if( nprocs < 2 )
{
return EXIT_SUCCESS;
}
ASSERT (gaspi_segment_create(seg_id, nprocs * 2 * transfer_size, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED));
gaspi_number_t max_notifications;
ASSERT(gaspi_notification_num(&max_notifications));
gaspi_number_t avail_notifications = max_notifications / nprocs;
max_notifications = avail_notifications * nprocs;
gaspi_pointer_t _vptr;
ASSERT (gaspi_segment_ptr(0, &_vptr));
int *mem = (int *) _vptr;
for(i = 0; i < nprocs; i++)
{
mem[i] = (int) rank;
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
for(i = 0; i < nprocs; i++)
{
gaspi_notification_id_t not;
for(not = 0; not < avail_notifications; not++)
{
ASSERT (gaspi_queue_size(0, &queue_size));
if( queue_size > queue_max - 1 )
{
ASSERT (gaspi_wait(0, GASPI_BLOCK));
}
gaspi_notification_id_t the_notification = (gaspi_notification_id_t) (rank * avail_notifications + not);
ASSERT( gaspi_write_notify( seg_id, offset, i,
seg_id, offset, transfer_size,
the_notification, 1,
0, GASPI_BLOCK));
}
}
gaspi_notification_id_t n = 0;
do
{
gaspi_notification_id_t id;
ASSERT (gaspi_notify_waitsome(seg_id, 0, max_notifications - 1 , &id, GASPI_BLOCK));
gaspi_notification_t notification_val;
ASSERT( gaspi_notify_reset(seg_id, id, ¬ification_val));
assert(notification_val == 1);
n++;
}
while(n < max_notifications - 1);
ASSERT(gaspi_wait(0, GASPI_BLOCK));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
