void work(int tid)
{
gaspi_rank_t rankSend;
gaspi_offset_t localOff= 0;
gaspi_offset_t remOff = 0;
gaspi_number_t queueSize, qmax;
gaspi_size_t commSize ;
ASSERT (gaspi_queue_size_max(&qmax));
for(commSize= 1; commSize < _500MB; commSize*=2 )
for(rankSend = 0; rankSend < numranks; rankSend++)
{
gaspi_printf("thread %d rank to send: %d - %lu bytes\n", tid, rankSend, commSize);
gaspi_queue_size(1, &queueSize);
if (queueSize > qmax - 100)
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_write(0, localOff, rankSend, 0, remOff, commSize, 1, GASPI_BLOCK));
}
ASSERT (gaspi_wait(1, GASPI_BLOCK));
gaspi_threads_sync();
}
void work(int tid)
{
gaspi_rank_t rankSend;
gaspi_offset_t localOff = 81478066;
gaspi_offset_t remOff = 81478246;
gaspi_offset_t size = 1800;
gaspi_number_t queueSize, qmax;
ASSERT (gaspi_queue_size_max(&qmax));
for(rankSend = 0; rankSend < numranks; rankSend++)
{
gaspi_printf("thread %d rank to send: %d\n", tid, rankSend);
gaspi_queue_size(1, &queueSize);
if (queueSize > qmax - 24)
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_write(0, localOff, rankSend, 0, remOff, size, 1, GASPI_BLOCK));
}
ASSERT (gaspi_wait(1, GASPI_BLOCK));
gaspi_threads_sync();
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_notification_id_t n=0;
gaspi_number_t notif_num;
gaspi_rank_t rank, nprocs, i;
const gaspi_segment_id_t seg_id = 0;
ASSERT(gaspi_proc_num(&nprocs));
ASSERT (gaspi_proc_rank(&rank));
ASSERT (gaspi_segment_create(seg_id, 1024, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED));
ASSERT( gaspi_notification_num(¬if_num));
gaspi_printf("max num notifications %u\n", notif_num);
if(rank == 0)
{
gaspi_number_t queue_size;
gaspi_number_t queue_max;
ASSERT (gaspi_queue_size_max(&queue_max));
for(n = 0; n < notif_num; n++)
{
ASSERT (gaspi_queue_size(0, &queue_size));
if(queue_size > queue_max - 1)
ASSERT (gaspi_wait(0, GASPI_BLOCK));
for(i = 1; i < nprocs; i++)
{
ASSERT (gaspi_notify( seg_id, i, n, 1, 0, GASPI_BLOCK));
}
}
}
else
{
do
{
gaspi_notification_id_t id;
ASSERT (gaspi_notify_waitsome(seg_id, 0, notif_num, &id, GASPI_BLOCK));
gaspi_notification_t notification_val;
ASSERT( gaspi_notify_reset(seg_id, id, ¬ification_val));
assert(notification_val == 1);
n++;
}
while(n < notif_num);
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
gaspi_rank_t numranks, myrank;
gaspi_rank_t rankSend;
gaspi_size_t segSize;
const gaspi_offset_t localOff= 0;
const gaspi_offset_t remOff = 0;
gaspi_number_t queueSize, qmax;
gaspi_size_t commSize ;
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_segment_create(0, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
ASSERT( gaspi_segment_size(0, myrank, &segSize));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_queue_size_max(&qmax));
for(commSize= 4; commSize < _8MB; commSize*=2 )
{
for(rankSend = 0; rankSend < numranks; rankSend++)
{
if(rankSend == myrank)
continue;
printf("partner rank: %d - %lu bytes\n", rankSend, commSize);
//FAILS with or without outstanding requests
gaspi_queue_size(1, &queueSize);
if (queueSize > qmax - 24)
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_read(0, localOff, rankSend, 0, remOff, commSize, 1, GASPI_BLOCK));
}
}
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
static void wait_for_queue_entries (gaspi_queue_id_t* queue, int wanted_entries)
{
gaspi_number_t queue_size_max;
gaspi_number_t queue_size;
gaspi_number_t queue_num;
SUCCESS_OR_DIE (gaspi_queue_size_max (&queue_size_max));
SUCCESS_OR_DIE (gaspi_queue_size (*queue, &queue_size));
SUCCESS_OR_DIE (gaspi_queue_num (&queue_num));
if (! (queue_size + wanted_entries <= queue_size_max))
{
*queue = (*queue + 1) % queue_num;
SUCCESS_OR_DIE (gaspi_wait (*queue, GASPI_BLOCK));
}
}
int main(int argc, char *argv[])
{
gaspi_rank_t numranks, myrank;
gaspi_rank_t rankSend;
gaspi_size_t segSize;
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_segment_create(0, _2GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
ASSERT( gaspi_segment_size(0, myrank, &segSize));
gaspi_printf("seg size %lu MB \n", segSize/1024/1024);
// ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
gaspi_offset_t localOff= 814780664;
gaspi_offset_t remOff = 81478246;
gaspi_offset_t size = 1800;
gaspi_number_t queueSize, qmax;
ASSERT (gaspi_queue_size_max(&qmax));
for(rankSend = 0; rankSend < numranks; rankSend++)
{
gaspi_printf("rank to send: %d\n", rankSend);
gaspi_queue_size(1, &queueSize);
if (queueSize > qmax - 24)
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_write(0, localOff, rankSend, 0, remOff, size, 1, GASPI_BLOCK));
}
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
gaspi_rank_t numranks, myrank;
gaspi_rank_t rankSend;
gaspi_size_t segSize;
const gaspi_offset_t localOff= 0;
const gaspi_offset_t remOff = 0;
gaspi_number_t queueSize, qmax;
gaspi_size_t commSize ;
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_segment_create(0, _8MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_queue_size_max(&qmax));
for(commSize= 1; commSize <= _8MB; commSize*=2 )
{
for(rankSend = 0; rankSend < numranks; rankSend++)
{
gaspi_printf("rank to send: %d - %lu bytes\n", rankSend, commSize);
gaspi_queue_size(1, &queueSize);
if (queueSize > qmax - 24)
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_write(0, localOff, rankSend, 0, remOff, commSize, 1, GASPI_BLOCK));
}
}
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_rank_t numranks, myrank;
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_segment_create(0, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
//prepare memory segment
gaspi_pointer_t _vptr;
ASSERT (gaspi_segment_ptr(0, &_vptr));
int *mem = (int *) _vptr;
unsigned long i;
const unsigned long maxInts = _128MB / sizeof(int);
for(i = 0; i < maxInts; i++)
{
mem[i] = (int) myrank;
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
//construct list of n elems
gaspi_number_t queue_size = 0;
const gaspi_number_t nListElems = 255;
gaspi_number_t n;
gaspi_segment_id_t localSegs[nListElems];
gaspi_offset_t localOffs[nListElems];
const gaspi_rank_t rank2read = (myrank + 1) % numranks;
gaspi_segment_id_t remSegs[nListElems];
gaspi_offset_t remOffs[nListElems];
gaspi_size_t sizes[nListElems];
const unsigned int bytes = sizeof(int);
gaspi_offset_t initLocOff = 0;
gaspi_offset_t initRemOff = (bytes * nListElems + 64);
for(n = 0; n < nListElems; n++)
{
sizes[n] = bytes;
localSegs[n] = 0;
localOffs[n] = initLocOff;
initLocOff += bytes;
remSegs[n] = 0;
remOffs[n] = initRemOff;
initRemOff += bytes;
}
ASSERT (gaspi_read_list(nListElems, localSegs,localOffs, rank2read, remSegs, remOffs, sizes, 0, GASPI_BLOCK));
ASSERT (gaspi_queue_size(0, &queue_size));
assert (queue_size == nListElems);
ASSERT (gaspi_wait(0, GASPI_BLOCK));
//sync
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
//check
gaspi_number_t l;
gaspi_offset_t off2check = 0;
char * chPtr = (char *) _vptr;
mem = (int *) (chPtr + off2check);
for(l = 0; l < nListElems; l++)
{
assert(mem[l] == (int) rank2read);
}
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
int k = 0;
int ret = 0;
unsigned long j;
const gaspi_size_t size = 4096;
const gaspi_size_t memSize = _4GB;
gaspi_offset_t offset_write = 0;
gaspi_offset_t offset_read = _2GB;
gaspi_offset_t offset_check = 3221225472;
gaspi_number_t qmax ;
gaspi_number_t queueSize;
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT (gaspi_queue_size_max(&qmax));
ASSERT (gaspi_segment_create(0, memSize, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
gaspi_pointer_t _vptr;
ASSERT (gaspi_segment_ptr(0, &_vptr));
/* get memory area pointer */
float *mptr_f = (float *) _vptr;
char *mptr_c = (char *) _vptr;
gaspi_rank_t myrank, highestnode;
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_proc_num(&highestnode));
while(k <= RUNS)
{
//generate random
srand((unsigned)time(0));
srand48((unsigned) time(0));
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
//clean
for(j = 0; j < memSize; j++)
mptr_c[j]= 0;
/* fill randoms up to 1GB */
for(j = 0; j < (GB / sizeof(float)); j++)
{
mptr_f[j]= drand48() + (myrank * 1.0);
}
#ifdef DEBUG
gaspi_printf("random value in pos 0 %f\n", mptr_f[0]);
#endif
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
gaspi_printf("\n....Running iteration %d of %d...\n",k, RUNS);
const unsigned long packets = (GB / size);
for(j = 0; j < packets; j++)
{
ASSERT(gaspi_queue_size(0, &queueSize));
if (queueSize > qmax - 24)
{
ASSERT(gaspi_wait(0, GASPI_BLOCK));
}
ASSERT (gaspi_write(0, offset_write, (myrank + 1) % highestnode,
0, offset_read, size,
0, GASPI_BLOCK));
offset_write += size;
offset_read += size;
}
offset_write=0;
offset_read = _2GB;
#ifdef DEBUG
gaspi_printf("%d bytes written!\n", packets * size);
#endif
/* notify remote that data is written */
ASSERT (gaspi_notify( 0, (myrank + 1) % highestnode, 0, 1, 0, GASPI_BLOCK));
gaspi_notification_id_t recv_id;
ASSERT(gaspi_notify_waitsome(0, 0, 1, &recv_id, GASPI_BLOCK));
assert(recv_id == 0);
gaspi_notification_t notification_val;
ASSERT( gaspi_notify_reset(0, recv_id, ¬ification_val));
/* notify remote that data has arrived */
ASSERT (gaspi_notify( 0, (myrank + highestnode - 1) % highestnode, 1, 1, 0, GASPI_BLOCK));
gaspi_notification_id_t ack_id;
ASSERT(gaspi_notify_waitsome(0, 1, 1, &ack_id, GASPI_BLOCK));
assert(ack_id == 1);
ASSERT( gaspi_notify_reset(0, ack_id, ¬ification_val));
/* check if data was written successfully */
ASSERT (gaspi_read(0, offset_check, (myrank + 1) % highestnode,
0, offset_read, GB / 2,
0, GASPI_BLOCK));
ASSERT (gaspi_read(0, offset_check + (GB / 2), (myrank + 1) % highestnode,
0, offset_read + (GB / 2), GB / 2,
0, GASPI_BLOCK));
ASSERT (gaspi_wait(0, GASPI_BLOCK));
#ifdef DEBUG
gaspi_printf("Values %f %f %f \n", mptr_f[0], mptr_f[offset_read / sizeof(float)], mptr_f[offset_check / sizeof(float)]);
#endif
j = 0;
while(j < GB / sizeof(float) )
{
if(mptr_f[j] != mptr_f[offset_check / sizeof(float) + j]){
gaspi_printf("value incorrect %f-%f at %d \n",
mptr_f[j],
mptr_f[offset_check / sizeof(float) + j],
j);
ret = -1;
goto out;
}
j++;
}
#ifdef DEBUG
gaspi_printf("Check!\n");
#endif
k++;
}
out:
gaspi_printf("Waiting to finish...\n");
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return ret;
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_rank_t numranks, myrank;
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
int rankSend = (myrank + 1) % numranks;
ASSERT(gaspi_segment_create(0, _1MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
gaspi_size_t segSize;
ASSERT( gaspi_segment_size(0, myrank, &segSize));
unsigned char * pGlbMem;
gaspi_pointer_t _vptr;
ASSERT(gaspi_segment_ptr(0, &_vptr));
pGlbMem = ( unsigned char *) _vptr;
gaspi_number_t queueSize, qmax ;
ASSERT (gaspi_queue_size_max(&qmax));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
const unsigned long localOff = 0;
const unsigned long remOff = 0;
/* write_notify */
do
{
ASSERT(gaspi_write_notify(0, localOff, rankSend,
0, remOff, 1,
(gaspi_notification_id_t) myrank, 1,
1, GASPI_BLOCK));
gaspi_queue_size(1, &queueSize);
}
while(queueSize < qmax);
EXPECT_FAIL_WITH(gaspi_write_notify(0, localOff, rankSend,
0, remOff, 1,
(gaspi_notification_id_t) myrank, 1,
1, GASPI_BLOCK),
GASPI_QUEUE_FULL);
ASSERT (gaspi_wait(1, GASPI_BLOCK));
/* write */
do
{
ASSERT(gaspi_write(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK));
gaspi_queue_size(1, &queueSize);
}
while(queueSize < qmax);
EXPECT_FAIL_WITH(gaspi_write(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK),
GASPI_QUEUE_FULL);
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT(gaspi_write(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK));
/* write + write_notify */
do
{
ASSERT(gaspi_write(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK));
gaspi_queue_size(1, &queueSize);
}
while(queueSize < qmax - 1);
EXPECT_FAIL_WITH(gaspi_write_notify(0, localOff, rankSend,
0, remOff, 1,
(gaspi_notification_id_t) myrank, 1,
1, GASPI_BLOCK),
GASPI_QUEUE_FULL);
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT(gaspi_write_notify(0, localOff, rankSend,
0, remOff, 1,
(gaspi_notification_id_t) myrank, 1,
1, GASPI_BLOCK));
/* read */
do
{
ASSERT(gaspi_read(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK));
gaspi_queue_size(1, &queueSize);
}
while(queueSize < qmax);
EXPECT_FAIL_WITH(gaspi_read(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK),
GASPI_QUEUE_FULL);
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT(gaspi_read(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK));
/* write_list_notify */
{
const gaspi_number_t nListElems = 255;
gaspi_number_t n;
gaspi_segment_id_t localSegs[nListElems];
gaspi_offset_t localOffs[nListElems];
const gaspi_rank_t rank2send = (myrank + 1) % numranks;
gaspi_segment_id_t remSegs[nListElems];
gaspi_offset_t remOffs[nListElems];
gaspi_size_t sizes[nListElems];
const unsigned int bytes = sizeof(int);
gaspi_offset_t initLocOff = 0;
gaspi_offset_t initRemOff = (bytes * nListElems + 64);
for(n = 0; n < nListElems; n++)
{
sizes[n] = bytes;
localSegs[n] = 0;
localOffs[n] = initLocOff;
initLocOff += bytes;
remSegs[n] = 0;
remOffs[n] = initRemOff;
initRemOff += bytes;
}
do
{
ASSERT( gaspi_write_list_notify( nListElems,
localSegs, localOffs, rank2send,
remSegs, remOffs, sizes,
0, myrank, 1,
0, GASPI_BLOCK));
gaspi_queue_size(0, &queueSize);
}
while(queueSize < qmax);
EXPECT_FAIL_WITH( gaspi_write_list_notify( nListElems,
localSegs, localOffs, rank2send,
remSegs, remOffs, sizes,
0, myrank, 1,
0, GASPI_BLOCK),
GASPI_QUEUE_FULL);
ASSERT (gaspi_wait(0, GASPI_BLOCK));
ASSERT( gaspi_write_list_notify( nListElems,
localSegs, localOffs, rank2send,
remSegs, remOffs, sizes,
0, myrank, 1,
0, GASPI_BLOCK));
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int
main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_rank_t rank, nprocs, i;
const gaspi_segment_id_t seg_id = 0;
const gaspi_offset_t offset = 0;
const gaspi_size_t transfer_size = 8192;
gaspi_number_t queue_size;
gaspi_number_t queue_max;
ASSERT (gaspi_queue_size_max(&queue_max));
ASSERT(gaspi_proc_num(&nprocs));
ASSERT (gaspi_proc_rank(&rank));
if( nprocs < 2 )
{
return EXIT_SUCCESS;
}
ASSERT (gaspi_segment_create(seg_id, nprocs * 2 * transfer_size, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED));
gaspi_number_t max_notifications;
ASSERT(gaspi_notification_num(&max_notifications));
gaspi_number_t avail_notifications = max_notifications / nprocs;
max_notifications = avail_notifications * nprocs;
gaspi_pointer_t _vptr;
ASSERT (gaspi_segment_ptr(0, &_vptr));
int *mem = (int *) _vptr;
for(i = 0; i < nprocs; i++)
{
mem[i] = (int) rank;
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
for(i = 0; i < nprocs; i++)
{
gaspi_notification_id_t not;
for(not = 0; not < avail_notifications; not++)
{
ASSERT (gaspi_queue_size(0, &queue_size));
if( queue_size > queue_max - 1 )
{
ASSERT (gaspi_wait(0, GASPI_BLOCK));
}
gaspi_notification_id_t the_notification = (gaspi_notification_id_t) (rank * avail_notifications + not);
ASSERT( gaspi_write_notify( seg_id, offset, i,
seg_id, offset, transfer_size,
the_notification, 1,
0, GASPI_BLOCK));
}
}
gaspi_notification_id_t n = 0;
do
{
gaspi_notification_id_t id;
ASSERT (gaspi_notify_waitsome(seg_id, 0, max_notifications - 1 , &id, GASPI_BLOCK));
gaspi_notification_t notification_val;
ASSERT( gaspi_notify_reset(seg_id, id, ¬ification_val));
assert(notification_val == 1);
n++;
}
while(n < max_notifications - 1);
ASSERT(gaspi_wait(0, GASPI_BLOCK));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main (int argc, char *argv[])
{
gaspi_proc_init(GASPI_BLOCK);
gaspi_rank_t myRank;
gaspi_rank_t nProc;
gaspi_proc_rank(&myRank);
gaspi_proc_num(&nProc);
if(nProc < 2)
goto end;
gaspi_number_t queue_size;
gaspi_number_t queue_max;
gaspi_queue_size_max(&queue_max);
if (myRank == 0)
gaspi_printf("Queue max is %d\n", queue_max);
gaspi_printf("Rank %i of %i started.\n", myRank, nProc);
const gaspi_segment_id_t segment_id = 0;
const gaspi_size_t nrReads = NR_OF_READS;
gaspi_group_commit(GASPI_GROUP_ALL,GASPI_BLOCK);
gaspi_segment_create(segment_id, nrReads * (RAWREADLENGTH) * sizeof(gaspi_char),GASPI_GROUP_ALL,GASPI_BLOCK,GASPI_ALLOC_DEFAULT);
gaspi_pointer_t _vptr; //pointer to the segment
if(gaspi_segment_ptr(segment_id, &_vptr) != GASPI_SUCCESS)
printf("gaspi_segment_ptr failed\n");
gaspi_char * shared_ptr = (gaspi_char *) _vptr;
// initialize and print segment
initReads(shared_ptr, nrReads, READLENGTH, myRank);
gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK);
//push the reads from the master to the slaves
gaspi_size_t r = 0;
int rawReadSize = RAWREADLENGTH * sizeof(gaspi_char);
int nrWorkers = nProc - 1;
int toRank;
gaspi_notification_id_t notif_id;
if (myRank == 0) {
for (r = 0; r < nrReads; r++) {
gaspi_queue_size(0, &queue_size);
if(queue_size > queue_max - 1)
gaspi_wait(0, GASPI_BLOCK); //wait for queue to become free again... (note: max is 1024)
toRank = (r % nrWorkers) + 1;
// notif_id = r + 1;
notif_id = ((r / nrWorkers) + 1);
if ( gaspi_write_notify( segment_id, // from segment
r*rawReadSize, // from offset
toRank, // to-rank
segment_id, // to segment
// ((int)(r/nrWorkers))*rawReadSize, // to-offset
r * rawReadSize,
rawReadSize, // size
notif_id, // notification id
r+1, // notification value (> 0!)
(gaspi_queue_id_t) 0, // notification queue
GASPI_BLOCK) == GASPI_SUCCESS) // block until written
gaspi_printf("Sending read %d from %d to rank %d with id %d\n", r, myRank, toRank, notif_id);
if (toRank == 2)
print_read(shared_ptr, r, READLENGTH, myRank);
}
}
//ranks receive reads from the master rank
if (myRank != 0) {
gaspi_notification_id_t fid;
gaspi_notification_t notification_value;
int nrOfReceives = (int)(nrReads / (nProc-1));
if (myRank <= nrReads % nrWorkers)
nrOfReceives++;
gaspi_printf("Rank %d -- listening for %d events...\n", myRank, nrOfReceives);
int complete = 0;
while (complete < nrOfReceives) {
if(gaspi_notify_waitsome( segment_id, // segment
1, // id of first notification to wait for
// nrReads,
nrOfReceives, // id of last notification to wait for (alternative)
&fid, // identifier (output parameter with the identifier of a received notification (?))
GASPI_TEST // immediately return (GASPI_TEST)
) == GASPI_SUCCESS) {
if(gaspi_notify_reset( segment_id, // segment
fid, // notification identifier
¬ification_value // notification value
) == GASPI_SUCCESS) {
complete++ ;
gaspi_printf("Rank %d -- got notification: read %d received (%d completed)\n", myRank, notification_value-1, complete);
if (myRank == 2)
print_read(shared_ptr, notification_value-1, READLENGTH, myRank);
}
}
}
}
// all values received ! print !
gaspi_barrier(GASPI_GROUP_ALL,GASPI_BLOCK);
gaspi_printf("Printing reads\n");
print_char_array_segment(shared_ptr, nrReads, READLENGTH, myRank);
// print_read(shared_ptr, 0, READLENGTH, myRank);
gaspi_barrier(GASPI_GROUP_ALL,GASPI_BLOCK);
gaspi_printf("Rank %d done\n", myRank);
//block and exit
end:
gaspi_barrier(GASPI_GROUP_ALL,GASPI_BLOCK);
gaspi_proc_term(GASPI_BLOCK);
return EXIT_SUCCESS;
}
//create GPU segemnts on all nodes
//and then tranfer data between all segments
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
//need the barrier to make sn is up
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
gaspi_rank_t rank, nprocs, i,j;
gaspi_number_t seg_max;
gaspi_gpu_id_t gpus[8];
gaspi_number_t nGPUs;
ASSERT(gaspi_proc_num(&nprocs));
ASSERT (gaspi_proc_rank(&rank));
ASSERT(gaspi_gpu_init());
seg_max = 1;
ASSERT (gaspi_gpu_number(&nGPUs));
ASSERT (gaspi_gpu_ids(gpus));
for(i = 0; i < nGPUs; i++)
{
cudaSetDevice(gpus[i]);
ASSERT (gaspi_segment_create(i, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED|GASPI_MEM_GPU));
}
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
gaspi_number_t queueSize;
int rankSend = (rank + 1) % nprocs;
for (j =0; j<nGPUs; j++)
{
//sleep(1);
if (gaspi_gpu_write(j, //seg
1024, //local
rankSend, //rank
j, //seg rem
1024, //remote
32768, //size
1, //queue
GASPI_BLOCK) != GASPI_SUCCESS)
{
gaspi_queue_size(1, &queueSize);
gaspi_printf (" failed with i = %d queue %u\n", j, queueSize);
exit(-1);
}
}
ASSERT(gaspi_wait(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
for (i =0; i<nGPUs; i++)
{
ASSERT (gaspi_segment_delete(i));
}
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_rank_t numranks, myrank;
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
//2 segments
ASSERT (gaspi_segment_create(0, _2GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
ASSERT (gaspi_segment_create(1, _2GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
//construct list of n elems
gaspi_number_t i, n, max;
gaspi_number_t queue_size = 0;
ASSERT( gaspi_rw_list_elem_max(&max));
for(n = 1; n < max; n++)
{
gaspi_number_t nListElems = n;
gaspi_segment_id_t localSegs[n];
gaspi_offset_t localOffs[n];
const gaspi_rank_t rank2send = (myrank + 1) % numranks;
gaspi_segment_id_t remSegs[n];
gaspi_offset_t remOffs[n];
gaspi_size_t sizes[n];
unsigned char flip = 0;
unsigned int bytes = 1;
gaspi_offset_t off = 0;
gaspi_size_t total_size = 0;
for(i = 0; i < nListElems; i++)
{
sizes[i] = bytes;
localSegs[i] = flip;
localOffs[i] = off;
remSegs[i] = flip;
remOffs[i] = off;
off+=sizes[i];
total_size+= bytes;
flip ^= 0x1;
bytes += 2;
}
ASSERT(gaspi_write_list(nListElems, localSegs, localOffs, rank2send, remSegs, remOffs, sizes, 0, GASPI_BLOCK));
ASSERT(gaspi_queue_size(0, &queue_size));
assert(queue_size == nListElems);
ASSERT(gaspi_wait(0, GASPI_BLOCK));
}
gaspi_pointer_t _vptr;
ASSERT (gaspi_segment_ptr(0, &_vptr));
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_notification_id_t n=0;
gaspi_rank_t rank, nprocs, i;
const gaspi_segment_id_t seg_id = 0;
gaspi_offset_t offset;
gaspi_number_t queue_size;
gaspi_number_t queue_max;
ASSERT (gaspi_queue_size_max(&queue_max));
ASSERT(gaspi_proc_num(&nprocs));
ASSERT (gaspi_proc_rank(&rank));
ASSERT (gaspi_segment_create(seg_id, nprocs * sizeof(int), GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED));
offset = rank * sizeof(int);
//set memory
gaspi_pointer_t _vptr;
ASSERT (gaspi_segment_ptr(0, &_vptr));
int *mem = (int *) _vptr;
for(i = 0; i < nprocs; i++)
{
mem[i] = (int) rank;
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
//go
for(i = 0; i < nprocs; i++)
{
if (i == rank)
continue;
ASSERT (gaspi_queue_size(0, &queue_size));
if(queue_size > queue_max - 1)
ASSERT (gaspi_wait(0, GASPI_BLOCK));
ASSERT (gaspi_write_notify( seg_id, offset, i,
seg_id, offset, sizeof(int),
(gaspi_notification_id_t) rank, 1,
0, GASPI_BLOCK));
}
do
{
gaspi_notification_id_t id;
ASSERT (gaspi_notify_waitsome(seg_id, 0, (gaspi_notification_id_t) nprocs , &id, GASPI_BLOCK));
gaspi_notification_t notification_val;
ASSERT( gaspi_notify_reset(seg_id, id, ¬ification_val));
assert(notification_val == 1);
n++;
}
while(n < (nprocs - 1));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
const unsigned long N = (1 << 13);
gaspi_rank_t P, myrank;
ASSERT (gaspi_proc_num(&P));
ASSERT (gaspi_proc_rank(&myrank));
gaspi_printf("P = %d N = %lu\n", P, N);
gaspi_printf("Seg size: %lu MB\n", MAX (_4GB, 2 * ((N/P) * N * 2 * sizeof (double)))/1024/1024);
if(gaspi_segment_create(0, _1GB,
GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED) != GASPI_SUCCESS){
gaspi_printf("Failed to create segment\n");
return -1;
}
gaspi_pointer_t _vptr;
if(gaspi_segment_ptr(0, &_vptr) != GASPI_SUCCESS)
printf("gaspi_segment_ptr failed\n");
gaspi_number_t qmax ;
ASSERT (gaspi_queue_size_max(&qmax));
gaspi_printf("Queue max: %lu\n", qmax);
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
int i;
gaspi_number_t queueSize;
int rankSend = (myrank + 1) % P;
gaspi_printf("rank to: %d\n", rankSend);
for (i = 0; i < 2 * N; i ++)
{
gaspi_queue_size(1, &queueSize);
if (queueSize > qmax - 24)
{
gaspi_return_t ret;
do
{
ret = gaspi_wait(1, GASPI_TEST);
assert (ret != GASPI_ERROR);
}
while(ret != GASPI_SUCCESS);
gaspi_queue_size(1, &queueSize);
assert(queueSize == 0);
}
ASSERT (gaspi_write(0, 4, rankSend, 0, 6, 32768, 1, GASPI_TEST));
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
gaspi_rank_t numranks, myrank;
gaspi_rank_t rankSend;
gaspi_size_t segSize;
const gaspi_offset_t localOff_r= 0;
const gaspi_offset_t remOff_r = 0;
const gaspi_offset_t localOff_w = _128MB / 2 ;
const gaspi_offset_t remOff_w = _128MB / 2;
gaspi_number_t queueSize, qmax;
const gaspi_size_t commSize = _8MB;
int i;
gaspi_gpu_t gpus[8];
gaspi_gpu_num nGPUs;
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_init_GPUs());
ASSERT (gaspi_number_of_GPUs(&nGPUs));
ASSERT (gaspi_GPU_ids(gpus));
ASSERT (gaspi_segment_create(0, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED|GASPI_MEM_GPU));
ASSERT( gaspi_segment_size(0, myrank, &segSize));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_queue_size_max(&qmax));
for(i = 0; i < 100; i++ )
{
for(rankSend = 0; rankSend < numranks; rankSend++)
{
if(rankSend == myrank)
continue;
gaspi_printf("partner rank: %d - %lu bytes (%d)\n", rankSend, commSize, i);
ASSERT (gaspi_queue_size(1, &queueSize));
if (queueSize > qmax - 24)
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_read(0, localOff_r, rankSend, 0, remOff_r, commSize, 1, GASPI_BLOCK));
}
}
for(i = 0; i < 100; i++ )
{
for(rankSend = 0; rankSend < numranks; rankSend++)
{
if(rankSend == myrank)
continue;
ASSERT (gaspi_gpu_write(0, localOff_r, rankSend, 0, remOff_r, commSize, 1, GASPI_BLOCK));
}
}
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_rank_t numranks, myrank;
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_segment_create(0, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
ASSERT (gaspi_segment_create(1, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
gaspi_number_t queueSize, qmax;
ASSERT (gaspi_queue_size_max(&qmax));
gaspi_pointer_t segPtr, segPtr1;
ASSERT( gaspi_segment_ptr(0, &segPtr));
int* segInt = (int *) segPtr;
ASSERT( gaspi_segment_ptr(1, &segPtr1));
int* segInt1 = (int *) segPtr1;
gaspi_size_t segSize;
ASSERT( gaspi_segment_size(0, myrank, &segSize));
int i;
for(i = 0; i < segSize / sizeof(int); i++)
{
segInt[i] = myrank;
segInt1[i] = -1;
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
gaspi_size_t commSize ;
gaspi_offset_t localOff= 0;
const gaspi_offset_t remOff = 0;
for(commSize = sizeof(int); commSize < _8MB; commSize *= 2 )
{
gaspi_rank_t rankSend;
for(rankSend = 0; rankSend < numranks; rankSend++)
{
if(rankSend == myrank)
continue;
gaspi_queue_size(1, &queueSize);
if (queueSize > qmax - 24)
{
ASSERT (gaspi_wait(1, GASPI_BLOCK));
}
ASSERT (gaspi_read(1, localOff, rankSend, 0, remOff, commSize, 1, GASPI_BLOCK));
localOff+= commSize;
}
ASSERT (gaspi_wait(1, GASPI_BLOCK));
const int elems_per_rank = commSize / sizeof(int);
int c, pos = 0;
int* seg_read = (int *) segPtr1;
for(rankSend = 0; rankSend < numranks; rankSend++)
{
if(rankSend == myrank)
continue;
for(c = 0; c < elems_per_rank; c++)
{
assert (seg_read[pos] == rankSend);
pos++;
}
}
localOff = 0;
}
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}