int
main(int argc, char *argv[])
{
int i;
int num_threads = 0;
gaspi_size_t segSize;
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT(gaspi_threads_init(&num_threads));
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_segment_create(0, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
ASSERT( gaspi_segment_size(0, myrank, &segSize));
for(i = 1; i < num_threads; i++)
ASSERT(gaspi_threads_run(thread_fun, NULL));
thread_fun(NULL);
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
gaspi_rank_t numranks, myrank;
gaspi_rank_t rankSend;
gaspi_size_t segSize;
const gaspi_offset_t localOff= 0;
const gaspi_offset_t remOff = 0;
gaspi_number_t queueSize, qmax;
gaspi_size_t commSize ;
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_segment_create(0, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
ASSERT( gaspi_segment_size(0, myrank, &segSize));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_queue_size_max(&qmax));
for(commSize= 4; commSize < _8MB; commSize*=2 )
{
for(rankSend = 0; rankSend < numranks; rankSend++)
{
if(rankSend == myrank)
continue;
printf("partner rank: %d - %lu bytes\n", rankSend, commSize);
//FAILS with or without outstanding requests
gaspi_queue_size(1, &queueSize);
if (queueSize > qmax - 24)
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_read(0, localOff, rankSend, 0, remOff, commSize, 1, GASPI_BLOCK));
}
}
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
gaspi_return_t
init_segment (gaspi_segment_id_t seg_id, gaspi_size_t seg_size)
{
gaspi_rank_t myrank;
gaspi_size_t segSize;
gaspi_proc_rank(&myrank);
ASSERT(gaspi_segment_create(seg_id, seg_size, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED));
ASSERT(gaspi_segment_size(seg_id, myrank, &segSize));
gaspi_printf("segSize is: %lu bytes\n", segSize);
return GASPI_SUCCESS;
}
int main(int argc, char *argv[])
{
gaspi_rank_t numranks, myrank;
gaspi_rank_t rankSend;
gaspi_size_t segSize;
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_segment_create(0, _2GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
ASSERT( gaspi_segment_size(0, myrank, &segSize));
gaspi_printf("seg size %lu MB \n", segSize/1024/1024);
// ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
gaspi_offset_t localOff= 814780664;
gaspi_offset_t remOff = 81478246;
gaspi_offset_t size = 1800;
gaspi_number_t queueSize, qmax;
ASSERT (gaspi_queue_size_max(&qmax));
for(rankSend = 0; rankSend < numranks; rankSend++)
{
gaspi_printf("rank to send: %d\n", rankSend);
gaspi_queue_size(1, &queueSize);
if (queueSize > qmax - 24)
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_write(0, localOff, rankSend, 0, remOff, size, 1, GASPI_BLOCK));
}
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
//alloc a segment of different size
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
gaspi_rank_t rank, nprocs, i;
gaspi_size_t seg_size;
ASSERT (gaspi_proc_num(&nprocs));
ASSERT (gaspi_proc_rank(&rank));
if (rank == 0)
{ ASSERT (gaspi_segment_alloc(0, 1024, GASPI_MEM_INITIALIZED)); }
else
ASSERT (gaspi_segment_alloc(0, 2048, GASPI_MEM_INITIALIZED));
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT(gaspi_segment_size(0, rank, &seg_size));
for (i = 0; i < nprocs; i++)
{
gaspi_printf("register seg of size %lu with %u\n", seg_size, i);
if(i == rank)
continue;
ASSERT( gaspi_segment_register(0, i, GASPI_BLOCK));
// sleep(1);
}
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_segment_delete(0));
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
gaspi_rank_t numranks, myrank;
gaspi_rank_t rankSend;
gaspi_size_t segSize;
const gaspi_offset_t localOff_r= 0;
const gaspi_offset_t remOff_r = 0;
const gaspi_offset_t localOff_w = _128MB / 2 ;
const gaspi_offset_t remOff_w = _128MB / 2;
gaspi_number_t queueSize, qmax;
const gaspi_size_t commSize = _8MB;
int i;
gaspi_gpu_t gpus[8];
gaspi_gpu_num nGPUs;
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_init_GPUs());
ASSERT (gaspi_number_of_GPUs(&nGPUs));
ASSERT (gaspi_GPU_ids(gpus));
ASSERT (gaspi_segment_create(0, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED|GASPI_MEM_GPU));
ASSERT( gaspi_segment_size(0, myrank, &segSize));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_queue_size_max(&qmax));
for(i = 0; i < 100; i++ )
{
for(rankSend = 0; rankSend < numranks; rankSend++)
{
if(rankSend == myrank)
continue;
gaspi_printf("partner rank: %d - %lu bytes (%d)\n", rankSend, commSize, i);
ASSERT (gaspi_queue_size(1, &queueSize));
if (queueSize > qmax - 24)
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_read(0, localOff_r, rankSend, 0, remOff_r, commSize, 1, GASPI_BLOCK));
}
}
for(i = 0; i < 100; i++ )
{
for(rankSend = 0; rankSend < numranks; rankSend++)
{
if(rankSend == myrank)
continue;
ASSERT (gaspi_gpu_write(0, localOff_r, rankSend, 0, remOff_r, commSize, 1, GASPI_BLOCK));
}
}
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_rank_t numranks, myrank;
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
int rankSend = (myrank + 1) % numranks;
ASSERT(gaspi_segment_create(0, _1MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
gaspi_size_t segSize;
ASSERT( gaspi_segment_size(0, myrank, &segSize));
unsigned char * pGlbMem;
gaspi_pointer_t _vptr;
ASSERT(gaspi_segment_ptr(0, &_vptr));
pGlbMem = ( unsigned char *) _vptr;
gaspi_number_t queueSize, qmax ;
ASSERT (gaspi_queue_size_max(&qmax));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
const unsigned long localOff = 0;
const unsigned long remOff = 0;
/* write_notify */
do
{
ASSERT(gaspi_write_notify(0, localOff, rankSend,
0, remOff, 1,
(gaspi_notification_id_t) myrank, 1,
1, GASPI_BLOCK));
gaspi_queue_size(1, &queueSize);
}
while(queueSize < qmax);
EXPECT_FAIL_WITH(gaspi_write_notify(0, localOff, rankSend,
0, remOff, 1,
(gaspi_notification_id_t) myrank, 1,
1, GASPI_BLOCK),
GASPI_QUEUE_FULL);
ASSERT (gaspi_wait(1, GASPI_BLOCK));
/* write */
do
{
ASSERT(gaspi_write(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK));
gaspi_queue_size(1, &queueSize);
}
while(queueSize < qmax);
EXPECT_FAIL_WITH(gaspi_write(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK),
GASPI_QUEUE_FULL);
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT(gaspi_write(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK));
/* write + write_notify */
do
{
ASSERT(gaspi_write(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK));
gaspi_queue_size(1, &queueSize);
}
while(queueSize < qmax - 1);
EXPECT_FAIL_WITH(gaspi_write_notify(0, localOff, rankSend,
0, remOff, 1,
(gaspi_notification_id_t) myrank, 1,
1, GASPI_BLOCK),
GASPI_QUEUE_FULL);
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT(gaspi_write_notify(0, localOff, rankSend,
0, remOff, 1,
(gaspi_notification_id_t) myrank, 1,
1, GASPI_BLOCK));
/* read */
do
{
ASSERT(gaspi_read(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK));
gaspi_queue_size(1, &queueSize);
}
while(queueSize < qmax);
EXPECT_FAIL_WITH(gaspi_read(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK),
GASPI_QUEUE_FULL);
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT(gaspi_read(0, localOff, rankSend,
0, remOff, 1,
1, GASPI_BLOCK));
/* write_list_notify */
{
const gaspi_number_t nListElems = 255;
gaspi_number_t n;
gaspi_segment_id_t localSegs[nListElems];
gaspi_offset_t localOffs[nListElems];
const gaspi_rank_t rank2send = (myrank + 1) % numranks;
gaspi_segment_id_t remSegs[nListElems];
gaspi_offset_t remOffs[nListElems];
gaspi_size_t sizes[nListElems];
const unsigned int bytes = sizeof(int);
gaspi_offset_t initLocOff = 0;
gaspi_offset_t initRemOff = (bytes * nListElems + 64);
for(n = 0; n < nListElems; n++)
{
sizes[n] = bytes;
localSegs[n] = 0;
localOffs[n] = initLocOff;
initLocOff += bytes;
remSegs[n] = 0;
remOffs[n] = initRemOff;
initRemOff += bytes;
}
do
{
ASSERT( gaspi_write_list_notify( nListElems,
localSegs, localOffs, rank2send,
remSegs, remOffs, sizes,
0, myrank, 1,
0, GASPI_BLOCK));
gaspi_queue_size(0, &queueSize);
}
while(queueSize < qmax);
EXPECT_FAIL_WITH( gaspi_write_list_notify( nListElems,
localSegs, localOffs, rank2send,
remSegs, remOffs, sizes,
0, myrank, 1,
0, GASPI_BLOCK),
GASPI_QUEUE_FULL);
ASSERT (gaspi_wait(0, GASPI_BLOCK));
ASSERT( gaspi_write_list_notify( nListElems,
localSegs, localOffs, rank2send,
remSegs, remOffs, sizes,
0, myrank, 1,
0, GASPI_BLOCK));
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_rank_t numranks, myrank;
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
int rankSend = (myrank + 1) % numranks;
gaspi_printf("Seg size: %lu MB\n", _2GB / 1024 / 1024);
ASSERT(gaspi_segment_create(0, _2GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
gaspi_size_t segSize;
ASSERT( gaspi_segment_size(0, myrank, &segSize));
unsigned char * pGlbMem;
gaspi_pointer_t _vptr;
ASSERT(gaspi_segment_ptr(0, &_vptr));
pGlbMem = ( unsigned char *) _vptr;
gaspi_number_t qmax ;
ASSERT (gaspi_queue_size_max(&qmax));
unsigned long i;
unsigned long size = 1800;
for(i = 0; i < size / sizeof(unsigned char); i++)
pGlbMem[i] = myrank;
gaspi_printf("Queue max: %lu\n", qmax);
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
unsigned long localOff = 0;
unsigned long remOff = size;
ASSERT(gaspi_write_notify(0, localOff, rankSend,
0, remOff, size,
(gaspi_notification_id_t) myrank, 1,
1, GASPI_BLOCK));
gaspi_rank_t rankGet = (myrank + numranks - 1) % numranks;
gaspi_notification_t got_val;
gaspi_notification_id_t got;
ASSERT(gaspi_notify_waitsome(0, (gaspi_notification_id_t) rankGet, 1, &got, GASPI_BLOCK));
ASSERT(gaspi_notify_reset(0, got, &got_val));
ASSERT (gaspi_wait(1, GASPI_BLOCK));
/* check */
for(i = size; i < 2 * size / sizeof(unsigned char); i++)
assert(pGlbMem[i] == rankGet);
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_rank_t numranks, myrank;
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_segment_create(0, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
ASSERT (gaspi_segment_create(1, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
gaspi_number_t queueSize, qmax;
ASSERT (gaspi_queue_size_max(&qmax));
gaspi_pointer_t segPtr, segPtr1;
ASSERT( gaspi_segment_ptr(0, &segPtr));
int* segInt = (int *) segPtr;
ASSERT( gaspi_segment_ptr(1, &segPtr1));
int* segInt1 = (int *) segPtr1;
gaspi_size_t segSize;
ASSERT( gaspi_segment_size(0, myrank, &segSize));
int i;
for(i = 0; i < segSize / sizeof(int); i++)
{
segInt[i] = myrank;
segInt1[i] = -1;
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
gaspi_size_t commSize ;
gaspi_offset_t localOff= 0;
const gaspi_offset_t remOff = 0;
for(commSize = sizeof(int); commSize < _8MB; commSize *= 2 )
{
gaspi_rank_t rankSend;
for(rankSend = 0; rankSend < numranks; rankSend++)
{
if(rankSend == myrank)
continue;
gaspi_queue_size(1, &queueSize);
if (queueSize > qmax - 24)
{
ASSERT (gaspi_wait(1, GASPI_BLOCK));
}
ASSERT (gaspi_read(1, localOff, rankSend, 0, remOff, commSize, 1, GASPI_BLOCK));
localOff+= commSize;
}
ASSERT (gaspi_wait(1, GASPI_BLOCK));
const int elems_per_rank = commSize / sizeof(int);
int c, pos = 0;
int* seg_read = (int *) segPtr1;
for(rankSend = 0; rankSend < numranks; rankSend++)
{
if(rankSend == myrank)
continue;
for(c = 0; c < elems_per_rank; c++)
{
assert (seg_read[pos] == rankSend);
pos++;
}
}
localOff = 0;
}
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}