int main(int argc, char *argv[]) { int ntimes = 10; gaspi_rank_t rank, nprocs; gaspi_notification_id_t id; gaspi_notification_t val; TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); ASSERT(gaspi_proc_num(&nprocs)); ASSERT (gaspi_proc_rank(&rank)); const gaspi_rank_t right = (rank + nprocs + 1) % nprocs; do { ASSERT (gaspi_segment_create(0, 1024, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED)); ASSERT( gaspi_write_notify(0, 0, right, 0, 0, 8, 0, 1, 0, GASPI_BLOCK) ); ASSERT( gaspi_wait( 0, GASPI_BLOCK) ); ASSERT(gaspi_notify_waitsome(0, 0, 1, &id, GASPI_BLOCK)); ASSERT( gaspi_notify_reset(0, id, &val)); ASSERT (gaspi_segment_delete(0)); ASSERT (gaspi_segment_create(0, 2048, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED)); ASSERT( gaspi_write_notify(0, 0, right, 0, 0, 8, 0, 1, 0, GASPI_BLOCK) ); ASSERT( gaspi_wait( 0, GASPI_BLOCK) ); ASSERT(gaspi_notify_waitsome(0, 0, 1, &id, GASPI_BLOCK)); ASSERT( gaspi_notify_reset(0, id, &val)); ASSERT (gaspi_segment_delete(0)); ntimes--; } while(ntimes > 0); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { gaspi_rank_t r; gaspi_notification_id_t id; TSUITE_INIT(argc, argv); ASSERT( gaspi_proc_init(GASPI_BLOCK)); ASSERT( gaspi_proc_rank(&r)); ASSERT( gaspi_segment_create(0, 1, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_ALLOC_DEFAULT)); ASSERT(gaspi_notify(0, r, r, 1, 0, GASPI_BLOCK)); ASSERT( gaspi_notify_waitsome(0, r, 1, &id, GASPI_BLOCK)); // wait for zero notifications ASSERT( gaspi_notify_waitsome(0, 0, 0, &id, GASPI_BLOCK)); ASSERT( gaspi_segment_delete(0 )); ASSERT( gaspi_proc_term(GASPI_BLOCK) ); return 0; }
//alloc GPU segemnt //and then register it with all nodes //then delete it int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); gaspi_rank_t rank, nprocs, i; gaspi_number_t seg_max; gaspi_gpu_id_t gpus[8]; gaspi_number_t nGPUs; ASSERT(gaspi_proc_num(&nprocs)); ASSERT (gaspi_proc_rank(&rank)); ASSERT(gaspi_gpu_init()); seg_max = 1; ASSERT (gaspi_gpu_number(&nGPUs)); ASSERT (gaspi_gpu_ids(gpus)); cudaSetDevice(gpus[0]); ASSERT (gaspi_segment_alloc(0, 1024, GASPI_MEM_INITIALIZED|GASPI_MEM_GPU)); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); for (i = 0; i < nprocs; i++) { if(i == rank) continue; ASSERT( gaspi_segment_register(0, i, GASPI_BLOCK)); } ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_segment_delete(0)); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
//alloc a segment of different size int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); gaspi_rank_t rank, nprocs, i; gaspi_size_t seg_size; ASSERT (gaspi_proc_num(&nprocs)); ASSERT (gaspi_proc_rank(&rank)); if (rank == 0) { ASSERT (gaspi_segment_alloc(0, 1024, GASPI_MEM_INITIALIZED)); } else ASSERT (gaspi_segment_alloc(0, 2048, GASPI_MEM_INITIALIZED)); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT(gaspi_segment_size(0, rank, &seg_size)); for (i = 0; i < nprocs; i++) { gaspi_printf("register seg of size %lu with %u\n", seg_size, i); if(i == rank) continue; ASSERT( gaspi_segment_register(0, i, GASPI_BLOCK)); // sleep(1); } ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_segment_delete(0)); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
//alloc max number of segs, fail after that //and them register them all with all nodes //then delete them int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); //need the barrier to make sn is up ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); gaspi_rank_t rank, nprocs, i; gaspi_number_t seg_max; ASSERT(gaspi_proc_num(&nprocs)); ASSERT (gaspi_proc_rank(&rank)); seg_max = 1; ASSERT (gaspi_segment_alloc(0, 1024, GASPI_MEM_INITIALIZED)); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); for (i = 0; i < nprocs; i++) { gaspi_printf("register with %u\n", i); if(i == rank) continue; ASSERT( gaspi_segment_register(0, i, GASPI_BLOCK)); // sleep(1); } ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_segment_delete(0)); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { const int num_elems = 1024; TSUITE_INIT( argc, argv ); ASSERT( gaspi_proc_init(GASPI_BLOCK) ); gaspi_rank_t rank, nprocs; ASSERT( gaspi_proc_num(&nprocs) ); ASSERT( gaspi_proc_rank(&rank) ); const gaspi_rank_t left = (rank + nprocs - 1 ) % nprocs; const gaspi_rank_t right = (rank + nprocs + 1) % nprocs; /* Create and fill buffer */ int * const buf = (int *) malloc(num_elems * sizeof(int)); assert( buf != NULL); int i; for (i = 0; i < num_elems; i++) { buf[i] = rank; } ASSERT( gaspi_segment_use( 0, buf, num_elems * sizeof(int), GASPI_GROUP_ALL, GASPI_BLOCK, 0) ); ASSERT( gaspi_segment_create( 1, num_elems * sizeof(int), GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED) ); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); /* write data to neighbour ( from seg 0 to seg 1) */ ASSERT( gaspi_write_notify( 0, 0, right, 1, 0, num_elems * sizeof(int), 0, 1, 0, GASPI_BLOCK) ); gaspi_notification_id_t id; ASSERT( gaspi_notify_waitsome( 1, 0, 1, &id, GASPI_BLOCK ) ); ASSERT( gaspi_wait( 0, GASPI_BLOCK ) ); /* Check data as segment */ gaspi_pointer_t seg1_ptr; ASSERT( gaspi_segment_ptr( 1, &seg1_ptr ) ); int * recv_buf = (int *) seg1_ptr; for (i = 0; i < num_elems; i++) { assert(recv_buf[i] == left); } ASSERT( gaspi_segment_delete(0)); ASSERT( gaspi_segment_delete(1)); /* Check data in buffer */ for (i = 0; i < num_elems; i++) { assert(buf[i] == rank); } ASSERT( gaspi_barrier( GASPI_GROUP_ALL, GASPI_BLOCK ) ); ASSERT( gaspi_proc_term( GASPI_BLOCK ) ); return EXIT_SUCCESS; }
//create GPU segemnts on all nodes //and then tranfer data between all segments int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); //need the barrier to make sn is up ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); gaspi_rank_t rank, nprocs, i,j; gaspi_number_t seg_max; gaspi_gpu_id_t gpus[8]; gaspi_number_t nGPUs; ASSERT(gaspi_proc_num(&nprocs)); ASSERT (gaspi_proc_rank(&rank)); ASSERT(gaspi_gpu_init()); seg_max = 1; ASSERT (gaspi_gpu_number(&nGPUs)); ASSERT (gaspi_gpu_ids(gpus)); for(i = 0; i < nGPUs; i++) { cudaSetDevice(gpus[i]); ASSERT (gaspi_segment_create(i, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED|GASPI_MEM_GPU)); } ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); gaspi_number_t queueSize; int rankSend = (rank + 1) % nprocs; for (j =0; j<nGPUs; j++) { //sleep(1); if (gaspi_gpu_write(j, //seg 1024, //local rankSend, //rank j, //seg rem 1024, //remote 32768, //size 1, //queue GASPI_BLOCK) != GASPI_SUCCESS) { gaspi_queue_size(1, &queueSize); gaspi_printf (" failed with i = %d queue %u\n", j, queueSize); exit(-1); } } ASSERT(gaspi_wait(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); for (i =0; i<nGPUs; i++) { ASSERT (gaspi_segment_delete(i)); } ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }