void work(int tid) { gaspi_rank_t rankSend; gaspi_offset_t localOff = 81478066; gaspi_offset_t remOff = 81478246; gaspi_offset_t size = 1800; gaspi_number_t queueSize, qmax; ASSERT (gaspi_queue_size_max(&qmax)); for(rankSend = 0; rankSend < numranks; rankSend++) { gaspi_printf("thread %d rank to send: %d\n", tid, rankSend); gaspi_queue_size(1, &queueSize); if (queueSize > qmax - 24) ASSERT (gaspi_wait(1, GASPI_BLOCK)); ASSERT (gaspi_write(0, localOff, rankSend, 0, remOff, size, 1, GASPI_BLOCK)); } ASSERT (gaspi_wait(1, GASPI_BLOCK)); gaspi_threads_sync(); }
int main(int argc, char *argv[]) { int ntimes = 10; gaspi_rank_t rank, nprocs; gaspi_notification_id_t id; gaspi_notification_t val; TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); ASSERT(gaspi_proc_num(&nprocs)); ASSERT (gaspi_proc_rank(&rank)); const gaspi_rank_t right = (rank + nprocs + 1) % nprocs; do { ASSERT (gaspi_segment_create(0, 1024, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED)); ASSERT( gaspi_write_notify(0, 0, right, 0, 0, 8, 0, 1, 0, GASPI_BLOCK) ); ASSERT( gaspi_wait( 0, GASPI_BLOCK) ); ASSERT(gaspi_notify_waitsome(0, 0, 1, &id, GASPI_BLOCK)); ASSERT( gaspi_notify_reset(0, id, &val)); ASSERT (gaspi_segment_delete(0)); ASSERT (gaspi_segment_create(0, 2048, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED)); ASSERT( gaspi_write_notify(0, 0, right, 0, 0, 8, 0, 1, 0, GASPI_BLOCK) ); ASSERT( gaspi_wait( 0, GASPI_BLOCK) ); ASSERT(gaspi_notify_waitsome(0, 0, 1, &id, GASPI_BLOCK)); ASSERT( gaspi_notify_reset(0, id, &val)); ASSERT (gaspi_segment_delete(0)); ntimes--; } while(ntimes > 0); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
void work(int tid) { gaspi_rank_t rankSend; gaspi_offset_t localOff= 0; gaspi_offset_t remOff = 0; gaspi_number_t queueSize, qmax; gaspi_size_t commSize ; ASSERT (gaspi_queue_size_max(&qmax)); for(commSize= 1; commSize < _500MB; commSize*=2 ) for(rankSend = 0; rankSend < numranks; rankSend++) { gaspi_printf("thread %d rank to send: %d - %lu bytes\n", tid, rankSend, commSize); gaspi_queue_size(1, &queueSize); if (queueSize > qmax - 100) ASSERT (gaspi_wait(1, GASPI_BLOCK)); ASSERT (gaspi_write(0, localOff, rankSend, 0, remOff, commSize, 1, GASPI_BLOCK)); } ASSERT (gaspi_wait(1, GASPI_BLOCK)); gaspi_threads_sync(); }
int main(int argc, char *argv[]) { gaspi_rank_t numranks, myrank; gaspi_rank_t rankSend; gaspi_size_t segSize; const gaspi_offset_t localOff= 0; const gaspi_offset_t remOff = 0; gaspi_number_t queueSize, qmax; gaspi_size_t commSize ; TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); ASSERT (gaspi_proc_num(&numranks)); ASSERT (gaspi_proc_rank(&myrank)); ASSERT (gaspi_segment_create(0, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); ASSERT( gaspi_segment_size(0, myrank, &segSize)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_queue_size_max(&qmax)); for(commSize= 4; commSize < _8MB; commSize*=2 ) { for(rankSend = 0; rankSend < numranks; rankSend++) { if(rankSend == myrank) continue; printf("partner rank: %d - %lu bytes\n", rankSend, commSize); //FAILS with or without outstanding requests gaspi_queue_size(1, &queueSize); if (queueSize > qmax - 24) ASSERT (gaspi_wait(1, GASPI_BLOCK)); ASSERT (gaspi_read(0, localOff, rankSend, 0, remOff, commSize, 1, GASPI_BLOCK)); } } ASSERT (gaspi_wait(1, GASPI_BLOCK)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { gaspi_rank_t numranks, myrank; gaspi_rank_t rankSend; gaspi_size_t segSize; TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); ASSERT (gaspi_proc_num(&numranks)); ASSERT (gaspi_proc_rank(&myrank)); ASSERT (gaspi_segment_create(0, _2GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); ASSERT( gaspi_segment_size(0, myrank, &segSize)); gaspi_printf("seg size %lu MB \n", segSize/1024/1024); // ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); gaspi_offset_t localOff= 814780664; gaspi_offset_t remOff = 81478246; gaspi_offset_t size = 1800; gaspi_number_t queueSize, qmax; ASSERT (gaspi_queue_size_max(&qmax)); for(rankSend = 0; rankSend < numranks; rankSend++) { gaspi_printf("rank to send: %d\n", rankSend); gaspi_queue_size(1, &queueSize); if (queueSize > qmax - 24) ASSERT (gaspi_wait(1, GASPI_BLOCK)); ASSERT (gaspi_write(0, localOff, rankSend, 0, remOff, size, 1, GASPI_BLOCK)); } ASSERT (gaspi_wait(1, GASPI_BLOCK)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { unsigned long i; gaspi_pointer_t _vptr; gaspi_rank_t num_ranks, myrank; gaspi_number_t qmax ; gaspi_number_t queueSize; gaspi_rank_t left_rank, right_rank; const unsigned long N = (1 << 13); TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); ASSERT (gaspi_proc_num(&num_ranks)); ASSERT (gaspi_proc_rank(&myrank)); ASSERT(gaspi_segment_create(0, _2MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); ASSERT(gaspi_segment_ptr(0, &_vptr)); ASSERT (gaspi_queue_size_max(&qmax)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); left_rank = (myrank + num_ranks - 1 ) % num_ranks; right_rank = (myrank + num_ranks + 1) % num_ranks; ASSERT( gaspi_write(0, //seg 0, //local off left_rank, //rank 0, //seg rem 0, //remote off 1, //size 32KB 0, //queue GASPI_BLOCK)); ASSERT( gaspi_write(0, //seg 0, //local off right_rank, //rank 0, //seg rem 0, //remote off 1, //size 32KB 0, //queue GASPI_BLOCK)); ASSERT (gaspi_wait(0, GASPI_BLOCK)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, 5000)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); printf("Rank %d: Finish\n", myrank); fflush(stdout); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { gaspi_rank_t numranks, myrank; gaspi_rank_t rankSend; gaspi_size_t segSize; const gaspi_offset_t localOff= 0; const gaspi_offset_t remOff = 0; gaspi_number_t queueSize, qmax; gaspi_size_t commSize ; TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); ASSERT (gaspi_proc_num(&numranks)); ASSERT (gaspi_proc_rank(&myrank)); ASSERT (gaspi_segment_create(0, _8MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_queue_size_max(&qmax)); for(commSize= 1; commSize <= _8MB; commSize*=2 ) { for(rankSend = 0; rankSend < numranks; rankSend++) { gaspi_printf("rank to send: %d - %lu bytes\n", rankSend, commSize); gaspi_queue_size(1, &queueSize); if (queueSize > qmax - 24) ASSERT (gaspi_wait(1, GASPI_BLOCK)); ASSERT (gaspi_write(0, localOff, rankSend, 0, remOff, commSize, 1, GASPI_BLOCK)); } } ASSERT (gaspi_wait(1, GASPI_BLOCK)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); gaspi_notification_id_t n=0; gaspi_number_t notif_num; gaspi_rank_t rank, nprocs, i; const gaspi_segment_id_t seg_id = 0; ASSERT(gaspi_proc_num(&nprocs)); ASSERT (gaspi_proc_rank(&rank)); ASSERT (gaspi_segment_create(seg_id, 1024, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED)); ASSERT( gaspi_notification_num(¬if_num)); gaspi_printf("max num notifications %u\n", notif_num); if(rank == 0) { gaspi_number_t queue_size; gaspi_number_t queue_max; ASSERT (gaspi_queue_size_max(&queue_max)); for(n = 0; n < notif_num; n++) { ASSERT (gaspi_queue_size(0, &queue_size)); if(queue_size > queue_max - 1) ASSERT (gaspi_wait(0, GASPI_BLOCK)); for(i = 1; i < nprocs; i++) { ASSERT (gaspi_notify( seg_id, i, n, 1, 0, GASPI_BLOCK)); } } } else { do { gaspi_notification_id_t id; ASSERT (gaspi_notify_waitsome(seg_id, 0, notif_num, &id, GASPI_BLOCK)); gaspi_notification_t notification_val; ASSERT( gaspi_notify_reset(seg_id, id, ¬ification_val)); assert(notification_val == 1); n++; } while(n < notif_num); } ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
void wait_for_queue_entries_for_read_notify (gaspi_queue_id_t* queue_id) { gaspi_number_t queue_size_max; gaspi_number_t queue_size; gaspi_number_t queue_num; SUCCESS_OR_DIE (gaspi_queue_num (&queue_num)); *queue_id = (*queue_id + 1) % queue_num; SUCCESS_OR_DIE (gaspi_wait (*queue_id, GASPI_BLOCK)); }
//recover => clean the queue gaspi_return_t recover() { gaspi_return_t ret = GASPI_ERROR; gaspi_printf("Enter recover\n"); while(ret != GASPI_SUCCESS) { ret = gaspi_wait(0, GASPI_BLOCK); } gaspi_printf("Exit recover with %d\n", ret); return ret; }
void wait_for_flush_queues () { gaspi_number_t queue_num; SUCCESS_OR_DIE (gaspi_queue_num (&queue_num)); gaspi_queue_id_t queue = 0; while( queue < queue_num ) { SUCCESS_OR_DIE (gaspi_wait (queue, GASPI_BLOCK)); ++queue; } }
static void wait_for_queue_entries (gaspi_queue_id_t* queue, int wanted_entries) { gaspi_number_t queue_size_max; gaspi_number_t queue_size; gaspi_number_t queue_num; SUCCESS_OR_DIE (gaspi_queue_size_max (&queue_size_max)); SUCCESS_OR_DIE (gaspi_queue_size (*queue, &queue_size)); SUCCESS_OR_DIE (gaspi_queue_num (&queue_num)); if (! (queue_size + wanted_entries <= queue_size_max)) { *queue = (*queue + 1) % queue_num; SUCCESS_OR_DIE (gaspi_wait (*queue, GASPI_BLOCK)); } }
int main (int argc, char *argv[]) { gaspi_configuration_t config = { 0 }; //argc, argv, "", (1UL << 30) gaspi_proc_init (config, GASPI_BLOCK); // 1 GiB DMA enabled memory per node gaspi_rank_t iProc, nProc; gaspi_proc_rank (&iProc); gaspi_proc_num (&nProc); void* temp_ptr; gaspi_segment_ptr(GPI_SEGMENT, &temp_ptr); int *mem = (int *) temp_ptr; // begin of DMA enabled memory int *src = mem; // offset 0 int *dst = mem + nProc; // offset nProc * sizeof(int) for (gaspi_rank_t p = 0; p < nProc; ++p) { src[p] = iProc * nProc + p; const unsigned long locOff = p * sizeof (int); const unsigned long remOff = (nProc + iProc) * sizeof (int); gaspi_write(GPI_SEGMENT, locOff, p, GPI_SEGMENT, remOff, sizeof (int), 0, GASPI_BLOCK); } gaspi_wait (0, GASPI_BLOCK); gaspi_barrier (GASPI_GROUP_ALL, GASPI_BLOCK); dump (src, iProc, nProc, "src"); dump (dst, iProc, nProc, "dst"); gaspi_proc_term (GASPI_BLOCK); return EXIT_SUCCESS; }
void send_global_msg_to_check_state(gaspi_state_vector_t health_vec, gaspi_rank_t *avoid_list) { int i, j; int num_simultaneous_fail_checks = 1; gaspi_timeout_t HEALTH_CHECK_TIMEOUT_TIME = GASPI_BLOCK; gaspi_printf("Checking global health state\n"); /* in order to check multiple simultaneous fail, health check has to be performed multiple times */ for(j = 0 ; j < num_simultaneous_fail_checks; ++j ) { for(i = 0; i < numprocs; ++i) { if(avoid_list[i] != 1) { ASSERT(gaspi_write(gm_seg_health_chk_array_id, myrank, i, gm_seg_health_chk_array_id, myrank, sizeof(int), queue_id, HEALTH_CHECK_TIMEOUT_TIME)); } } gaspi_wait(queue_id, HEALTH_CHECK_TIMEOUT_TIME); ASSERT(gaspi_state_vec_get(health_vec)); /* adding the dead processes to avoid_list */ /* so that message for health test is not sent to them next time. */ for(i = 0; i < numprocs; ++i) { if(health_vec[i] == 1) { avoid_list[i] = 1; } } } print_health_vec(health_vec); }
int main(int argc, char *argv[]) { int i, iterations = 1; gaspi_size_t max_msg_size = (1 << 30); const gaspi_offset_t offset = 0; gaspi_rank_t P, myrank, rank2send; TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); ASSERT (gaspi_proc_num(&P)); ASSERT (gaspi_proc_rank(&myrank)); rank2send = (myrank + 1) % P; assert(rank2send < P); gaspi_printf("Segment and msg size %lu MB\n", max_msg_size / 1024 / 1024); ASSERT( gaspi_segment_create(0, max_msg_size, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); for(i = 0; i < iterations; i++) { ASSERT(gaspi_read(0, offset, rank2send, 0, offset, max_msg_size, 0, GASPI_BLOCK)); } ASSERT (gaspi_wait(0, GASPI_BLOCK)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { int j,i,k=0; int ret=0; const gaspi_size_t size=4096;//4k const gaspi_size_t memSize = 4294967296; //4GB gaspi_offset_t offset_write=0, offset_read = memSize / 2, offset_check = 3221225472 ; ASSERT (gaspi_proc_init(GASPI_BLOCK)); ASSERT (gaspi_segment_create(0, memSize, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); gaspi_pointer_t _vptr; ASSERT (gaspi_segment_ptr(0, &_vptr)); /* get memory area pointer */ #ifdef FLOAT float *mptr = (float *) _vptr; #else int *mptr = (int *) _vptr; #endif gaspi_rank_t myrank, highestnode; ASSERT (gaspi_proc_rank(&myrank)); ASSERT (gaspi_proc_num(&highestnode)); while(k <= RUNS) { //generate random srand((unsigned)time(0)); #ifdef FLOAT srand48((unsigned) time(0)); #endif //clean for(j = 0; j < (memSize / 4); j++) mptr[j]= 0; //fill randoms up to 1GB for(j = 0; j < (memSize / 16); j++) { #ifdef FLOAT mptr[j]= drand48() + (myrank*1.0); #else mptr[j]= rand() + myrank; #endif } #ifdef DEBUG #ifdef FLOAT gaspi_printf("random value in pos 0 %f\n", mptr[0]); #else gaspi_printf("random value in pos 0 %d\n", mptr[0]); #endif #endif //DEBUG ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); gaspi_printf("\n....Running iteration %d of %d...\n",k, RUNS); for(i = 0; i < ITERATIONS; i++) { for(j = 0; j < ITERATIONS; j++) { ASSERT (gaspi_write(0, offset_write, (myrank + 1) % highestnode, 0, offset_read, size, 0, GASPI_BLOCK)); offset_write += size; offset_read += size; } ASSERT (gaspi_wait(0, GASPI_BLOCK)); } #ifdef DEBUG gaspi_printf("%d bytes written!\n", ITERATIONS * ITERATIONS * size); #endif //check if data was written successfully ASSERT (gaspi_read(0, offset_check, (myrank + 1) % highestnode, 0, memSize/2, GB, 0, GASPI_BLOCK)); ASSERT (gaspi_wait(0, GASPI_BLOCK)); #ifdef DEBUG gaspi_printf("%d bytes read!\n",GB); #endif j=0; #ifdef DEBUG #ifdef FLOAT gaspi_printf("Values %f %f %f \n", mptr[0], mptr[memSize/8], mptr[offset_check/4]); #else gaspi_printf("Values %d %d %d \n", mptr[0], mptr[memSize/8], mptr[offset_check/4]); #endif #endif//DEBUG while(j < GB / 4 ) { if(mptr[j] != mptr[offset_check / 4 + j]){ #ifdef FLOAT gaspi_printf("value incorrect %f-%f at %d \n",mptr[j],mptr[offset_check / 4],j); #else gaspi_printf("value incorrect %d-%d at %d \n",mptr[j],mptr[offset_check / 4],j); #endif ret = -1; goto out; } j++; } offset_write=0; offset_read = memSize / 2; #ifdef DEBUG gaspi_printf("Check!\n"); #endif k++; } out: gaspi_printf("Waiting to finish...\n"); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return ret; }
int main(int argc, char *argv[]) { const int num_elems = 1024; TSUITE_INIT( argc, argv ); ASSERT( gaspi_proc_init(GASPI_BLOCK) ); gaspi_rank_t rank, nprocs; ASSERT( gaspi_proc_num(&nprocs) ); ASSERT( gaspi_proc_rank(&rank) ); const gaspi_rank_t left = (rank + nprocs - 1 ) % nprocs; const gaspi_rank_t right = (rank + nprocs + 1) % nprocs; /* Create and fill buffer */ int * const buf = (int *) malloc(num_elems * sizeof(int)); assert( buf != NULL); int i; for (i = 0; i < num_elems; i++) { buf[i] = rank; } ASSERT( gaspi_segment_use( 0, buf, num_elems * sizeof(int), GASPI_GROUP_ALL, GASPI_BLOCK, 0) ); ASSERT( gaspi_segment_create( 1, num_elems * sizeof(int), GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED) ); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); /* write data to neighbour ( from seg 0 to seg 1) */ ASSERT( gaspi_write_notify( 0, 0, right, 1, 0, num_elems * sizeof(int), 0, 1, 0, GASPI_BLOCK) ); gaspi_notification_id_t id; ASSERT( gaspi_notify_waitsome( 1, 0, 1, &id, GASPI_BLOCK ) ); ASSERT( gaspi_wait( 0, GASPI_BLOCK ) ); /* Check data as segment */ gaspi_pointer_t seg1_ptr; ASSERT( gaspi_segment_ptr( 1, &seg1_ptr ) ); int * recv_buf = (int *) seg1_ptr; for (i = 0; i < num_elems; i++) { assert(recv_buf[i] == left); } ASSERT( gaspi_segment_delete(0)); ASSERT( gaspi_segment_delete(1)); /* Check data in buffer */ for (i = 0; i < num_elems; i++) { assert(buf[i] == rank); } ASSERT( gaspi_barrier( GASPI_GROUP_ALL, GASPI_BLOCK ) ); ASSERT( gaspi_proc_term( GASPI_BLOCK ) ); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); gaspi_rank_t numranks, myrank; ASSERT (gaspi_proc_num(&numranks)); ASSERT (gaspi_proc_rank(&myrank)); ASSERT (gaspi_segment_create(0, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); //prepare memory segment gaspi_pointer_t _vptr; ASSERT (gaspi_segment_ptr(0, &_vptr)); int *mem = (int *) _vptr; unsigned long i; const unsigned long maxInts = _128MB / sizeof(int); for(i = 0; i < maxInts; i++) { mem[i] = (int) myrank; } ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); //construct list of n elems gaspi_number_t queue_size = 0; const gaspi_number_t nListElems = 255; gaspi_number_t n; gaspi_segment_id_t localSegs[nListElems]; gaspi_offset_t localOffs[nListElems]; const gaspi_rank_t rank2read = (myrank + 1) % numranks; gaspi_segment_id_t remSegs[nListElems]; gaspi_offset_t remOffs[nListElems]; gaspi_size_t sizes[nListElems]; const unsigned int bytes = sizeof(int); gaspi_offset_t initLocOff = 0; gaspi_offset_t initRemOff = (bytes * nListElems + 64); for(n = 0; n < nListElems; n++) { sizes[n] = bytes; localSegs[n] = 0; localOffs[n] = initLocOff; initLocOff += bytes; remSegs[n] = 0; remOffs[n] = initRemOff; initRemOff += bytes; } ASSERT (gaspi_read_list(nListElems, localSegs,localOffs, rank2read, remSegs, remOffs, sizes, 0, GASPI_BLOCK)); ASSERT (gaspi_queue_size(0, &queue_size)); assert (queue_size == nListElems); ASSERT (gaspi_wait(0, GASPI_BLOCK)); //sync ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); //check gaspi_number_t l; gaspi_offset_t off2check = 0; char * chPtr = (char *) _vptr; mem = (int *) (chPtr + off2check); for(l = 0; l < nListElems; l++) { assert(mem[l] == (int) rank2read); } ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { int k = 0; int ret = 0; unsigned long j; const gaspi_size_t size = 4096; const gaspi_size_t memSize = _4GB; gaspi_offset_t offset_write = 0; gaspi_offset_t offset_read = _2GB; gaspi_offset_t offset_check = 3221225472; gaspi_number_t qmax ; gaspi_number_t queueSize; TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); ASSERT (gaspi_queue_size_max(&qmax)); ASSERT (gaspi_segment_create(0, memSize, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); gaspi_pointer_t _vptr; ASSERT (gaspi_segment_ptr(0, &_vptr)); /* get memory area pointer */ float *mptr_f = (float *) _vptr; char *mptr_c = (char *) _vptr; gaspi_rank_t myrank, highestnode; ASSERT (gaspi_proc_rank(&myrank)); ASSERT (gaspi_proc_num(&highestnode)); while(k <= RUNS) { //generate random srand((unsigned)time(0)); srand48((unsigned) time(0)); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); //clean for(j = 0; j < memSize; j++) mptr_c[j]= 0; /* fill randoms up to 1GB */ for(j = 0; j < (GB / sizeof(float)); j++) { mptr_f[j]= drand48() + (myrank * 1.0); } #ifdef DEBUG gaspi_printf("random value in pos 0 %f\n", mptr_f[0]); #endif ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); gaspi_printf("\n....Running iteration %d of %d...\n",k, RUNS); const unsigned long packets = (GB / size); for(j = 0; j < packets; j++) { ASSERT(gaspi_queue_size(0, &queueSize)); if (queueSize > qmax - 24) { ASSERT(gaspi_wait(0, GASPI_BLOCK)); } ASSERT (gaspi_write(0, offset_write, (myrank + 1) % highestnode, 0, offset_read, size, 0, GASPI_BLOCK)); offset_write += size; offset_read += size; } offset_write=0; offset_read = _2GB; #ifdef DEBUG gaspi_printf("%d bytes written!\n", packets * size); #endif /* notify remote that data is written */ ASSERT (gaspi_notify( 0, (myrank + 1) % highestnode, 0, 1, 0, GASPI_BLOCK)); gaspi_notification_id_t recv_id; ASSERT(gaspi_notify_waitsome(0, 0, 1, &recv_id, GASPI_BLOCK)); assert(recv_id == 0); gaspi_notification_t notification_val; ASSERT( gaspi_notify_reset(0, recv_id, ¬ification_val)); /* notify remote that data has arrived */ ASSERT (gaspi_notify( 0, (myrank + highestnode - 1) % highestnode, 1, 1, 0, GASPI_BLOCK)); gaspi_notification_id_t ack_id; ASSERT(gaspi_notify_waitsome(0, 1, 1, &ack_id, GASPI_BLOCK)); assert(ack_id == 1); ASSERT( gaspi_notify_reset(0, ack_id, ¬ification_val)); /* check if data was written successfully */ ASSERT (gaspi_read(0, offset_check, (myrank + 1) % highestnode, 0, offset_read, GB / 2, 0, GASPI_BLOCK)); ASSERT (gaspi_read(0, offset_check + (GB / 2), (myrank + 1) % highestnode, 0, offset_read + (GB / 2), GB / 2, 0, GASPI_BLOCK)); ASSERT (gaspi_wait(0, GASPI_BLOCK)); #ifdef DEBUG gaspi_printf("Values %f %f %f \n", mptr_f[0], mptr_f[offset_read / sizeof(float)], mptr_f[offset_check / sizeof(float)]); #endif j = 0; while(j < GB / sizeof(float) ) { if(mptr_f[j] != mptr_f[offset_check / sizeof(float) + j]){ gaspi_printf("value incorrect %f-%f at %d \n", mptr_f[j], mptr_f[offset_check / sizeof(float) + j], j); ret = -1; goto out; } j++; } #ifdef DEBUG gaspi_printf("Check!\n"); #endif k++; } out: gaspi_printf("Waiting to finish...\n"); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return ret; }
int main(int argc, char *argv[]) { static const int VLEN = 1 << 2; SUCCESS_OR_DIE(gaspi_proc_init (GASPI_BLOCK)); gaspi_rank_t iProc, nProc; SUCCESS_OR_DIE(gaspi_proc_rank (&iProc)); SUCCESS_OR_DIE(gaspi_proc_num (&nProc)); gaspi_segment_id_t const segment_id = 0; gaspi_size_t const segment_size = 2 * VLEN * sizeof(double); SUCCESS_OR_DIE(gaspi_segment_create (segment_id, segment_size, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED )); gaspi_pointer_t array; SUCCESS_OR_DIE(gaspi_segment_ptr (segment_id, &array)); double * src_array = (double *) (array); double * rcv_array = src_array + VLEN; for (int j = 0; j < VLEN; ++j) { src_array[j] = (double) (iProc * VLEN + j); } gaspi_notification_id_t data_available = 0; gaspi_queue_id_t queue_id = 0; gaspi_offset_t loc_off = 0; gaspi_offset_t rem_off = VLEN * sizeof(double); /* TODO ==== - check queue. - gaspi_write_notify 1-dim array of size VLEN to right neighbour - wait for notification from left neighbour */ for (int j = 0; j < VLEN; ++j) { printf ("rank %d rcv elem %d: %f \n", iProc, j, rcv_array[j]); } /* TODO ==== - why do we have to wait for the queue here ? */ SUCCESS_OR_DIE(gaspi_wait (queue_id, GASPI_BLOCK)); SUCCESS_OR_DIE(gaspi_proc_term (GASPI_BLOCK)); return EXIT_SUCCESS; }
//create GPU segemnts on all nodes //and then tranfer data between all segments int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); //need the barrier to make sn is up ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); gaspi_rank_t rank, nprocs, i,j; gaspi_number_t seg_max; gaspi_gpu_id_t gpus[8]; gaspi_number_t nGPUs; ASSERT(gaspi_proc_num(&nprocs)); ASSERT (gaspi_proc_rank(&rank)); ASSERT(gaspi_gpu_init()); seg_max = 1; ASSERT (gaspi_gpu_number(&nGPUs)); ASSERT (gaspi_gpu_ids(gpus)); for(i = 0; i < nGPUs; i++) { cudaSetDevice(gpus[i]); ASSERT (gaspi_segment_create(i, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED|GASPI_MEM_GPU)); } ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); gaspi_number_t queueSize; int rankSend = (rank + 1) % nprocs; for (j =0; j<nGPUs; j++) { //sleep(1); if (gaspi_gpu_write(j, //seg 1024, //local rankSend, //rank j, //seg rem 1024, //remote 32768, //size 1, //queue GASPI_BLOCK) != GASPI_SUCCESS) { gaspi_queue_size(1, &queueSize); gaspi_printf (" failed with i = %d queue %u\n", j, queueSize); exit(-1); } } ASSERT(gaspi_wait(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); for (i =0; i<nGPUs; i++) { ASSERT (gaspi_segment_delete(i)); } ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); gaspi_notification_id_t n=0; gaspi_rank_t rank, nprocs, i; const gaspi_segment_id_t seg_id = 0; gaspi_offset_t offset; gaspi_number_t queue_size; gaspi_number_t queue_max; ASSERT (gaspi_queue_size_max(&queue_max)); ASSERT(gaspi_proc_num(&nprocs)); ASSERT (gaspi_proc_rank(&rank)); ASSERT (gaspi_segment_create(seg_id, nprocs * sizeof(int), GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED)); offset = rank * sizeof(int); //set memory gaspi_pointer_t _vptr; ASSERT (gaspi_segment_ptr(0, &_vptr)); int *mem = (int *) _vptr; for(i = 0; i < nprocs; i++) { mem[i] = (int) rank; } ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); //go for(i = 0; i < nprocs; i++) { if (i == rank) continue; ASSERT (gaspi_queue_size(0, &queue_size)); if(queue_size > queue_max - 1) ASSERT (gaspi_wait(0, GASPI_BLOCK)); ASSERT (gaspi_write_notify( seg_id, offset, i, seg_id, offset, sizeof(int), (gaspi_notification_id_t) rank, 1, 0, GASPI_BLOCK)); } do { gaspi_notification_id_t id; ASSERT (gaspi_notify_waitsome(seg_id, 0, (gaspi_notification_id_t) nprocs , &id, GASPI_BLOCK)); gaspi_notification_t notification_val; ASSERT( gaspi_notify_reset(seg_id, id, ¬ification_val)); assert(notification_val == 1); n++; } while(n < (nprocs - 1)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main (int argc, char *argv[]) { int i, j, t; gaspi_rank_t myrank; char *ptr0; //on numa architectures you have to map this process to the numa node where nic is installed if (start_bench (2) != 0) { printf ("Initialization failed\n"); exit (-1); } // BENCH // gaspi_proc_rank (&myrank); if (gaspi_segment_ptr (0, (void **) &ptr0) != GASPI_SUCCESS) { printf ("gaspi_segment_ptr failed !\n"); exit (-1); } gaspi_float cpu_freq; gaspi_cpu_frequency(&cpu_freq); if (myrank < 2) { if(myrank == 0) { printf("-----------------------------------\n"); printf ("%12s\t%5s\n", "Bytes", "Lat(usecs)"); printf("-----------------------------------\n"); } int bytes = 2; volatile char *postBuf = (volatile char *) ptr0; for (i = 1; i < 24; i++) { volatile char *pollBuf = (volatile char *) (ptr0 + ( 2 * bytes -1 )); int rcnt = 0; int cnt = 0; gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK); for (j = 0; j < ITERATIONS; j++) { if (rcnt < ITERATIONS && !(cnt < 1 && myrank == 1)) { rcnt++; while (*pollBuf != (char) rcnt) { #ifdef MIC _mm_delay_32(32); #else _mm_pause(); #endif } } stamp[j] = get_mcycles (); postBuf[bytes - 1] = (char) ++cnt; gaspi_write (0, 0, myrank ^ 0x1, 0, bytes, bytes, 0, GASPI_BLOCK); gaspi_wait (0, GASPI_BLOCK); } for (t = 0; t < (ITERATIONS - 1); t++) delta[t] = stamp[t + 1] - stamp[t]; qsort (delta, (ITERATIONS - 1), sizeof *delta, mcycles_compare); const double div = 1.0 / cpu_freq; const double ts = (double) delta[ITERATIONS / 2] * div * 0.5; if(myrank == 0) printf ("%12d\t%4.2f\n", bytes, ts); bytes <<= 1; } } end_bench (); return 0; }
int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT( gaspi_proc_init(GASPI_BLOCK) ); gaspi_rank_t numranks, myrank; ASSERT( gaspi_proc_num(&numranks) ); ASSERT( gaspi_proc_rank(&myrank) ); ASSERT( gaspi_segment_create(0, _8MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED) ); ASSERT( gaspi_segment_create(1, _8MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED) ); //prepare memory segment gaspi_pointer_t _vptr, _vptr1; ASSERT (gaspi_segment_ptr(0, &_vptr)); ASSERT (gaspi_segment_ptr(1, &_vptr1)); int *mem = (int *) _vptr; int *mem_read = (int *) _vptr1; unsigned long i; const unsigned long maxInts = _8MB / sizeof(int); for(i = 0; i < maxInts; i++) { mem[i] = (int) myrank; } ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); //construct list of n elems gaspi_number_t queue_size = 0; const gaspi_number_t nListElems = 255; gaspi_number_t n; gaspi_segment_id_t localSegs[nListElems]; gaspi_offset_t localOffs[nListElems]; const gaspi_rank_t rank2read = (myrank + 1) % numranks; gaspi_segment_id_t remSegs[nListElems]; gaspi_offset_t remOffs[nListElems]; gaspi_size_t sizes[nListElems]; const unsigned int bytes = sizeof(int); gaspi_offset_t initLocOff = 0; gaspi_offset_t initRemOff = 0; for(n = 0; n < nListElems; n++) { sizes[n] = bytes; localSegs[n] = 1; localOffs[n] = initLocOff; initLocOff += bytes; remSegs[n] = 0; remOffs[n] = initRemOff; initRemOff += bytes; } ASSERT( gaspi_read_list_notify( nListElems, localSegs, localOffs, rank2read, remSegs, remOffs, sizes, 1, myrank, 0, GASPI_BLOCK)); gaspi_notification_id_t id; ASSERT (gaspi_notify_waitsome(1, myrank, 1, &id, GASPI_BLOCK)); gaspi_notification_t notification_val; ASSERT( gaspi_notify_reset(1, id, ¬ification_val)); assert(notification_val == 1); //check gaspi_number_t l; for(l = 0; l < nListElems; l++) { assert(mem_read[l] == (int) rank2read); } ASSERT (gaspi_wait(0, GASPI_BLOCK)); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { int i, iter; int ret = 0; gaspi_rank_t myrank, numranks; gaspi_size_t mem_size = 0UL, j; TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); ASSERT (gaspi_proc_rank(&myrank)); ASSERT (gaspi_proc_num(&numranks)); if( numranks < 2 ) { return EXIT_SUCCESS; } mem_size = 2 * SLOT_SIZE * (numranks - 1); if(myrank == 0) { printf("Mem size: %lu (%.2f MB)\nProcs: %u Max Slot size %lu Iterations %d\n", mem_size, mem_size * 1.0f / 1024/ 1024, numranks, (gaspi_size_t) SLOT_SIZE, MAX_ITERATIONS); #ifdef WITH_SYNC printf("Using notifications only\n"); #endif } ASSERT (gaspi_segment_create(0, mem_size, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); gaspi_pointer_t _vptr; ASSERT (gaspi_segment_ptr(0, &_vptr)); float *mptr = (float *) _vptr; //generate random srand((unsigned)time(0)); srand48((unsigned) time(0)); gaspi_size_t cur_slot_size = SLOT_SIZE; for(cur_slot_size = SLOT_SIZE; cur_slot_size >= sizeof(float); cur_slot_size/=2) { if(myrank == 0) printf("===== Slot Size %lu ====\n", cur_slot_size); for(iter = 0; iter < MAX_ITERATIONS; iter++) { if(myrank == 0) printf("iteration %3d... ", iter); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); /* fill slots with randoms */ for(j = 0; j < (mem_size / sizeof(float) / 2); j++) { mptr[j]= drand48() + (myrank*1.0); } ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); gaspi_offset_t offset_in = 0, offset_out = mem_size / 2; /* rank 0 write to all others */ if(myrank == 0) { for (i = 1; i < numranks; i++) { offset_in = (i - 1) * cur_slot_size; #ifdef WITH_SYNC ASSERT (gaspi_write_notify(0, offset_in, i, 0, 0, cur_slot_size, 0, 1, 0, GASPI_BLOCK)); #else ASSERT (gaspi_write(0, offset_in, i, 0, 0, cur_slot_size, 0, GASPI_BLOCK)); #endif } ASSERT(gaspi_wait(0, GASPI_BLOCK)); } #ifdef WITH_SYNC else { gaspi_notification_id_t id; gaspi_notification_t val; ASSERT(gaspi_notify_waitsome(0, 0, 1, &id, GASPI_BLOCK)); ASSERT(gaspi_notify_reset(0, id, &val)); assert(val == 1); } #endif #ifndef WITH_SYNC ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); #endif /* other ranks all write back to 0 */ if(myrank != 0) { offset_in = 0; offset_out = (mem_size / 2) + (cur_slot_size * (myrank - 1)); #ifdef WITH_SYNC ASSERT (gaspi_write_notify(0, offset_in, 0, 0, offset_out, cur_slot_size, myrank, 1, 0, GASPI_BLOCK)); #else ASSERT (gaspi_write(0, offset_in, i, 0, offset_out, cur_slot_size, 0, GASPI_BLOCK)); #endif ASSERT(gaspi_wait(0, GASPI_BLOCK)); } #ifdef WITH_SYNC else { gaspi_notification_id_t id; gaspi_notification_t val; int notification_counter = 0; do { ASSERT(gaspi_notify_waitsome(0, 1, numranks - 1, &id, GASPI_BLOCK)); ASSERT(gaspi_notify_reset(0, id, &val)); assert(val == 1); notification_counter++; } while( notification_counter < numranks - 1); } #endif #ifndef WITH_SYNC ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); #endif if(myrank == 0) { /* check correctness */ float *in = (float *) _vptr; float *out = (float *) ((char *) _vptr + mem_size / 2); const gaspi_size_t total_elems = (cur_slot_size * (numranks - 1) / sizeof(float)); for(j = 0; j < total_elems; j++) { if(in[j] != out[j]) { printf("Different values at pos %lu: %f %f (iterations %d)\n", j, in[j], out[j], iter); ret = -1; goto end; } } printf("All fine!\n"); } } } end: ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return ret; }
int main (int argc, char *argv[]) { gaspi_proc_init(GASPI_BLOCK); gaspi_rank_t myRank; gaspi_rank_t nProc; gaspi_proc_rank(&myRank); gaspi_proc_num(&nProc); if(nProc < 2) goto end; gaspi_number_t queue_size; gaspi_number_t queue_max; gaspi_queue_size_max(&queue_max); if (myRank == 0) gaspi_printf("Queue max is %d\n", queue_max); gaspi_printf("Rank %i of %i started.\n", myRank, nProc); const gaspi_segment_id_t segment_id = 0; const gaspi_size_t nrReads = NR_OF_READS; gaspi_group_commit(GASPI_GROUP_ALL,GASPI_BLOCK); gaspi_segment_create(segment_id, nrReads * (RAWREADLENGTH) * sizeof(gaspi_char),GASPI_GROUP_ALL,GASPI_BLOCK,GASPI_ALLOC_DEFAULT); gaspi_pointer_t _vptr; //pointer to the segment if(gaspi_segment_ptr(segment_id, &_vptr) != GASPI_SUCCESS) printf("gaspi_segment_ptr failed\n"); gaspi_char * shared_ptr = (gaspi_char *) _vptr; // initialize and print segment initReads(shared_ptr, nrReads, READLENGTH, myRank); gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK); //push the reads from the master to the slaves gaspi_size_t r = 0; int rawReadSize = RAWREADLENGTH * sizeof(gaspi_char); int nrWorkers = nProc - 1; int toRank; gaspi_notification_id_t notif_id; if (myRank == 0) { for (r = 0; r < nrReads; r++) { gaspi_queue_size(0, &queue_size); if(queue_size > queue_max - 1) gaspi_wait(0, GASPI_BLOCK); //wait for queue to become free again... (note: max is 1024) toRank = (r % nrWorkers) + 1; // notif_id = r + 1; notif_id = ((r / nrWorkers) + 1); if ( gaspi_write_notify( segment_id, // from segment r*rawReadSize, // from offset toRank, // to-rank segment_id, // to segment // ((int)(r/nrWorkers))*rawReadSize, // to-offset r * rawReadSize, rawReadSize, // size notif_id, // notification id r+1, // notification value (> 0!) (gaspi_queue_id_t) 0, // notification queue GASPI_BLOCK) == GASPI_SUCCESS) // block until written gaspi_printf("Sending read %d from %d to rank %d with id %d\n", r, myRank, toRank, notif_id); if (toRank == 2) print_read(shared_ptr, r, READLENGTH, myRank); } } //ranks receive reads from the master rank if (myRank != 0) { gaspi_notification_id_t fid; gaspi_notification_t notification_value; int nrOfReceives = (int)(nrReads / (nProc-1)); if (myRank <= nrReads % nrWorkers) nrOfReceives++; gaspi_printf("Rank %d -- listening for %d events...\n", myRank, nrOfReceives); int complete = 0; while (complete < nrOfReceives) { if(gaspi_notify_waitsome( segment_id, // segment 1, // id of first notification to wait for // nrReads, nrOfReceives, // id of last notification to wait for (alternative) &fid, // identifier (output parameter with the identifier of a received notification (?)) GASPI_TEST // immediately return (GASPI_TEST) ) == GASPI_SUCCESS) { if(gaspi_notify_reset( segment_id, // segment fid, // notification identifier ¬ification_value // notification value ) == GASPI_SUCCESS) { complete++ ; gaspi_printf("Rank %d -- got notification: read %d received (%d completed)\n", myRank, notification_value-1, complete); if (myRank == 2) print_read(shared_ptr, notification_value-1, READLENGTH, myRank); } } } } // all values received ! print ! gaspi_barrier(GASPI_GROUP_ALL,GASPI_BLOCK); gaspi_printf("Printing reads\n"); print_char_array_segment(shared_ptr, nrReads, READLENGTH, myRank); // print_read(shared_ptr, 0, READLENGTH, myRank); gaspi_barrier(GASPI_GROUP_ALL,GASPI_BLOCK); gaspi_printf("Rank %d done\n", myRank); //block and exit end: gaspi_barrier(GASPI_GROUP_ALL,GASPI_BLOCK); gaspi_proc_term(GASPI_BLOCK); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { gaspi_rank_t nprocs, myrank, i; int j, n; gaspi_rank_t *avoid_list; gaspi_group_t survivors; TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); ASSERT(gaspi_proc_num(&nprocs)); ASSERT(gaspi_proc_rank(&myrank)); ASSERT(gaspi_segment_create(0, _4MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); avoid_list = (gaspi_rank_t *) malloc(nprocs * sizeof(gaspi_rank_t)); assert (avoid_list != NULL); memset(avoid_list, 0, nprocs * sizeof(gaspi_rank_t)); gaspi_state_vector_t vec = (gaspi_state_vector_t) malloc(nprocs); ASSERT(gaspi_state_vec_get(vec)); //check that everyone is healthy for(i = 0; i < nprocs; i++) { assert(vec[i] == 0); } //sync ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); //now last rank dies if(myrank == nprocs - 1) exit(-1); else { //create group of survivors ASSERT(gaspi_group_create(&survivors)); for(i = 0; i < nprocs - 1; i++) ASSERT(gaspi_group_add(survivors, i)); ASSERT(gaspi_group_commit(survivors, GASPI_BLOCK)); gaspi_printf("Done with groups\n"); sleep(2); } //the others communicate gaspi_return_t retval; for(j = 0; j < 10; j++) { gaspi_printf("Iteration %d\n", j); for(i = 0; i < nprocs; i++) { if( avoid_list[i] != 1 ) ASSERT(gaspi_write(0, 0, i, 0, 0, sizeof(int), 0, GASPI_BLOCK)); } retval = gaspi_wait(0, GASPI_BLOCK); //problem found -> recover if(retval != GASPI_SUCCESS) { ASSERT(gaspi_state_vec_get(vec)); for(n = 0; n < nprocs; n++) { if(vec[n] != GASPI_STATE_HEALTHY) { gaspi_printf("Problem with node %d detected\n", n); assert(n == (nprocs - 1)); ASSERT(recover()); avoid_list[n] = 1; } } } } ASSERT (gaspi_barrier(survivors, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); gaspi_printf("exiting\n"); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); const unsigned long N = (1 << 13); gaspi_rank_t P, myrank; ASSERT (gaspi_proc_num(&P)); ASSERT (gaspi_proc_rank(&myrank)); gaspi_printf("P = %d N = %lu\n", P, N); gaspi_printf("Seg size: %lu MB\n", MAX (_4GB, 2 * ((N/P) * N * 2 * sizeof (double)))/1024/1024); if(gaspi_segment_create(0, _1GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED) != GASPI_SUCCESS){ gaspi_printf("Failed to create segment\n"); return -1; } gaspi_pointer_t _vptr; if(gaspi_segment_ptr(0, &_vptr) != GASPI_SUCCESS) printf("gaspi_segment_ptr failed\n"); gaspi_number_t qmax ; ASSERT (gaspi_queue_size_max(&qmax)); gaspi_printf("Queue max: %lu\n", qmax); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); int i; gaspi_number_t queueSize; int rankSend = (myrank + 1) % P; gaspi_printf("rank to: %d\n", rankSend); for (i = 0; i < 2 * N; i ++) { gaspi_queue_size(1, &queueSize); if (queueSize > qmax - 24) { gaspi_return_t ret; do { ret = gaspi_wait(1, GASPI_TEST); assert (ret != GASPI_ERROR); } while(ret != GASPI_SUCCESS); gaspi_queue_size(1, &queueSize); assert(queueSize == 0); } ASSERT (gaspi_write(0, 4, rankSend, 0, 6, 32768, 1, GASPI_TEST)); } ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { gaspi_rank_t numranks, myrank; gaspi_rank_t rankSend; gaspi_size_t segSize; const gaspi_offset_t localOff_r= 0; const gaspi_offset_t remOff_r = 0; const gaspi_offset_t localOff_w = _128MB / 2 ; const gaspi_offset_t remOff_w = _128MB / 2; gaspi_number_t queueSize, qmax; const gaspi_size_t commSize = _8MB; int i; gaspi_gpu_t gpus[8]; gaspi_gpu_num nGPUs; TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); ASSERT (gaspi_proc_num(&numranks)); ASSERT (gaspi_proc_rank(&myrank)); ASSERT (gaspi_init_GPUs()); ASSERT (gaspi_number_of_GPUs(&nGPUs)); ASSERT (gaspi_GPU_ids(gpus)); ASSERT (gaspi_segment_create(0, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED|GASPI_MEM_GPU)); ASSERT( gaspi_segment_size(0, myrank, &segSize)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_queue_size_max(&qmax)); for(i = 0; i < 100; i++ ) { for(rankSend = 0; rankSend < numranks; rankSend++) { if(rankSend == myrank) continue; gaspi_printf("partner rank: %d - %lu bytes (%d)\n", rankSend, commSize, i); ASSERT (gaspi_queue_size(1, &queueSize)); if (queueSize > qmax - 24) ASSERT (gaspi_wait(1, GASPI_BLOCK)); ASSERT (gaspi_read(0, localOff_r, rankSend, 0, remOff_r, commSize, 1, GASPI_BLOCK)); } } for(i = 0; i < 100; i++ ) { for(rankSend = 0; rankSend < numranks; rankSend++) { if(rankSend == myrank) continue; ASSERT (gaspi_gpu_write(0, localOff_r, rankSend, 0, remOff_r, commSize, 1, GASPI_BLOCK)); } } ASSERT (gaspi_wait(1, GASPI_BLOCK)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); gaspi_rank_t numranks, myrank; ASSERT (gaspi_proc_num(&numranks)); ASSERT (gaspi_proc_rank(&myrank)); int rankSend = (myrank + 1) % numranks; ASSERT(gaspi_segment_create(0, _1MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); gaspi_size_t segSize; ASSERT( gaspi_segment_size(0, myrank, &segSize)); unsigned char * pGlbMem; gaspi_pointer_t _vptr; ASSERT(gaspi_segment_ptr(0, &_vptr)); pGlbMem = ( unsigned char *) _vptr; gaspi_number_t queueSize, qmax ; ASSERT (gaspi_queue_size_max(&qmax)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); const unsigned long localOff = 0; const unsigned long remOff = 0; /* write_notify */ do { ASSERT(gaspi_write_notify(0, localOff, rankSend, 0, remOff, 1, (gaspi_notification_id_t) myrank, 1, 1, GASPI_BLOCK)); gaspi_queue_size(1, &queueSize); } while(queueSize < qmax); EXPECT_FAIL_WITH(gaspi_write_notify(0, localOff, rankSend, 0, remOff, 1, (gaspi_notification_id_t) myrank, 1, 1, GASPI_BLOCK), GASPI_QUEUE_FULL); ASSERT (gaspi_wait(1, GASPI_BLOCK)); /* write */ do { ASSERT(gaspi_write(0, localOff, rankSend, 0, remOff, 1, 1, GASPI_BLOCK)); gaspi_queue_size(1, &queueSize); } while(queueSize < qmax); EXPECT_FAIL_WITH(gaspi_write(0, localOff, rankSend, 0, remOff, 1, 1, GASPI_BLOCK), GASPI_QUEUE_FULL); ASSERT (gaspi_wait(1, GASPI_BLOCK)); ASSERT(gaspi_write(0, localOff, rankSend, 0, remOff, 1, 1, GASPI_BLOCK)); /* write + write_notify */ do { ASSERT(gaspi_write(0, localOff, rankSend, 0, remOff, 1, 1, GASPI_BLOCK)); gaspi_queue_size(1, &queueSize); } while(queueSize < qmax - 1); EXPECT_FAIL_WITH(gaspi_write_notify(0, localOff, rankSend, 0, remOff, 1, (gaspi_notification_id_t) myrank, 1, 1, GASPI_BLOCK), GASPI_QUEUE_FULL); ASSERT (gaspi_wait(1, GASPI_BLOCK)); ASSERT(gaspi_write_notify(0, localOff, rankSend, 0, remOff, 1, (gaspi_notification_id_t) myrank, 1, 1, GASPI_BLOCK)); /* read */ do { ASSERT(gaspi_read(0, localOff, rankSend, 0, remOff, 1, 1, GASPI_BLOCK)); gaspi_queue_size(1, &queueSize); } while(queueSize < qmax); EXPECT_FAIL_WITH(gaspi_read(0, localOff, rankSend, 0, remOff, 1, 1, GASPI_BLOCK), GASPI_QUEUE_FULL); ASSERT (gaspi_wait(1, GASPI_BLOCK)); ASSERT(gaspi_read(0, localOff, rankSend, 0, remOff, 1, 1, GASPI_BLOCK)); /* write_list_notify */ { const gaspi_number_t nListElems = 255; gaspi_number_t n; gaspi_segment_id_t localSegs[nListElems]; gaspi_offset_t localOffs[nListElems]; const gaspi_rank_t rank2send = (myrank + 1) % numranks; gaspi_segment_id_t remSegs[nListElems]; gaspi_offset_t remOffs[nListElems]; gaspi_size_t sizes[nListElems]; const unsigned int bytes = sizeof(int); gaspi_offset_t initLocOff = 0; gaspi_offset_t initRemOff = (bytes * nListElems + 64); for(n = 0; n < nListElems; n++) { sizes[n] = bytes; localSegs[n] = 0; localOffs[n] = initLocOff; initLocOff += bytes; remSegs[n] = 0; remOffs[n] = initRemOff; initRemOff += bytes; } do { ASSERT( gaspi_write_list_notify( nListElems, localSegs, localOffs, rank2send, remSegs, remOffs, sizes, 0, myrank, 1, 0, GASPI_BLOCK)); gaspi_queue_size(0, &queueSize); } while(queueSize < qmax); EXPECT_FAIL_WITH( gaspi_write_list_notify( nListElems, localSegs, localOffs, rank2send, remSegs, remOffs, sizes, 0, myrank, 1, 0, GASPI_BLOCK), GASPI_QUEUE_FULL); ASSERT (gaspi_wait(0, GASPI_BLOCK)); ASSERT( gaspi_write_list_notify( nListElems, localSegs, localOffs, rank2send, remSegs, remOffs, sizes, 0, myrank, 1, 0, GASPI_BLOCK)); } ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }