int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); gaspi_rank_t numranks, myrank, n; ASSERT (gaspi_proc_num(&numranks)); ASSERT (gaspi_proc_rank(&myrank)); ASSERT (gaspi_segment_create(0, _2GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); gaspi_atomic_value_t val; gaspi_atomic_value_t compare = 0; if(myrank == 0) for(n = 0; n < numranks; n++) ASSERT(gaspi_atomic_compare_swap(0, 0, n, compare, (gaspi_atomic_value_t) 42, &val, GASPI_BLOCK)); gaspi_pointer_t _vptr; ASSERT (gaspi_segment_ptr(0, &_vptr)); //sync ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); gaspi_atomic_value_t *end_val; end_val = (gaspi_atomic_value_t *)_vptr; assert(*end_val == 42); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { static const int VLEN = 1 << 2; SUCCESS_OR_DIE( gaspi_proc_init(GASPI_BLOCK) ); gaspi_rank_t iProc, nProc; SUCCESS_OR_DIE( gaspi_proc_rank(&iProc) ); SUCCESS_OR_DIE( gaspi_proc_num(&nProc) ); gaspi_segment_id_t const segment_id = 0; gaspi_size_t const segment_size = VLEN * sizeof (double); SUCCESS_OR_DIE ( gaspi_segment_create ( segment_id, segment_size , GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED ) ); gaspi_pointer_t array; SUCCESS_OR_DIE( gaspi_segment_ptr (segment_id, &array) ); for (int j = 0; j < VLEN; ++j) { ( (double *)array )[j]= (double)( iProc * VLEN + j ); printf("rank %d elem %d: %f \n", iProc,j,( (double *)array )[j] ); } SUCCESS_OR_DIE( gaspi_proc_term(GASPI_BLOCK) ); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { unsigned long i; gaspi_pointer_t _vptr; gaspi_rank_t num_ranks, myrank; gaspi_number_t qmax ; gaspi_number_t queueSize; gaspi_rank_t left_rank, right_rank; const unsigned long N = (1 << 13); TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); ASSERT (gaspi_proc_num(&num_ranks)); ASSERT (gaspi_proc_rank(&myrank)); ASSERT(gaspi_segment_create(0, _2MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); ASSERT(gaspi_segment_ptr(0, &_vptr)); ASSERT (gaspi_queue_size_max(&qmax)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); left_rank = (myrank + num_ranks - 1 ) % num_ranks; right_rank = (myrank + num_ranks + 1) % num_ranks; ASSERT( gaspi_write(0, //seg 0, //local off left_rank, //rank 0, //seg rem 0, //remote off 1, //size 32KB 0, //queue GASPI_BLOCK)); ASSERT( gaspi_write(0, //seg 0, //local off right_rank, //rank 0, //seg rem 0, //remote off 1, //size 32KB 0, //queue GASPI_BLOCK)); ASSERT (gaspi_wait(0, GASPI_BLOCK)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, 5000)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); printf("Rank %d: Finish\n", myrank); fflush(stdout); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); gaspi_rank_t P, myrank; ASSERT (gaspi_proc_num(&P)); ASSERT (gaspi_proc_rank(&myrank)); ASSERT (gaspi_segment_create(0, _2MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); int * int_GlbMem; gaspi_pointer_t _vptr; ASSERT(gaspi_segment_ptr(0, &_vptr)); int_GlbMem = ( int *) _vptr; gaspi_size_t msgSize; gaspi_size_t i; for(msgSize = 4; msgSize < 8192; msgSize *=2) if(myrank == 0) { for(i = 0; i < msgSize / sizeof(int); i++) int_GlbMem[i] = (int) msgSize; gaspi_rank_t n; for(n = 1; n < P; n++) ASSERT(gaspi_passive_send(0, 0, n, msgSize, GASPI_BLOCK)); } else { gaspi_rank_t sender; ASSERT(gaspi_passive_receive(0, 0, &sender, msgSize, GASPI_BLOCK)); for(i = 0; i < msgSize / sizeof(int); i++) assert( int_GlbMem[i] == (int) msgSize ); } ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { const int ntimes = 1000; TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); gaspi_rank_t numranks, myrank, n,m; ASSERT (gaspi_proc_num(&numranks)); ASSERT (gaspi_proc_rank(&myrank)); ASSERT (gaspi_segment_create(0, _2MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); gaspi_atomic_value_t val; for(m = 0; m < ntimes; m++) for(n = 0; n < numranks; n++) { gaspi_return_t ret; do { ret = gaspi_atomic_fetch_add(0, 0, n, 1, &val, GASPI_BLOCK); } while(ret == GASPI_TIMEOUT); } gaspi_pointer_t _vptr; ASSERT (gaspi_segment_ptr(0, &_vptr)); //sync to make sure everyone did it ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); gaspi_atomic_value_t *end_val; end_val = (gaspi_atomic_value_t *)_vptr; assert(*end_val == (numranks * ntimes)); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); mctpInitUser(2); int tid = mctpRegisterThread(); gaspi_rank_t P, myrank; ASSERT (gaspi_proc_num(&P)); ASSERT (gaspi_proc_rank(&myrank)); ASSERT (gaspi_segment_create(0, _4GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); int * int_GlbMem; gaspi_pointer_t _vptr; ASSERT(gaspi_segment_ptr(0, &_vptr)); int_GlbMem = ( int *) _vptr; mctpStartSingleThread(recvThread, int_GlbMem); int_GlbMem[0] = 11223344; //send to myself (need trick to avoid blocking) gaspi_return_t ret = GASPI_ERROR; do { ret = gaspi_passive_send(0, 0, myrank, sizeof(int), GASPI_TEST); assert (ret != GASPI_ERROR); sleep(1); } while (ret != GASPI_SUCCESS); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); gaspi_rank_t P, myrank; ASSERT (gaspi_proc_num(&P)); ASSERT (gaspi_proc_rank(&myrank)); ASSERT (gaspi_segment_create(0, _4GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); int * int_GlbMem; gaspi_pointer_t _vptr; ASSERT(gaspi_segment_ptr(0, &_vptr)); int_GlbMem = ( int *) _vptr; const gaspi_size_t msgSize = sizeof(int); if(myrank == 0) { int_GlbMem[0] = 11223344; gaspi_rank_t n; for(n = 1; n < P; n++) ASSERT(gaspi_passive_send(0, 0, n, msgSize, GASPI_BLOCK)); } else { gaspi_rank_t sender; ASSERT(gaspi_passive_receive(0, 0, &sender, msgSize, GASPI_BLOCK)); gaspi_printf("Received msg from %d\n", sender); assert( int_GlbMem[0] == 11223344 ); } ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); ASSERT(gaspi_threads_init_user(2)); int tid; ASSERT (gaspi_threads_register(&tid)); gaspi_rank_t P, myrank; ASSERT (gaspi_proc_num(&P)); ASSERT (gaspi_proc_rank(&myrank)); ASSERT (gaspi_segment_create(0, _2MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); int * int_GlbMem; gaspi_pointer_t _vptr; ASSERT(gaspi_segment_ptr(0, &_vptr)); int_GlbMem = ( int *) _vptr; ASSERT(gaspi_threads_run(recvThread, int_GlbMem)); int_GlbMem[0] = 11223344; ASSERT(gaspi_passive_send(0, 0, myrank, sizeof(int), GASPI_BLOCK)); gaspi_threads_sync(); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main (int argc, char *argv[]) { gaspi_configuration_t config = { 0 }; //argc, argv, "", (1UL << 30) gaspi_proc_init (config, GASPI_BLOCK); // 1 GiB DMA enabled memory per node gaspi_rank_t iProc, nProc; gaspi_proc_rank (&iProc); gaspi_proc_num (&nProc); void* temp_ptr; gaspi_segment_ptr(GPI_SEGMENT, &temp_ptr); int *mem = (int *) temp_ptr; // begin of DMA enabled memory int *src = mem; // offset 0 int *dst = mem + nProc; // offset nProc * sizeof(int) for (gaspi_rank_t p = 0; p < nProc; ++p) { src[p] = iProc * nProc + p; const unsigned long locOff = p * sizeof (int); const unsigned long remOff = (nProc + iProc) * sizeof (int); gaspi_write(GPI_SEGMENT, locOff, p, GPI_SEGMENT, remOff, sizeof (int), 0, GASPI_BLOCK); } gaspi_wait (0, GASPI_BLOCK); gaspi_barrier (GASPI_GROUP_ALL, GASPI_BLOCK); dump (src, iProc, nProc, "src"); dump (dst, iProc, nProc, "dst"); gaspi_proc_term (GASPI_BLOCK); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { int k = 0; int ret = 0; unsigned long j; const gaspi_size_t size = 4096; const gaspi_size_t memSize = _4GB; gaspi_offset_t offset_write = 0; gaspi_offset_t offset_read = _2GB; gaspi_offset_t offset_check = 3221225472; gaspi_number_t qmax ; gaspi_number_t queueSize; TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); ASSERT (gaspi_queue_size_max(&qmax)); ASSERT (gaspi_segment_create(0, memSize, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); gaspi_pointer_t _vptr; ASSERT (gaspi_segment_ptr(0, &_vptr)); /* get memory area pointer */ float *mptr_f = (float *) _vptr; char *mptr_c = (char *) _vptr; gaspi_rank_t myrank, highestnode; ASSERT (gaspi_proc_rank(&myrank)); ASSERT (gaspi_proc_num(&highestnode)); while(k <= RUNS) { //generate random srand((unsigned)time(0)); srand48((unsigned) time(0)); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); //clean for(j = 0; j < memSize; j++) mptr_c[j]= 0; /* fill randoms up to 1GB */ for(j = 0; j < (GB / sizeof(float)); j++) { mptr_f[j]= drand48() + (myrank * 1.0); } #ifdef DEBUG gaspi_printf("random value in pos 0 %f\n", mptr_f[0]); #endif ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); gaspi_printf("\n....Running iteration %d of %d...\n",k, RUNS); const unsigned long packets = (GB / size); for(j = 0; j < packets; j++) { ASSERT(gaspi_queue_size(0, &queueSize)); if (queueSize > qmax - 24) { ASSERT(gaspi_wait(0, GASPI_BLOCK)); } ASSERT (gaspi_write(0, offset_write, (myrank + 1) % highestnode, 0, offset_read, size, 0, GASPI_BLOCK)); offset_write += size; offset_read += size; } offset_write=0; offset_read = _2GB; #ifdef DEBUG gaspi_printf("%d bytes written!\n", packets * size); #endif /* notify remote that data is written */ ASSERT (gaspi_notify( 0, (myrank + 1) % highestnode, 0, 1, 0, GASPI_BLOCK)); gaspi_notification_id_t recv_id; ASSERT(gaspi_notify_waitsome(0, 0, 1, &recv_id, GASPI_BLOCK)); assert(recv_id == 0); gaspi_notification_t notification_val; ASSERT( gaspi_notify_reset(0, recv_id, ¬ification_val)); /* notify remote that data has arrived */ ASSERT (gaspi_notify( 0, (myrank + highestnode - 1) % highestnode, 1, 1, 0, GASPI_BLOCK)); gaspi_notification_id_t ack_id; ASSERT(gaspi_notify_waitsome(0, 1, 1, &ack_id, GASPI_BLOCK)); assert(ack_id == 1); ASSERT( gaspi_notify_reset(0, ack_id, ¬ification_val)); /* check if data was written successfully */ ASSERT (gaspi_read(0, offset_check, (myrank + 1) % highestnode, 0, offset_read, GB / 2, 0, GASPI_BLOCK)); ASSERT (gaspi_read(0, offset_check + (GB / 2), (myrank + 1) % highestnode, 0, offset_read + (GB / 2), GB / 2, 0, GASPI_BLOCK)); ASSERT (gaspi_wait(0, GASPI_BLOCK)); #ifdef DEBUG gaspi_printf("Values %f %f %f \n", mptr_f[0], mptr_f[offset_read / sizeof(float)], mptr_f[offset_check / sizeof(float)]); #endif j = 0; while(j < GB / sizeof(float) ) { if(mptr_f[j] != mptr_f[offset_check / sizeof(float) + j]){ gaspi_printf("value incorrect %f-%f at %d \n", mptr_f[j], mptr_f[offset_check / sizeof(float) + j], j); ret = -1; goto out; } j++; } #ifdef DEBUG gaspi_printf("Check!\n"); #endif k++; } out: gaspi_printf("Waiting to finish...\n"); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return ret; }
int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); gaspi_rank_t numranks, myrank; ASSERT (gaspi_proc_num(&numranks)); ASSERT (gaspi_proc_rank(&myrank)); int rankSend = (myrank + 1) % numranks; ASSERT(gaspi_segment_create(0, _1MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); gaspi_size_t segSize; ASSERT( gaspi_segment_size(0, myrank, &segSize)); unsigned char * pGlbMem; gaspi_pointer_t _vptr; ASSERT(gaspi_segment_ptr(0, &_vptr)); pGlbMem = ( unsigned char *) _vptr; gaspi_number_t queueSize, qmax ; ASSERT (gaspi_queue_size_max(&qmax)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); const unsigned long localOff = 0; const unsigned long remOff = 0; /* write_notify */ do { ASSERT(gaspi_write_notify(0, localOff, rankSend, 0, remOff, 1, (gaspi_notification_id_t) myrank, 1, 1, GASPI_BLOCK)); gaspi_queue_size(1, &queueSize); } while(queueSize < qmax); EXPECT_FAIL_WITH(gaspi_write_notify(0, localOff, rankSend, 0, remOff, 1, (gaspi_notification_id_t) myrank, 1, 1, GASPI_BLOCK), GASPI_QUEUE_FULL); ASSERT (gaspi_wait(1, GASPI_BLOCK)); /* write */ do { ASSERT(gaspi_write(0, localOff, rankSend, 0, remOff, 1, 1, GASPI_BLOCK)); gaspi_queue_size(1, &queueSize); } while(queueSize < qmax); EXPECT_FAIL_WITH(gaspi_write(0, localOff, rankSend, 0, remOff, 1, 1, GASPI_BLOCK), GASPI_QUEUE_FULL); ASSERT (gaspi_wait(1, GASPI_BLOCK)); ASSERT(gaspi_write(0, localOff, rankSend, 0, remOff, 1, 1, GASPI_BLOCK)); /* write + write_notify */ do { ASSERT(gaspi_write(0, localOff, rankSend, 0, remOff, 1, 1, GASPI_BLOCK)); gaspi_queue_size(1, &queueSize); } while(queueSize < qmax - 1); EXPECT_FAIL_WITH(gaspi_write_notify(0, localOff, rankSend, 0, remOff, 1, (gaspi_notification_id_t) myrank, 1, 1, GASPI_BLOCK), GASPI_QUEUE_FULL); ASSERT (gaspi_wait(1, GASPI_BLOCK)); ASSERT(gaspi_write_notify(0, localOff, rankSend, 0, remOff, 1, (gaspi_notification_id_t) myrank, 1, 1, GASPI_BLOCK)); /* read */ do { ASSERT(gaspi_read(0, localOff, rankSend, 0, remOff, 1, 1, GASPI_BLOCK)); gaspi_queue_size(1, &queueSize); } while(queueSize < qmax); EXPECT_FAIL_WITH(gaspi_read(0, localOff, rankSend, 0, remOff, 1, 1, GASPI_BLOCK), GASPI_QUEUE_FULL); ASSERT (gaspi_wait(1, GASPI_BLOCK)); ASSERT(gaspi_read(0, localOff, rankSend, 0, remOff, 1, 1, GASPI_BLOCK)); /* write_list_notify */ { const gaspi_number_t nListElems = 255; gaspi_number_t n; gaspi_segment_id_t localSegs[nListElems]; gaspi_offset_t localOffs[nListElems]; const gaspi_rank_t rank2send = (myrank + 1) % numranks; gaspi_segment_id_t remSegs[nListElems]; gaspi_offset_t remOffs[nListElems]; gaspi_size_t sizes[nListElems]; const unsigned int bytes = sizeof(int); gaspi_offset_t initLocOff = 0; gaspi_offset_t initRemOff = (bytes * nListElems + 64); for(n = 0; n < nListElems; n++) { sizes[n] = bytes; localSegs[n] = 0; localOffs[n] = initLocOff; initLocOff += bytes; remSegs[n] = 0; remOffs[n] = initRemOff; initRemOff += bytes; } do { ASSERT( gaspi_write_list_notify( nListElems, localSegs, localOffs, rank2send, remSegs, remOffs, sizes, 0, myrank, 1, 0, GASPI_BLOCK)); gaspi_queue_size(0, &queueSize); } while(queueSize < qmax); EXPECT_FAIL_WITH( gaspi_write_list_notify( nListElems, localSegs, localOffs, rank2send, remSegs, remOffs, sizes, 0, myrank, 1, 0, GASPI_BLOCK), GASPI_QUEUE_FULL); ASSERT (gaspi_wait(0, GASPI_BLOCK)); ASSERT( gaspi_write_list_notify( nListElems, localSegs, localOffs, rank2send, remSegs, remOffs, sizes, 0, myrank, 1, 0, GASPI_BLOCK)); } ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); const unsigned long N = (1 << 13); gaspi_rank_t P, myrank; ASSERT (gaspi_proc_num(&P)); ASSERT (gaspi_proc_rank(&myrank)); gaspi_printf("P = %d N = %lu\n", P, N); gaspi_printf("Seg size: %lu MB\n", MAX (_4GB, 2 * ((N/P) * N * 2 * sizeof (double)))/1024/1024); if(gaspi_segment_create(0, _1GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED) != GASPI_SUCCESS){ gaspi_printf("Failed to create segment\n"); return -1; } gaspi_pointer_t _vptr; if(gaspi_segment_ptr(0, &_vptr) != GASPI_SUCCESS) printf("gaspi_segment_ptr failed\n"); gaspi_number_t qmax ; ASSERT (gaspi_queue_size_max(&qmax)); gaspi_printf("Queue max: %lu\n", qmax); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); int i; gaspi_number_t queueSize; int rankSend = (myrank + 1) % P; gaspi_printf("rank to: %d\n", rankSend); for (i = 0; i < 2 * N; i ++) { gaspi_queue_size(1, &queueSize); if (queueSize > qmax - 24) { gaspi_return_t ret; do { ret = gaspi_wait(1, GASPI_TEST); assert (ret != GASPI_ERROR); } while(ret != GASPI_SUCCESS); gaspi_queue_size(1, &queueSize); assert(queueSize == 0); } ASSERT (gaspi_write(0, 4, rankSend, 0, 6, 32768, 1, GASPI_TEST)); } ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main (int argc, char *argv[]) { gaspi_proc_init(GASPI_BLOCK); gaspi_rank_t myRank; gaspi_rank_t nProc; gaspi_proc_rank(&myRank); gaspi_proc_num(&nProc); if(nProc < 2) goto end; gaspi_number_t queue_size; gaspi_number_t queue_max; gaspi_queue_size_max(&queue_max); if (myRank == 0) gaspi_printf("Queue max is %d\n", queue_max); gaspi_printf("Rank %i of %i started.\n", myRank, nProc); const gaspi_segment_id_t segment_id = 0; const gaspi_size_t nrReads = NR_OF_READS; gaspi_group_commit(GASPI_GROUP_ALL,GASPI_BLOCK); gaspi_segment_create(segment_id, nrReads * (RAWREADLENGTH) * sizeof(gaspi_char),GASPI_GROUP_ALL,GASPI_BLOCK,GASPI_ALLOC_DEFAULT); gaspi_pointer_t _vptr; //pointer to the segment if(gaspi_segment_ptr(segment_id, &_vptr) != GASPI_SUCCESS) printf("gaspi_segment_ptr failed\n"); gaspi_char * shared_ptr = (gaspi_char *) _vptr; // initialize and print segment initReads(shared_ptr, nrReads, READLENGTH, myRank); gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK); //push the reads from the master to the slaves gaspi_size_t r = 0; int rawReadSize = RAWREADLENGTH * sizeof(gaspi_char); int nrWorkers = nProc - 1; int toRank; gaspi_notification_id_t notif_id; if (myRank == 0) { for (r = 0; r < nrReads; r++) { gaspi_queue_size(0, &queue_size); if(queue_size > queue_max - 1) gaspi_wait(0, GASPI_BLOCK); //wait for queue to become free again... (note: max is 1024) toRank = (r % nrWorkers) + 1; // notif_id = r + 1; notif_id = ((r / nrWorkers) + 1); if ( gaspi_write_notify( segment_id, // from segment r*rawReadSize, // from offset toRank, // to-rank segment_id, // to segment // ((int)(r/nrWorkers))*rawReadSize, // to-offset r * rawReadSize, rawReadSize, // size notif_id, // notification id r+1, // notification value (> 0!) (gaspi_queue_id_t) 0, // notification queue GASPI_BLOCK) == GASPI_SUCCESS) // block until written gaspi_printf("Sending read %d from %d to rank %d with id %d\n", r, myRank, toRank, notif_id); if (toRank == 2) print_read(shared_ptr, r, READLENGTH, myRank); } } //ranks receive reads from the master rank if (myRank != 0) { gaspi_notification_id_t fid; gaspi_notification_t notification_value; int nrOfReceives = (int)(nrReads / (nProc-1)); if (myRank <= nrReads % nrWorkers) nrOfReceives++; gaspi_printf("Rank %d -- listening for %d events...\n", myRank, nrOfReceives); int complete = 0; while (complete < nrOfReceives) { if(gaspi_notify_waitsome( segment_id, // segment 1, // id of first notification to wait for // nrReads, nrOfReceives, // id of last notification to wait for (alternative) &fid, // identifier (output parameter with the identifier of a received notification (?)) GASPI_TEST // immediately return (GASPI_TEST) ) == GASPI_SUCCESS) { if(gaspi_notify_reset( segment_id, // segment fid, // notification identifier ¬ification_value // notification value ) == GASPI_SUCCESS) { complete++ ; gaspi_printf("Rank %d -- got notification: read %d received (%d completed)\n", myRank, notification_value-1, complete); if (myRank == 2) print_read(shared_ptr, notification_value-1, READLENGTH, myRank); } } } } // all values received ! print ! gaspi_barrier(GASPI_GROUP_ALL,GASPI_BLOCK); gaspi_printf("Printing reads\n"); print_char_array_segment(shared_ptr, nrReads, READLENGTH, myRank); // print_read(shared_ptr, 0, READLENGTH, myRank); gaspi_barrier(GASPI_GROUP_ALL,GASPI_BLOCK); gaspi_printf("Rank %d done\n", myRank); //block and exit end: gaspi_barrier(GASPI_GROUP_ALL,GASPI_BLOCK); gaspi_proc_term(GASPI_BLOCK); return EXIT_SUCCESS; }
int main(int argc, char* argv[]) { int i, j; gaspi_number_t gsize; int comm_state = WORKING; int num_failures = 0; int timesteps = 0; ASSERT (gaspi_proc_init(GASPI_BLOCK)); ASSERT (gaspi_proc_rank(&myrank)); ASSERT (gaspi_proc_num(&numprocs)); read_params(argc, argv, ×teps, &numprocs_idle); numprocs_working = numprocs - numprocs_idle; numprocs_working_and_idle = numprocs_working + numprocs_idle; gaspi_rank_t *comm_main_ranks = malloc( numprocs_idle * sizeof(gaspi_rank_t)); init_array_2(comm_main_ranks, numprocs_working); /* contains info of all processes: which are working(0), broken(1) and idle(2). keeps updated all the time(iterations) */ int * status_processes = (int *) malloc(numprocs * sizeof(int)); init_array_3(status_processes, numprocs, WORKING); for(i = numprocs-1, j=0; j < numprocs_idle;--i,++j) { status_processes[i] = IDLE; // putting last processes to IDLE } // ===== GASPI group creation ===== if(status_processes[myrank]==WORKING) { ASSERT(gaspi_group_create(&COMM_MAIN)); gaspi_number_t i; for(i=0; i<numprocs; i++) { if(status_processes[i]==WORKING) { ASSERT(gaspi_group_add(COMM_MAIN, i)); ASSERT(gaspi_group_size(COMM_MAIN, &gsize)); } } ASSERT(gaspi_group_ranks (COMM_MAIN, comm_main_ranks)); ASSERT(gaspi_group_commit (COMM_MAIN, GASPI_BLOCK)); } /* ====== Init a SYNC FLAGS Segment ====== */ /* used to communicate the WORKING, BROKEN, or FINISHED_WORK status between the working and idle processes. */ gaspi_size_t SYNC_global_mem_size; SYNC_global_mem_size = numprocs * sizeof(int); gaspi_pointer_t gm_ptr_sync=NULL; ASSERT(init_segment (gm_seg_sync_flags_id, SYNC_global_mem_size)); ASSERT(gaspi_segment_ptr (gm_seg_sync_flags_id, &gm_ptr_sync)); int * sync_flags = (int *) gm_ptr_sync; init_array_3(sync_flags, numprocs, WORKING); /* ====== Init a health check write FLAGS Segment ====== */ /* This array is used to send the gaspi_write message write before health_chk routine, which will then update the gaspi internal health vector */ gaspi_size_t health_chk_global_mem_size; health_chk_global_mem_size = numprocs*sizeof(int); gaspi_pointer_t gm_ptr_health_chk=NULL; ASSERT(init_segment (gm_seg_health_chk_array_id, health_chk_global_mem_size)); ASSERT(gaspi_segment_ptr (gm_seg_health_chk_array_id, &gm_ptr_health_chk)); gaspi_state_vector_t health_vec = (gaspi_state_vector_t) malloc(numprocs); ASSERT(gaspi_state_vec_get(health_vec)); gaspi_rank_t * avoid_list= (gaspi_rank_t *) malloc(numprocs * sizeof(gaspi_rank_t)); for(i = 0;i < numprocs; ++i) avoid_list[i] = (gaspi_rank_t) 0; gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK); /* ===== TIME-STEP LOOP ===== */ if(status_processes[myrank]==IDLE) { /* IDLE processes remain in this loop */ while(1) { gaspi_printf("%d.", myrank); if(sync_flags[0] == WORKING) { /* NO FAILURE REPORTED */ usleep(1000000); } if(sync_flags[0] == BROKEN) { /* FAILURE REPORTED */ gaspi_printf("myrank: %d Broken reported\n", myrank); comm_state=BROKEN; break; } if(sync_flags[0] == WORKFINISHED) { /* WORKFINISHED REPORTED */ gaspi_printf("myrank: %d WorkFinished reported\n", myrank); comm_state = WORKFINISHED; break; } } } int time_step; for(time_step=1; time_step <= timesteps && comm_state!=WORKFINISHED; time_step++) { gaspi_printf("== time_step: %d ==\n", time_step); if(comm_state==WORKING && status_processes[myrank]==WORKING) { gaspi_barrier(COMM_MAIN, GASPI_TIMEOUT_TIME); sleep(1); // NOTE: this is the work section. if(time_step == 5 && myrank== 1) { exit (-1); } } if(time_step<timesteps ) { send_global_msg_to_check_state(health_vec, avoid_list); num_failures = check_comm_health(status_processes, health_vec); gaspi_printf("%d NUM_FAILURES at timestep %d = %d\n", myrank, time_step, num_failures); if( num_failures != 0 ) { rescue_process = numprocs_working; if(myrank==0) { // message the IDLE process sync_flags[0]=BROKEN; for(i = 0 ; i < num_failures ; ++i) { /* TODO: multiple failures at the same time. */ gaspi_printf("messaging rescue_process: %d\n", rescue_process); ASSERT(gaspi_write(gm_seg_sync_flags_id, 0, rescue_process, gm_seg_sync_flags_id, 0, sizeof(int), 0, GASPI_BLOCK)); rescue_process++; } } if(myrank==0 || myrank==rescue_process) gaspi_printf("%d REPAIRING COMM_MAIN FLAG 1\n", myrank); update_status_processes_array(status_processes, health_vec); numprocs_working_and_idle = refresh_numprocs_working_and_idle(status_processes); if(myrank != rescue_process) { ASSERT(gaspi_group_delete(COMM_MAIN)); ASSERT(recover()); } ASSERT(gaspi_group_create(&COMM_MAIN_NEW)); for(i = 0; i < numprocs; i++) { if(status_processes[i]==WORKING) { ASSERT(gaspi_group_add(COMM_MAIN_NEW, i)); ASSERT(gaspi_group_size(COMM_MAIN_NEW, &gsize)); if(gsize == numprocs_working) break; } } gaspi_printf("%d: COMM_MAIN_NEW size is: %hi\n", myrank, gsize); ASSERT(gaspi_group_commit (COMM_MAIN_NEW, GASPI_BLOCK)); init_array_2(comm_main_ranks, numprocs_working); ASSERT(gaspi_group_ranks (COMM_MAIN_NEW, comm_main_ranks)); gaspi_printf("printing group_ranks_main: \n"); gaspi_printf_array(comm_main_ranks, numprocs_working); comm_state = WORKING; gaspi_printf("%d REPAIRING COMM_MAIN_NEW FLAG 2\n", myrank); if(status_processes[myrank] == WORKING) { ASSERT(gaspi_barrier(COMM_MAIN_NEW, GASPI_BLOCK)); ASSERT(gaspi_barrier(COMM_MAIN_NEW, GASPI_BLOCK)); } /* set things to work again */ COMM_MAIN = COMM_MAIN_NEW; time_step = 5; } } } if(myrank == 0) { gaspi_printf("finished successfully\n"); } gaspi_proc_term(10000); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { int i, iter; int ret = 0; gaspi_rank_t myrank, numranks; gaspi_size_t mem_size = 0UL, j; TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); ASSERT (gaspi_proc_rank(&myrank)); ASSERT (gaspi_proc_num(&numranks)); if( numranks < 2 ) { return EXIT_SUCCESS; } mem_size = 2 * SLOT_SIZE * (numranks - 1); if(myrank == 0) { printf("Mem size: %lu (%.2f MB)\nProcs: %u Max Slot size %lu Iterations %d\n", mem_size, mem_size * 1.0f / 1024/ 1024, numranks, (gaspi_size_t) SLOT_SIZE, MAX_ITERATIONS); #ifdef WITH_SYNC printf("Using notifications only\n"); #endif } ASSERT (gaspi_segment_create(0, mem_size, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); gaspi_pointer_t _vptr; ASSERT (gaspi_segment_ptr(0, &_vptr)); float *mptr = (float *) _vptr; //generate random srand((unsigned)time(0)); srand48((unsigned) time(0)); gaspi_size_t cur_slot_size = SLOT_SIZE; for(cur_slot_size = SLOT_SIZE; cur_slot_size >= sizeof(float); cur_slot_size/=2) { if(myrank == 0) printf("===== Slot Size %lu ====\n", cur_slot_size); for(iter = 0; iter < MAX_ITERATIONS; iter++) { if(myrank == 0) printf("iteration %3d... ", iter); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); /* fill slots with randoms */ for(j = 0; j < (mem_size / sizeof(float) / 2); j++) { mptr[j]= drand48() + (myrank*1.0); } ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); gaspi_offset_t offset_in = 0, offset_out = mem_size / 2; /* rank 0 write to all others */ if(myrank == 0) { for (i = 1; i < numranks; i++) { offset_in = (i - 1) * cur_slot_size; #ifdef WITH_SYNC ASSERT (gaspi_write_notify(0, offset_in, i, 0, 0, cur_slot_size, 0, 1, 0, GASPI_BLOCK)); #else ASSERT (gaspi_write(0, offset_in, i, 0, 0, cur_slot_size, 0, GASPI_BLOCK)); #endif } ASSERT(gaspi_wait(0, GASPI_BLOCK)); } #ifdef WITH_SYNC else { gaspi_notification_id_t id; gaspi_notification_t val; ASSERT(gaspi_notify_waitsome(0, 0, 1, &id, GASPI_BLOCK)); ASSERT(gaspi_notify_reset(0, id, &val)); assert(val == 1); } #endif #ifndef WITH_SYNC ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); #endif /* other ranks all write back to 0 */ if(myrank != 0) { offset_in = 0; offset_out = (mem_size / 2) + (cur_slot_size * (myrank - 1)); #ifdef WITH_SYNC ASSERT (gaspi_write_notify(0, offset_in, 0, 0, offset_out, cur_slot_size, myrank, 1, 0, GASPI_BLOCK)); #else ASSERT (gaspi_write(0, offset_in, i, 0, offset_out, cur_slot_size, 0, GASPI_BLOCK)); #endif ASSERT(gaspi_wait(0, GASPI_BLOCK)); } #ifdef WITH_SYNC else { gaspi_notification_id_t id; gaspi_notification_t val; int notification_counter = 0; do { ASSERT(gaspi_notify_waitsome(0, 1, numranks - 1, &id, GASPI_BLOCK)); ASSERT(gaspi_notify_reset(0, id, &val)); assert(val == 1); notification_counter++; } while( notification_counter < numranks - 1); } #endif #ifndef WITH_SYNC ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); #endif if(myrank == 0) { /* check correctness */ float *in = (float *) _vptr; float *out = (float *) ((char *) _vptr + mem_size / 2); const gaspi_size_t total_elems = (cur_slot_size * (numranks - 1) / sizeof(float)); for(j = 0; j < total_elems; j++) { if(in[j] != out[j]) { printf("Different values at pos %lu: %f %f (iterations %d)\n", j, in[j], out[j], iter); ret = -1; goto end; } } printf("All fine!\n"); } } } end: ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return ret; }
int main(int argc, char *argv[]) { const int num_elems = 1024; TSUITE_INIT( argc, argv ); ASSERT( gaspi_proc_init(GASPI_BLOCK) ); gaspi_rank_t rank, nprocs; ASSERT( gaspi_proc_num(&nprocs) ); ASSERT( gaspi_proc_rank(&rank) ); const gaspi_rank_t left = (rank + nprocs - 1 ) % nprocs; const gaspi_rank_t right = (rank + nprocs + 1) % nprocs; /* Create and fill buffer */ int * const buf = (int *) malloc(num_elems * sizeof(int)); assert( buf != NULL); int i; for (i = 0; i < num_elems; i++) { buf[i] = rank; } ASSERT( gaspi_segment_use( 0, buf, num_elems * sizeof(int), GASPI_GROUP_ALL, GASPI_BLOCK, 0) ); ASSERT( gaspi_segment_create( 1, num_elems * sizeof(int), GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED) ); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); /* write data to neighbour ( from seg 0 to seg 1) */ ASSERT( gaspi_write_notify( 0, 0, right, 1, 0, num_elems * sizeof(int), 0, 1, 0, GASPI_BLOCK) ); gaspi_notification_id_t id; ASSERT( gaspi_notify_waitsome( 1, 0, 1, &id, GASPI_BLOCK ) ); ASSERT( gaspi_wait( 0, GASPI_BLOCK ) ); /* Check data as segment */ gaspi_pointer_t seg1_ptr; ASSERT( gaspi_segment_ptr( 1, &seg1_ptr ) ); int * recv_buf = (int *) seg1_ptr; for (i = 0; i < num_elems; i++) { assert(recv_buf[i] == left); } ASSERT( gaspi_segment_delete(0)); ASSERT( gaspi_segment_delete(1)); /* Check data in buffer */ for (i = 0; i < num_elems; i++) { assert(buf[i] == rank); } ASSERT( gaspi_barrier( GASPI_GROUP_ALL, GASPI_BLOCK ) ); ASSERT( gaspi_proc_term( GASPI_BLOCK ) ); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); gaspi_rank_t numranks, myrank; ASSERT (gaspi_proc_num(&numranks)); ASSERT (gaspi_proc_rank(&myrank)); ASSERT (gaspi_segment_create(0, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); ASSERT (gaspi_segment_create(1, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); gaspi_number_t queueSize, qmax; ASSERT (gaspi_queue_size_max(&qmax)); gaspi_pointer_t segPtr, segPtr1; ASSERT( gaspi_segment_ptr(0, &segPtr)); int* segInt = (int *) segPtr; ASSERT( gaspi_segment_ptr(1, &segPtr1)); int* segInt1 = (int *) segPtr1; gaspi_size_t segSize; ASSERT( gaspi_segment_size(0, myrank, &segSize)); int i; for(i = 0; i < segSize / sizeof(int); i++) { segInt[i] = myrank; segInt1[i] = -1; } ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); gaspi_size_t commSize ; gaspi_offset_t localOff= 0; const gaspi_offset_t remOff = 0; for(commSize = sizeof(int); commSize < _8MB; commSize *= 2 ) { gaspi_rank_t rankSend; for(rankSend = 0; rankSend < numranks; rankSend++) { if(rankSend == myrank) continue; gaspi_queue_size(1, &queueSize); if (queueSize > qmax - 24) { ASSERT (gaspi_wait(1, GASPI_BLOCK)); } ASSERT (gaspi_read(1, localOff, rankSend, 0, remOff, commSize, 1, GASPI_BLOCK)); localOff+= commSize; } ASSERT (gaspi_wait(1, GASPI_BLOCK)); const int elems_per_rank = commSize / sizeof(int); int c, pos = 0; int* seg_read = (int *) segPtr1; for(rankSend = 0; rankSend < numranks; rankSend++) { if(rankSend == myrank) continue; for(c = 0; c < elems_per_rank; c++) { assert (seg_read[pos] == rankSend); pos++; } } localOff = 0; } ASSERT (gaspi_wait(1, GASPI_BLOCK)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main (int argc, char *argv[]) { int i, j, t; gaspi_rank_t myrank; char *ptr0; //on numa architectures you have to map this process to the numa node where nic is installed if (start_bench (2) != 0) { printf ("Initialization failed\n"); exit (-1); } // BENCH // gaspi_proc_rank (&myrank); if (gaspi_segment_ptr (0, (void **) &ptr0) != GASPI_SUCCESS) { printf ("gaspi_segment_ptr failed !\n"); exit (-1); } gaspi_float cpu_freq; gaspi_cpu_frequency(&cpu_freq); if (myrank < 2) { if(myrank == 0) { printf("-----------------------------------\n"); printf ("%12s\t%5s\n", "Bytes", "Lat(usecs)"); printf("-----------------------------------\n"); } int bytes = 2; volatile char *postBuf = (volatile char *) ptr0; for (i = 1; i < 24; i++) { volatile char *pollBuf = (volatile char *) (ptr0 + ( 2 * bytes -1 )); int rcnt = 0; int cnt = 0; gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK); for (j = 0; j < ITERATIONS; j++) { if (rcnt < ITERATIONS && !(cnt < 1 && myrank == 1)) { rcnt++; while (*pollBuf != (char) rcnt) { #ifdef MIC _mm_delay_32(32); #else _mm_pause(); #endif } } stamp[j] = get_mcycles (); postBuf[bytes - 1] = (char) ++cnt; gaspi_write (0, 0, myrank ^ 0x1, 0, bytes, bytes, 0, GASPI_BLOCK); gaspi_wait (0, GASPI_BLOCK); } for (t = 0; t < (ITERATIONS - 1); t++) delta[t] = stamp[t + 1] - stamp[t]; qsort (delta, (ITERATIONS - 1), sizeof *delta, mcycles_compare); const double div = 1.0 / cpu_freq; const double ts = (double) delta[ITERATIONS / 2] * div * 0.5; if(myrank == 0) printf ("%12d\t%4.2f\n", bytes, ts); bytes <<= 1; } } end_bench (); return 0; }
int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT( gaspi_proc_init(GASPI_BLOCK) ); gaspi_rank_t numranks, myrank; ASSERT( gaspi_proc_num(&numranks) ); ASSERT( gaspi_proc_rank(&myrank) ); ASSERT( gaspi_segment_create(0, _8MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED) ); ASSERT( gaspi_segment_create(1, _8MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED) ); //prepare memory segment gaspi_pointer_t _vptr, _vptr1; ASSERT (gaspi_segment_ptr(0, &_vptr)); ASSERT (gaspi_segment_ptr(1, &_vptr1)); int *mem = (int *) _vptr; int *mem_read = (int *) _vptr1; unsigned long i; const unsigned long maxInts = _8MB / sizeof(int); for(i = 0; i < maxInts; i++) { mem[i] = (int) myrank; } ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); //construct list of n elems gaspi_number_t queue_size = 0; const gaspi_number_t nListElems = 255; gaspi_number_t n; gaspi_segment_id_t localSegs[nListElems]; gaspi_offset_t localOffs[nListElems]; const gaspi_rank_t rank2read = (myrank + 1) % numranks; gaspi_segment_id_t remSegs[nListElems]; gaspi_offset_t remOffs[nListElems]; gaspi_size_t sizes[nListElems]; const unsigned int bytes = sizeof(int); gaspi_offset_t initLocOff = 0; gaspi_offset_t initRemOff = 0; for(n = 0; n < nListElems; n++) { sizes[n] = bytes; localSegs[n] = 1; localOffs[n] = initLocOff; initLocOff += bytes; remSegs[n] = 0; remOffs[n] = initRemOff; initRemOff += bytes; } ASSERT( gaspi_read_list_notify( nListElems, localSegs, localOffs, rank2read, remSegs, remOffs, sizes, 1, myrank, 0, GASPI_BLOCK)); gaspi_notification_id_t id; ASSERT (gaspi_notify_waitsome(1, myrank, 1, &id, GASPI_BLOCK)); gaspi_notification_t notification_val; ASSERT( gaspi_notify_reset(1, id, ¬ification_val)); assert(notification_val == 1); //check gaspi_number_t l; for(l = 0; l < nListElems; l++) { assert(mem_read[l] == (int) rank2read); } ASSERT (gaspi_wait(0, GASPI_BLOCK)); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main (int argc, char *argv[]) { SUCCESS_OR_DIE (gaspi_proc_init (GASPI_BLOCK)); gaspi_rank_t iProc, nProc; SUCCESS_OR_DIE (gaspi_proc_rank (&iProc)); SUCCESS_OR_DIE (gaspi_proc_num (&nProc)); // number of threads const int NTHREADS = 2; // number of buffers const int NWAY = 2; // allocate segment for array for local vector, left halo and right halo gaspi_segment_id_t const segment_id = 0; SUCCESS_OR_DIE ( gaspi_segment_create ( segment_id, NWAY * (NTHREADS + 2) * 2 * VLEN * sizeof (double) , GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED)); gaspi_pointer_t array; SUCCESS_OR_DIE ( gaspi_segment_ptr ( segment_id, &array) ); // initial buffer id int buffer_id = 0; // set notification values gaspi_notification_id_t left_data_available[NWAY]; gaspi_notification_id_t right_data_available[NWAY]; for (gaspi_notification_id_t id = 0; id < NWAY; ++id) { left_data_available[id] = id; right_data_available[id] = NWAY + id; } // set queue id gaspi_queue_id_t queue_id = 0; // initialize slice data structures slice *ssl = (slice *) malloc (NTHREADS * sizeof (slice)); ASSERT (ssl); init_slices (ssl, NTHREADS); // initialize data data_init (NTHREADS,iProc, buffer_id, array); const int right_halo = NTHREADS+1; const int left_halo = 0; // issue initial write to left ngb wait_for_queue_max_half (&queue_id); SUCCESS_OR_DIE ( gaspi_write_notify ( segment_id, array_OFFSET_left (buffer_id, left_halo + 1, 0), LEFT(iProc, nProc) , segment_id, array_OFFSET_left (buffer_id, right_halo, 0), VLEN * sizeof (double) , right_data_available[buffer_id], 1, queue_id, GASPI_BLOCK)); // issue initial write to right ngb wait_for_queue_max_half (&queue_id); SUCCESS_OR_DIE ( gaspi_write_notify ( segment_id, array_OFFSET_right (buffer_id, right_halo - 1, 0), RIGHT(iProc, nProc) , segment_id, array_OFFSET_right (buffer_id, left_halo, 0), VLEN * sizeof (double) , left_data_available[buffer_id], 1, queue_id, GASPI_BLOCK)); // set total number of iterations per slice const int num = nProc * NTHREADS * NITER; omp_set_num_threads (NTHREADS); double time = -now(); #pragma omp parallel default (none) firstprivate (buffer_id, queue_id) \ shared (array, left_data_available, right_data_available, ssl, stderr) { slice* sl; while ((sl = get_slice_and_lock (ssl, NTHREADS, num))) { handle_slice ( sl, array, left_data_available, right_data_available , segment_id, queue_id, NWAY, NTHREADS, num); /* TODO ==== - Which functionality do we need in 'handle_slice' ? (asynchronous dataflow for 1-D halo-exchange) - Discuss. - Bonus question: Can we be at different iteration stages for left and right halo ? if yes: Why ? */ omp_unset_lock (&sl->lock); } #pragma omp barrier } time += now(); data_verify (NTHREADS, iProc, (NITER * nProc * NTHREADS) % NWAY, array); printf ("# gaspi %s nProc %d vlen %i niter %d nthreads %i nway %i time %g\n" , argv[0], nProc, VLEN, NITER, NTHREADS, NWAY, time ); gaspi_proc_term (GASPI_BLOCK); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { static const int VLEN = 1 << 2; SUCCESS_OR_DIE(gaspi_proc_init (GASPI_BLOCK)); gaspi_rank_t iProc, nProc; SUCCESS_OR_DIE(gaspi_proc_rank (&iProc)); SUCCESS_OR_DIE(gaspi_proc_num (&nProc)); gaspi_segment_id_t const segment_id = 0; gaspi_size_t const segment_size = 2 * VLEN * sizeof(double); SUCCESS_OR_DIE(gaspi_segment_create (segment_id, segment_size, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED )); gaspi_pointer_t array; SUCCESS_OR_DIE(gaspi_segment_ptr (segment_id, &array)); double * src_array = (double *) (array); double * rcv_array = src_array + VLEN; for (int j = 0; j < VLEN; ++j) { src_array[j] = (double) (iProc * VLEN + j); } gaspi_notification_id_t data_available = 0; gaspi_queue_id_t queue_id = 0; gaspi_offset_t loc_off = 0; gaspi_offset_t rem_off = VLEN * sizeof(double); /* TODO ==== - check queue. - gaspi_write_notify 1-dim array of size VLEN to right neighbour - wait for notification from left neighbour */ for (int j = 0; j < VLEN; ++j) { printf ("rank %d rcv elem %d: %f \n", iProc, j, rcv_array[j]); } /* TODO ==== - why do we have to wait for the queue here ? */ SUCCESS_OR_DIE(gaspi_wait (queue_id, GASPI_BLOCK)); SUCCESS_OR_DIE(gaspi_proc_term (GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); gaspi_rank_t numranks, myrank; ASSERT (gaspi_proc_num(&numranks)); ASSERT (gaspi_proc_rank(&myrank)); ASSERT (gaspi_segment_create(0, _128MB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); //prepare memory segment gaspi_pointer_t _vptr; ASSERT (gaspi_segment_ptr(0, &_vptr)); int *mem = (int *) _vptr; unsigned long i; const unsigned long maxInts = _128MB / sizeof(int); for(i = 0; i < maxInts; i++) { mem[i] = (int) myrank; } ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); //construct list of n elems gaspi_number_t queue_size = 0; const gaspi_number_t nListElems = 255; gaspi_number_t n; gaspi_segment_id_t localSegs[nListElems]; gaspi_offset_t localOffs[nListElems]; const gaspi_rank_t rank2read = (myrank + 1) % numranks; gaspi_segment_id_t remSegs[nListElems]; gaspi_offset_t remOffs[nListElems]; gaspi_size_t sizes[nListElems]; const unsigned int bytes = sizeof(int); gaspi_offset_t initLocOff = 0; gaspi_offset_t initRemOff = (bytes * nListElems + 64); for(n = 0; n < nListElems; n++) { sizes[n] = bytes; localSegs[n] = 0; localOffs[n] = initLocOff; initLocOff += bytes; remSegs[n] = 0; remOffs[n] = initRemOff; initRemOff += bytes; } ASSERT (gaspi_read_list(nListElems, localSegs,localOffs, rank2read, remSegs, remOffs, sizes, 0, GASPI_BLOCK)); ASSERT (gaspi_queue_size(0, &queue_size)); assert (queue_size == nListElems); ASSERT (gaspi_wait(0, GASPI_BLOCK)); //sync ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); //check gaspi_number_t l; gaspi_offset_t off2check = 0; char * chPtr = (char *) _vptr; mem = (int *) (chPtr + off2check); for(l = 0; l < nListElems; l++) { assert(mem[l] == (int) rank2read); } ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); gaspi_rank_t numranks, myrank; ASSERT (gaspi_proc_num(&numranks)); ASSERT (gaspi_proc_rank(&myrank)); int rankSend = (myrank + 1) % numranks; gaspi_printf("Seg size: %lu MB\n", _2GB / 1024 / 1024); ASSERT(gaspi_segment_create(0, _2GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); gaspi_size_t segSize; ASSERT( gaspi_segment_size(0, myrank, &segSize)); unsigned char * pGlbMem; gaspi_pointer_t _vptr; ASSERT(gaspi_segment_ptr(0, &_vptr)); pGlbMem = ( unsigned char *) _vptr; gaspi_number_t qmax ; ASSERT (gaspi_queue_size_max(&qmax)); unsigned long i; unsigned long size = 1800; for(i = 0; i < size / sizeof(unsigned char); i++) pGlbMem[i] = myrank; gaspi_printf("Queue max: %lu\n", qmax); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); unsigned long localOff = 0; unsigned long remOff = size; ASSERT(gaspi_write_notify(0, localOff, rankSend, 0, remOff, size, (gaspi_notification_id_t) myrank, 1, 1, GASPI_BLOCK)); gaspi_rank_t rankGet = (myrank + numranks - 1) % numranks; gaspi_notification_t got_val; gaspi_notification_id_t got; ASSERT(gaspi_notify_waitsome(0, (gaspi_notification_id_t) rankGet, 1, &got, GASPI_BLOCK)); ASSERT(gaspi_notify_reset(0, got, &got_val)); ASSERT (gaspi_wait(1, GASPI_BLOCK)); /* check */ for(i = size; i < 2 * size / sizeof(unsigned char); i++) assert(pGlbMem[i] == rankGet); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { int j,i,k=0; int ret=0; const gaspi_size_t size=4096;//4k const gaspi_size_t memSize = 4294967296; //4GB gaspi_offset_t offset_write=0, offset_read = memSize / 2, offset_check = 3221225472 ; ASSERT (gaspi_proc_init(GASPI_BLOCK)); ASSERT (gaspi_segment_create(0, memSize, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); gaspi_pointer_t _vptr; ASSERT (gaspi_segment_ptr(0, &_vptr)); /* get memory area pointer */ #ifdef FLOAT float *mptr = (float *) _vptr; #else int *mptr = (int *) _vptr; #endif gaspi_rank_t myrank, highestnode; ASSERT (gaspi_proc_rank(&myrank)); ASSERT (gaspi_proc_num(&highestnode)); while(k <= RUNS) { //generate random srand((unsigned)time(0)); #ifdef FLOAT srand48((unsigned) time(0)); #endif //clean for(j = 0; j < (memSize / 4); j++) mptr[j]= 0; //fill randoms up to 1GB for(j = 0; j < (memSize / 16); j++) { #ifdef FLOAT mptr[j]= drand48() + (myrank*1.0); #else mptr[j]= rand() + myrank; #endif } #ifdef DEBUG #ifdef FLOAT gaspi_printf("random value in pos 0 %f\n", mptr[0]); #else gaspi_printf("random value in pos 0 %d\n", mptr[0]); #endif #endif //DEBUG ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); gaspi_printf("\n....Running iteration %d of %d...\n",k, RUNS); for(i = 0; i < ITERATIONS; i++) { for(j = 0; j < ITERATIONS; j++) { ASSERT (gaspi_write(0, offset_write, (myrank + 1) % highestnode, 0, offset_read, size, 0, GASPI_BLOCK)); offset_write += size; offset_read += size; } ASSERT (gaspi_wait(0, GASPI_BLOCK)); } #ifdef DEBUG gaspi_printf("%d bytes written!\n", ITERATIONS * ITERATIONS * size); #endif //check if data was written successfully ASSERT (gaspi_read(0, offset_check, (myrank + 1) % highestnode, 0, memSize/2, GB, 0, GASPI_BLOCK)); ASSERT (gaspi_wait(0, GASPI_BLOCK)); #ifdef DEBUG gaspi_printf("%d bytes read!\n",GB); #endif j=0; #ifdef DEBUG #ifdef FLOAT gaspi_printf("Values %f %f %f \n", mptr[0], mptr[memSize/8], mptr[offset_check/4]); #else gaspi_printf("Values %d %d %d \n", mptr[0], mptr[memSize/8], mptr[offset_check/4]); #endif #endif//DEBUG while(j < GB / 4 ) { if(mptr[j] != mptr[offset_check / 4 + j]){ #ifdef FLOAT gaspi_printf("value incorrect %f-%f at %d \n",mptr[j],mptr[offset_check / 4],j); #else gaspi_printf("value incorrect %d-%d at %d \n",mptr[j],mptr[offset_check / 4],j); #endif ret = -1; goto out; } j++; } offset_write=0; offset_read = memSize / 2; #ifdef DEBUG gaspi_printf("Check!\n"); #endif k++; } out: gaspi_printf("Waiting to finish...\n"); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return ret; }
int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); gaspi_notification_id_t n=0; gaspi_rank_t rank, nprocs, i; const gaspi_segment_id_t seg_id = 0; gaspi_offset_t offset; gaspi_number_t queue_size; gaspi_number_t queue_max; ASSERT (gaspi_queue_size_max(&queue_max)); ASSERT(gaspi_proc_num(&nprocs)); ASSERT (gaspi_proc_rank(&rank)); ASSERT (gaspi_segment_create(seg_id, nprocs * sizeof(int), GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED)); offset = rank * sizeof(int); //set memory gaspi_pointer_t _vptr; ASSERT (gaspi_segment_ptr(0, &_vptr)); int *mem = (int *) _vptr; for(i = 0; i < nprocs; i++) { mem[i] = (int) rank; } ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); //go for(i = 0; i < nprocs; i++) { if (i == rank) continue; ASSERT (gaspi_queue_size(0, &queue_size)); if(queue_size > queue_max - 1) ASSERT (gaspi_wait(0, GASPI_BLOCK)); ASSERT (gaspi_write_notify( seg_id, offset, i, seg_id, offset, sizeof(int), (gaspi_notification_id_t) rank, 1, 0, GASPI_BLOCK)); } do { gaspi_notification_id_t id; ASSERT (gaspi_notify_waitsome(seg_id, 0, (gaspi_notification_id_t) nprocs , &id, GASPI_BLOCK)); gaspi_notification_t notification_val; ASSERT( gaspi_notify_reset(seg_id, id, ¬ification_val)); assert(notification_val == 1); n++; } while(n < (nprocs - 1)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main (int argc, char *argv[]) { SUCCESS_OR_DIE (gaspi_proc_init (GASPI_BLOCK)); gaspi_rank_t iProc, nProc; SUCCESS_OR_DIE (gaspi_proc_rank (&iProc)); SUCCESS_OR_DIE (gaspi_proc_num (&nProc)); // number of threads const int NTHREADS = 2; // number of buffers const int NWAY = 2; gaspi_segment_id_t const segment_id = 0; // allocate segment for array for local vector, left halo and right halo SUCCESS_OR_DIE ( gaspi_segment_create ( segment_id, NWAY * (NTHREADS + 2) * 2 * VLEN * sizeof (double) , GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED)); gaspi_pointer_t array; SUCCESS_OR_DIE ( gaspi_segment_ptr ( segment_id, &array) ); // initial buffer id int buffer_id = 0; // set notification values gaspi_notification_id_t left_data_available[NWAY]; gaspi_notification_id_t right_data_available[NWAY]; for (gaspi_notification_id_t id = 0; id < NWAY; ++id) { left_data_available[id] = id; right_data_available[id] = NWAY + id; } // set queue id gaspi_queue_id_t queue_id = 0; // initialize data data_init (NTHREADS, iProc, buffer_id, array); omp_set_num_threads (NTHREADS); double time = -now(); #pragma omp parallel default (shared) firstprivate (buffer_id) { const int tid = omp_get_thread_num(); for (int k = 0; k < NITER; ++k) { for ( int i = 0; i < nProc * NTHREADS; ++i ) { const int left_halo = 0; const int slice_id = tid + 1; const int right_halo = NTHREADS+1; if (tid == 0) { // issue write wait_for_queue_max_half (&queue_id); SUCCESS_OR_DIE ( gaspi_write_notify ( segment_id, array_OFFSET_left (buffer_id, left_halo + 1, 0), LEFT(iProc, nProc) , segment_id, array_OFFSET_left (buffer_id, right_halo, 0), VLEN * sizeof (double) , right_data_available[buffer_id], 1 + i, queue_id, GASPI_BLOCK)); // issue write wait_for_queue_max_half (&queue_id); SUCCESS_OR_DIE ( gaspi_write_notify ( segment_id, array_OFFSET_right (buffer_id, right_halo - 1, 0), RIGHT(iProc, nProc) , segment_id, array_OFFSET_right (buffer_id, left_halo, 0), VLEN * sizeof (double) , left_data_available[buffer_id], 1 + i, queue_id, GASPI_BLOCK)); // wait for data notification wait_or_die (segment_id, right_data_available[buffer_id], 1 + i); // wait for data notification wait_or_die (segment_id, left_data_available[buffer_id], 1 + i); } #pragma omp barrier // compute data, read from id "buffer_id", write to id "1 - buffer_id" data_compute ( NTHREADS, array, 1 - buffer_id, buffer_id, slice_id); #pragma omp barrier // alternate the buffer buffer_id = 1 - buffer_id; } } } time += now(); data_verify (NTHREADS, iProc, (NITER * nProc * NTHREADS) % NWAY, array); printf ("# gaspi %s nProc %d vlen %i niter %d nthreads %i nway %i time %g\n" , argv[0], nProc, VLEN, NITER, NTHREADS, NWAY, time ); gaspi_proc_term (GASPI_BLOCK); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); gaspi_rank_t numranks, myrank; ASSERT (gaspi_proc_num(&numranks)); ASSERT (gaspi_proc_rank(&myrank)); //2 segments ASSERT (gaspi_segment_create(0, _2GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); ASSERT (gaspi_segment_create(1, _2GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED)); //construct list of n elems gaspi_number_t i, n, max; gaspi_number_t queue_size = 0; ASSERT( gaspi_rw_list_elem_max(&max)); for(n = 1; n < max; n++) { gaspi_number_t nListElems = n; gaspi_segment_id_t localSegs[n]; gaspi_offset_t localOffs[n]; const gaspi_rank_t rank2send = (myrank + 1) % numranks; gaspi_segment_id_t remSegs[n]; gaspi_offset_t remOffs[n]; gaspi_size_t sizes[n]; unsigned char flip = 0; unsigned int bytes = 1; gaspi_offset_t off = 0; gaspi_size_t total_size = 0; for(i = 0; i < nListElems; i++) { sizes[i] = bytes; localSegs[i] = flip; localOffs[i] = off; remSegs[i] = flip; remOffs[i] = off; off+=sizes[i]; total_size+= bytes; flip ^= 0x1; bytes += 2; } ASSERT(gaspi_write_list(nListElems, localSegs, localOffs, rank2send, remSegs, remOffs, sizes, 0, GASPI_BLOCK)); ASSERT(gaspi_queue_size(0, &queue_size)); assert(queue_size == nListElems); ASSERT(gaspi_wait(0, GASPI_BLOCK)); } gaspi_pointer_t _vptr; ASSERT (gaspi_segment_ptr(0, &_vptr)); ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }
int main(int argc, char *argv[]) { TSUITE_INIT(argc, argv); ASSERT (gaspi_proc_init(GASPI_BLOCK)); gaspi_rank_t rank, nprocs, i; const gaspi_segment_id_t seg_id = 0; const gaspi_offset_t offset = 0; const gaspi_size_t transfer_size = 8192; gaspi_number_t queue_size; gaspi_number_t queue_max; ASSERT (gaspi_queue_size_max(&queue_max)); ASSERT(gaspi_proc_num(&nprocs)); ASSERT (gaspi_proc_rank(&rank)); if( nprocs < 2 ) { return EXIT_SUCCESS; } ASSERT (gaspi_segment_create(seg_id, nprocs * 2 * transfer_size, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED)); gaspi_number_t max_notifications; ASSERT(gaspi_notification_num(&max_notifications)); gaspi_number_t avail_notifications = max_notifications / nprocs; max_notifications = avail_notifications * nprocs; gaspi_pointer_t _vptr; ASSERT (gaspi_segment_ptr(0, &_vptr)); int *mem = (int *) _vptr; for(i = 0; i < nprocs; i++) { mem[i] = (int) rank; } ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); for(i = 0; i < nprocs; i++) { gaspi_notification_id_t not; for(not = 0; not < avail_notifications; not++) { ASSERT (gaspi_queue_size(0, &queue_size)); if( queue_size > queue_max - 1 ) { ASSERT (gaspi_wait(0, GASPI_BLOCK)); } gaspi_notification_id_t the_notification = (gaspi_notification_id_t) (rank * avail_notifications + not); ASSERT( gaspi_write_notify( seg_id, offset, i, seg_id, offset, transfer_size, the_notification, 1, 0, GASPI_BLOCK)); } } gaspi_notification_id_t n = 0; do { gaspi_notification_id_t id; ASSERT (gaspi_notify_waitsome(seg_id, 0, max_notifications - 1 , &id, GASPI_BLOCK)); gaspi_notification_t notification_val; ASSERT( gaspi_notify_reset(seg_id, id, ¬ification_val)); assert(notification_val == 1); n++; } while(n < max_notifications - 1); ASSERT(gaspi_wait(0, GASPI_BLOCK)); ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK)); ASSERT (gaspi_proc_term(GASPI_BLOCK)); return EXIT_SUCCESS; }