int main(int argc, char *argv[])
{
gaspi_size_t mem;
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
mem = get_system_mem();
if(mem > 0)
{
mem *= 1024; //to bytes
mem *= 45; //45%
mem /= 100;
}
else
{
gaspi_printf("Failed to get mem (%lu)\n", mem);
exit(-1);
}
gaspi_printf("Segment size %lu MB\n", mem / 1024 / 1024);
ASSERT( gaspi_segment_create(0, mem, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
gaspi_rank_t nprocs, i;
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT(gaspi_proc_num(&nprocs));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
gaspi_state_vector_t vec = (gaspi_state_vector_t) malloc(nprocs);
gaspi_printf("vec out %p\n", vec);
ASSERT(gaspi_state_vec_get(vec));
gaspi_printf("vec out %p\n", vec);
for(i = 0; i < nprocs; i++)
{
assert(vec[i] == 0);
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
gaspi_rank_t gsize, nprocs, n;
gaspi_number_t max_groups;
gaspi_rank_t *partners;
TSUITE_INIT(argc, argv);
gaspi_group_max(&max_groups);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT(gaspi_proc_num(&nprocs));
ASSERT(gaspi_group_size(GASPI_GROUP_ALL,&gsize));
partners = malloc(gsize * sizeof(gaspi_rank_t));
ASSERT(gaspi_group_ranks(GASPI_GROUP_ALL,partners));
gaspi_printf("%d partners\n", gsize);
for(n=0; n < gsize; n++)
gaspi_printf("%d ", partners[n]);
gaspi_printf("\n");
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
gaspi_return_t
gaspi_threads_init_user(const unsigned int use_nr_of_threads)
{
if(__gaspiThreadsActivated)
{
gaspi_printf("gaspi_threads: re-initialization !\n");
return GASPI_ERROR;
}
if( use_nr_of_threads < 1)
{
gaspi_printf("gaspi_threads: Invalid num of threads (%u)\n", use_nr_of_threads);
return GASPI_ERROR;
}
__gaspiThreadsGlobalIDCnt = 0;
__gaspiThreadsActivated = use_nr_of_threads;
__gaspiThreadsActivated = (__gaspiThreadsActivated < GASPI_MAX_THREADS) ? __gaspiThreadsActivated : GASPI_MAX_THREADS;
__gaspiThreadsMode = 0;
__gaspiThreadsFlag0 = 0;
__gaspiThreadsFlag1 = 0;
__gaspiThreadsCount0[0] = 0;
__gaspiThreadsCount1[0] = 0;
return GASPI_SUCCESS;
}
int main(int argc, char *argv[])
{
gaspi_group_t g;
gaspi_rank_t nprocs, myrank;
gaspi_number_t gsize;
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT(gaspi_proc_num(&nprocs));
ASSERT(gaspi_proc_rank(&myrank));
ASSERT (gaspi_group_create(&g));
gaspi_rank_t i;
for(i = 0; i < nprocs; i++)
{
ASSERT(gaspi_group_add(g, i));
}
ASSERT(gaspi_group_size(g, &gsize));
assert((gsize == nprocs));
if(myrank > 0 )
sleep(10); //simulate delay
//should fail since other ranks are still sleeping
/* if(myrank == 0 ) */
/* EXPECT_TIMEOUT(gaspi_group_commit(g, 1000)); */
gaspi_return_t ret;
do
{
ret = gaspi_group_commit(g, 1000);
gaspi_printf("commit returned %d\n", ret);
}
while (ret == GASPI_TIMEOUT || ret == GASPI_ERROR);
assert((ret != GASPI_ERROR));
gaspi_printf("group barrier %d \n", ret);
//group barrier -> should fail due to timeout of commit
ASSERT(gaspi_barrier(g, 5000));
gaspi_printf("all barrier\n");
//all barrier
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
//sync
ASSERT (gaspi_proc_term(GASPI_BLOCK));
gaspi_printf("finish\n");
return EXIT_SUCCESS;
}
//recover => clean the queue
gaspi_return_t recover()
{
gaspi_return_t ret = GASPI_ERROR;
gaspi_printf("Enter recover\n");
while(ret != GASPI_SUCCESS)
{
ret = gaspi_wait(0, GASPI_BLOCK);
}
gaspi_printf("Exit recover with %d\n", ret);
return ret;
}
void
gaspi_printf_array(int * array_ptr, int num_elem)
{
int i = 0;
for(i = 0;i < num_elem ;++i)
gaspi_printf("%d\n", array_ptr[i]);
}
int main(int argc,char *argv[]){
gaspi_rank_t rank,num;
gaspi_return_t ret;
float vers;
signal(SIGINT,signal_handler);
gaspi_print_affinity_mask();
ret = gaspi_proc_init(GPI2_TOUT);
if(ret!=GASPI_SUCCESS){
printf("gaspi_init failed ! [%s]\n",ret==-1 ? "GASPI_ERROR":"GASPI_TIMEOUT");
gaspi_proc_term(GPI2_TOUT);
exit(-1);
}
gaspi_version(&vers);
gaspi_proc_rank(&rank);
gaspi_proc_num(&num);
gaspi_printf("rank: %d num: %d (vers: %.2f)\n",rank,num,vers);
srand(time(NULL)*(rank+1));
const int dst = rand();
if( gaspi_sn_ping(dst%num,1000) != GASPI_SUCCESS) printf("gaspi_sn_ping failed ! [%s]\n",ret==-1 ? "GASPI_ERROR":"GASPI_TIMEOUT");
//tmp workaround, we have no barrier yet
gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK);
gaspi_proc_term(GPI2_TOUT);
return 0;
}
gaspi_size_t gaspi_get_system_mem()
{
FILE *fp;
char line[1024];
gaspi_size_t memory = 0;
fp = fopen("/proc/meminfo", "r");
if (fp == NULL)
{
gaspi_printf("Cannot open file /proc/meminfo\n");
return 0;
}
while (fgets(line, sizeof(line), fp))
{
if (!strncmp("MemTotal", line, 8))
{
strtok(line, ":");
memory = strtol((char*) strtok(NULL, " kB\n"), (char**) NULL, 0);
}
}
fclose(fp);
return memory;
}
int main(int argc, char *argv[])
{
// gaspi_group_t g;
gaspi_rank_t nprocs, myrank;
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT(gaspi_proc_num(&nprocs));
ASSERT(gaspi_proc_rank(&myrank));
int n;
for(n = 1; n <= 255; n++)
{
gaspi_datatype_t type;
for(type = GASPI_TYPE_INT; type <= GASPI_TYPE_ULONG; type++)
{
gaspi_operation_t op;
for(op = GASPI_OP_MIN; op <= GASPI_OP_SUM; op++)
ASSERT(testOP(op, type, n, GASPI_GROUP_ALL));
}
}
gaspi_printf("done\n");
//sync
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
gaspi_gpu_id_t gpus[8];
gaspi_number_t nGPUs;
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_rank_t rank, num;
ASSERT (gaspi_proc_rank(&rank));
ASSERT (gaspi_proc_num(&num));
ASSERT (gaspi_gpu_init());
ASSERT (gaspi_gpu_number(&nGPUs));
ASSERT (gaspi_gpu_ids(gpus));
printf("Hello from rank %d of %d with %d GPUs:\n",
rank, num, nGPUs );
int i;
for (i = 0; i < gpus; i++)
{
gaspi_printf("device with Id %d \n", gpus[i]);
}
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
void work(int tid)
{
gaspi_rank_t rankSend;
gaspi_offset_t localOff = 81478066;
gaspi_offset_t remOff = 81478246;
gaspi_offset_t size = 1800;
gaspi_number_t queueSize, qmax;
ASSERT (gaspi_queue_size_max(&qmax));
for(rankSend = 0; rankSend < numranks; rankSend++)
{
gaspi_printf("thread %d rank to send: %d\n", tid, rankSend);
gaspi_queue_size(1, &queueSize);
if (queueSize > qmax - 24)
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_write(0, localOff, rankSend, 0, remOff, size, 1, GASPI_BLOCK));
}
ASSERT (gaspi_wait(1, GASPI_BLOCK));
gaspi_threads_sync();
}
void work(int tid)
{
gaspi_rank_t rankSend;
gaspi_offset_t localOff= 0;
gaspi_offset_t remOff = 0;
gaspi_number_t queueSize, qmax;
gaspi_size_t commSize ;
ASSERT (gaspi_queue_size_max(&qmax));
for(commSize= 1; commSize < _500MB; commSize*=2 )
for(rankSend = 0; rankSend < numranks; rankSend++)
{
gaspi_printf("thread %d rank to send: %d - %lu bytes\n", tid, rankSend, commSize);
gaspi_queue_size(1, &queueSize);
if (queueSize > qmax - 100)
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_write(0, localOff, rankSend, 0, remOff, commSize, 1, GASPI_BLOCK));
}
ASSERT (gaspi_wait(1, GASPI_BLOCK));
gaspi_threads_sync();
}
gaspi_return_t
gaspi_threads_get_num_cores(gaspi_int * const cores)
{
int i,n;
cpu_set_t tmask;
gaspi_verify_null_ptr(cores);
if(sched_getaffinity(0, sizeof(cpu_set_t), &tmask) < 0)
{
gaspi_printf("sched_getaffinity failed !\n");
return GASPI_ERROR;
}
for(i = n = 0; i < CPU_SETSIZE; i++)
{
if(CPU_ISSET(i,&tmask))
{
n++;
}
}
*cores = n;
return GASPI_SUCCESS;
}
void print_char_array_segment(gaspi_char* segment,
const gaspi_size_t nrReads,
const gaspi_size_t readlength,
gaspi_rank_t myRank)
{
char *bp;
size_t size;
FILE *stream;
stream = open_memstream (&bp, &size);
gaspi_size_t r = 0, i = 0, offset;
fprintf(stream, "rank %d:\n", myRank);
for(r = 0; r < nrReads; r++) {
fprintf(stream, " [%lu]: ", r);
offset = r * (readlength + 1);
for(i = 0; i < readlength; i++)
fprintf(stream, "%c", segment[offset + i]);
fprintf(stream, "\n");
}
fclose(stream);
if (myRank == 0)
printf("%s", bp);
else
gaspi_printf("%s", bp);
}
int main(int argc, char *argv[])
{
gaspi_rank_t numranks, myrank;
gaspi_rank_t rankSend;
gaspi_size_t segSize;
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT (gaspi_proc_num(&numranks));
ASSERT (gaspi_proc_rank(&myrank));
ASSERT (gaspi_segment_create(0, _2GB, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_INITIALIZED));
ASSERT( gaspi_segment_size(0, myrank, &segSize));
gaspi_printf("seg size %lu MB \n", segSize/1024/1024);
// ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
gaspi_offset_t localOff= 814780664;
gaspi_offset_t remOff = 81478246;
gaspi_offset_t size = 1800;
gaspi_number_t queueSize, qmax;
ASSERT (gaspi_queue_size_max(&qmax));
for(rankSend = 0; rankSend < numranks; rankSend++)
{
gaspi_printf("rank to send: %d\n", rankSend);
gaspi_queue_size(1, &queueSize);
if (queueSize > qmax - 24)
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_write(0, localOff, rankSend, 0, remOff, size, 1, GASPI_BLOCK));
}
ASSERT (gaspi_wait(1, GASPI_BLOCK));
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
void success_or_exit ( const char* file, const int line, const gaspi_return_t ec)
{
if (ec != GASPI_SUCCESS)
{
gaspi_printf ("Assertion failed in %s[%i]: Return %d: %s\n", file, line, ec, gaspi_error_str(ec));
exit (EXIT_FAILURE);
}
}
gaspi_return_t
pgaspi_gpu_write_notify(const gaspi_segment_id_t segment_id_local,
const gaspi_offset_t offset_local,
const gaspi_rank_t rank,
const gaspi_segment_id_t segment_id_remote,
const gaspi_offset_t offset_remote,
const gaspi_size_t size,
const gaspi_notification_id_t notification_id,
const gaspi_notification_t notification_value,
const gaspi_queue_id_t queue,
const gaspi_timeout_t timeout_ms)
{
gaspi_verify_init("gaspi_gpu_write_notify");
gaspi_verify_local_off(offset_local, segment_id_local, size);
gaspi_verify_remote_off(offset_remote, segment_id_remote, rank, size);
gaspi_verify_queue(queue);
gaspi_verify_comm_size(size, segment_id_local, segment_id_remote, rank, GASPI_MAX_TSIZE_C);
if( notification_value == 0 )
{
gaspi_printf("Zero is not allowed as notification value.");
return GASPI_ERR_INV_NOTIF_VAL;
}
gaspi_return_t eret = GASPI_ERROR;
gaspi_context_t * const gctx = &glb_gaspi_ctx;
if(lock_gaspi_tout (&gctx->lockC[queue], timeout_ms))
return GASPI_TIMEOUT;
if( GASPI_ENDPOINT_DISCONNECTED == gctx->ep_conn[rank].cstat )
{
eret = pgaspi_connect((gaspi_rank_t) rank, timeout_ms);
if ( eret != GASPI_SUCCESS)
{
goto endL;
}
}
eret = pgaspi_dev_gpu_write_notify(segment_id_local, offset_local, rank,
segment_id_remote, offset_remote, size,
notification_id, notification_value,
queue, timeout_ms);
if( eret != GASPI_SUCCESS )
{
/* gctx->qp_state_vec[queue][rank] = GASPI_STATE_CORRUPT; */
goto endL;
}
/* GPI2_STATS_INC_COUNT(GASPI_STATS_COUNTER_NUM_WRITE_NOT, 1); */
/* GPI2_STATS_INC_COUNT(GASPI_STATS_COUNTER_BYTES_WRITE, size); */
endL:
unlock_gaspi (&gctx->lockC[queue]);
return eret;
}
void must_fail ( const char* file, const int line, const gaspi_return_t ec)
{
if (ec == GASPI_SUCCESS || ec == GASPI_TIMEOUT)
{
gaspi_printf ("Non-expected success in %s[%i]\n", file, line);
exit (EXIT_FAILURE);
}
}
void must_timeout ( const char* file, const int line, const gaspi_return_t ec)
{
if (ec != GASPI_TIMEOUT)
{
gaspi_printf ("Expected timeout but got %d in %s[%i]\n", ec, file, line);
exit (EXIT_FAILURE);
}
}
int main(int argc, char *argv[])
{
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
gaspi_notification_id_t n=0;
gaspi_number_t notif_num;
gaspi_rank_t rank, nprocs, i;
const gaspi_segment_id_t seg_id = 0;
ASSERT(gaspi_proc_num(&nprocs));
ASSERT (gaspi_proc_rank(&rank));
ASSERT (gaspi_segment_create(seg_id, 1024, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED));
ASSERT( gaspi_notification_num(¬if_num));
gaspi_printf("max num notifications %u\n", notif_num);
if(rank == 0)
{
gaspi_number_t queue_size;
gaspi_number_t queue_max;
ASSERT (gaspi_queue_size_max(&queue_max));
for(n = 0; n < notif_num; n++)
{
ASSERT (gaspi_queue_size(0, &queue_size));
if(queue_size > queue_max - 1)
ASSERT (gaspi_wait(0, GASPI_BLOCK));
for(i = 1; i < nprocs; i++)
{
ASSERT (gaspi_notify( seg_id, i, n, 1, 0, GASPI_BLOCK));
}
}
}
else
{
do
{
gaspi_notification_id_t id;
ASSERT (gaspi_notify_waitsome(seg_id, 0, notif_num, &id, GASPI_BLOCK));
gaspi_notification_t notification_val;
ASSERT( gaspi_notify_reset(seg_id, id, ¬ification_val));
assert(notification_val == 1);
n++;
}
while(n < notif_num);
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
void
read_params(int argc, char* argv[], int * timesteps, gaspi_rank_t * numprocs_idle)
{
int i;
for (i = 1; i < argc; ++i)
{
if(ARG_IS("timesteps"))
{
*timesteps = atoi(argv[++i]);
gaspi_printf("Timesteps: %d\n", *timesteps);
}
if(ARG_IS("numprocs_idle"))
{
*numprocs_idle = atoi(argv[++i]);
gaspi_printf("numprocs_idle : %d\n", *numprocs_idle);
}
}
}
void
print_health_vec(gaspi_state_vector_t health_vec)
{
int i;
ASSERT(gaspi_state_vec_get(health_vec));
for(i = 0;i < numprocs; ++i)
{
gaspi_printf("%d_health is: %d \n", myrank, health_vec[i]);
}
}
int main(int argc, char *argv[])
{
gaspi_rank_t nproc, rank;
gaspi_printf_to(0, "Before proc_init printf_to 0\n");
gaspi_printf("before proc_init printf\n", rank);
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT (gaspi_proc_num(&nproc));
ASSERT (gaspi_proc_rank(&rank));
gaspi_printf_to(rank, "Node %d writing specific\n", rank);
gaspi_printf("Node %d writing normal\n", rank);
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}
gaspi_return_t
init_segment (gaspi_segment_id_t seg_id, gaspi_size_t seg_size)
{
gaspi_rank_t myrank;
gaspi_size_t segSize;
gaspi_proc_rank(&myrank);
ASSERT(gaspi_segment_create(seg_id, seg_size, GASPI_GROUP_ALL, GASPI_BLOCK, GASPI_MEM_UNINITIALIZED));
ASSERT(gaspi_segment_size(seg_id, myrank, &segSize));
gaspi_printf("segSize is: %lu bytes\n", segSize);
return GASPI_SUCCESS;
}
void * recvThread(void * arg)
{
int tid = mctpRegisterThread();
gaspi_rank_t sender;
gaspi_offset_t recvPos = (RECV_OFF / sizeof(int));
int * memArray = (int *) arg;
gaspi_return_t ret = GASPI_ERROR;
do
{
ret = gaspi_passive_receive(0, RECV_OFF, &sender, sizeof(int), GASPI_TEST);
assert (ret != GASPI_ERROR);
sleep(1);
}
while(ret != GASPI_SUCCESS);
gaspi_printf("Received msg from %d\n", sender);
if( memArray[recvPos] != 11223344 )
gaspi_printf("Wrong value!\n");
return NULL;
}
int main(int argc, char *argv[])
{
int i;
int mpi_rank, mpi_sz, mpi_ret;
gaspi_rank_t rank, nnodes;
if(MPI_Init(&argc, &argv) != MPI_SUCCESS)
return EXIT_FAILURE;
ASSERT(gaspi_proc_init(GASPI_BLOCK));
ASSERT(gaspi_proc_rank(&rank));
ASSERT(gaspi_proc_num(&nnodes));
mpi_ret = MPI_Comm_size(MPI_COMM_WORLD, &mpi_sz);
assert(mpi_ret == 0);
mpi_ret = MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
assert(mpi_ret == 0);
/* the test starts here */
int vec_elems = 1;
int max_vec_elems = 255;
if (argc > 1)
max_vec_elems = atoi(argv[1]);
double * vector = (double *) malloc (max_vec_elems * sizeof(double));
for(i = 0; i < max_vec_elems; i++)
vector[i] = 1.0f;
for(i = vec_elems; i <= max_vec_elems; i++)
{
gaspi_printf("Norm for %d elems: %.2f\n", i, normP_GPI(vector, i));
}
/* sync and finish */
ASSERT(gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
assert(MPI_Barrier(MPI_COMM_WORLD) == MPI_SUCCESS);
ASSERT(gaspi_proc_term(GASPI_BLOCK));
if(MPI_Finalize() != MPI_SUCCESS)
return EXIT_FAILURE;
free(vector);
return 0;
}
gaspi_return_t
gaspi_threads_get_tid(gaspi_int *const tid)
{
gaspi_verify_null_ptr(tid);
if(__gaspiThreadsGlobalIDCnt == -1)
{
gaspi_printf("gaspi_threads: not initialized !\n");
return GASPI_ERROR;
}
*tid = __gaspi_thread_tid;
return GASPI_SUCCESS;
}
gaspi_return_t
gaspi_threads_get_total(gaspi_int *const num)
{
gaspi_verify_null_ptr(num);
if(__gaspiThreadsGlobalIDCnt == -1)
{
gaspi_printf("gaspi_threads: not initialized !\n");
return GASPI_ERROR;
}
*num = __gaspi_thread_tnc;
return GASPI_SUCCESS;
}
int main(int argc, char *argv[])
{
// gaspi_group_t g;
gaspi_rank_t nprocs, myrank;
TSUITE_INIT(argc, argv);
ASSERT (gaspi_proc_init(GASPI_BLOCK));
ASSERT(gaspi_proc_num(&nprocs));
ASSERT(gaspi_proc_rank(&myrank));
int n;
double * a = (double *) malloc(255 * sizeof(double));
double * b = (double *) malloc(255 * sizeof(double));
if(a == NULL || b == NULL)
return EXIT_FAILURE;
for(n = 0; n < 255; n++)
{
a[n] = b[n] = myrank * 1.0;
}
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
for(n = 1; n <= 255; n++)
{
int i;
ASSERT(gaspi_allreduce_user(a, b, n, sizeof(double),
(gaspi_reduce_operation_t) my_fun, NULL,
GASPI_GROUP_ALL, GASPI_BLOCK));
for(i = 0; i < n; i++)
assert(b[i] == nprocs - 1);
}
free(a);
free(b);
gaspi_printf("done\n");
//sync
ASSERT (gaspi_barrier(GASPI_GROUP_ALL, GASPI_BLOCK));
ASSERT (gaspi_proc_term(GASPI_BLOCK));
return EXIT_SUCCESS;
}