int main(int argc, char* argv[]) { pami_result_t result = PAMI_ERROR; /* initialize the client */ pami_client_t client; char * clientname = ""; result = PAMI_Client_create(clientname, &client, NULL, 0); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_create"); /* query properties of the client */ pami_configuration_t config; size_t num_contexts; config.name = PAMI_CLIENT_NUM_TASKS; result = PAMI_Client_query( client, &config,1); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_query"); world_size = config.value.intval; config.name = PAMI_CLIENT_TASK_ID; result = PAMI_Client_query( client, &config,1); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_query"); world_rank = config.value.intval; printf("hello world from rank %ld of %ld \n", world_rank, world_size ); fflush(stdout); config.name = PAMI_CLIENT_NUM_CONTEXTS; result = PAMI_Client_query( client, &config, 1); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_query"); num_contexts = config.value.intval; /* initialize the contexts */ pami_context_t * contexts; contexts = (pami_context_t *) malloc( num_contexts * sizeof(pami_context_t) ); TEST_ASSERT(contexts!=NULL,"malloc"); result = PAMI_Context_createv( client, &config, 0, contexts, num_contexts ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_createv"); printf("%ld contexts were created by rank %ld \n", num_contexts, world_rank ); fflush(stdout); /* setup the world geometry */ pami_geometry_t world_geometry; result = PAMI_Geometry_world( client, &world_geometry ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Geometry_world"); barrier(); /****************************************************************/ int n = 1000; char * in = (char *) malloc(n); char * out = (char *) malloc(n); memset(in, '\0', n); memset(out, '\a', n); allgather(n, in, out); free(out); free(in); /****************************************************************/ barrier(); /* finalize the contexts */ result = PAMI_Context_destroyv( contexts, num_contexts ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_destroyv"); free(contexts); /* finalize the client */ result = PAMI_Client_destroy( &client ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_destroy"); printf("%ld: end of test \n", world_rank ); fflush(stdout); return 0; }
int main(int argc, char* argv[]) { pami_result_t result = PAMI_ERROR; /* initialize the second client */ char * clientname = ""; pami_client_t client; result = PAMI_Client_create(clientname, &client, NULL, 0); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_create"); /* query properties of the client */ pami_configuration_t config[4]; config[0].name = PAMI_CLIENT_NUM_TASKS; config[1].name = PAMI_CLIENT_TASK_ID; config[2].name = PAMI_CLIENT_NUM_CONTEXTS; config[3].name = PAMI_CLIENT_NUM_LOCAL_TASKS; result = PAMI_Client_query(client, config, 4); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_query"); const size_t world_size = config[0].value.intval; const size_t world_rank = config[1].value.intval; const size_t num_contexts = (config[2].value.intval > 32) ? 32 : config[2].value.intval; /* because I only need 16+16 contexts in c1 mode */ const size_t num_local_tasks = config[3].value.intval; TEST_ASSERT(num_contexts>1,"num_contexts>1"); const int ppn = (int)num_local_tasks; const int nnodes = world_size/ppn; const int mycore = world_size%nnodes; const int mynode = (world_rank-mycore)/ppn; const int num_sync = num_contexts/2; const int num_async = num_contexts/2; const int async_context_begin = num_sync+1; const int async_context_end = num_contexts; if (world_rank==0) { printf("hello world from rank %ld of %ld, node %d of %d, core %d of %d \n", world_rank, world_size, mynode, nnodes, mycore, ppn ); printf("num_contexts = %ld, async_context_begin = %d, async_context_end = %d \n", num_contexts, async_context_begin, async_context_end); fflush(stdout); } /* initialize the contexts */ contexts = (pami_context_t *) safemalloc( num_contexts * sizeof(pami_context_t) ); result = PAMI_Context_createv( client, NULL, 0, contexts, num_contexts ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_createv"); /* setup the world geometry */ pami_geometry_t world_geometry; result = PAMI_Geometry_world(client, &world_geometry ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Geometry_world"); /************************************************************************/ for (int n=1; n<=(256*1024); n*=2) { if (world_rank==0) { printf("starting n = %d \n", n); fflush(stdout); } result = barrier(world_geometry, contexts[0]); TEST_ASSERT(result == PAMI_SUCCESS,"barrier"); double * sbuf = safemalloc(world_size*n*sizeof(double)); double * rbuf = safemalloc(world_size*n*sizeof(double)); for (int s=0; s<world_size; s++ ) for (int k=0; k<n; k++) sbuf[s*n+k] = world_rank*n+k; for (int s=0; s<world_size; s++ ) for (int k=0; k<n; k++) rbuf[s*n+k] = -1.0; result = barrier(world_geometry, contexts[0]); TEST_ASSERT(result == PAMI_SUCCESS,"barrier"); size_t bytes = world_size * n * sizeof(double), bytes_out; pami_memregion_t * local_mr = safemalloc(num_sync * sizeof(pami_memregion_t) ); pami_memregion_t * shared_mr = safemalloc(num_sync * sizeof(pami_memregion_t) ); for (int i=0; i<num_sync; i++) { result = PAMI_Memregion_create(contexts[i], rbuf, bytes, &bytes_out, &(local_mr[i])); TEST_ASSERT(result == PAMI_SUCCESS && bytes==bytes_out,"PAMI_Memregion_create"); result = PAMI_Memregion_create(contexts[async_context_begin+i], sbuf, bytes, &bytes_out, &(shared_mr[i])); TEST_ASSERT(result == PAMI_SUCCESS && bytes==bytes_out,"PAMI_Memregion_create"); } result = barrier(world_geometry, contexts[0]); TEST_ASSERT(result == PAMI_SUCCESS,"barrier"); pami_endpoint_t * target_eps = (pami_endpoint_t *) safemalloc( num_async * world_size * sizeof(pami_endpoint_t) ); for (int target=0; target<world_size; target++) for (int i=0; i<num_async; i++) { result = PAMI_Endpoint_create(client, (pami_task_t) target, i, &(target_eps[target*num_async+i]) ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Endpoint_create"); } result = barrier(world_geometry, contexts[0]); TEST_ASSERT(result == PAMI_SUCCESS,"barrier"); pami_memregion_t * shmrs = (pami_memregion_t *) safemalloc( num_async * world_size * sizeof(pami_memregion_t) ); result = allgather(world_geometry, contexts[0], num_async * sizeof(pami_memregion_t), shared_mr, shmrs); TEST_ASSERT(result == PAMI_SUCCESS,"allgather"); /* check now that count will not iterate over an incomplete iteration space */ int remote_targets_per_thread = world_size/num_sync; assert((world_size%num_sync)==0); if (world_rank==0) { printf("starting A2A \n"); fflush(stdout); } result = barrier(world_geometry, contexts[0]); TEST_ASSERT(result == PAMI_SUCCESS,"barrier"); int active = world_size; uint64_t t0 = GetTimeBase(); result = barrier(world_geometry, contexts[0]); TEST_ASSERT(result == PAMI_SUCCESS,"barrier"); /* GCC prior to 4.7 will not permit const variables to be private i.e. firstprivate */ #ifdef _OPENMP #pragma omp parallel default(shared) firstprivate(n, num_async, num_sync) #endif { #ifdef _OPENMP int tid = omp_get_thread_num(); #else int tid = 0; #endif for (int count=0; count<remote_targets_per_thread; count++) { int target = remote_targets_per_thread*tid + count; target += world_rank; target = target % world_size; //printf("%ld: attempting Rget to %ld \n", (long)world_rank, (long)target); //fflush(stdout); int local_context = tid; /* each thread uses its own context so this is thread-safe */ int remote_context = target % num_async; pami_rget_simple_t parameters; parameters.rma.dest = target_eps[target*num_async+remote_context]; //parameters.rma.hints = ; parameters.rma.bytes = n*sizeof(double); parameters.rma.cookie = &active; parameters.rma.done_fn = cb_done; parameters.rdma.local.mr = &local_mr[local_context]; parameters.rdma.local.offset = target*n*sizeof(double); parameters.rdma.remote.mr = &shmrs[target*num_async+remote_context]; parameters.rdma.remote.offset = world_rank*n*sizeof(double); result = PAMI_Rget(contexts[local_context], ¶meters); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Rget"); } } uint64_t t1 = GetTimeBase(); double dt1 = (t1-t0)*tic; while (active>0) { result = PAMI_Context_trylock_advancev(&(contexts[0]), num_sync+num_async, 1000); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_trylock_advancev"); } result = barrier(world_geometry, contexts[0]); TEST_ASSERT(result == PAMI_SUCCESS,"barrier"); uint64_t t2 = GetTimeBase(); double dt2 = (t2-t0)*tic; //result = barrier(world_geometry, contexts[0]); //TEST_ASSERT(result == PAMI_SUCCESS,"barrier"); double megabytes = 1.e-6*bytes; printf("%ld: PAMI_Rget A2A: %ld bytes per rank, local %lf seconds (%lf MB/s), remote %lf seconds (%lf MB/s) \n", (long)world_rank, n*sizeof(double), dt1, megabytes/dt1, dt2, megabytes/dt2 ); fflush(stdout); result = barrier(world_geometry, contexts[0]); TEST_ASSERT(result == PAMI_SUCCESS,"barrier"); for (int s=0; s<world_size; s++ ) for (int k=0; k<n; k++) { if (rbuf[s*n+k]!=(1.0*s*n+1.0*k)) printf("%4d: rbuf[%d] = %lf (%lf) \n", (int)world_rank, s*n+k, rbuf[s*n+k], (1.0*s*n+1.0*k) ); } fflush(stdout); result = barrier(world_geometry, contexts[0]); TEST_ASSERT(result == PAMI_SUCCESS,"barrier"); for (int i=0; i<num_async; i++) { result = PAMI_Memregion_destroy(contexts[i], &(local_mr[i]) ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Memregion_destroy"); result = PAMI_Memregion_destroy(contexts[async_context_begin+i], &(shared_mr[i]) ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Memregion_destroy"); } free(shared_mr); free(local_mr); free(target_eps); free(shmrs); free(rbuf); free(sbuf); } /************************************************************************/ result = barrier(world_geometry, contexts[0]); TEST_ASSERT(result == PAMI_SUCCESS,"barrier"); /* finalize the contexts */ result = PAMI_Context_destroyv( contexts, num_contexts ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_destroyv"); free(contexts); /* finalize the client */ result = PAMI_Client_destroy( &client ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_destroy"); if (world_rank==0) printf("%ld: end of test \n", world_rank ); fflush(stdout); return 0; }
int main(int argc, char* argv[]) { int status = MPI_SUCCESS; pami_result_t result = PAMI_ERROR; int provided = MPI_THREAD_SINGLE; MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); /* IBM: --ranks-per-node 64 fails to init threads but this */ /* IBM: testcase doesn't really care so don't exit */ TEST_ASSERT((provided>=MPI_THREAD_MULTIPLE),"MPI_Init_thread"); /* initialize the second client */ char * clientname = "test"; /* IBM: PE PAMI requires a client name */ pami_client_t client; result = PAMI_Client_create(clientname, &client, NULL, 0); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_create"); /* query properties of the client */ pami_configuration_t config[3]; size_t num_contexts; config[0].name = PAMI_CLIENT_NUM_TASKS; config[1].name = PAMI_CLIENT_TASK_ID; config[2].name = PAMI_CLIENT_NUM_CONTEXTS; result = PAMI_Client_query(client, config, 3); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_query"); world_size = config[0].value.intval; world_rank = config[1].value.intval; num_contexts = config[2].value.intval; if (world_rank==0) printf("hello world from rank %ld of %ld, number of contexts %zu \n", world_rank, world_size, num_contexts );/*IBM: debug num_contexts */ fflush(stdout); /* initialize the contexts */ pami_context_t * contexts; contexts = (pami_context_t *) safemalloc( num_contexts * sizeof(pami_context_t) ); result = PAMI_Context_createv( client, NULL, 0, contexts, num_contexts ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_createv"); /************************************************************************/ /* IBM: Updating the test with the assumption that we will Rput the */ /* IBM: local byte array to our neighbor's shared byte array. */ int n = (argc>1 ? atoi(argv[1]) : 1000); size_t bytes = n * sizeof(int), bytes_out;/* IBM: debug - scale up testing */ int * shared = (int *) safemalloc(bytes); for (int i=0; i<n; i++) shared[i] = -1; /*IBM: initialize with -1, replaced with neighbor's rank */ pami_memregion_t shared_mr; result = PAMI_Memregion_create(contexts[0], shared, bytes, &bytes_out, &shared_mr); TEST_ASSERT(result == PAMI_SUCCESS && bytes==bytes_out,"PAMI_Memregion_create"); int * local = (int *) safemalloc(bytes); for (int i=0; i<n; i++) local[i] = world_rank; /*IBM: initialize with our rank */ pami_memregion_t local_mr; result = PAMI_Memregion_create(contexts[0], local, bytes, &bytes_out, &local_mr); /* IBM: local */ TEST_ASSERT(result == PAMI_SUCCESS && bytes==bytes_out,"PAMI_Memregion_create"); status = MPI_Barrier(MPI_COMM_WORLD); TEST_ASSERT(result == MPI_SUCCESS,"MPI_Barrier"); pami_memregion_t * shmrs = (pami_memregion_t *) safemalloc( world_size * sizeof(pami_memregion_t) ); status = MPI_Allgather(&shared_mr, sizeof(pami_memregion_t), MPI_BYTE, shmrs, sizeof(pami_memregion_t), MPI_BYTE, MPI_COMM_WORLD); TEST_ASSERT(result == MPI_SUCCESS,"MPI_Allgather"); int target = (world_rank>0 ? world_rank-1 : world_size-1); pami_endpoint_t target_ep; result = PAMI_Endpoint_create(client, (pami_task_t) target, 0, &target_ep); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Endpoint_create"); int active = 2; pami_rput_simple_t parameters; parameters.rma.dest = target_ep; parameters.rma.bytes = bytes; parameters.rma.cookie = &active; parameters.rma.done_fn = cb_done; parameters.rdma.local.mr = &local_mr; parameters.rdma.local.offset = 0; parameters.rdma.remote.mr = &shmrs[target]; /*IBM: target's mem region */ parameters.rdma.remote.offset = 0; parameters.put.rdone_fn = cb_done; result = PAMI_Rput(contexts[0], ¶meters); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Rput"); while (active) { //result = PAMI_Context_advance( contexts[0], 100); //TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_advance"); result = PAMI_Context_trylock_advancev(contexts, 1, 1000); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_trylock_advancev"); } /* IBM: I'm done with Rput but my world_rank + 1 neighbor might not be so need to advance */ /* IBM: Could do a barrier or send/recv a completion message instead ....*/ active = 10; /* IBM: Arbitrary - advance some more - 10*10000 good enough? */ while (--active) /* IBM*/ { /* IBM*/ result = PAMI_Context_trylock_advancev(contexts, 1, 10000); /* IBM*/ /*TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_trylock_advancev");*/ /* IBM*/ } /* IBM*/ int errors = 0; target = (world_rank<(world_size-1) ? world_rank+1 : 0); for (int i=0; i<n; i++) if ((shared[i] != target) || (local[i] != world_rank)) /*IBM: also verify didn't change local */ errors++; if (errors>0) { printf("%ld: %d errors :-( \n", (long)world_rank, errors); /*IBM: grep "errors" in scaled up output */ for (int i=0; i<n; i++) printf("%ld: local[%d] = %d , shared[%d] = %d (%d) \n", (long)world_rank, i, local[i], i, shared[i], target); /*IBM: print both arrays */ } else printf("%ld: no errors :-) \n", (long)world_rank); MPI_Barrier(MPI_COMM_WORLD); result = PAMI_Memregion_destroy(contexts[0], &shared_mr); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Memregion_destroy"); result = PAMI_Memregion_destroy(contexts[0], &local_mr); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Memregion_destroy"); free(shmrs); free(local); free(shared); /************************************************************************/ /* finalize the contexts */ result = PAMI_Context_destroyv( contexts, num_contexts ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_destroyv"); free(contexts); /* finalize the client */ result = PAMI_Client_destroy( &client ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_destroy"); status = MPI_Barrier(MPI_COMM_WORLD); TEST_ASSERT(result == MPI_SUCCESS,"MPI_Barrier"); MPI_Finalize(); if (world_rank==0) printf("%ld: end of test \n", world_rank ); fflush(stdout); return 0; }
int main(int argc, char* argv[]) { pami_result_t result = PAMI_ERROR; /* initialize the second client */ char * clientname = ""; pami_client_t client; result = PAMI_Client_create(clientname, &client, NULL, 0); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_create"); /* query properties of the client */ pami_configuration_t config[3]; size_t num_contexts; config[0].name = PAMI_CLIENT_NUM_TASKS; config[1].name = PAMI_CLIENT_TASK_ID; config[2].name = PAMI_CLIENT_NUM_CONTEXTS; result = PAMI_Client_query(client, config, 3); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_query"); world_size = config[0].value.intval; world_rank = config[1].value.intval; num_contexts = config[2].value.intval; TEST_ASSERT(num_contexts>1,"num_contexts>1"); if (world_rank==0) { printf("hello world from rank %ld of %ld \n", world_rank, world_size ); fflush(stdout); } /* initialize the contexts */ contexts = (pami_context_t *) safemalloc( num_contexts * sizeof(pami_context_t) ); result = PAMI_Context_createv( client, NULL, 0, contexts, num_contexts ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_createv"); /* setup the world geometry */ pami_geometry_t world_geometry; result = PAMI_Geometry_world(client, &world_geometry ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Geometry_world"); int status = pthread_create(&Progress_thread, NULL, &Progress_function, NULL); TEST_ASSERT(status==0, "pthread_create"); /************************************************************************/ int n = (argc>1 ? atoi(argv[1]) : 1000); size_t bytes = n * sizeof(int); int * local = (int *) safemalloc(bytes); for (int i=0; i<n; i++) local[i] = world_rank; /* create the endpoint */ int target = (world_rank>0 ? world_rank-1 : world_size-1); pami_endpoint_t target_ep; result = PAMI_Endpoint_create(client, (pami_task_t) target, 1, &target_ep); //result = PAMI_Endpoint_create(client, (pami_task_t) target, 0, &target_ep); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Endpoint_create"); /* register the dispatch function */ pami_dispatch_callback_function dispatch_cb; size_t dispatch_id = 37; dispatch_cb.p2p = dispatch_recv_cb; pami_dispatch_hint_t dispatch_hint = {0}; int dispatch_cookie = 1000000+world_rank; //dispatch_hint.recv_immediate = PAMI_HINT_DISABLE; result = PAMI_Dispatch_set(contexts[0], dispatch_id, dispatch_cb, &dispatch_cookie, dispatch_hint); result = PAMI_Dispatch_set(contexts[1], dispatch_id, dispatch_cb, &dispatch_cookie, dispatch_hint); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Dispatch_set"); /* make sure everything is ready */ result = barrier(world_geometry, contexts[0]); TEST_ASSERT(result == PAMI_SUCCESS,"barrier"); // The iovec structure describes a buffer. It contains two fields: // void *iov_base - Contains the address of a buffer. // size_t iov_len - Contains the length of the buffer. int header = 37373; int active = 1; pami_send_t parameters; parameters.send.header.iov_base = &header; parameters.send.header.iov_len = sizeof(int); parameters.send.data.iov_base = local; parameters.send.data.iov_len = bytes; parameters.send.dispatch = dispatch_id; //parameters.send.hints = ; parameters.send.dest = target_ep; parameters.events.cookie = &active; parameters.events.local_fn = cb_done; parameters.events.remote_fn = NULL;//cb_done; uint64_t t0 = GetTimeBase(); result = PAMI_Send(contexts[0], ¶meters); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Send"); while (active) { //result = PAMI_Context_advance( contexts[0], 100); //TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_advance"); result = PAMI_Context_trylock_advancev(&(contexts[0]), 1, 1000); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_trylock_advancev"); } uint64_t t1 = GetTimeBase(); uint64_t dt = t1-t0; /* barrier on non-progressing context to make sure CHT does its job */ barrier(world_geometry, contexts[0]); printf("%ld: PAMI_Send of %ld bytes achieves %lf MB/s \n", (long)world_rank, bytes, 1.6e9*1e-6*(double)bytes/(double)dt ); fflush(stdout); result = barrier(world_geometry, contexts[0]); TEST_ASSERT(result == PAMI_SUCCESS,"barrier"); free(local); /************************************************************************/ void * rv; status = pthread_cancel(Progress_thread); TEST_ASSERT(status==0, "pthread_cancel"); status = pthread_join(Progress_thread, &rv); TEST_ASSERT(status==0, "pthread_join"); result = barrier(world_geometry, contexts[0]); TEST_ASSERT(result == PAMI_SUCCESS,"barrier"); /* finalize the contexts */ result = PAMI_Context_destroyv( contexts, num_contexts ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_destroyv"); free(contexts); /* finalize the client */ result = PAMI_Client_destroy( &client ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_destroy"); if (world_rank==0) printf("%ld: end of test \n", world_rank ); fflush(stdout); return 0; }
int main(int argc, char* argv[]) { pami_result_t result = PAMI_ERROR; /* initialize the client */ char * clientname = ""; pami_client_t client; result = PAMI_Client_create (clientname, &client, NULL, 0); assert(result == PAMI_SUCCESS); /* query properties of the client */ pami_configuration_t config; config.name = PAMI_CLIENT_TASK_ID; result = PAMI_Client_query( client, &config,1); assert(result == PAMI_SUCCESS); printf("PAMI_CLIENT_TASK_ID = %ld \n", config.value.intval); fflush(stdout); SLEEP(1); config.name = PAMI_CLIENT_NUM_TASKS; result = PAMI_Client_query( client, &config,1); assert(result == PAMI_SUCCESS); printf("PAMI_CLIENT_NUM_TASKS = %ld \n", config.value.intval); fflush(stdout); SLEEP(1); config.name = PAMI_CLIENT_NUM_CONTEXTS; result = PAMI_Client_query( client, &config,1); assert(result == PAMI_SUCCESS); printf("PAMI_CLIENT_NUM_CONTEXTS = %ld \n", config.value.intval); fflush(stdout); SLEEP(1); config.name = PAMI_CLIENT_CONST_CONTEXTS; result = PAMI_Client_query( client, &config,1); assert(result == PAMI_SUCCESS); printf("PAMI_CLIENT_CONST_CONTEXTS = %ld \n", config.value.intval); fflush(stdout); SLEEP(1); config.name = PAMI_CLIENT_NUM_LOCAL_TASKS; result = PAMI_Client_query( client, &config,1); assert(result == PAMI_SUCCESS); size_t num_local_tasks = config.value.intval; printf("PAMI_CLIENT_NUM_LOCAL_TASKS = %ld \n", config.value.intval); fflush(stdout); SLEEP(1); config.name = PAMI_CLIENT_LOCAL_TASKS; result = PAMI_Client_query( client, &config,1); assert(result == PAMI_SUCCESS); if ( num_local_tasks == 1 ) printf("PAMI_CLIENT_LOCAL_TASKS = %ld \n", config.value.intarray[0] ); else if ( num_local_tasks > 1 ) printf("PAMI_CLIENT_LOCAL_TASKS = %ld .. %ld \n", config.value.intarray[0], config.value.intarray[num_local_tasks-1] ); fflush(stdout); SLEEP(1); config.name = PAMI_CLIENT_HWTHREADS_AVAILABLE; result = PAMI_Client_query( client, &config,1); assert(result == PAMI_SUCCESS); printf("PAMI_CLIENT_HWTHREADS_AVAILABLE = %ld \n", config.value.intval); fflush(stdout); SLEEP(1); config.name = PAMI_CLIENT_MEMREGION_SIZE; result = PAMI_Client_query( client, &config,1); assert(result == PAMI_SUCCESS); printf("PAMI_CLIENT_MEMREGION_SIZE = %ld \n", config.value.intval); fflush(stdout); SLEEP(1); config.name = PAMI_CLIENT_MEM_SIZE; result = PAMI_Client_query( client, &config,1); assert(result == PAMI_SUCCESS); printf("PAMI_CLIENT_MEM_SIZE = %ld \n", config.value.intval); fflush(stdout); SLEEP(1); config.name = PAMI_CLIENT_PROCESSOR_NAME; result = PAMI_Client_query( client, &config,1); assert(result == PAMI_SUCCESS); printf("PAMI_CLIENT_PROCESSOR_NAME = %s \n", config.value.chararray); fflush(stdout); SLEEP(1); config.name = PAMI_CLIENT_WTICK; result = PAMI_Client_query( client, &config,1); assert(result == PAMI_SUCCESS); printf("PAMI_CLIENT_WTICK = %e \n", config.value.doubleval); fflush(stdout); SLEEP(1); config.name = PAMI_CLIENT_WTIMEBASE_MHZ; result = PAMI_Client_query( client, &config,1); assert(result == PAMI_SUCCESS); printf("PAMI_CLIENT_WTIMEBASE_MHZ = %ld \n", config.value.intval); fflush(stdout); SLEEP(1); config.name = PAMI_CLIENT_CLOCK_MHZ; result = PAMI_Client_query( client, &config,1); assert(result == PAMI_SUCCESS); printf("PAMI_CLIENT_CLOCK_MHZ = %ld \n", config.value.intval); fflush(stdout); SLEEP(1); /* finalize the client */ result = PAMI_Client_destroy(&client); assert(result == PAMI_SUCCESS); printf("end of test \n"); fflush(stdout); SLEEP(1); return 0; }
int main (int argc, char ** argv) { pami_client_t client; pami_context_t context[2]; pami_task_t task_id; size_t num_tasks = 0; size_t ncontexts = 0; size_t errors = 0; pami_result_t result = PAMI_ERROR; pami_type_t subtype; pami_type_t compound_type; pami_type_t simple_type; info_t exchange[MAX_TASKS]; double data[BUFFERSIZE]; volatile unsigned ready; { /* init */ ready = 0; unsigned i; for (i = 0; i < MAX_TASKS; i++) exchange[i].active = 0; for (i = 0; i < BUFFERSIZE; i++) data[i] = E; result = PAMI_Client_create ("TEST", &client, NULL, 0); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to create pami client. result = %d\n", result); return 1; } pami_configuration_t configuration; configuration.name = PAMI_CLIENT_TASK_ID; result = PAMI_Client_query(client, &configuration, 1); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable query configuration (%d). result = %d\n", configuration.name, result); return 1; } task_id = configuration.value.intval; /*fprintf (stderr, "My task id = %d\n", task_id);*/ configuration.name = PAMI_CLIENT_NUM_TASKS; result = PAMI_Client_query(client, &configuration, 1); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable query configuration (%d). result = %d\n", configuration.name, result); return 1; } num_tasks = configuration.value.intval; /*if (task_id == 0) fprintf (stderr, "Number of tasks = %zu\n", num_tasks);*/ if ((num_tasks < 2) || (num_tasks > MAX_TASKS)) { fprintf (stderr, "Error. This test requires 2-%d tasks. Number of tasks in this job: %zu\n", MAX_TASKS, num_tasks); return 1; } if (task_id == num_tasks - 1) { for (i = 0; i < 320; i++) data[i] = PI; } configuration.name = PAMI_CLIENT_NUM_CONTEXTS; result = PAMI_Client_query(client, &configuration, 1); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable query configuration (%d). result = %d\n", configuration.name, result); return 1; } ncontexts = (configuration.value.intval < 2) ? 1 : 2; /*if (task_id == 0) fprintf (stderr, "maximum contexts = %zu, number of contexts used in this test = %zu\n", configuration.value.intval, ncontexts);*/ result = PAMI_Context_createv(client, NULL, 0, context, ncontexts); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to create pami context(s). result = %d\n", result); return 1; } pami_dispatch_hint_t options = {}; for (i = 0; i < ncontexts; i++) { pami_dispatch_callback_function fn; fn.p2p = dispatch_exchange; result = PAMI_Dispatch_set (context[i], EXCHANGE_DISPATCH_ID, fn, (void *) exchange, options); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable register pami 'exchange' dispatch. result = %d\n", result); return 1; } fn.p2p = dispatch_notify; result = PAMI_Dispatch_set (context[i], NOTIFY_DISPATCH_ID, fn, (void *) & ready, options); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable register pami 'notify' dispatch. result = %d\n", result); return 1; } } /* *********************************** * Create the pami types ************************************/ /* This compound noncontiguous type is composed of one double, skips a double, * two doubles, skips a double, three doubles, skips a double, five doubles, * skips a double, six doubles, skips a double, seven doubles, skips a double, * eight doubles, then skips two doubles. * * This results in a type with 32 doubles that is 40 doubles * 'wide'. */ PAMI_Type_create (&subtype); PAMI_Type_add_simple (subtype, sizeof(double), /* bytes */ 0, /* offset */ 1, /* count */ sizeof(double) * 2); /* stride */ PAMI_Type_add_simple (subtype, sizeof(double) * 2, /* bytes */ 0, /* offset */ 1, /* count */ sizeof(double) * 3); /* stride */ PAMI_Type_add_simple (subtype, sizeof(double) * 3, /* bytes */ 0, /* offset */ 1, /* count */ sizeof(double) * 4); /* stride */ PAMI_Type_add_simple (subtype, sizeof(double) * 5, /* bytes */ 0, /* offset */ 1, /* count */ sizeof(double) * 6); /* stride */ PAMI_Type_add_simple (subtype, sizeof(double) * 6, /* bytes */ 0, /* offset */ 1, /* count */ sizeof(double) * 7); /* stride */ PAMI_Type_add_simple (subtype, sizeof(double) * 7, /* bytes */ 0, /* offset */ 1, /* count */ sizeof(double) * 8);/* stride */ PAMI_Type_add_simple (subtype, sizeof(double) * 8, /* bytes */ 0, /* offset */ 1, /* count */ sizeof(double) * 10);/* stride */ PAMI_Type_complete (subtype, sizeof(double)); /* This noncontiguous type is composed of the above compound type, repeated * ten times with no stride. * * This results in a type with 320 doubles that is 400 doubles * 'wide'. */ PAMI_Type_create (&compound_type); PAMI_Type_add_typed (compound_type, subtype, /* subtype */ 0, /* offset */ 10, /* count */ sizeof(double) * 32); /* stride */ PAMI_Type_complete (compound_type, sizeof(double)); /* This simple noncontiguous type is composed of eight contiguous doubles, * then skips a _single_ double, repeated 40 times. * * This results in a type with 320 doubles that is 360 doubles 'wide'. */ PAMI_Type_create (&simple_type); PAMI_Type_add_simple (simple_type, sizeof(double) * 8, /* bytes */ 0, /* offset */ 40, /* count */ sizeof(double) * 9); /* stride */ PAMI_Type_complete (simple_type, sizeof(double)); /* Create a memory region for the local data buffer. */ size_t bytes; result = PAMI_Memregion_create (context[0], (void *) data, BUFFERSIZE, &bytes, &exchange[task_id].mr); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to create memory region. result = %d\n", result); return 1; } else if (bytes < BUFFERSIZE) { fprintf (stderr, "Error. Unable to create memory region of a large enough size. result = %d\n", result); return 1; } /* Broadcast the memory region to all tasks - including self. */ for (i = 0; i < num_tasks; i++) { pami_send_immediate_t parameters; parameters.dispatch = EXCHANGE_DISPATCH_ID; parameters.header.iov_base = (void *) & bytes; parameters.header.iov_len = sizeof(size_t); parameters.data.iov_base = (void *) & exchange[task_id].mr; parameters.data.iov_len = sizeof(pami_memregion_t); PAMI_Endpoint_create (client, i, 0, ¶meters.dest); result = PAMI_Send_immediate (context[0], ¶meters); } /* Advance until all memory regions have been received. */ for (i = 0; i < num_tasks; i++) { while (exchange[i].active == 0) PAMI_Context_advance (context[0], 100); } } /* init done */ pami_send_immediate_t notify; notify.dispatch = NOTIFY_DISPATCH_ID; notify.header.iov_base = NULL; notify.header.iov_len = 0; notify.data.iov_base = NULL; notify.data.iov_len = 0; volatile size_t active = 1; pami_rget_typed_t parameters; parameters.rma.hints = (pami_send_hint_t) {0}; parameters.rma.cookie = (void *) & active; parameters.rma.done_fn = decrement; parameters.rma.bytes = 320 * sizeof(double); if (task_id == 0) { fprintf (stdout, "PAMI_Rget('typed') functional test %s\n", (ncontexts < 2) ? "" : "[crosstalk]"); fprintf (stdout, "\n"); parameters.rdma.local.mr = &exchange[0].mr; parameters.rdma.remote.mr = &exchange[num_tasks - 1].mr; PAMI_Endpoint_create (client, num_tasks - 1, ncontexts - 1, ¶meters.rma.dest); PAMI_Endpoint_create (client, num_tasks - 1, ncontexts - 1, ¬ify.dest); } else { parameters.rdma.local.mr = &exchange[num_tasks - 1].mr; parameters.rdma.remote.mr = &exchange[0].mr; PAMI_Endpoint_create (client, 0, 0, ¶meters.rma.dest); PAMI_Endpoint_create (client, 0, 0, ¬ify.dest); } /* ******************************************************************** */ /* contiguous -> contiguous transfer test */ /* ******************************************************************** */ if (task_id == 0) { parameters.rdma.local.offset = 0; parameters.rdma.remote.offset = 0; parameters.type.local = PAMI_TYPE_DOUBLE; parameters.type.remote = PAMI_TYPE_DOUBLE; active = 1; PAMI_Rget_typed (context[0], ¶meters); while (active > 0) PAMI_Context_advance (context[0], 100); /* Notify the remote task that the data has been transfered. */ PAMI_Send_immediate (context[0], ¬ify); } else if (task_id == num_tasks - 1) { /* Wait for notification that the data has been transfered. */ while (ready == 0) PAMI_Context_advance (context[ncontexts - 1], 100); ready = 0; } /* ******************************************************************** */ /* contiguous -> non-contiguous transfer test */ /* ******************************************************************** */ if (task_id == num_tasks - 1) { parameters.rdma.local.offset = 4 * 1024; parameters.rdma.remote.offset = 0; parameters.type.local = simple_type; parameters.type.remote = PAMI_TYPE_DOUBLE; active = 1; PAMI_Rget_typed (context[ncontexts - 1], ¶meters); while (active > 0) PAMI_Context_advance (context[ncontexts - 1], 100); /* Notify the remote task that the data has been transfered. */ PAMI_Send_immediate (context[ncontexts - 1], ¬ify); } else if (task_id == 0) { /* Wait for notification that the data has been transfered. */ while (ready == 0) PAMI_Context_advance (context[0], 100); ready = 0; } /* ******************************************************************** */ /* non-contiguous -> non-contiguous transfer test */ /* ******************************************************************** */ if (task_id == 0) { parameters.rdma.local.offset = 4 * 1024; parameters.rdma.remote.offset = 4 * 1024; parameters.type.local = compound_type; parameters.type.remote = simple_type; active = 1; PAMI_Rget_typed (context[0], ¶meters); while (active > 0) PAMI_Context_advance (context[0], 100); /* Notify the remote task that the data has been transfered. */ PAMI_Send_immediate (context[0], ¬ify); } else if (task_id == num_tasks - 1) { /* Wait for notification that the data has been transfered. */ while (ready == 0) PAMI_Context_advance (context[ncontexts - 1], 100); ready = 0; } /* ******************************************************************** */ /* non-contiguous -> contiguous transfer test */ /* ******************************************************************** */ if (task_id == num_tasks - 1) { parameters.rdma.local.offset = 8 * 1024; parameters.rdma.remote.offset = 4 * 1024; parameters.type.local = PAMI_TYPE_DOUBLE; parameters.type.remote = compound_type; active = 1; PAMI_Rget_typed (context[ncontexts - 1], ¶meters); while (active > 0) PAMI_Context_advance (context[ncontexts - 1], 100); /* Notify the remote task that the data has been transfered. */ PAMI_Send_immediate (context[ncontexts - 1], ¬ify); } else if (task_id == 0) { /* Wait for notification that the data has been transfered. */ while (ready == 0) PAMI_Context_advance (context[0], 100); ready = 0; } /* ******************************************************************** */ /* VERIFY data buffers */ /* ******************************************************************** */ if (task_id == num_tasks - 1) { if (task_id == 0) { unsigned i = 0; for (; i < 320; i++) { if (data[i] != PI) { errors++; fprintf (stderr, "Error. data[%d] != %g ..... (%g)\n", i, PI, data[i]); } } for (; i < 512; i++) { if (data[i] != E) { errors++; fprintf (stderr, "Error. data[%d] != %g ..... (%g)\n", i, E, data[i]); } } unsigned j = 0; for (; j < 40; j++) { unsigned n = 0; for (; n < 8; n++) { if (data[i] != PI) { errors++; fprintf (stderr, "Error. data[%d] != %g ..... (%g)\n", i, PI, data[i]); } i++; } if (data[i] != E) { errors++; fprintf (stderr, "Error. data[%d] != %g ..... (%g)\n", i, E, data[i]); } i++; } for (; i < 1024; i++) { if (data[i] != E) { errors++; fprintf (stderr, "Error. data[%d] != %g ..... (%g)\n", i, E, data[i]); } } for (; i < 1024 + 320; i++) { if (data[i] != PI) { errors++; fprintf (stderr, "Error. data[%d] != %g ..... (%g)\n", i, PI, data[i]); } } for (; i < BUFFERSIZE; i++) { if (data[i] != E) { errors++; fprintf (stderr, "Error. data[%d] != %g ..... (%g)\n", i, E, data[i]); } } } } { /* cleanup */ result = PAMI_Context_destroyv(context, ncontexts); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to destroy pami context. result = %d\n", result); return 1; } result = PAMI_Client_destroy(&client); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to destroy pami client. result = %d\n", result); return 1; } if (task_id == num_tasks - 1) { if (errors) fprintf (stdout, "Test completed with errors (%zu)\n", errors); else fprintf (stdout, "Test completed with success\n"); } } /* cleanup done */ return (errors != 0); };
int main (int argc, char ** argv) { volatile size_t _rts_active = 1; volatile size_t _ack_active = 1; pami_client_t client; pami_context_t context[2]; char cl_string[] = "TEST"; pami_result_t result = PAMI_ERROR; result = PAMI_Client_create (cl_string, &client, NULL, 0); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to create pami client. result = %d\n", result); return 1; } #ifdef TEST_CROSSTALK size_t num = 2; #else size_t num = 1; #endif result = PAMI_Context_createv(client, NULL, 0, context, num); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to create pami context(s). result = %d\n", result); return 1; } pami_configuration_t configuration; configuration.name = PAMI_CLIENT_TASK_ID; result = PAMI_Client_query(client, &configuration, 1); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable query configuration (%d). result = %d\n", configuration.name, result); return 1; } pami_task_t task_id = configuration.value.intval; fprintf (stderr, "My task id = %d\n", task_id); configuration.name = PAMI_CLIENT_NUM_TASKS; result = PAMI_Client_query(client, &configuration, 1); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable query configuration (%d). result = %d\n", configuration.name, result); return 1; } size_t num_tasks = configuration.value.intval; fprintf (stderr, "Number of tasks = %zu\n", num_tasks); if (num_tasks != 2) { fprintf (stderr, "Error. This test requires 2 tasks. Number of tasks in this job: %zu\n", num_tasks); return 1; } pami_dispatch_hint_t options={}; #ifdef USE_SHMEM_OPTION options.use_shmem = PAMI_HINT_ENABLE; fprintf (stderr, "##########################################\n"); fprintf (stderr, "shared memory optimizations forced ON\n"); fprintf (stderr, "##########################################\n"); #elif defined(NO_SHMEM_OPTION) options.use_shmem = PAMI_HINT_DISABLE; fprintf (stderr, "##########################################\n"); fprintf (stderr, "shared memory optimizations forced OFF\n"); fprintf (stderr, "##########################################\n"); #endif size_t i = 0; #ifdef TEST_CROSSTALK for (i=0; i<2; i++) #endif { pami_dispatch_callback_function fn; fprintf (stderr, "Before PAMI_Dispatch_set(%d) .. &_rts_active = %p, _rts_active = %zu\n", DISPATCH_ID_RTS, &_rts_active, _rts_active); fn.p2p = dispatch_rts; result = PAMI_Dispatch_set (context[i], DISPATCH_ID_RTS, fn, (void *)&_rts_active, options); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable register pami dispatch. result = %d\n", result); return 1; } fprintf (stderr, "Before PAMI_Dispatch_set(%d) .. &_ack_active = %p, _ack_active = %zu\n", DISPATCH_ID_ACK, &_ack_active, _ack_active); fn.p2p = dispatch_ack; result = PAMI_Dispatch_set (context[i], DISPATCH_ID_ACK, fn, (void *)&_ack_active, options); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable register pami dispatch. result = %d\n", result); return 1; } } if (task_id == 0) { pami_send_immediate_t parameters; #ifdef TEST_CROSSTALK fprintf (stdout, "PAMI_Get('simple') functional test [crosstalk]\n"); fprintf (stdout, "\n"); PAMI_Endpoint_create (client, 1, 1, ¶meters.dest); #else fprintf (stdout, "PAMI_Get('simple') functional test\n"); fprintf (stdout, "\n"); PAMI_Endpoint_create (client, 1, 0, ¶meters.dest); #endif /* Allocate some memory from the heap. */ void * send_buffer = malloc (BUFFERSIZE); /* Initialize the memory for validation. */ initialize_data ((uint32_t *)send_buffer, BUFFERSIZE, 0); print_data (send_buffer, BUFFERSIZE); /* Send an 'rts' message to the target task */ rts_info_t rts_info; PAMI_Endpoint_create (client, 0, 0, &rts_info.origin); rts_info.bytes = BUFFERSIZE; rts_info.source = send_buffer; parameters.dispatch = DISPATCH_ID_RTS; parameters.header.iov_base = &rts_info; parameters.header.iov_len = sizeof(rts_info_t); parameters.data.iov_base = NULL; parameters.data.iov_len = 0; parameters.hints = null_send_hint; fprintf (stderr, "Before PAMI_Send_immediate()\n"); PAMI_Send_immediate (context[0], ¶meters); /* wait for the 'ack' */ fprintf (stderr, "Wait for 'ack', _ack_active = %zu\n", _ack_active); while (_ack_active != 0) result = PAMI_Context_advance (context[0], 100); free (send_buffer); switch (_ack_status) { case 0: fprintf (stdout, "Test PASSED\n"); break; case 1: fprintf (stdout, "Test FAILED (get error)\n"); break; case 2: fprintf (stdout, "Test FAILED (data error)\n"); break; default: fprintf (stdout, "Test FAILED (unknown error)\n"); break; } } else if (task_id == 1) { #ifdef TEST_CROSSTALK size_t contextid = 1; #else size_t contextid = 0; #endif /* wait for the 'rts' */ fprintf (stderr, "Wait for 'rts', _rts_active = %zu, contextid = %zu\n", _rts_active, contextid); while (_rts_active != 0) result = PAMI_Context_advance (context[contextid], 100); } fprintf (stderr, "Test completed .. cleanup\n"); result = PAMI_Context_destroyv(context, num); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to destroy pami context. result = %d\n", result); return 1; } result = PAMI_Client_destroy(&client); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to destroy pami client. result = %d\n", result); return 1; } /*fprintf (stdout, "Success (%d)\n", task_id); */ return 0; };