static pami_result_t MPIDI_Accumulate(pami_context_t context, void * _req) { MPIDI_Win_request *req = (MPIDI_Win_request*)_req; pami_result_t rc; void *map; pami_send_t params; params = zero_send_parms; params.send.header.iov_len = sizeof(MPIDI_Win_MsgInfo); params.send.dispatch = MPIDI_Protocols_WinAccum; params.send.dest = req->dest; params.events.cookie = req; params.events.remote_fn = MPIDI_Win_DoneCB; struct MPIDI_Win_sync* sync = &req->win->mpid.sync; TRACE_ERR("Start index=%u/%d l-addr=%p r-base=%p r-offset=%zu (sync->started=%u sync->complete=%u)\n", req->state.index, req->target.dt.num_contig, req->buffer, req->win->mpid.info[req->target.rank].base_addr, req->offset, sync->started, sync->complete); while (req->state.index < req->target.dt.num_contig) { if (sync->started > sync->complete + MPIDI_Process.rma_pending) { TRACE_ERR("Bailing out; index=%u/%d sync->started=%u sync->complete=%u\n", req->state.index, req->target.dt.num_contig, sync->started, sync->complete); return PAMI_EAGAIN; } ++sync->started; params.send.header.iov_base = &(((MPIDI_Win_MsgInfo *)req->accum_headers)[req->state.index]); params.send.data.iov_len = req->target.dt.map[req->state.index].DLOOP_VECTOR_LEN; params.send.data.iov_base = req->buffer + req->state.local_offset; #ifdef TRACE_ON void * buf = params.send.data.iov_base; unsigned* ibuf = (unsigned*)buf; double * dbuf = (double *)buf; TRACE_ERR(" Sub index=%u bytes=%zu l-offset=%zu r-addr=%p l-buf=%p *(int*)buf=0x%08x *(double*)buf=%g\n", req->state.index, params.send.data.iov_len, req->state.local_offset, req->accum_headers[req->state.index].addr, buf, *ibuf, *dbuf); #endif /** sync->total will be updated with every RMA and the complete will not change till that RMA has completed. In the meanwhile the rest of the RMAs will have memory leaks */ if (req->target.dt.num_contig - req->state.index == 1) { rc = PAMI_Send(context, ¶ms); MPID_assert(rc == PAMI_SUCCESS); return PAMI_SUCCESS; } else { rc = PAMI_Send(context, ¶ms); MPID_assert(rc == PAMI_SUCCESS); req->state.local_offset += params.send.data.iov_len; ++req->state.index; } } return PAMI_SUCCESS; }
void send_strided ( pami_context_t context, size_t dispatchc, size_t dispatchs, A1PAMI_Info_t * ainfo, size_t hdrsize, char * buffer, pami_endpoint_t target ) { //fprintf(stderr, "send_strided dispatch %ld, hdrsize %ld\n", // dispatch, hdrsize); int i = 0; _PackState pstate; pami_send_t parameters; parameters.send.header.iov_base = ainfo; parameters.send.header.iov_len = hdrsize; parameters.send.data.iov_base = NULL; parameters.events.cookie = (void *) & _send_active; parameters.events.local_fn = NULL; parameters.events.remote_fn = NULL; parameters.send.dest = target; memset(¶meters.send.hints, 0, sizeof(parameters.send.hints)); if (ainfo->count[0] >= 512) { parameters.send.dispatch = dispatchc; for (i = 0; i < ainfo->count[1]; ++i) { parameters.send.data.iov_base = buffer + ainfo->sstride*i; parameters.send.data.iov_len = ainfo->count[0]; if (i == ainfo->count[1] - 1) parameters.events.remote_fn = cb_done; RC( PAMI_Send (context, ¶meters) ); } while (_send_active) PAMI_Context_advance (context, POLL_CNT); _send_active = 1; } else { parameters.send.dispatch = dispatchs; int bytes = ainfo->count[0] * ainfo->count[1]; void *packbuf = malloc (bytes); packStrided (ainfo, packbuf, buffer); pstate.buffer = packbuf; pstate.counter = & _send_active; parameters.send.data.iov_base = packbuf; parameters.send.data.iov_len = bytes; parameters.events.remote_fn = cb_pack_done; parameters.events.cookie = (void*)&pstate; //fprintf (stderr, "Calling pami_send\n"); RC( PAMI_Send (context, ¶meters) ); while (_send_active) PAMI_Context_advance (context, POLL_CNT); _send_active = 1; //fprintf (stderr, "After pami_send\n"); } }
/** * This does the actual long-send using PAMI_Send. */ static pami_result_t SendLongHandoff(pami_context_t context, void * cookie) { assert(cookie != NULL); unsigned* quad = (unsigned*)cookie; pami_task_t remote_task = 1-task; size_t remote_context = (task+LONG_DISPATCH)&(num_contexts-1); pami_endpoint_t dest; PAMI_Endpoint_create(client, remote_task, remote_context, &dest); pami_send_t parameters; bzero(¶meters, sizeof(pami_send_t)); parameters.send.dispatch = LONG_DISPATCH; /*parameters.send.hints = {0}; */ parameters.send.dest = dest; parameters.send.header.iov_base = quad; parameters.send.header.iov_len = 4*sizeof(unsigned); parameters.send.data.iov_base = lbuf; parameters.send.data.iov_len = LSIZE; parameters.events.cookie = lbuf; parameters.events.local_fn = SendLongDoneCB; parameters.events.remote_fn = NULL; PAMI_Send(context, ¶meters); return PAMI_SUCCESS; }
void send_contig ( pami_context_t context, size_t dispatch, void * metadata, size_t hdrsize, char * buffer, size_t sndlen, pami_endpoint_t target ) { //fprintf(stderr, "Calling send dispatch %ld, hdrsize %ld, bytes %ld\n", //dispatch, hdrsize, sndlen); pami_send_t parameters; parameters.send.dispatch = dispatch; parameters.send.header.iov_base = metadata; parameters.send.header.iov_len = hdrsize; parameters.send.data.iov_base = buffer; parameters.send.data.iov_len = sndlen; parameters.events.cookie = (void *) & _send_active; parameters.events.local_fn = NULL; //cb_done; parameters.events.remote_fn = cb_done; parameters.send.dest = target; memset(¶meters.send.hints, 0, sizeof(parameters.send.hints)); #if ENABLE_PROGRESS pami_work_t work; PAMI_Context_post (context, &work, (pami_work_function)PAMI_Send, (void*)¶meters); while (_send_active == 1); #else RC( PAMI_Send (context, ¶meters) ); while (_send_active) PAMI_Context_advance (context, POLL_CNT); #endif _send_active = 1; }
static pami_result_t send_endpoint (pami_context_t context, size_t target, pami_send_t * parameters) { parameters->send.dest = _endpoint[target]; fprintf (stderr, "send message to endpoint 0x%08x.\n", parameters->send.dest); return PAMI_Send (context, parameters); };
int optiq_pami_transport_actual_send(struct optiq_transport *self, struct optiq_message *message) { #ifdef __bgq__ pami_result_t result; struct optiq_pami_transport *pami_transport = (struct optiq_pami_transport *)optiq_transport_get_concrete_transport(self); struct optiq_send_cookie *send_cookie = optiq_pami_transport_get_send_cookie(pami_transport); send_cookie->message = message; if (message->length <= MAX_SHORT_MESSAGE_LENGTH) { pami_send_immediate_t parameter; parameter.dispatch = RECV_MESSAGE_DISPATCH_ID; parameter.header.iov_base = (void *)&message->header; parameter.header.iov_len = sizeof(struct optiq_message_header); parameter.data.iov_base = (void *)message->buffer; parameter.data.iov_len = message->length; parameter.dest = pami_transport->endpoints[message->next_dest]; result = PAMI_Send_immediate (pami_transport->context, ¶meter); assert(result == PAMI_SUCCESS); if (result != PAMI_SUCCESS) { return 1; } /*Add the cookie to the vector of in-use send cookies*/ pami_transport->in_use_send_cookies.push_back(send_cookie); } else { pami_send_t param_send; param_send.send.dest = message->next_dest; param_send.send.dispatch = RECV_MESSAGE_DISPATCH_ID; param_send.send.header.iov_base = (void *)&message->header; param_send.send.header.iov_len = sizeof(struct optiq_message_header); param_send.send.data.iov_base = (void *)message->buffer; param_send.send.data.iov_len = message->length; param_send.events.cookie = (void *)send_cookie; param_send.events.local_fn = optiq_send_done_fn; param_send.events.remote_fn = NULL; result = PAMI_Send(pami_transport->context, ¶m_send); assert(result == PAMI_SUCCESS); if (result != PAMI_SUCCESS) { return 1; } } #ifdef DEBUG printf("Rank %d is sending data of size %d to Rank %d with flow_id = %d, original_offset = %d\n", self->rank, message->length, message->next_dest, message->header.flow_id, message->header.original_offset); #endif #endif return 0; }
pami_send_t params = { .send = { .dispatch = MPIDI_Protocols_WinCtrl, .dest = dest, .header = { .iov_base = control, .iov_len = sizeof(MPIDI_Win_control_t), }, }, .events = { .cookie = win, .local_fn = NULL, .remote_fn= MPIDI_WinUnlockDoneCB, }, }; rc = PAMI_Send(context, ¶ms); } else { pami_send_immediate_t params = { .dispatch = MPIDI_Protocols_WinCtrl, .dest = dest, .header = { .iov_base = control, .iov_len = sizeof(MPIDI_Win_control_t), }, .data = { .iov_base = NULL, .iov_len = 0, }, }; rc = PAMI_Send_immediate(context, ¶ms); }
int main (int argc, char ** argv) { /*volatile size_t send_active = 2; */ volatile size_t send_active = 1; volatile size_t recv_active = 1; pami_client_t client; pami_context_t context; char cl_string[] = "TEST"; pami_result_t result = PAMI_ERROR; fprintf (stderr, "Before Client initialize\n"); result = PAMI_Client_create (cl_string, &client, NULL, 0); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to initialize pami client. result = %d\n", result); return 1; } fprintf (stderr, "After Client initialize\n"); fprintf (stderr, "before context createv\n"); { result = PAMI_Context_createv(client, NULL, 0, &context, 1); } if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to create pami context. result = %d\n", result); return 1; } fprintf (stderr, "after context createv\n"); pami_configuration_t configuration; configuration.name = PAMI_CLIENT_TASK_ID; result = PAMI_Client_query(client, &configuration,1); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable query configuration (%d). result = %d\n", configuration.name, result); return 1; } size_t task_id = configuration.value.intval; fprintf (stderr, "My task id = %zu\n", task_id); configuration.name = PAMI_CLIENT_NUM_TASKS; result = PAMI_Client_query(client, &configuration,1); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable query configuration (%d). result = %d\n", configuration.name, result); return 1; } /*size_t num_tasks = configuration.value.intval; */ size_t dispatch = 0; pami_dispatch_callback_function fn; fn.p2p = test_dispatch; pami_dispatch_hint_t options={}; fprintf (stderr, "Before PAMI_Dispatch_set() .. &recv_active = %p, recv_active = %zu\n", &recv_active, recv_active); result = PAMI_Dispatch_set (context, dispatch, fn, (void *)&recv_active, options); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable register pami dispatch. result = %d\n", result); return 1; } pami_send_t parameters; parameters.send.dispatch = dispatch; parameters.send.header.iov_base = (void *)&dispatch; /* send *something* */ parameters.send.header.iov_len = sizeof(size_t); parameters.send.data.iov_base = (void *)&dispatch; /* send *something* */ parameters.send.data.iov_len = sizeof(size_t); parameters.events.cookie = (void *) &send_active; parameters.events.local_fn = send_done_local; /*parameters.events.remote_fn = send_done_remote; */ parameters.events.remote_fn = NULL; #if 1 int iter; for (iter=0; iter < 100; iter++) { fprintf (stderr, "before send ...\n"); PAMI_Endpoint_create (client, task_id, 0, ¶meters.send.dest); result = PAMI_Send (context, ¶meters); fprintf (stderr, "... after send.\n"); fprintf (stderr, "before send-recv advance loop (send_active = %zu, recv_active = %zu) ...\n", send_active, recv_active); while (send_active || recv_active) { result = PAMI_Context_advance (context, 100); if ( (result != PAMI_SUCCESS) && (result != PAMI_EAGAIN) ) { fprintf (stderr, "Error. Unable to advance pami context. result = %d\n", result); return 1; } } fprintf (stderr, "... after send-recv advance loop\n"); send_active = recv_active = 1; } #endif result = PAMI_Context_destroyv(&context, 1); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to destroy pami context. result = %d\n", result); return 1; } result = PAMI_Client_destroy(&client); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to finalize pami client. result = %d\n", result); return 1; } return 0; };
void test_fn (int argc, char * argv[], pami_client_t client, pami_context_t context[]) { int num_doubles = 16; if (argc > 1) num_doubles = atoi (argv[1]); size_t num_tasks = size (client); pami_task_t my_task_id = task (client); pami_task_t target_task_id = num_tasks - 1; pami_task_t origin_task_id = 0; /* * Allocate a 'window' of memory region information, one for each task in the * client. Only the 'local' memory region information for this task will * contain a valid data buffer. The memory region information is marked * 'active' when the memory regions are received from each remote task. */ memregion_information_t * mr_info = (memregion_information_t *) malloc (sizeof(memregion_information_t) * num_tasks); unsigned i; for (i = 0; i < num_tasks; i++) { mr_info[i].data.iov_len = 0; mr_info[i].data.iov_base = NULL; mr_info[i].active = 0; } /* * Create a local memregion for each context. * * Note that both memregions will describe the same memory location. This is * necessary when writing portable, platform independent code as the physical * hardware underlying the contexts may, or may not, require separate memory * pinning. */ size_t actual_memregion_bytes = 0; mr_info[my_task_id].data.iov_base = malloc (sizeof(double) * num_doubles); mr_info[my_task_id].data.iov_len = sizeof(double) * num_doubles; PAMI_Memregion_create (context[0], mr_info[my_task_id].data.iov_base, mr_info[my_task_id].data.iov_len, & actual_memregion_bytes, & mr_info[my_task_id].memregion[0]); PAMI_Memregion_create (context[1], mr_info[my_task_id].data.iov_base, mr_info[my_task_id].data.iov_len, & actual_memregion_bytes, & mr_info[my_task_id].memregion[1]); mr_info[my_task_id].active = 1; /* * Register the memory region exchange dispatch; only needed on the * first context of each task. */ pami_dispatch_hint_t mr_hint = {0}; pami_dispatch_callback_function mr_dispatch; mr_dispatch.p2p = exchange_memregion_recv_cb; PAMI_Dispatch_set (context[0], MEMREGION_EXCHANGE_DISPATCH_ID, mr_dispatch, (void *) mr_info, mr_hint); accumulate_test_information_t test_info; test_info.data_buffer.iov_base = malloc (sizeof(double) * num_doubles); test_info.data_buffer.iov_len = sizeof(double) * num_doubles; test_info.scalar = 1.2; test_info.data_fn[ACCUMULATE_TEST_SCALAR_SUM] = accumulate_scalar_sum_data_function; test_info.data_cookie[ACCUMULATE_TEST_SCALAR_SUM] = (void *) & test_info.scalar; test_info.data_fn[ACCUMULATE_TEST_VECTOR_SUM] = accumulate_vector_sum_data_function; test_info.data_cookie[ACCUMULATE_TEST_VECTOR_SUM] = malloc (sizeof(double) * num_doubles); test_info.data_fn[ACCUMULATE_TEST_SCALAR_SUBTRACT] = accumulate_scalar_subtract_data_function; test_info.data_cookie[ACCUMULATE_TEST_SCALAR_SUBTRACT] = (void *) & test_info.scalar; test_info.data_fn[ACCUMULATE_TEST_VECTOR_SUBTRACT] = accumulate_vector_subtract_data_function; test_info.data_cookie[ACCUMULATE_TEST_VECTOR_SUBTRACT] = malloc (sizeof(double) * num_doubles); test_info.data_fn[ACCUMULATE_TEST_VECTOR_MAX_SUM] = accumulate_vector_max_sum_data_function; test_info.data_cookie[ACCUMULATE_TEST_VECTOR_MAX_SUM] = malloc (sizeof(double) * num_doubles); test_info.data_fn[ACCUMULATE_TEST_VECTOR_MIN_SUM] = accumulate_vector_min_sum_data_function; test_info.data_cookie[ACCUMULATE_TEST_VECTOR_MIN_SUM] = malloc (sizeof(double) * num_doubles); /* * Register the accumulate dispatch; needed on both * contexts to enable "crosstalk". */ pami_dispatch_hint_t acc_hint = {0}; acc_hint.recv_immediate = PAMI_HINT_DISABLE; pami_dispatch_callback_function acc_dispatch; acc_dispatch.p2p = accumulate_test_recv_cb; PAMI_Dispatch_set (context[0], ACCUMULATE_TEST_DISPATCH_ID, acc_dispatch, (void *) & test_info, acc_hint); PAMI_Dispatch_set (context[1], ACCUMULATE_TEST_DISPATCH_ID, acc_dispatch, (void *) & test_info, acc_hint); simple_barrier(client, context[0]); /* * Exchange the memory regions */ volatile unsigned mr_exchange_active = 0; pami_send_t mr_exchange_parameters = {0}; mr_exchange_parameters.send.dispatch = MEMREGION_EXCHANGE_DISPATCH_ID; mr_exchange_parameters.send.header.iov_base = NULL; mr_exchange_parameters.send.header.iov_len = 0; mr_exchange_parameters.send.data.iov_base = (void *) mr_info[my_task_id].memregion; mr_exchange_parameters.send.data.iov_len = sizeof(pami_memregion_t) * 2; mr_exchange_parameters.events.cookie = (void *) & mr_exchange_active; mr_exchange_parameters.events.local_fn = decrement; for (i = 0; i < num_tasks; i++) { if (i == my_task_id) continue; PAMI_Endpoint_create (client, i, 0, & mr_exchange_parameters.send.dest); mr_exchange_active++; PAMI_Send (context[0], & mr_exchange_parameters); } /* * Advance until local memory regions have been sent and * all memory regions have been received. */ unsigned num_memregions_active; do { num_memregions_active = 0; for (i = 0; i < num_tasks; i++) num_memregions_active += mr_info[i].active; PAMI_Context_advance (context[0], 1); } while (num_memregions_active < num_tasks); while (mr_exchange_active > 0) PAMI_Context_advance (context[0], 1); #ifdef ASYNC_PROGRESS async_progress_t async_progress; async_progress_open (client, &async_progress); async_progress_enable (&async_progress, context[1]); #endif if (my_task_id == target_task_id) { /* * This is the "passive target" task. */ #ifdef ASYNC_PROGRESS /* * Do "something" besides communication for a little bit. */ sleep(1); #else /* * Advance the second context for a little bit. */ fprintf (stdout, "(%03d) spoofing async progress\n", __LINE__); for (i=0; i<10; i++) { fprintf (stdout, "(%03d) 'async progress context' advancing\n", __LINE__); PAMI_Context_advance (context[1], 100000); fprintf (stdout, "(%03d) 'async progress context' sleeping\n", __LINE__); sleep(1); } #endif } else if (my_task_id == origin_task_id) { /* * This is the "active origin" task. */ { /* * Use rdma put to initialize the remote buffer with the local data. */ volatile unsigned rput_active = 1; pami_rput_simple_t rput_parameters = {0}; PAMI_Endpoint_create (client, target_task_id, 1, & rput_parameters.rma.dest); rput_parameters.rma.bytes = num_doubles * sizeof(double); rput_parameters.rdma.local.mr = mr_info[origin_task_id].memregion; rput_parameters.rdma.local.offset = 0; rput_parameters.rdma.remote.mr = mr_info[target_task_id].memregion; rput_parameters.rdma.remote.offset = 0; rput_parameters.put.rdone_fn = decrement; rput_parameters.rma.cookie = (void *) & rput_active; PAMI_Rput (context[0], & rput_parameters); while (rput_active > 0) PAMI_Context_advance (context[0], 1); } { volatile unsigned send_active = 0; accumulate_test_t test_id = ACCUMULATE_TEST_SCALAR_SUM; pami_send_t send_parameters = {0}; PAMI_Endpoint_create (client, target_task_id, 1, & send_parameters.send.dest); send_parameters.send.dispatch = ACCUMULATE_TEST_DISPATCH_ID; send_parameters.send.header.iov_len = sizeof (accumulate_test_t); send_parameters.send.header.iov_base = (void *) & test_id; send_parameters.send.data.iov_base = test_info.data_buffer.iov_base; send_parameters.send.data.iov_len = test_info.data_buffer.iov_len; send_parameters.events.remote_fn = decrement; send_parameters.events.cookie = (void *) & send_active; for (test_id = ACCUMULATE_TEST_SCALAR_SUM; test_id < ACCUMULATE_TEST_COUNT; test_id++) { send_active = 1; fprintf (stdout, "(%03d) sending data buffer for accumulate test \"%s\"\n", __LINE__, accumulate_test_name[test_id]); PAMI_Send (context[0], & send_parameters); fprintf (stdout, "(%03d) waiting for remote completion of data buffer sent for accumulate test \"%s\"\n", __LINE__, accumulate_test_name[test_id]); while (send_active > 0) PAMI_Context_advance (context[0], 1); fprintf (stdout, "(%03d) data buffer received on remote for accumulate test \"%s\"\n", __LINE__, accumulate_test_name[test_id]); } } { /* * Use rdma get to retrieve the remote buffer and compare results. */ volatile unsigned rget_active = 1; pami_rget_simple_t rget_parameters = {0}; PAMI_Endpoint_create (client, target_task_id, 1, & rget_parameters.rma.dest); rget_parameters.rma.done_fn = decrement; rget_parameters.rma.cookie = (void *) & rget_active; rget_parameters.rma.bytes = sizeof(double) * num_doubles; rget_parameters.rdma.local.mr = mr_info[origin_task_id].memregion; rget_parameters.rdma.local.offset = 0; rget_parameters.rdma.remote.mr = mr_info[target_task_id].memregion; rget_parameters.rdma.remote.offset = 0; PAMI_Rget (context[0], & rget_parameters); while (rget_active > 0) PAMI_Context_advance (context[0], 1); } } else { /* * All other tasks, if any, do nothing and simply enter the barrier. */ } simple_barrier (client, context[0]); #ifdef ASYNC_PROGRESS async_progress_disable (&async_progress, context[1]); async_progress_close (&async_progress); #endif /* * Do cleanup ? */ return; }
int main (int argc, char ** argv) { pami_client_t client; pami_context_t context; pami_configuration_t * configuration = NULL; char cl_string[] = "TEST"; pami_result_t result = PAMI_ERROR; result = PAMI_Client_create (cl_string, &client, NULL, 0); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to initialize pami client. result = %d\n", result); return 1; } pami_task_t task = PAMIX_Client_task (client); size_t size = PAMIX_Client_size (client); result = PAMI_Context_createv (client, configuration, 0, &context, 1); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to create pami context. result = %d\n", result); return 1; } /* Attempt to send using a dispatch id that has not been registered. */ char metadata[1024]; char buffer[1024]; volatile unsigned recv_active = 1; volatile unsigned send_active = 1; pami_send_t parameters; parameters.send.dispatch = 10; parameters.send.header.iov_base = metadata; parameters.send.header.iov_len = 8; parameters.send.data.iov_base = buffer; parameters.send.data.iov_len = 8; parameters.events.cookie = (void *) & send_active; parameters.events.local_fn = decrement; parameters.events.remote_fn = NULL; PAMI_Endpoint_create (client, size-1, 0, ¶meters.send.dest); if (task == 0) { result = PAMI_Send (context, ¶meters); if (result == PAMI_SUCCESS) { fprintf (stderr, "Test failure. Expected error when using an unregistered dispatch id.\n"); return 1; } } size_t dispatch = 10; pami_dispatch_callback_function fn; fn.p2p = test_dispatch; pami_dispatch_hint_t options = {}; result = PAMI_Dispatch_set (context, dispatch, fn, (void *) & recv_active, options); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable register pami dispatch. result = %d\n", result); return 1; } if (task == 0) { PAMI_Endpoint_create (client, size-1, 0, ¶meters.send.dest); result = PAMI_Send (context, ¶meters); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to send using a registered dispatch id.\n"); return 1; } while (send_active) { result = PAMI_Context_advance (context, 1000); if ( (result != PAMI_SUCCESS) && (result != PAMI_EAGAIN) ) { fprintf (stderr, "Error. Unable to advance pami context. result = %d\n", result); return 1; } } while (recv_active) { result = PAMI_Context_advance (context, 1000); if ( (result != PAMI_SUCCESS) && (result != PAMI_EAGAIN) ) { fprintf (stderr, "Error. Unable to advance pami context. result = %d\n", result); return 1; } } } else if (task == (size-1)) { PAMI_Endpoint_create (client, 0, 0, ¶meters.send.dest); while (recv_active) { result = PAMI_Context_advance (context, 1000); if ( (result != PAMI_SUCCESS) && (result != PAMI_EAGAIN) ) { fprintf (stderr, "Error. Unable to advance pami context. result = %d\n", result); return 1; } } result = PAMI_Send (context, ¶meters); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to send using a registered dispatch id.\n"); return 1; } while (send_active) { result = PAMI_Context_advance (context, 1000); if ( (result != PAMI_SUCCESS) && (result != PAMI_EAGAIN) ) { fprintf (stderr, "Error. Unable to advance pami context. result = %d\n", result); return 1; } } } result = PAMI_Context_destroyv(&context, 1); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to destroy pami context. result = %d\n", result); return 1; } result = PAMI_Client_destroy(&client); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to finalize pami client. result = %d\n", result); return 1; } return 0; };
int main (int argc, char ** argv) { pami_client_t client; pami_context_t context; pami_task_t task; size_t size; pami_dispatch_callback_function fn; pami_dispatch_hint_t options; pami_result_t result = PAMI_ERROR; /* ====== INITIALIZE ====== */ result = PAMI_Client_create ("TEST", &client, NULL, 0); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to initialize pami client. result = %d\n", result); return 1; } task = client_task (client); size = client_size (client); result = PAMI_Context_createv (client, NULL, 0, &context, 1); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to create pami context. result = %d\n", result); return 1; } fn.p2p = dispatch_fn; options.recv_immediate = PAMI_HINT_DEFAULT; result = PAMI_Dispatch_set (context, DISPATCH_ID_DEFAULT_EXPECT_IMMEDIATE, fn, (void *) EXPECT_IMMEDIATE, options); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to register DISPATCH_ID_DEFAULT_EXPECT_IMMEDIATE. result = %d\n", result); return 1; } options.recv_immediate = PAMI_HINT_DEFAULT; result = PAMI_Dispatch_set (context, DISPATCH_ID_DEFAULT_EXPECT_ASYNC, fn, (void *) EXPECT_ASYNC, options); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to register DISPATCH_ID_DEFAULT_EXPECT_ASYNC. result = %d\n", result); return 1; } options.recv_immediate = PAMI_HINT_ENABLE; result = PAMI_Dispatch_set (context, DISPATCH_ID_ENABLE, fn, (void *) EXPECT_IMMEDIATE, options); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to register DISPATCH_ID_ENABLE. result = %d\n", result); return 1; } options.recv_immediate = PAMI_HINT_DISABLE; result = PAMI_Dispatch_set (context, DISPATCH_ID_DISABLE, fn, (void *) EXPECT_ASYNC, options); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to register DISPATCH_ID_DISABLE. result = %d\n", result); return 1; } /* ====== START TEST ====== */ __test_errors = 0; __test_recvs = 0; size_t test_count = 0; volatile size_t send_active = 0; pami_send_t parameters; parameters.send.header.iov_base = __junk; parameters.send.header.iov_len = 0; parameters.send.data.iov_base = __junk; parameters.send.data.iov_len = 0; parameters.send.dispatch = 0; parameters.events.cookie = (void *) & send_active; parameters.events.local_fn = decrement; parameters.events.remote_fn = NULL; result = PAMI_Endpoint_create (client, 0, 0, ¶meters.send.dest); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error: PAMI_Endpoint_create() failed for task 0, context 0 with %d.\n", result); return 1; } /* =================================================================== * 'recv_immediate' default * * (header+data) > recv_immediate_max MUST be an asynchronous receive * * A zero-byte send will \b always result in an immediate receive. * \see pami_dispatch_p2p_function * * Data sizes to test: * - recv_immediate_max + 1 * - 0 */ parameters.send.data.iov_len = recv_immediate_max (context, DISPATCH_ID_DEFAULT_EXPECT_ASYNC) + 1; parameters.send.dispatch = DISPATCH_ID_DEFAULT_EXPECT_ASYNC; test_count++; if (task == 1) { send_active++; result = PAMI_Send (context, ¶meters); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error: Unable to send to 0x%08x using dispatch %zu with %d.\n", parameters.send.dest, parameters.send.dispatch, result); return 1; } } parameters.send.data.iov_len = 0; parameters.send.dispatch = DISPATCH_ID_DEFAULT_EXPECT_IMMEDIATE; test_count++; if (task == 1) { send_active++; result = PAMI_Send (context, ¶meters); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error: Unable to send to 0x%08x using dispatch %zu with %d.\n", parameters.send.dest, parameters.send.dispatch, result); return 1; } } /* =================================================================== * 'recv_immediate' enabled * * All receives are 'immediate'. (header+data) > recv_immediate_max is * invalid, but may not neccesarily return an error. * * Data sizes to test: * - 0 * - recv_immediate_max * - recv_immediate_max + 1 ...... ? */ parameters.send.data.iov_len = 0; parameters.send.dispatch = DISPATCH_ID_ENABLE; test_count++; if (task == 1) { send_active++; result = PAMI_Send (context, ¶meters); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error: Unable to send to 0x%08x using dispatch %zu with %d.\n", parameters.send.dest, parameters.send.dispatch, result); return 1; } } parameters.send.data.iov_len = recv_immediate_max (context, DISPATCH_ID_DEFAULT_EXPECT_ASYNC); parameters.send.dispatch = DISPATCH_ID_ENABLE; test_count++; if (task == 1) { send_active++; result = PAMI_Send (context, ¶meters); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error: Unable to send to 0x%08x using dispatch %zu with %d.\n", parameters.send.dest, parameters.send.dispatch, result); return 1; } } #if 0 parameters.send.data.iov_len = recv_immediate_max (context, DISPATCH_ID_DEFAULT_EXPECT_ASYNC) + 1; parameters.send.dispatch = DISPATCH_ID_ENABLE; test_count++; if (task == 1) { send_active++; result = PAMI_Send (context, ¶meters); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error: Unable to send to 0x%08x using dispatch %d with %d.\n", parameters.send.dest, parameters.send.dispatch, result); return 1; } } #endif /* =================================================================== * 'recv_immediate' disabled * * All receives are NOT 'immediate' - even "zero byte data" * * Data sizes to test: * - 0 * - recv_immediate_max * - recv_immediate_max + 1 */ parameters.send.data.iov_len = 0; parameters.send.dispatch = DISPATCH_ID_DISABLE; test_count++; if (task == 1) { send_active++; result = PAMI_Send (context, ¶meters); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error: Unable to send to 0x%08x using dispatch %zu with %d.\n", parameters.send.dest, parameters.send.dispatch, result); return 1; } } parameters.send.data.iov_len = recv_immediate_max (context, DISPATCH_ID_DEFAULT_EXPECT_ASYNC); parameters.send.dispatch = DISPATCH_ID_DISABLE; test_count++; if (task == 1) { send_active++; result = PAMI_Send (context, ¶meters); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error: Unable to send to 0x%08x using dispatch %zu with %d.\n", parameters.send.dest, parameters.send.dispatch, result); return 1; } } parameters.send.data.iov_len = recv_immediate_max (context, DISPATCH_ID_DEFAULT_EXPECT_ASYNC) + 1; parameters.send.dispatch = DISPATCH_ID_DISABLE; test_count++; if (task == 1) { send_active++; result = PAMI_Send (context, ¶meters); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error: Unable to send to 0x%08x using dispatch %zu with %d.\n", parameters.send.dest, parameters.send.dispatch, result); return 1; } } /* ====== WAIT FOR COMMUNICATION COMPLETION ====== */ if (task == 0) { while (__test_recvs < test_count) PAMI_Context_advance (context, 1000); } else if (task == 1) { while (send_active) PAMI_Context_advance (context, 1000); } /* ====== CLEANUP ====== */ result = PAMI_Context_destroyv (&context, 1); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to destroy context, result = %d\n", result); return 1; } result = PAMI_Client_destroy (&client); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to destroy pami client. result = %d\n", result); return 1; } /* ====== REPORT ERRORS ====== */ if (__test_errors > 0) { fprintf (stderr, "Error. Non-compliant PAMI receive immediate implementation! error count = %zu\n", __test_errors); return 1; } return 0; }
int main(int argc, char* argv[]) { pami_result_t result = PAMI_ERROR; /* initialize the second client */ char * clientname = ""; pami_client_t client; result = PAMI_Client_create(clientname, &client, NULL, 0); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_create"); /* query properties of the client */ pami_configuration_t config[3]; size_t num_contexts; config[0].name = PAMI_CLIENT_NUM_TASKS; config[1].name = PAMI_CLIENT_TASK_ID; config[2].name = PAMI_CLIENT_NUM_CONTEXTS; result = PAMI_Client_query(client, config, 3); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_query"); world_size = config[0].value.intval; world_rank = config[1].value.intval; num_contexts = config[2].value.intval; TEST_ASSERT(num_contexts>1,"num_contexts>1"); if (world_rank==0) { printf("hello world from rank %ld of %ld \n", world_rank, world_size ); fflush(stdout); } /* initialize the contexts */ contexts = (pami_context_t *) safemalloc( num_contexts * sizeof(pami_context_t) ); result = PAMI_Context_createv( client, NULL, 0, contexts, num_contexts ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_createv"); /* setup the world geometry */ pami_geometry_t world_geometry; result = PAMI_Geometry_world(client, &world_geometry ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Geometry_world"); int status = pthread_create(&Progress_thread, NULL, &Progress_function, NULL); TEST_ASSERT(status==0, "pthread_create"); /************************************************************************/ int n = (argc>1 ? atoi(argv[1]) : 1000); size_t bytes = n * sizeof(int); int * local = (int *) safemalloc(bytes); for (int i=0; i<n; i++) local[i] = world_rank; /* create the endpoint */ int target = (world_rank>0 ? world_rank-1 : world_size-1); pami_endpoint_t target_ep; result = PAMI_Endpoint_create(client, (pami_task_t) target, 1, &target_ep); //result = PAMI_Endpoint_create(client, (pami_task_t) target, 0, &target_ep); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Endpoint_create"); /* register the dispatch function */ pami_dispatch_callback_function dispatch_cb; size_t dispatch_id = 37; dispatch_cb.p2p = dispatch_recv_cb; pami_dispatch_hint_t dispatch_hint = {0}; int dispatch_cookie = 1000000+world_rank; //dispatch_hint.recv_immediate = PAMI_HINT_DISABLE; result = PAMI_Dispatch_set(contexts[0], dispatch_id, dispatch_cb, &dispatch_cookie, dispatch_hint); result = PAMI_Dispatch_set(contexts[1], dispatch_id, dispatch_cb, &dispatch_cookie, dispatch_hint); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Dispatch_set"); /* make sure everything is ready */ result = barrier(world_geometry, contexts[0]); TEST_ASSERT(result == PAMI_SUCCESS,"barrier"); // The iovec structure describes a buffer. It contains two fields: // void *iov_base - Contains the address of a buffer. // size_t iov_len - Contains the length of the buffer. int header = 37373; int active = 1; pami_send_t parameters; parameters.send.header.iov_base = &header; parameters.send.header.iov_len = sizeof(int); parameters.send.data.iov_base = local; parameters.send.data.iov_len = bytes; parameters.send.dispatch = dispatch_id; //parameters.send.hints = ; parameters.send.dest = target_ep; parameters.events.cookie = &active; parameters.events.local_fn = cb_done; parameters.events.remote_fn = NULL;//cb_done; uint64_t t0 = GetTimeBase(); result = PAMI_Send(contexts[0], ¶meters); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Send"); while (active) { //result = PAMI_Context_advance( contexts[0], 100); //TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_advance"); result = PAMI_Context_trylock_advancev(&(contexts[0]), 1, 1000); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_trylock_advancev"); } uint64_t t1 = GetTimeBase(); uint64_t dt = t1-t0; /* barrier on non-progressing context to make sure CHT does its job */ barrier(world_geometry, contexts[0]); printf("%ld: PAMI_Send of %ld bytes achieves %lf MB/s \n", (long)world_rank, bytes, 1.6e9*1e-6*(double)bytes/(double)dt ); fflush(stdout); result = barrier(world_geometry, contexts[0]); TEST_ASSERT(result == PAMI_SUCCESS,"barrier"); free(local); /************************************************************************/ void * rv; status = pthread_cancel(Progress_thread); TEST_ASSERT(status==0, "pthread_cancel"); status = pthread_join(Progress_thread, &rv); TEST_ASSERT(status==0, "pthread_join"); result = barrier(world_geometry, contexts[0]); TEST_ASSERT(result == PAMI_SUCCESS,"barrier"); /* finalize the contexts */ result = PAMI_Context_destroyv( contexts, num_contexts ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_destroyv"); free(contexts); /* finalize the client */ result = PAMI_Client_destroy( &client ); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_destroy"); if (world_rank==0) printf("%ld: end of test \n", world_rank ); fflush(stdout); return 0; }