void exchange_memregion_recv_cb (pami_context_t context, void * cookie, const void * header_addr, size_t header_size, const void * pipe_addr, size_t data_size, pami_endpoint_t origin, pami_recv_t * recv) { memregion_information_t * info = (memregion_information_t *) cookie; pami_task_t origin_task; size_t origin_offset; PAMI_Endpoint_query (origin, & origin_task, & origin_offset); pami_memregion_t * memregion = (pami_memregion_t *) pipe_addr; memcpy ((void *) & info[origin_task].memregion[0], (void *) & memregion[0], sizeof(pami_memregion_t)); memcpy ((void *) & info[origin_task].memregion[1], (void *) & memregion[1], sizeof(pami_memregion_t)); info[origin_task].active = 1; return; }
void dispatch_recv_cb(pami_context_t context, void * cookie, const void * header_addr, size_t header_size, const void * pipe_addr, size_t data_size, pami_endpoint_t origin, pami_recv_t * recv) { pami_result_t result = PAMI_ERROR; size_t task; size_t ctxoff; result = PAMI_Endpoint_query(origin, &task, &ctxoff); TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Endpoint_query"); int * c = (int *) cookie; printf("dispatch_recv_cb: origin = (%ld,%ld) cookie = %d \n", task, ctxoff, *c); printf("dispatch_recv_cb: header_size = %ld \n", header_size); //const char * h = header_addr; //printf("dispatch_recv_cb: header_addr[] = %s \n", h); const int * h = header_addr; printf("dispatch_recv_cb: header_addr[] = %d \n", *h); fflush(stdout); if (pipe_addr!=NULL) { printf("dispatch_recv_cb: immediate protocol \n"); fflush(stdout); printf("dispatch_recv_cb: data_size = %ld \n", data_size); const char * p = pipe_addr; printf("dispatch_recv_cb: pipe_addr[] = %s \n", p); fflush(stdout); } else { printf("dispatch_recv_cb: asychronous protocol \n"); printf("dispatch_recv_cb: data_size = %ld \n", data_size); fflush(stdout); char * buffer = safemalloc(data_size); recv->cookie = buffer; recv->local_fn = dispatch_done_cb; recv->addr = buffer; recv->type = PAMI_TYPE_BYTE; recv->offset = 0; recv->data_fn = PAMI_DATA_COPY; recv->data_cookie = NULL; } return; }
/** * User dispatch function */ void cb_amreduce_send(pami_context_t context, /**< IN: communication context which invoked the dispatch function */ void * cookie, /**< IN: dispatch cookie */ const void * header_addr, /**< IN: header address */ size_t header_size, /**< IN: header size */ size_t data_size, /**< IN: data size */ pami_endpoint_t origin, /**< IN: root initiating endpoint */ pami_geometry_t geometry, /**< IN: Geometry */ pami_data_function * op, /**< OUT: PAMI math operation to perform on the datatype */ pami_recv_t * send) /**< OUT: receive message structure */ { user_header_t *hdr; hdr = (user_header_t *) header_addr; DEBUG((stderr,"cb_amreduce_send(): header_addr=%p header_size=%zu op=%d dt=%d cookie=%p\n", header_addr, header_size, hdr->op, hdr->dt, cookie)); if(gVerbose) { check_context(context); } pami_task_t task; size_t offset; _gRc |= PAMI_Endpoint_query (origin, &task, &offset); validation_t *v = _g_val_buffer + task; v->cookie = cookie; v->bytes = data_size; v->op = hdr->op; v->dt = hdr->dt; v->root = task; send->cookie = (void*)v; send->local_fn = cb_amreduce_done; send->addr = _g_send_buffer; send->type = dt_array[hdr->dt]; send->offset = 0; send->data_fn = PAMI_DATA_COPY; send->data_cookie = NULL; *op = op_array[hdr->op]; }
static void dispatch_exchange ( pami_context_t context, /**< IN: origin context */ void * cookie, /**< IN: dispatch cookie */ const void * header_addr, /**< IN: header address */ size_t header_size, /**< IN: header size */ const void * pipe_addr, /**< IN: address of PAMI pipe buffer */ size_t pipe_size, /**< IN: size of PAMI pipe buffer */ pami_endpoint_t origin, /**< IN: endpoint that originated the send */ pami_recv_t * recv) /**< OUT: receive message structure */ { info_t * exchange = (info_t *) cookie; pami_task_t task; size_t offset; PAMI_Endpoint_query (origin, &task, &offset); memcpy (&exchange[task].mr, pipe_addr, pipe_size); exchange[task].active = 1; return; }
/** * User dispatch function */ void cb_amscatter_recv(pami_context_t context, /**< IN: communication context which invoked the dispatch function */ void * cookie, /**< IN: dispatch cookie */ const void * header_addr, /**< IN: header address */ size_t header_size, /**< IN: header size */ const void * pipe_addr, /**< IN: address of PAMI pipe buffer, valid only if non-NULL */ size_t data_size, /**< IN: data size */ pami_endpoint_t origin, /**< IN: root initiating endpoint */ pami_geometry_t geometry, /**< IN: Geometry */ pami_recv_t * recv) /**< OUT: receive message structure */ { DEBUG((stderr,"cb_amscatter_recv(): header_addr=%p header_size=%zu dst_rank=%u cookie=%p\n", header_addr, header_size,((user_header_t *)header_addr)->dst_rank, cookie)); if(gVerbose) { check_context(context); } pami_task_t task; size_t offset; _gRc |= PAMI_Endpoint_query (origin, &task, &offset); validation_t *v = _g_val_buffer + task; v->rbuf = _g_recv_buffer; v->cookie = cookie; v->bytes = data_size; v->root = task; recv->cookie = (void*)v; recv->local_fn = cb_amscatter_done; recv->addr = v->rbuf; recv->type = PAMI_TYPE_BYTE; recv->offset = 0; recv->data_fn = PAMI_DATA_COPY; recv->data_cookie = NULL; }
int main () { pami_client_t client; pami_context_t *context; pami_result_t result; pami_configuration_t configuration; PAMI_Client_create ("TEST", &client, NULL, 0); configuration.name = PAMI_CLIENT_NUM_CONTEXTS; result = PAMI_Client_query(client, &configuration, 1); size_t num = configuration.value.intval; context = (pami_context_t*) malloc (num*sizeof(pami_context_t)); /* Create four contexts - every task creates the same number */ PAMI_Context_createv (client, NULL, 0, context, num); createEndpointTable (client, num); pami_dispatch_callback_function fn; fn.p2p = test_dispatch; pami_dispatch_hint_t options = {0}; pami_send_hint_t hints = {0}; volatile size_t expect = 0; size_t i; for (i=0; i<num; i++) { PAMI_Context_lock (context[i]); result = PAMI_Dispatch_set (context[i], 0, fn, (void *)&expect, options); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable register pami dispatch. result = %d\n", result); return 1; } } configuration.name = PAMI_CLIENT_TASK_ID; result = PAMI_Client_query(client, &configuration, 1); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable query configuration (%d). result = %d\n", configuration.name, result); return 1; } pami_task_t task_id = configuration.value.intval; fprintf (stderr, "My task id = %d\n", task_id); configuration.name = PAMI_CLIENT_NUM_TASKS; result = PAMI_Client_query(client, &configuration, 1); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable query configuration (%d). result = %d\n", configuration.name, result); return 1; } size_t num_tasks = configuration.value.intval; fprintf (stderr, "Number of tasks = %zu\n", num_tasks); uint8_t header[16]; uint8_t data[1024]; volatile size_t active = 1; pami_send_t parameters; parameters.send.dispatch = 0; parameters.send.header.iov_base = header; parameters.send.header.iov_len = 16; parameters.send.data.iov_base = data; parameters.send.data.iov_len = 1024; parameters.send.hints = hints; parameters.events.cookie = (void *) &active; parameters.events.local_fn = decrement; parameters.events.remote_fn = NULL; /* Send a message to endpoint "num_tasks * num_contexts - 1" */ pami_task_t target_task = (pami_task_t) -1; size_t target_offset = (size_t) -1; PAMI_Endpoint_query (_endpoint[num*num_tasks-1], &target_task, &target_offset); if (task_id == target_task) expect += num_tasks; send_endpoint (context[0], num*num_tasks-1, ¶meters); fprintf (stdout, "before advance, active = %zu, expect = %zu\n", active, expect); while ((active + expect) > 0) PAMI_Context_advancev (context, num, 100); for (i=0; i<num; i++) PAMI_Context_unlock (context[i]); result = PAMI_Context_destroyv (context, num); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to destroy pami context. result = %d\n", result); return 1; } result = PAMI_Client_destroy (&client); if (result != PAMI_SUCCESS) { fprintf (stderr, "Error. Unable to finalize pami client. result = %d\n", result); return 1; } fprintf (stdout, "Success (%d)\n", task_id); return 0; };