void exchange_memregion_recv_cb (pami_context_t context,
void * cookie,
const void * header_addr,
size_t header_size,
const void * pipe_addr,
size_t data_size,
pami_endpoint_t origin,
pami_recv_t * recv)
{
memregion_information_t * info = (memregion_information_t *) cookie;
pami_task_t origin_task;
size_t origin_offset;
PAMI_Endpoint_query (origin, & origin_task, & origin_offset);
pami_memregion_t * memregion = (pami_memregion_t *) pipe_addr;
memcpy ((void *) & info[origin_task].memregion[0], (void *) & memregion[0], sizeof(pami_memregion_t));
memcpy ((void *) & info[origin_task].memregion[1], (void *) & memregion[1], sizeof(pami_memregion_t));
info[origin_task].active = 1;
return;
}
void dispatch_recv_cb(pami_context_t context,
void * cookie,
const void * header_addr, size_t header_size,
const void * pipe_addr,
size_t data_size,
pami_endpoint_t origin,
pami_recv_t * recv)
{
pami_result_t result = PAMI_ERROR;
size_t task;
size_t ctxoff;
result = PAMI_Endpoint_query(origin, &task, &ctxoff);
TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Endpoint_query");
int * c = (int *) cookie;
printf("dispatch_recv_cb: origin = (%ld,%ld) cookie = %d \n", task, ctxoff, *c);
printf("dispatch_recv_cb: header_size = %ld \n", header_size);
//const char * h = header_addr;
//printf("dispatch_recv_cb: header_addr[] = %s \n", h);
const int * h = header_addr;
printf("dispatch_recv_cb: header_addr[] = %d \n", *h);
fflush(stdout);
if (pipe_addr!=NULL)
{
printf("dispatch_recv_cb: immediate protocol \n");
fflush(stdout);
printf("dispatch_recv_cb: data_size = %ld \n", data_size);
const char * p = pipe_addr;
printf("dispatch_recv_cb: pipe_addr[] = %s \n", p);
fflush(stdout);
}
else
{
printf("dispatch_recv_cb: asychronous protocol \n");
printf("dispatch_recv_cb: data_size = %ld \n", data_size);
fflush(stdout);
char * buffer = safemalloc(data_size);
recv->cookie = buffer;
recv->local_fn = dispatch_done_cb;
recv->addr = buffer;
recv->type = PAMI_TYPE_BYTE;
recv->offset = 0;
recv->data_fn = PAMI_DATA_COPY;
recv->data_cookie = NULL;
}
return;
}
/**
* User dispatch function
*/
void cb_amreduce_send(pami_context_t context, /**< IN: communication context which invoked the dispatch function */
void * cookie, /**< IN: dispatch cookie */
const void * header_addr, /**< IN: header address */
size_t header_size, /**< IN: header size */
size_t data_size, /**< IN: data size */
pami_endpoint_t origin, /**< IN: root initiating endpoint */
pami_geometry_t geometry, /**< IN: Geometry */
pami_data_function * op, /**< OUT: PAMI math operation to perform on the datatype */
pami_recv_t * send) /**< OUT: receive message structure */
{
user_header_t *hdr;
hdr = (user_header_t *) header_addr;
DEBUG((stderr,"cb_amreduce_send(): header_addr=%p header_size=%zu op=%d dt=%d cookie=%p\n",
header_addr, header_size, hdr->op, hdr->dt, cookie));
if(gVerbose)
{
check_context(context);
}
pami_task_t task;
size_t offset;
_gRc |= PAMI_Endpoint_query (origin,
&task,
&offset);
validation_t *v = _g_val_buffer + task;
v->cookie = cookie;
v->bytes = data_size;
v->op = hdr->op;
v->dt = hdr->dt;
v->root = task;
send->cookie = (void*)v;
send->local_fn = cb_amreduce_done;
send->addr = _g_send_buffer;
send->type = dt_array[hdr->dt];
send->offset = 0;
send->data_fn = PAMI_DATA_COPY;
send->data_cookie = NULL;
*op = op_array[hdr->op];
}
static void dispatch_exchange (
pami_context_t context, /**< IN: origin context */
void * cookie, /**< IN: dispatch cookie */
const void * header_addr, /**< IN: header address */
size_t header_size, /**< IN: header size */
const void * pipe_addr, /**< IN: address of PAMI pipe buffer */
size_t pipe_size, /**< IN: size of PAMI pipe buffer */
pami_endpoint_t origin, /**< IN: endpoint that originated the send */
pami_recv_t * recv) /**< OUT: receive message structure */
{
info_t * exchange = (info_t *) cookie;
pami_task_t task;
size_t offset;
PAMI_Endpoint_query (origin, &task, &offset);
memcpy (&exchange[task].mr, pipe_addr, pipe_size);
exchange[task].active = 1;
return;
}
/**
* User dispatch function
*/
void cb_amscatter_recv(pami_context_t context, /**< IN: communication context which invoked the dispatch function */
void * cookie, /**< IN: dispatch cookie */
const void * header_addr, /**< IN: header address */
size_t header_size, /**< IN: header size */
const void * pipe_addr, /**< IN: address of PAMI pipe buffer, valid only if non-NULL */
size_t data_size, /**< IN: data size */
pami_endpoint_t origin, /**< IN: root initiating endpoint */
pami_geometry_t geometry, /**< IN: Geometry */
pami_recv_t * recv) /**< OUT: receive message structure */
{
DEBUG((stderr,"cb_amscatter_recv(): header_addr=%p header_size=%zu dst_rank=%u cookie=%p\n",
header_addr, header_size,((user_header_t *)header_addr)->dst_rank, cookie));
if(gVerbose)
{
check_context(context);
}
pami_task_t task;
size_t offset;
_gRc |= PAMI_Endpoint_query (origin,
&task,
&offset);
validation_t *v = _g_val_buffer + task;
v->rbuf = _g_recv_buffer;
v->cookie = cookie;
v->bytes = data_size;
v->root = task;
recv->cookie = (void*)v;
recv->local_fn = cb_amscatter_done;
recv->addr = v->rbuf;
recv->type = PAMI_TYPE_BYTE;
recv->offset = 0;
recv->data_fn = PAMI_DATA_COPY;
recv->data_cookie = NULL;
}
int main ()
{
pami_client_t client;
pami_context_t *context;
pami_result_t result;
pami_configuration_t configuration;
PAMI_Client_create ("TEST", &client, NULL, 0);
configuration.name = PAMI_CLIENT_NUM_CONTEXTS;
result = PAMI_Client_query(client, &configuration, 1);
size_t num = configuration.value.intval;
context = (pami_context_t*) malloc (num*sizeof(pami_context_t));
/* Create four contexts - every task creates the same number */
PAMI_Context_createv (client, NULL, 0, context, num);
createEndpointTable (client, num);
pami_dispatch_callback_function fn;
fn.p2p = test_dispatch;
pami_dispatch_hint_t options = {0};
pami_send_hint_t hints = {0};
volatile size_t expect = 0;
size_t i;
for (i=0; i<num; i++)
{
PAMI_Context_lock (context[i]);
result = PAMI_Dispatch_set (context[i], 0, fn, (void *)&expect, options);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable register pami dispatch. result = %d\n", result);
return 1;
}
}
configuration.name = PAMI_CLIENT_TASK_ID;
result = PAMI_Client_query(client, &configuration, 1);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable query configuration (%d). result = %d\n", configuration.name, result);
return 1;
}
pami_task_t task_id = configuration.value.intval;
fprintf (stderr, "My task id = %d\n", task_id);
configuration.name = PAMI_CLIENT_NUM_TASKS;
result = PAMI_Client_query(client, &configuration, 1);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable query configuration (%d). result = %d\n", configuration.name, result);
return 1;
}
size_t num_tasks = configuration.value.intval;
fprintf (stderr, "Number of tasks = %zu\n", num_tasks);
uint8_t header[16];
uint8_t data[1024];
volatile size_t active = 1;
pami_send_t parameters;
parameters.send.dispatch = 0;
parameters.send.header.iov_base = header;
parameters.send.header.iov_len = 16;
parameters.send.data.iov_base = data;
parameters.send.data.iov_len = 1024;
parameters.send.hints = hints;
parameters.events.cookie = (void *) &active;
parameters.events.local_fn = decrement;
parameters.events.remote_fn = NULL;
/* Send a message to endpoint "num_tasks * num_contexts - 1" */
pami_task_t target_task = (pami_task_t) -1;
size_t target_offset = (size_t) -1;
PAMI_Endpoint_query (_endpoint[num*num_tasks-1], &target_task, &target_offset);
if (task_id == target_task) expect += num_tasks;
send_endpoint (context[0], num*num_tasks-1, ¶meters);
fprintf (stdout, "before advance, active = %zu, expect = %zu\n", active, expect);
while ((active + expect) > 0) PAMI_Context_advancev (context, num, 100);
for (i=0; i<num; i++) PAMI_Context_unlock (context[i]);
result = PAMI_Context_destroyv (context, num);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable to destroy pami context. result = %d\n", result);
return 1;
}
result = PAMI_Client_destroy (&client);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable to finalize pami client. result = %d\n", result);
return 1;
}
fprintf (stdout, "Success (%d)\n", task_id);
return 0;
};