/**
* This does the actual short-send using PAMI_Send_immediate. This
* will cause the data to be injected immediately, avoiding call-backs
* and allowing us to declare more info on the stack. We can mark it
* done at the end.
*/
static pami_result_t SendShortHandoff(pami_context_t context,
void * cookie)
{
assert(cookie == NULL);
unsigned quad[] = {(unsigned)task, 0x11, 0x22, 0x33};
pami_task_t remote_task = 1-task;
size_t remote_context = (task+SHORT_DISPATCH)&(num_contexts-1);
pami_endpoint_t dest;
PAMI_Endpoint_create(client, remote_task, remote_context, &dest);
pami_send_immediate_t parameters = { {0,0}, {0,0}, 0 };
parameters.dispatch = SHORT_DISPATCH;
/*parameters.hints = {0}; */
parameters.dest = dest;
parameters.header.iov_base = quad;
parameters.header.iov_len = sizeof(quad);
parameters.data.iov_base = sbuf;
parameters.data.iov_len = SSIZE;
PAMI_Send_immediate(context, ¶meters);
printf("Task=%zu Channel=%p <Sent short msg> data=%x\n", task, context, sbuf[0]);
done.sshort.send = 1;
return PAMI_SUCCESS;
}
int optiq_pami_transport_actual_send(struct optiq_transport *self, struct optiq_message *message)
{
#ifdef __bgq__
pami_result_t result;
struct optiq_pami_transport *pami_transport = (struct optiq_pami_transport *)optiq_transport_get_concrete_transport(self);
struct optiq_send_cookie *send_cookie = optiq_pami_transport_get_send_cookie(pami_transport);
send_cookie->message = message;
if (message->length <= MAX_SHORT_MESSAGE_LENGTH) {
pami_send_immediate_t parameter;
parameter.dispatch = RECV_MESSAGE_DISPATCH_ID;
parameter.header.iov_base = (void *)&message->header;
parameter.header.iov_len = sizeof(struct optiq_message_header);
parameter.data.iov_base = (void *)message->buffer;
parameter.data.iov_len = message->length;
parameter.dest = pami_transport->endpoints[message->next_dest];
result = PAMI_Send_immediate (pami_transport->context, ¶meter);
assert(result == PAMI_SUCCESS);
if (result != PAMI_SUCCESS) {
return 1;
}
/*Add the cookie to the vector of in-use send cookies*/
pami_transport->in_use_send_cookies.push_back(send_cookie);
} else {
pami_send_t param_send;
param_send.send.dest = message->next_dest;
param_send.send.dispatch = RECV_MESSAGE_DISPATCH_ID;
param_send.send.header.iov_base = (void *)&message->header;
param_send.send.header.iov_len = sizeof(struct optiq_message_header);
param_send.send.data.iov_base = (void *)message->buffer;
param_send.send.data.iov_len = message->length;
param_send.events.cookie = (void *)send_cookie;
param_send.events.local_fn = optiq_send_done_fn;
param_send.events.remote_fn = NULL;
result = PAMI_Send(pami_transport->context, ¶m_send);
assert(result == PAMI_SUCCESS);
if (result != PAMI_SUCCESS) {
return 1;
}
}
#ifdef DEBUG
printf("Rank %d is sending data of size %d to Rank %d with flow_id = %d, original_offset = %d\n", self->rank, message->length, message->next_dest, message->header.flow_id, message->header.original_offset);
#endif
#endif
return 0;
}
static void get_done (pami_context_t context,
void * cookie,
pami_result_t result)
{
get_info_t * info = (get_info_t *) cookie;
fprintf (stderr, ">> 'get_done' callback, cookie = %p (info->value = %zu => %zu), result = %d\n", cookie, *(info->value), *(info->value)-1, result);
size_t status = 0; /* success */
if (result != PAMI_SUCCESS)
{
fprintf (stderr, " 'get_done' callback, PAMI_Rget failed\n");
status = 1; /* get failed */
}
else
{
/* validate the data! */
print_data ((void *)info->buffer, 4 * 12);
if (!validate_data(info->buffer, info->bytes, 4))
{
fprintf (stderr, " 'get_done' callback,) PAMI_Rget data validation error.\n");
status = 2; /* get data validation failure */
}
}
/* Send an 'ack' to the origin */
pami_send_immediate_t parameters;
parameters.dispatch = DISPATCH_ID_ACK;
parameters.dest = info->origin;
parameters.header.iov_base = &status;
parameters.header.iov_len = sizeof(status);
parameters.data.iov_base = NULL;
parameters.data.iov_len = 0;
parameters.hints = null_send_hint;
PAMI_Send_immediate (context, ¶meters);
/* Destroy the local memory region */
PAMI_Memregion_destroy (context, &(info->memregion));
--*(info->value);
free (cookie);
fprintf (stderr, "<< 'get_done' callback\n");
}
static void get_done (pami_context_t context,
void * cookie,
pami_result_t result)
{
get_info_t * info = (get_info_t *) cookie;
TRACE_ERR((stderr, ">> get_done() cookie = %p (info->value = %d => %d), result = %zu\n", cookie, *(info->value), *(info->value)-1, result));
size_t status = 0; /* success */
if (result != PAMI_SUCCESS)
{
TRACE_ERR((stderr, " get_done() PAMI_Get failed\n"));
status = 1; /* get failed */
}
else
{
/* validate the data! */
print_data ((void *)info->buffer, 4*12);
if (!validate_data(info->buffer, info->bytes, 4))
{
TRACE_ERR((stderr, " get_done() PAMI_Get data validation error.\n"));
status = 2; /* get data validation failure */
}
}
/* Send an 'ack' to the origin */
pami_send_immediate_t parameters;
parameters.dispatch = DISPATCH_ID_ACK;
parameters.dest = info->origin;
parameters.header.iov_base = &status;
parameters.header.iov_len = sizeof(status);
parameters.data.iov_base = NULL;
parameters.data.iov_len = 0;
parameters.hints = null_send_hint;
PAMI_Send_immediate (context, ¶meters);
--*(info->value);
free (cookie);
TRACE_ERR((stderr, "<< get_done()\n"));
}
int optiq_notify_job_done(struct optiq_transport *self, int job_id, vector<int> *dests)
{
#ifdef __bgq__
pami_result_t result;
struct optiq_pami_transport *pami_transport = (struct optiq_pami_transport *)optiq_transport_get_concrete_transport(self);
for (int i = 0; i < dests->size(); i++) {
pami_send_immediate_t parameter;
parameter.dispatch = JOB_DONE_NOTIFICATION_DISPATCH_ID;
parameter.header.iov_base = &job_id;
parameter.header.iov_len = sizeof(int);
parameter.data.iov_base = NULL;
parameter.data.iov_len = 0;
parameter.dest = pami_transport->endpoints[(*dests)[i]];
result = PAMI_Send_immediate (pami_transport->context, ¶meter);
assert(result == PAMI_SUCCESS);
if (result != PAMI_SUCCESS) {
return 1;
}
}
#endif
return 0;
}
pami_send_immediate_t params = {
.dispatch = MPIDI_Protocols_WinAtomicAck,
.dest = sender,
.header = {
.iov_base = &ack_hdr,
.iov_len = sizeof(MPIDI_AtomicHeader_t),
},
.data = {
.iov_base = NULL,
.iov_len = 0,
},
.hints = {0},
};
pami_result_t rc = PAMI_Send_immediate(context, ¶ms);
MPID_assert(rc == PAMI_SUCCESS);
}
void
MPIDI_WinAtomicAckCB(pami_context_t context,
void * cookie,
const void * _hdr,
size_t size,
const void * sndbuf,
size_t sndlen,
pami_endpoint_t sender,
pami_recv_t * recv)
{
int len;
MPIDI_AtomicHeader_t *ahdr = (MPIDI_AtomicHeader_t *) _hdr;
TRACE_ERR("CtrlSend: type=%d local=%u remote=%u\n", msginfo->control, MPIR_Process.comm_world->rank, peerrank);
pami_send_immediate_t params = {
.dispatch = MPIDI_Protocols_Control,
.dest = dest,
.header = {
.iov_base = msginfo,
.iov_len = sizeof(MPIDI_MsgInfo),
},
.data = {
.iov_base = NULL,
.iov_len = 0,
},
};
pami_result_t rc;
rc = PAMI_Send_immediate(context, ¶ms);
MPID_assert(rc == PAMI_SUCCESS);
}
/**
* \brief Message layer callback which is invoked on the target node
* of a flow-control rendezvous operation.
*
* This callback is invoked when the data buffer on the origin node
* has been completely transfered to the target node. The target node
* must acknowledge the completion of the transfer to the origin node
* with a control message and then complete the receive by releasing
* the request object.
*
* \param[in,out] rreq MPI receive request object
int main (int argc, char ** argv)
{
volatile size_t _rts_active = 1;
volatile size_t _ack_active = 1;
memset(&null_send_hint, 0, sizeof(null_send_hint));
pami_client_t client;
pami_context_t context[2];
char cl_string[] = "TEST";
pami_result_t result = PAMI_ERROR;
result = PAMI_Client_create (cl_string, &client, NULL, 0);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable to create pami client. result = %d\n", result);
return 1;
}
#ifdef TEST_CROSSTALK
size_t num = 2;
#else
size_t num = 1;
#endif
result = PAMI_Context_createv(client, NULL, 0, context, num);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable to create pami context(s). result = %d\n", result);
return 1;
}
pami_configuration_t configuration;
configuration.name = PAMI_CLIENT_TASK_ID;
result = PAMI_Client_query(client, &configuration,1);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable query configuration (%d). result = %d\n", configuration.name, result);
return 1;
}
pami_task_t task_id = configuration.value.intval;
fprintf (stderr, "My task id = %d\n", task_id);
configuration.name = PAMI_CLIENT_NUM_TASKS;
result = PAMI_Client_query(client, &configuration,1);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable query configuration (%d). result = %d\n", configuration.name, result);
return 1;
}
size_t num_tasks = configuration.value.intval;
fprintf (stderr, "Number of tasks = %zu\n", num_tasks);
if (num_tasks < 2)
{
fprintf (stderr, "Error. This test requires at least 2 tasks. Number of tasks in this job: %zu\n", num_tasks);
return 1;
}
pami_dispatch_hint_t options={};
#ifdef USE_SHMEM_OPTION
options.use_shmem = PAMI_HINT_ENABLE;
fprintf (stderr, "##########################################\n");
fprintf (stderr, "shared memory optimizations forced ON\n");
fprintf (stderr, "##########################################\n");
#elif defined(NO_SHMEM_OPTION)
options.use_shmem = PAMI_HINT_DISABLE;
fprintf (stderr, "##########################################\n");
fprintf (stderr, "shared memory optimizations forced OFF\n");
fprintf (stderr, "##########################################\n");
#endif
size_t i = 0;
#ifdef TEST_CROSSTALK
for (i=0; i<2; i++)
#endif
{
pami_dispatch_callback_function fn;
fprintf (stderr, "Before PAMI_Dispatch_set(%d) .. &_rts_active = %p, _rts_active = %zu\n", DISPATCH_ID_RTS, &_rts_active, _rts_active);
fn.p2p = dispatch_rts;
result = PAMI_Dispatch_set (context[i],
DISPATCH_ID_RTS,
fn,
(void *)&_rts_active,
options);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable register pami dispatch. result = %d\n", result);
return 1;
}
fprintf (stderr, "Before PAMI_Dispatch_set(%d) .. &_ack_active = %p, _ack_active = %zu\n", DISPATCH_ID_ACK, &_ack_active, _ack_active);
fn.p2p = dispatch_ack;
result = PAMI_Dispatch_set (context[i],
DISPATCH_ID_ACK,
fn,
(void *)&_ack_active,
options);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable register pami dispatch. result = %d\n", result);
return 1;
}
}
if (task_id == 0)
{
pami_send_immediate_t parameters;
#ifdef TEST_CROSSTALK
fprintf (stdout, "PAMI_Rget('simple') functional test [crosstalk]\n");
fprintf (stdout, "\n");
PAMI_Endpoint_create (client, num_tasks-1, 1, ¶meters.dest);
#else
fprintf (stdout, "PAMI_Rget('simple') functional test\n");
fprintf (stdout, "\n");
PAMI_Endpoint_create (client, num_tasks-1, 0, ¶meters.dest);
#endif
/* Allocate some memory from the heap. */
void * send_buffer = malloc (BUFFERSIZE);
/* Initialize the memory for validation. */
initialize_data ((uint32_t *)send_buffer, BUFFERSIZE, 0);
print_data (send_buffer, BUFFERSIZE);
/* Send an 'rts' message to the target task and provide the memory region */
rts_info_t rts_info;
PAMI_Endpoint_create (client, 0, 0, &rts_info.origin);
rts_info.bytes = BUFFERSIZE;
/* Create a memory region for this memoru buffer */
size_t bytes = 0;
pami_result_t pami_rc = PAMI_Memregion_create (context[0], send_buffer, BUFFERSIZE, &bytes, &(rts_info.memregion));
if (PAMI_SUCCESS != pami_rc) {
fprintf (stderr, "PAMI_Memregion_create failed with rc = %d\n", pami_rc) ;
exit(1);
}
parameters.dispatch = DISPATCH_ID_RTS;
parameters.header.iov_base = &rts_info;
parameters.header.iov_len = sizeof(rts_info_t);
parameters.data.iov_base = NULL;
parameters.data.iov_len = 0;
fprintf (stderr, "Before PAMI_Send_immediate()\n");
PAMI_Send_immediate (context[0], ¶meters);
/* wait for the 'ack' */
fprintf (stderr, "Wait for 'ack', _ack_active = %zu\n", _ack_active);
while (_ack_active != 0)
{
result = PAMI_Context_advance (context[0], 100);
if (result != PAMI_SUCCESS && result != PAMI_EAGAIN)
{
fprintf (stderr, "Error. Unable to advance pami context. result = %d\n", result);
return 1;
}
}
/* Destroy the local memory region */
PAMI_Memregion_destroy (context[0], &(rts_info.memregion));
free (send_buffer);
switch (_ack_status)
{
case 0:
fprintf (stdout, "Test PASSED\n");
break;
case 1:
fprintf (stdout, "Test FAILED (rget error)\n");
break;
case 2:
fprintf (stdout, "Test FAILED (data error)\n");
break;
default:
fprintf (stdout, "Test FAILED (unknown error)\n");
break;
}
}
else if (task_id == num_tasks-1)
{
#ifdef TEST_CROSSTALK
size_t contextid = 1;
#else
size_t contextid = 0;
#endif
/* wait for the 'rts' */
fprintf (stderr, "Wait for 'rts', _rts_active = %zu, contextid = %zu\n", _rts_active, contextid);
while (_rts_active != 0)
{
result = PAMI_Context_advance (context[contextid], 100);
if (result != PAMI_SUCCESS && result != PAMI_EAGAIN)
{
fprintf (stderr, "Error. Unable to advance pami context. result = %d\n", result);
return 1;
}
}
}
fprintf (stderr, "Test completed .. cleanup\n");
result = PAMI_Context_destroyv (context, num);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable to destroy pami context. result = %d\n", result);
return 1;
}
result = PAMI_Client_destroy(&client);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable to destroy pami client. result = %d\n", result);
return 1;
}
/*fprintf (stdout, "Success (%d)\n", task_id); */
return 0;
};
int main (int argc, char ** argv)
{
pami_client_t client;
pami_context_t context[2];
pami_task_t task_id;
size_t num_tasks = 0;
size_t ncontexts = 0;
size_t errors = 0;
pami_result_t result = PAMI_ERROR;
pami_type_t subtype;
pami_type_t compound_type;
pami_type_t simple_type;
info_t exchange[MAX_TASKS];
double data[BUFFERSIZE];
volatile unsigned ready;
{ /* init */
ready = 0;
unsigned i;
for (i = 0; i < MAX_TASKS; i++)
exchange[i].active = 0;
for (i = 0; i < BUFFERSIZE; i++)
data[i] = E;
result = PAMI_Client_create ("TEST", &client, NULL, 0);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable to create pami client. result = %d\n", result);
return 1;
}
pami_configuration_t configuration;
configuration.name = PAMI_CLIENT_TASK_ID;
result = PAMI_Client_query(client, &configuration, 1);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable query configuration (%d). result = %d\n", configuration.name, result);
return 1;
}
task_id = configuration.value.intval;
/*fprintf (stderr, "My task id = %d\n", task_id);*/
configuration.name = PAMI_CLIENT_NUM_TASKS;
result = PAMI_Client_query(client, &configuration, 1);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable query configuration (%d). result = %d\n", configuration.name, result);
return 1;
}
num_tasks = configuration.value.intval;
/*if (task_id == 0)
fprintf (stderr, "Number of tasks = %zu\n", num_tasks);*/
if ((num_tasks < 2) || (num_tasks > MAX_TASKS))
{
fprintf (stderr, "Error. This test requires 2-%d tasks. Number of tasks in this job: %zu\n", MAX_TASKS, num_tasks);
return 1;
}
if (task_id == num_tasks - 1)
{
for (i = 0; i < 320; i++)
data[i] = PI;
}
configuration.name = PAMI_CLIENT_NUM_CONTEXTS;
result = PAMI_Client_query(client, &configuration, 1);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable query configuration (%d). result = %d\n", configuration.name, result);
return 1;
}
ncontexts = (configuration.value.intval < 2) ? 1 : 2;
/*if (task_id == 0)
fprintf (stderr, "maximum contexts = %zu, number of contexts used in this test = %zu\n", configuration.value.intval, ncontexts);*/
result = PAMI_Context_createv(client, NULL, 0, context, ncontexts);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable to create pami context(s). result = %d\n", result);
return 1;
}
pami_dispatch_hint_t options = {};
for (i = 0; i < ncontexts; i++)
{
pami_dispatch_callback_function fn;
fn.p2p = dispatch_exchange;
result = PAMI_Dispatch_set (context[i],
EXCHANGE_DISPATCH_ID,
fn,
(void *) exchange,
options);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable register pami 'exchange' dispatch. result = %d\n", result);
return 1;
}
fn.p2p = dispatch_notify;
result = PAMI_Dispatch_set (context[i],
NOTIFY_DISPATCH_ID,
fn,
(void *) & ready,
options);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable register pami 'notify' dispatch. result = %d\n", result);
return 1;
}
}
/* ***********************************
* Create the pami types
************************************/
/* This compound noncontiguous type is composed of one double, skips a double,
* two doubles, skips a double, three doubles, skips a double, five doubles,
* skips a double, six doubles, skips a double, seven doubles, skips a double,
* eight doubles, then skips two doubles.
*
* This results in a type with 32 doubles that is 40 doubles
* 'wide'.
*/
PAMI_Type_create (&subtype);
PAMI_Type_add_simple (subtype,
sizeof(double), /* bytes */
0, /* offset */
1, /* count */
sizeof(double) * 2); /* stride */
PAMI_Type_add_simple (subtype,
sizeof(double) * 2, /* bytes */
0, /* offset */
1, /* count */
sizeof(double) * 3); /* stride */
PAMI_Type_add_simple (subtype,
sizeof(double) * 3, /* bytes */
0, /* offset */
1, /* count */
sizeof(double) * 4); /* stride */
PAMI_Type_add_simple (subtype,
sizeof(double) * 5, /* bytes */
0, /* offset */
1, /* count */
sizeof(double) * 6); /* stride */
PAMI_Type_add_simple (subtype,
sizeof(double) * 6, /* bytes */
0, /* offset */
1, /* count */
sizeof(double) * 7); /* stride */
PAMI_Type_add_simple (subtype,
sizeof(double) * 7, /* bytes */
0, /* offset */
1, /* count */
sizeof(double) * 8);/* stride */
PAMI_Type_add_simple (subtype,
sizeof(double) * 8, /* bytes */
0, /* offset */
1, /* count */
sizeof(double) * 10);/* stride */
PAMI_Type_complete (subtype, sizeof(double));
/* This noncontiguous type is composed of the above compound type, repeated
* ten times with no stride.
*
* This results in a type with 320 doubles that is 400 doubles
* 'wide'.
*/
PAMI_Type_create (&compound_type);
PAMI_Type_add_typed (compound_type,
subtype, /* subtype */
0, /* offset */
10, /* count */
sizeof(double) * 32); /* stride */
PAMI_Type_complete (compound_type, sizeof(double));
/* This simple noncontiguous type is composed of eight contiguous doubles,
* then skips a _single_ double, repeated 40 times.
*
* This results in a type with 320 doubles that is 360 doubles 'wide'.
*/
PAMI_Type_create (&simple_type);
PAMI_Type_add_simple (simple_type,
sizeof(double) * 8, /* bytes */
0, /* offset */
40, /* count */
sizeof(double) * 9); /* stride */
PAMI_Type_complete (simple_type, sizeof(double));
/* Create a memory region for the local data buffer. */
size_t bytes;
result = PAMI_Memregion_create (context[0], (void *) data, BUFFERSIZE,
&bytes, &exchange[task_id].mr);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable to create memory region. result = %d\n", result);
return 1;
}
else if (bytes < BUFFERSIZE)
{
fprintf (stderr, "Error. Unable to create memory region of a large enough size. result = %d\n", result);
return 1;
}
/* Broadcast the memory region to all tasks - including self. */
for (i = 0; i < num_tasks; i++)
{
pami_send_immediate_t parameters;
parameters.dispatch = EXCHANGE_DISPATCH_ID;
parameters.header.iov_base = (void *) & bytes;
parameters.header.iov_len = sizeof(size_t);
parameters.data.iov_base = (void *) & exchange[task_id].mr;
parameters.data.iov_len = sizeof(pami_memregion_t);
PAMI_Endpoint_create (client, i, 0, ¶meters.dest);
result = PAMI_Send_immediate (context[0], ¶meters);
}
/* Advance until all memory regions have been received. */
for (i = 0; i < num_tasks; i++)
{
while (exchange[i].active == 0)
PAMI_Context_advance (context[0], 100);
}
} /* init done */
pami_send_immediate_t notify;
notify.dispatch = NOTIFY_DISPATCH_ID;
notify.header.iov_base = NULL;
notify.header.iov_len = 0;
notify.data.iov_base = NULL;
notify.data.iov_len = 0;
volatile size_t active = 1;
pami_rget_typed_t parameters;
parameters.rma.hints = (pami_send_hint_t) {0};
parameters.rma.cookie = (void *) & active;
parameters.rma.done_fn = decrement;
parameters.rma.bytes = 320 * sizeof(double);
if (task_id == 0)
{
fprintf (stdout, "PAMI_Rget('typed') functional test %s\n", (ncontexts < 2) ? "" : "[crosstalk]");
fprintf (stdout, "\n");
parameters.rdma.local.mr = &exchange[0].mr;
parameters.rdma.remote.mr = &exchange[num_tasks - 1].mr;
PAMI_Endpoint_create (client, num_tasks - 1, ncontexts - 1, ¶meters.rma.dest);
PAMI_Endpoint_create (client, num_tasks - 1, ncontexts - 1, ¬ify.dest);
}
else
{
parameters.rdma.local.mr = &exchange[num_tasks - 1].mr;
parameters.rdma.remote.mr = &exchange[0].mr;
PAMI_Endpoint_create (client, 0, 0, ¶meters.rma.dest);
PAMI_Endpoint_create (client, 0, 0, ¬ify.dest);
}
/* ******************************************************************** */
/* contiguous -> contiguous transfer test */
/* ******************************************************************** */
if (task_id == 0)
{
parameters.rdma.local.offset = 0;
parameters.rdma.remote.offset = 0;
parameters.type.local = PAMI_TYPE_DOUBLE;
parameters.type.remote = PAMI_TYPE_DOUBLE;
active = 1;
PAMI_Rget_typed (context[0], ¶meters);
while (active > 0)
PAMI_Context_advance (context[0], 100);
/* Notify the remote task that the data has been transfered. */
PAMI_Send_immediate (context[0], ¬ify);
}
else if (task_id == num_tasks - 1)
{
/* Wait for notification that the data has been transfered. */
while (ready == 0)
PAMI_Context_advance (context[ncontexts - 1], 100);
ready = 0;
}
/* ******************************************************************** */
/* contiguous -> non-contiguous transfer test */
/* ******************************************************************** */
if (task_id == num_tasks - 1)
{
parameters.rdma.local.offset = 4 * 1024;
parameters.rdma.remote.offset = 0;
parameters.type.local = simple_type;
parameters.type.remote = PAMI_TYPE_DOUBLE;
active = 1;
PAMI_Rget_typed (context[ncontexts - 1], ¶meters);
while (active > 0)
PAMI_Context_advance (context[ncontexts - 1], 100);
/* Notify the remote task that the data has been transfered. */
PAMI_Send_immediate (context[ncontexts - 1], ¬ify);
}
else if (task_id == 0)
{
/* Wait for notification that the data has been transfered. */
while (ready == 0)
PAMI_Context_advance (context[0], 100);
ready = 0;
}
/* ******************************************************************** */
/* non-contiguous -> non-contiguous transfer test */
/* ******************************************************************** */
if (task_id == 0)
{
parameters.rdma.local.offset = 4 * 1024;
parameters.rdma.remote.offset = 4 * 1024;
parameters.type.local = compound_type;
parameters.type.remote = simple_type;
active = 1;
PAMI_Rget_typed (context[0], ¶meters);
while (active > 0)
PAMI_Context_advance (context[0], 100);
/* Notify the remote task that the data has been transfered. */
PAMI_Send_immediate (context[0], ¬ify);
}
else if (task_id == num_tasks - 1)
{
/* Wait for notification that the data has been transfered. */
while (ready == 0)
PAMI_Context_advance (context[ncontexts - 1], 100);
ready = 0;
}
/* ******************************************************************** */
/* non-contiguous -> contiguous transfer test */
/* ******************************************************************** */
if (task_id == num_tasks - 1)
{
parameters.rdma.local.offset = 8 * 1024;
parameters.rdma.remote.offset = 4 * 1024;
parameters.type.local = PAMI_TYPE_DOUBLE;
parameters.type.remote = compound_type;
active = 1;
PAMI_Rget_typed (context[ncontexts - 1], ¶meters);
while (active > 0)
PAMI_Context_advance (context[ncontexts - 1], 100);
/* Notify the remote task that the data has been transfered. */
PAMI_Send_immediate (context[ncontexts - 1], ¬ify);
}
else if (task_id == 0)
{
/* Wait for notification that the data has been transfered. */
while (ready == 0)
PAMI_Context_advance (context[0], 100);
ready = 0;
}
/* ******************************************************************** */
/* VERIFY data buffers */
/* ******************************************************************** */
if (task_id == num_tasks - 1)
{
if (task_id == 0)
{
unsigned i = 0;
for (; i < 320; i++)
{
if (data[i] != PI)
{
errors++;
fprintf (stderr, "Error. data[%d] != %g ..... (%g)\n", i, PI, data[i]);
}
}
for (; i < 512; i++)
{
if (data[i] != E)
{
errors++;
fprintf (stderr, "Error. data[%d] != %g ..... (%g)\n", i, E, data[i]);
}
}
unsigned j = 0;
for (; j < 40; j++)
{
unsigned n = 0;
for (; n < 8; n++)
{
if (data[i] != PI)
{
errors++;
fprintf (stderr, "Error. data[%d] != %g ..... (%g)\n", i, PI, data[i]);
}
i++;
}
if (data[i] != E)
{
errors++;
fprintf (stderr, "Error. data[%d] != %g ..... (%g)\n", i, E, data[i]);
}
i++;
}
for (; i < 1024; i++)
{
if (data[i] != E)
{
errors++;
fprintf (stderr, "Error. data[%d] != %g ..... (%g)\n", i, E, data[i]);
}
}
for (; i < 1024 + 320; i++)
{
if (data[i] != PI)
{
errors++;
fprintf (stderr, "Error. data[%d] != %g ..... (%g)\n", i, PI, data[i]);
}
}
for (; i < BUFFERSIZE; i++)
{
if (data[i] != E)
{
errors++;
fprintf (stderr, "Error. data[%d] != %g ..... (%g)\n", i, E, data[i]);
}
}
}
}
{ /* cleanup */
result = PAMI_Context_destroyv(context, ncontexts);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable to destroy pami context. result = %d\n", result);
return 1;
}
result = PAMI_Client_destroy(&client);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable to destroy pami client. result = %d\n", result);
return 1;
}
if (task_id == num_tasks - 1)
{
if (errors)
fprintf (stdout, "Test completed with errors (%zu)\n", errors);
else
fprintf (stdout, "Test completed with success\n");
}
} /* cleanup done */
return (errors != 0);
};
