void* thread_routine(void* input)
{
pami_result_t res = PAMI_ERROR;
pami_context_t ctx = (pami_context_t)input;
int my_key;
// get the lock
do {
res = PAMI_Context_trylock (ctx);
} while (res != PAMI_SUCCESS);
// modify the common data
my_key = keys;
#if DBG
fprintf (stderr,
"%d: changed common data from %d to %d\n",
pthread_self(), my_key, my_key+1);
#endif
my_key ++;
keys = my_key;
// release the lock
res = PAMI_Context_unlock (ctx);
if (res != PAMI_SUCCESS)
{
fprintf (stderr,
"Error. Unable to unlock the pami context. result = %d\n",
res);
}
pthread_exit(NULL);
}
void async_progress_enable(async_progress_t * async_progress, pami_context_t context)
{
pami_result_t rc = PAMI_ERROR;
async_progress_impl_t * async = (async_progress_impl_t *) async_progress;
/* Register the async progress event handlers for this context. */
rc = PAMI_ERROR;
rc = async->register_fn(context,
NULL, /* progress function */
NULL, /* suspend function */
NULL, /* resume function */
NULL); /* cookie */
PAMID_ASSERT(rc==PAMI_SUCCESS,"async->register_fn");
/* Enable async progress for this context. */
rc = PAMI_ERROR;
rc = async->enable_fn(context, ASYNC_PROGRESS_EVENT_ALL);
PAMID_ASSERT(rc==PAMI_SUCCESS,"async->enable_fn");
/* Async progress is enabled when the async progress extension acquires the context lock. */
do
{
rc = PAMI_ERROR;
rc = PAMI_Context_trylock (context);
PAMID_ASSERT(rc!=PAMI_ERROR,"PAMI_Context_trylock");
if (rc == PAMI_SUCCESS)
PAMI_Context_unlock (context);
}
while (rc == PAMI_SUCCESS);
return;
}
void async_progress_enable (async_progress_t * async_progress,
pami_context_t context)
{
pami_result_t result;
async_progress_impl_t * async = (async_progress_impl_t *) async_progress;
/*
* Register the async progress event handlers for this context.
*/
result = PAMI_ERROR;
result = async->register_fn (context,
NULL, /* progress function */
NULL, /* suspend function */
NULL, /* resume function */
NULL); /* cookie */
assert (result == PAMI_SUCCESS);
/*
* Enable async progress for this context.
*/
result = PAMI_ERROR;
result = async->enable_fn (context, ASYNC_PROGRESS_EVENT_ALL);
assert (result == PAMI_SUCCESS);
/*
* Async progress is enabled when the async progress extension acquires the
* context lock.
*/
fprintf (stdout, "(%03d) Waiting for the async progress extension to acquire the context lock.\n", __LINE__);
do
{
result = PAMI_ERROR;
result = PAMI_Context_trylock (context);
assert (result != PAMI_ERROR);
if (result == PAMI_SUCCESS)
PAMI_Context_unlock (context);
}
while (result == PAMI_SUCCESS);
fprintf (stdout, "(%03d) Async progress enabled for the context.\n", __LINE__);
return;
}
int main ()
{
pami_client_t client;
pami_context_t *context;
pami_result_t result;
pami_configuration_t configuration;
PAMI_Client_create ("TEST", &client, NULL, 0);
configuration.name = PAMI_CLIENT_NUM_CONTEXTS;
result = PAMI_Client_query(client, &configuration, 1);
size_t num = configuration.value.intval;
context = (pami_context_t*) malloc (num*sizeof(pami_context_t));
/* Create four contexts - every task creates the same number */
PAMI_Context_createv (client, NULL, 0, context, num);
createEndpointTable (client, num);
pami_dispatch_callback_function fn;
fn.p2p = test_dispatch;
pami_dispatch_hint_t options = {0};
pami_send_hint_t hints = {0};
volatile size_t expect = 0;
size_t i;
for (i=0; i<num; i++)
{
PAMI_Context_lock (context[i]);
result = PAMI_Dispatch_set (context[i], 0, fn, (void *)&expect, options);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable register pami dispatch. result = %d\n", result);
return 1;
}
}
configuration.name = PAMI_CLIENT_TASK_ID;
result = PAMI_Client_query(client, &configuration, 1);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable query configuration (%d). result = %d\n", configuration.name, result);
return 1;
}
pami_task_t task_id = configuration.value.intval;
fprintf (stderr, "My task id = %d\n", task_id);
configuration.name = PAMI_CLIENT_NUM_TASKS;
result = PAMI_Client_query(client, &configuration, 1);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable query configuration (%d). result = %d\n", configuration.name, result);
return 1;
}
size_t num_tasks = configuration.value.intval;
fprintf (stderr, "Number of tasks = %zu\n", num_tasks);
uint8_t header[16];
uint8_t data[1024];
volatile size_t active = 1;
pami_send_t parameters;
parameters.send.dispatch = 0;
parameters.send.header.iov_base = header;
parameters.send.header.iov_len = 16;
parameters.send.data.iov_base = data;
parameters.send.data.iov_len = 1024;
parameters.send.hints = hints;
parameters.events.cookie = (void *) &active;
parameters.events.local_fn = decrement;
parameters.events.remote_fn = NULL;
/* Send a message to endpoint "num_tasks * num_contexts - 1" */
pami_task_t target_task = (pami_task_t) -1;
size_t target_offset = (size_t) -1;
PAMI_Endpoint_query (_endpoint[num*num_tasks-1], &target_task, &target_offset);
if (task_id == target_task) expect += num_tasks;
send_endpoint (context[0], num*num_tasks-1, ¶meters);
fprintf (stdout, "before advance, active = %zu, expect = %zu\n", active, expect);
while ((active + expect) > 0) PAMI_Context_advancev (context, num, 100);
for (i=0; i<num; i++) PAMI_Context_unlock (context[i]);
result = PAMI_Context_destroyv (context, num);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable to destroy pami context. result = %d\n", result);
return 1;
}
result = PAMI_Client_destroy (&client);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable to finalize pami client. result = %d\n", result);
return 1;
}
fprintf (stdout, "Success (%d)\n", task_id);
return 0;
};
int main (int argc, char ** argv)
{
pami_client_t client;
pami_context_t context;
pami_result_t result = PAMI_ERROR;
pami_configuration_t* configuration = NULL;
char cl_string[] = "TEST";
pthread_t threads[MAX_THREAD_NUM];
int i, rc;
result = PAMI_Client_create (cl_string, &client, NULL, 0);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable to initialize pami client. result = %d\n", result);
return 1;
}
result = PAMI_Context_createv(client, configuration, 0, &context, 1);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable to create the pami context. result = %d\n", result);
return 1;
}
/* Test a context lock */
result = PAMI_Context_lock (context);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable to lock the pami context. result = %d\n", result);
return 1;
}
/* Test a context unlock */
result = PAMI_Context_unlock (context);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable to unlock the pami context. result = %d\n", result);
return 1;
}
// test context trylock with multiple threads
for (i = 0; i < MAX_THREAD_NUM; i ++) {
rc = pthread_create(&threads[i], NULL, thread_routine, (void*)context);
if (rc) {
fprintf (stderr,
"Error. Unable to create %d-th thread with rc %d\n",
i, rc);
return 1;
}
}
// join all the created threads
for (i = 0; i < MAX_THREAD_NUM; i ++) {
do {
rc = pthread_join(threads[i], NULL);
if (rc) {
fprintf (stderr,
"Error. Unable to join %d-th thread with rc %d\n",
i, rc);
}
} while (0 != rc);
}
// check the common data
if ( MAX_THREAD_NUM == keys ) {
fprintf (stderr,
"Trylock with multiple threads passed. %d (%d expected)\n",
keys, MAX_THREAD_NUM);
} else {
fprintf (stderr,
"Error. Trylock with multiple threads failed. %d (%d expected)\n",
keys, MAX_THREAD_NUM);
}
/* Destroy the context */
result = PAMI_Context_destroyv(&context, 1);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable to destroy the pami context. result = %d\n", result);
return 1;
}
/* Finalize (destroy) the client */
result = PAMI_Client_destroy(&client);
if (result != PAMI_SUCCESS)
{
fprintf (stderr, "Error. Unable to finalize pami client. result = %d\n", result);
return 1;
}
fprintf (stderr, "Success.\n");
return 0;
};
int main(int argc, char ** argv)
{
pami_client_t client;
pami_context_t context;
pami_result_t status = PAMI_ERROR;
pami_configuration_t pami_config;
pami_geometry_t world_geo;
size_t barrier_alg_num[2];
pami_algorithm_t* bar_always_works_algo = NULL;
pami_metadata_t* bar_always_works_md = NULL;
pami_algorithm_t* bar_must_query_algo = NULL;
pami_metadata_t* bar_must_query_md = NULL;
pami_xfer_t barrier;
int my_id;
volatile int is_fence_done = 0;
volatile int is_barrier_done = 0;
/* create PAMI client */
RC( PAMI_Client_create("TEST", &client, NULL, 0) );
DBG_FPRINTF((stderr,"Client created successfully at 0x%p\n",client));
/* create PAMI context */
RC( PAMI_Context_createv(client, NULL, 0, &context, 1) );
DBG_FPRINTF((stderr,"Context created successfully at 0x%p\n",context));
/* query my task id */
bzero(&pami_config, sizeof(pami_configuration_t));
pami_config.name = PAMI_CLIENT_TASK_ID;
RC( PAMI_Client_query(client, &pami_config, 1) );
my_id = pami_config.value.intval;
DBG_FPRINTF((stderr,"My task id is %d\n", my_id));
/* get the world geometry */
RC( PAMI_Geometry_world(client, &world_geo) );
DBG_FPRINTF((stderr,"World geometry is at 0x%p\n",world_geo));
/* query number of barrier algorithms */
RC( PAMI_Geometry_algorithms_num(world_geo, PAMI_XFER_BARRIER,
barrier_alg_num) );
DBG_FPRINTF((stderr,"%d-%d algorithms are available for barrier op\n",
barrier_alg_num[0], barrier_alg_num[1]));
if (barrier_alg_num[0] <= 0) {
fprintf (stderr, "Error. No (%lu) algorithm is available for barrier op\n",
barrier_alg_num[0]);
return 1;
}
/* query barrier algorithm list */
bar_always_works_algo =
(pami_algorithm_t*)malloc(sizeof(pami_algorithm_t)*barrier_alg_num[0]);
bar_always_works_md =
(pami_metadata_t*)malloc(sizeof(pami_metadata_t)*barrier_alg_num[0]);
bar_must_query_algo =
(pami_algorithm_t*)malloc(sizeof(pami_algorithm_t)*barrier_alg_num[1]);
bar_must_query_md =
(pami_metadata_t*)malloc(sizeof(pami_metadata_t)*barrier_alg_num[1]);
RC( PAMI_Geometry_algorithms_query(world_geo, PAMI_XFER_BARRIER,
bar_always_works_algo, bar_always_works_md, barrier_alg_num[0],
bar_must_query_algo, bar_must_query_md, barrier_alg_num[1]) );
DBG_FPRINTF((stderr,"Algorithm [%s] at 0x%p will be used for barrier op\n",
bar_always_works_md[0].name, bar_always_works_algo[0]));
/* begin PAMI fence */
RC( PAMI_Fence_begin(context) );
DBG_FPRINTF((stderr,"PAMI fence begins\n"));
/* ------------------------------------------------------------------------ */
pami_extension_t extension;
const char ext_name[] = "EXT_hfi_extension";
const char sym_name[] = "hfi_remote_update";
hfi_remote_update_fn remote_update = NULL;
hfi_remote_update_info_t remote_info;
pami_memregion_t mem_region;
size_t mem_region_sz = 0;
unsigned long long operand = 1234;
unsigned long long orig_val = 0;
int offset = (operand)%MAX_TABLE_SZ;
/* initialize table for remote update operation */
int i;
for (i = 0; i < MAX_TABLE_SZ; i ++) {
table[i] = (unsigned long long) i;
}
orig_val = table[offset];
/* open PAMI extension */
RC( PAMI_Extension_open (client, ext_name, &extension) );
DBG_FPRINTF((stderr,"Open %s successfully.\n", ext_name));
/* load PAMI extension function */
remote_update = (hfi_remote_update_fn)
PAMI_Extension_symbol (extension, sym_name);
if (remote_update == (void *)NULL)
{
fprintf (stderr, "Error. Failed to load %s function in %s\n",
sym_name, ext_name);
return 1;
} else {
DBG_FPRINTF((stderr,"Loaded function %s in %s successfully.\n",
sym_name, ext_name));
}
/* create a memory region for remote update operation */
RC( PAMI_Memregion_create(context, table,
MAX_TABLE_SZ*sizeof(unsigned long long),
&mem_region_sz, &mem_region) );
DBG_FPRINTF((stderr,"%d-byte PAMI memory region created successfully.\n",
mem_region_sz));
/* perform a PAMI barrier */
is_barrier_done = 0;
barrier.cb_done = barrier_done;
barrier.cookie = (void*)&is_barrier_done;
barrier.algorithm = bar_always_works_algo[0];
RC( PAMI_Collective(context, &barrier) );
DBG_FPRINTF((stderr,"PAMI barrier op invoked successfully.\n"));
while (is_barrier_done == 0)
PAMI_Context_advance(context, 1000);
DBG_FPRINTF((stderr,"PAMI barrier op finished successfully.\n"));
RC( PAMI_Context_lock(context) );
/* prepare remote update info */
remote_info.dest = my_id^1;
remote_info.op = 0; /* op_add */
remote_info.atomic_operand = operand;
remote_info.dest_buf = (unsigned long long)(&(table[offset]));
/* invoke remote update PAMI extension function */
RC( remote_update(context, 1, &remote_info) );
DBG_FPRINTF((stderr,"Function %s invoked successfully.\n",
sym_name));
RC( PAMI_Context_unlock(context) );
/* perform a PAMI fence */
is_fence_done = 0;
RC( PAMI_Fence_all(context, fence_done, (void*)&is_fence_done) );
DBG_FPRINTF((stderr,"PAMI_Fence_all invoked successfully.\n"));
while (is_fence_done == 0)
PAMI_Context_advance(context, 1000);
DBG_FPRINTF((stderr,"PAMI_Fence_all finished successfully.\n"));
/* perform a PAMI barrier */
is_barrier_done = 0;
barrier.cb_done = barrier_done;
barrier.cookie = (void*)&is_barrier_done;
barrier.algorithm = bar_always_works_algo[0];
RC( PAMI_Collective(context, &barrier) );
DBG_FPRINTF((stderr,"PAMI barrier op invoked successfully.\n"));
while (is_barrier_done == 0)
PAMI_Context_advance(context, 1000);
DBG_FPRINTF((stderr,"PAMI barrier op finished successfully.\n"));
/* verify data after remote update operation */
if (table[offset] != orig_val + operand) {
printf("Data verification at offset %d with operand %lu failed: "
"[%lu expected with %lu updated]\n",
offset, operand, orig_val+operand, table[offset]);
} else {
printf("Data verification at offset %d with operand %lu passed: "
"[%lu expected with %lu updated].\n",
offset, operand, orig_val+operand, table[offset]);
}
/* destroy the memory region after remote update operation */
RC( PAMI_Memregion_destroy(context, &mem_region) );
DBG_FPRINTF((stderr,"PAMI memory region removed successfully.\n"));
/* close PAMI extension */
RC( PAMI_Extension_close (extension) );
DBG_FPRINTF((stderr,"Close %s successfully.\n", ext_name));
/* ------------------------------------------------------------------------ */
/* end PAMI fence */
RC( PAMI_Fence_end(context) );
DBG_FPRINTF((stderr,"PAMI fence ends\n"));
/* destroy PAMI context */
RC( PAMI_Context_destroyv(&context, 1) );
DBG_FPRINTF((stderr, "PAMI context destroyed successfully\n"));
/* destroy PAMI client */
RC( PAMI_Client_destroy(&client) );
DBG_FPRINTF((stderr, "PAMI client destroyed successfully\n"));
return 0;
}
