static void *thread_main(void *arg)
{
int rank = (int) (unsigned long) arg;
int i;
/* thread tests */
for (i = 0; i < nreps; i++) {
opal_atomic_add_32(&val32, 5);
#if OPAL_HAVE_ATOMIC_MATH_64
opal_atomic_add_64(&val64, 5);
#endif
opal_atomic_add(&valint, 5);
}
return (void *) (unsigned long) (rank + 1000);
}
void mca_mpool_hugepage_seg_free (void *ctx, void *addr)
{
mca_mpool_hugepage_module_t *hugepage_module = (mca_mpool_hugepage_module_t *) ctx;
size_t size;
opal_mutex_lock (&hugepage_module->lock);
size = (size_t) (intptr_t) opal_rb_tree_find (&hugepage_module->allocation_tree, addr);
if (size > 0) {
opal_rb_tree_delete (&hugepage_module->allocation_tree, addr);
OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_mpool_base_framework.framework_verbose,
"freeing segment %p of size %lu bytes", addr, size));
munmap (addr, size);
opal_atomic_add (&mca_mpool_hugepage_component.bytes_allocated, -(int64_t) size);
}
opal_mutex_unlock (&hugepage_module->lock);
}
/*
* Insert an item at a specific place in a list
*/
bool opal_list_insert(opal_list_t *list, opal_list_item_t *item, long long idx)
{
/* Adds item to list at index and retains item. */
int i;
volatile opal_list_item_t *ptr, *next;
if ( idx >= (long long)list->opal_list_length ) {
return false;
}
if ( 0 == idx )
{
opal_list_prepend(list, item);
} else {
#if OPAL_ENABLE_DEBUG
/* Spot check: ensure that this item is previously on no
lists */
assert(0 == item->opal_list_item_refcount);
#endif
/* pointer to element 0 */
ptr = list->opal_list_sentinel.opal_list_next;
for ( i = 0; i < idx-1; i++ )
ptr = ptr->opal_list_next;
next = ptr->opal_list_next;
item->opal_list_next = next;
item->opal_list_prev = ptr;
next->opal_list_prev = item;
ptr->opal_list_next = item;
#if OPAL_ENABLE_DEBUG
/* Spot check: ensure this item is only on the list that we
just insertted it into */
(void)opal_atomic_add( &(item->opal_list_item_refcount), 1 );
assert(1 == item->opal_list_item_refcount);
item->opal_list_item_belong_to = list;
#endif
}
list->opal_list_length++;
return true;
}
static int vader_check_reg (mca_rcache_base_registration_t *reg, void *ctx)
{
vader_check_reg_ctx_t *vader_ctx = (vader_check_reg_ctx_t *) ctx;
if ((intptr_t) reg->alloc_base != vader_ctx->ep->peer_smp_rank ||
(reg->flags & MCA_RCACHE_FLAGS_PERSIST)) {
/* ignore this registration */
return OPAL_SUCCESS;
}
vader_ctx->reg[0] = reg;
if (vader_ctx->bound <= (uintptr_t) reg->bound && vader_ctx->base >= (uintptr_t) reg->base) {
(void)opal_atomic_add (®->ref_count, 1);
return 1;
}
/* remove this pointer from the rcache and decrement its reference count
(so it is detached later) */
mca_rcache_base_vma_delete (vader_ctx->vma_module, reg);
return 2;
}
int main(int argc, char *argv[])
{
#if OPAL_HAVE_POSIX_THREADS
int tid;
pthread_t *th;
#endif
if (argc != 2) {
printf("*** Incorrect number of arguments. Skipping test\n");
return 77;
}
nthreads = atoi(argv[1]);
/* first test single-threaded functionality */
/* -- cmpset 32-bit tests -- */
vol32 = 42, old32 = 42, new32 = 50;
assert(opal_atomic_cmpset_32(&vol32, old32, new32) == 1);
opal_atomic_rmb();
assert(vol32 == new32);
vol32 = 42, old32 = 420, new32 = 50;
assert(opal_atomic_cmpset_32(&vol32, old32, new32) == 0);
opal_atomic_rmb();
assert(vol32 == 42);
vol32 = 42, old32 = 42, new32 = 50;
assert(opal_atomic_cmpset_acq_32(&vol32, old32, new32) == 1);
assert(vol32 == new32);
vol32 = 42, old32 = 420, new32 = 50;
assert(opal_atomic_cmpset_acq_32(&vol32, old32, new32) == 0);
assert(vol32 == 42);
vol32 = 42, old32 = 42, new32 = 50;
assert(opal_atomic_cmpset_rel_32(&vol32, old32, new32) == 1);
opal_atomic_rmb();
assert(vol32 == new32);
vol32 = 42, old32 = 420, new32 = 50;
assert(opal_atomic_cmpset_rel_32(&vol32, old32, new32) == 0);
opal_atomic_rmb();
assert(vol32 == 42);
/* -- cmpset 64-bit tests -- */
#if OPAL_HAVE_ATOMIC_MATH_64
vol64 = 42, old64 = 42, new64 = 50;
assert(1 == opal_atomic_cmpset_64(&vol64, old64, new64));
opal_atomic_rmb();
assert(new64 == vol64);
vol64 = 42, old64 = 420, new64 = 50;
assert(opal_atomic_cmpset_64(&vol64, old64, new64) == 0);
opal_atomic_rmb();
assert(vol64 == 42);
vol64 = 42, old64 = 42, new64 = 50;
assert(opal_atomic_cmpset_acq_64(&vol64, old64, new64) == 1);
assert(vol64 == new64);
vol64 = 42, old64 = 420, new64 = 50;
assert(opal_atomic_cmpset_acq_64(&vol64, old64, new64) == 0);
assert(vol64 == 42);
vol64 = 42, old64 = 42, new64 = 50;
assert(opal_atomic_cmpset_rel_64(&vol64, old64, new64) == 1);
opal_atomic_rmb();
assert(vol64 == new64);
vol64 = 42, old64 = 420, new64 = 50;
assert(opal_atomic_cmpset_rel_64(&vol64, old64, new64) == 0);
opal_atomic_rmb();
assert(vol64 == 42);
#endif
/* -- cmpset int tests -- */
volint = 42, oldint = 42, newint = 50;
assert(opal_atomic_cmpset(&volint, oldint, newint) == 1);
opal_atomic_rmb();
assert(volint ==newint);
volint = 42, oldint = 420, newint = 50;
assert(opal_atomic_cmpset(&volint, oldint, newint) == 0);
opal_atomic_rmb();
assert(volint == 42);
volint = 42, oldint = 42, newint = 50;
assert(opal_atomic_cmpset_acq(&volint, oldint, newint) == 1);
assert(volint == newint);
volint = 42, oldint = 420, newint = 50;
assert(opal_atomic_cmpset_acq(&volint, oldint, newint) == 0);
assert(volint == 42);
volint = 42, oldint = 42, newint = 50;
assert(opal_atomic_cmpset_rel(&volint, oldint, newint) == 1);
opal_atomic_rmb();
assert(volint == newint);
volint = 42, oldint = 420, newint = 50;
assert(opal_atomic_cmpset_rel(&volint, oldint, newint) == 0);
opal_atomic_rmb();
assert(volint == 42);
/* -- cmpset ptr tests -- */
volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50;
assert(opal_atomic_cmpset_ptr(&volptr, oldptr, newptr) == 1);
opal_atomic_rmb();
assert(volptr == newptr);
volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50;
assert(opal_atomic_cmpset_ptr(&volptr, oldptr, newptr) == 0);
opal_atomic_rmb();
assert(volptr == (void *) 42);
volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50;
assert(opal_atomic_cmpset_acq_ptr(&volptr, oldptr, newptr) == 1);
assert(volptr == newptr);
volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50;
assert(opal_atomic_cmpset_acq_ptr(&volptr, oldptr, newptr) == 0);
assert(volptr == (void *) 42);
volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50;
assert(opal_atomic_cmpset_rel_ptr(&volptr, oldptr, newptr) == 1);
opal_atomic_rmb();
assert(volptr == newptr);
volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50;
assert(opal_atomic_cmpset_rel_ptr(&volptr, oldptr, newptr) == 0);
opal_atomic_rmb();
assert(volptr == (void *) 42);
/* -- add_32 tests -- */
val32 = 42;
assert(opal_atomic_add_32(&val32, 5) == (42 + 5));
opal_atomic_rmb();
assert((42 + 5) == val32);
/* -- add_64 tests -- */
#if OPAL_HAVE_ATOMIC_MATH_64
val64 = 42;
assert(opal_atomic_add_64(&val64, 5) == (42 + 5));
opal_atomic_rmb();
assert((42 + 5) == val64);
#endif
/* -- add_int tests -- */
valint = 42;
opal_atomic_add(&valint, 5);
opal_atomic_rmb();
assert((42 + 5) == valint);
/* threaded tests */
val32 = 0;
#if OPAL_HAVE_ATOMIC_MATH_64
val64 = 0ul;
#endif
valint = 0;
/* -- create the thread set -- */
#if OPAL_HAVE_POSIX_THREADS
th = (pthread_t *) malloc(nthreads * sizeof(pthread_t));
if (!th) {
perror("malloc");
exit(EXIT_FAILURE);
}
for (tid = 0; tid < nthreads; tid++) {
if (pthread_create(&th[tid], NULL, thread_main, (void *) (unsigned long) tid) != 0) {
perror("pthread_create");
exit(EXIT_FAILURE);
}
}
/* -- wait for the thread set to finish -- */
for (tid = 0; tid < nthreads; tid++) {
void *thread_return;
if (pthread_join(th[tid], &thread_return) != 0) {
perror("pthread_join");
exit(EXIT_FAILURE);
}
}
free(th);
opal_atomic_rmb();
assert((5 * nthreads * nreps) == val32);
#if OPAL_HAVE_ATOMIC_MATH_64
opal_atomic_rmb();
assert((5 * nthreads * nreps) == val64);
#endif
opal_atomic_rmb();
assert((5 * nthreads * nreps) == valint);
#endif
return 0;
}
void *mca_mpool_hugepage_seg_alloc (void *ctx, size_t *sizep)
{
mca_mpool_hugepage_module_t *hugepage_module = (mca_mpool_hugepage_module_t *) ctx;
mca_mpool_hugepage_hugepage_t *huge_page = hugepage_module->huge_page;
size_t size = *sizep;
void *base = NULL;
char *path = NULL;
int flags = MAP_PRIVATE;
int fd = -1;
int rc;
size = OPAL_ALIGN(size, huge_page->page_size, size_t);
if (huge_page->path) {
int32_t count;
count = opal_atomic_add_32 (&huge_page->count, 1);
rc = asprintf (&path, "%s/hugepage.openmpi.%d.%d", huge_page->path,
getpid (), count);
if (0 > rc) {
return NULL;
}
fd = open (path, O_RDWR | O_CREAT, 0600);
if (-1 == fd) {
free (path);
return NULL;
}
if (0 != ftruncate (fd, size)) {
close (fd);
unlink (path);
free (path);
return NULL;
}
} else {
#if defined(MAP_ANONYMOUS)
flags |= MAP_ANONYMOUS;
#elif defined(MAP_ANON)
/* older versions of OS X do not define MAP_ANONYMOUS (10.9.x and older) */
flags |= MAP_ANON;
#endif
}
base = mmap (NULL, size, PROT_READ | PROT_WRITE, flags | huge_page->mmap_flags, fd, 0);
close (fd);
if (path) {
unlink (path);
free (path);
}
if (MAP_FAILED == base) {
opal_output_verbose (MCA_BASE_VERBOSE_WARN, opal_mpool_base_framework.framework_verbose,
"could not allocate huge page(s). falling back on standard pages");
/* fall back on regular pages */
base = mmap (NULL, size, PROT_READ | PROT_WRITE, flags, -1, 0);
}
if (MAP_FAILED == base) {
return NULL;
}
opal_mutex_lock (&hugepage_module->lock);
opal_rb_tree_insert (&hugepage_module->allocation_tree, base, (void *) (intptr_t) size);
opal_atomic_add (&mca_mpool_hugepage_component.bytes_allocated, (int64_t) size);
opal_mutex_unlock (&hugepage_module->lock);
OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_mpool_base_framework.framework_verbose,
"allocated segment %p of size %lu bytes", base, size));
*sizep = size;
return base;
}
int ompi_coll_sm_lazy_enable(mca_coll_base_module_t *module,
struct ompi_communicator_t *comm)
{
int i, j, root, ret;
int rank = ompi_comm_rank(comm);
int size = ompi_comm_size(comm);
mca_coll_sm_module_t *sm_module = (mca_coll_sm_module_t*) module;
mca_coll_sm_comm_t *data = NULL;
size_t control_size, frag_size;
mca_coll_sm_component_t *c = &mca_coll_sm_component;
#if OPAL_HAVE_HWLOC
opal_hwloc_base_memory_segment_t *maffinity;
#endif
int parent, min_child, max_child, num_children;
unsigned char *base = NULL;
const int num_barrier_buffers = 2;
/* Just make sure we haven't been here already */
if (sm_module->enabled) {
return OMPI_SUCCESS;
}
sm_module->enabled = true;
#if OPAL_HAVE_HWLOC
/* Get some space to setup memory affinity (just easier to try to
alloc here to handle the error case) */
maffinity = (opal_hwloc_base_memory_segment_t*)
malloc(sizeof(opal_hwloc_base_memory_segment_t) *
c->sm_comm_num_segments * 3);
if (NULL == maffinity) {
opal_output_verbose(10, ompi_coll_base_framework.framework_output,
"coll:sm:enable (%d/%s): malloc failed (1)",
comm->c_contextid, comm->c_name);
return OMPI_ERR_OUT_OF_RESOURCE;
}
#endif
/* Allocate data to hang off the communicator. The memory we
alloc will be laid out as follows:
1. mca_coll_base_comm_t
2. array of num_segments mca_coll_base_mpool_index_t instances
(pointed to by the array in 2)
3. array of ompi_comm_size(comm) mca_coll_sm_tree_node_t
instances
4. array of sm_tree_degree pointers to other tree nodes (i.e.,
this nodes' children) for each instance of
mca_coll_sm_tree_node_t
*/
sm_module->sm_comm_data = data = (mca_coll_sm_comm_t*)
malloc(sizeof(mca_coll_sm_comm_t) +
(c->sm_comm_num_segments *
sizeof(mca_coll_sm_data_index_t)) +
(size *
(sizeof(mca_coll_sm_tree_node_t) +
(sizeof(mca_coll_sm_tree_node_t*) * c->sm_tree_degree))));
if (NULL == data) {
#if OPAL_HAVE_HWLOC
free(maffinity);
#endif
opal_output_verbose(10, ompi_coll_base_framework.framework_output,
"coll:sm:enable (%d/%s): malloc failed (2)",
comm->c_contextid, comm->c_name);
return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
}
data->mcb_operation_count = 0;
/* Setup #2: set the array to point immediately beyond the
mca_coll_base_comm_t */
data->mcb_data_index = (mca_coll_sm_data_index_t*) (data + 1);
/* Setup array of pointers for #3 */
data->mcb_tree = (mca_coll_sm_tree_node_t*)
(data->mcb_data_index + c->sm_comm_num_segments);
/* Finally, setup the array of children pointers in the instances
in #5 to point to their corresponding arrays in #6 */
data->mcb_tree[0].mcstn_children = (mca_coll_sm_tree_node_t**)
(data->mcb_tree + size);
for (i = 1; i < size; ++i) {
data->mcb_tree[i].mcstn_children =
data->mcb_tree[i - 1].mcstn_children + c->sm_tree_degree;
}
/* Pre-compute a tree for a given number of processes and degree.
We'll re-use this tree for all possible values of root (i.e.,
shift everyone's process to be the "0"/root in this tree. */
for (root = 0; root < size; ++root) {
parent = (root - 1) / mca_coll_sm_component.sm_tree_degree;
num_children = mca_coll_sm_component.sm_tree_degree;
/* Do we have children? If so, how many? */
if ((root * num_children) + 1 >= size) {
/* Leaves */
min_child = -1;
max_child = -1;
num_children = 0;
} else {
/* Interior nodes */
min_child = root * num_children + 1;
max_child = root * num_children + num_children;
if (max_child >= size) {
max_child = size - 1;
}
num_children = max_child - min_child + 1;
}
/* Save the values */
data->mcb_tree[root].mcstn_id = root;
if (root == 0 && parent == 0) {
data->mcb_tree[root].mcstn_parent = NULL;
} else {
data->mcb_tree[root].mcstn_parent = &data->mcb_tree[parent];
}
data->mcb_tree[root].mcstn_num_children = num_children;
for (i = 0; i < c->sm_tree_degree; ++i) {
data->mcb_tree[root].mcstn_children[i] =
(i < num_children) ?
&data->mcb_tree[min_child + i] : NULL;
}
}
/* Attach to this communicator's shmem data segment */
if (OMPI_SUCCESS != (ret = bootstrap_comm(comm, sm_module))) {
free(data);
#if OPAL_HAVE_HWLOC
free(maffinity);
#endif
sm_module->sm_comm_data = NULL;
return ret;
}
/* Once the communicator is bootstrapped, setup the pointers into
the per-communicator shmem data segment. First, setup the
barrier buffers. There are 2 sets of barrier buffers (because
there can never be more than one outstanding barrier occuring
at any timie). Setup pointers to my control buffers, my
parents, and [the beginning of] my children (note that the
children are contiguous, so having the first pointer and the
num_children from the mcb_tree data is sufficient). */
control_size = c->sm_control_size;
base = data->sm_bootstrap_meta->module_data_addr;
data->mcb_barrier_control_me = (uint32_t*)
(base + (rank * control_size * num_barrier_buffers * 2));
if (data->mcb_tree[rank].mcstn_parent) {
data->mcb_barrier_control_parent = (uint32_t*)
(base +
(data->mcb_tree[rank].mcstn_parent->mcstn_id * control_size *
num_barrier_buffers * 2));
} else {
data->mcb_barrier_control_parent = NULL;
}
if (data->mcb_tree[rank].mcstn_num_children > 0) {
data->mcb_barrier_control_children = (uint32_t*)
(base +
(data->mcb_tree[rank].mcstn_children[0]->mcstn_id * control_size *
num_barrier_buffers * 2));
} else {
data->mcb_barrier_control_children = NULL;
}
data->mcb_barrier_count = 0;
/* Next, setup the pointer to the in-use flags. The number of
segments will be an even multiple of the number of in-use
flags. */
base += (c->sm_control_size * size * num_barrier_buffers * 2);
data->mcb_in_use_flags = (mca_coll_sm_in_use_flag_t*) base;
/* All things being equal, if we're rank 0, then make the in-use
flags be local (memory affinity). Then zero them all out so
that they're marked as unused. */
j = 0;
if (0 == rank) {
#if OPAL_HAVE_HWLOC
maffinity[j].mbs_start_addr = base;
maffinity[j].mbs_len = c->sm_control_size *
c->sm_comm_num_in_use_flags;
#endif
/* Set the op counts to 1 (actually any nonzero value will do)
so that the first time children/leaf processes come
through, they don't see a value of 0 and think that the
root/parent has already set the count to their op number
(i.e., 0 is the first op count value). */
for (i = 0; i < mca_coll_sm_component.sm_comm_num_in_use_flags; ++i) {
((mca_coll_sm_in_use_flag_t *)base)[i].mcsiuf_operation_count = 1;
((mca_coll_sm_in_use_flag_t *)base)[i].mcsiuf_num_procs_using = 0;
}
++j;
}
/* Next, setup pointers to the control and data portions of the
segments, as well as to the relevant in-use flags. */
base += (c->sm_comm_num_in_use_flags * c->sm_control_size);
control_size = size * c->sm_control_size;
frag_size = size * c->sm_fragment_size;
for (i = 0; i < c->sm_comm_num_segments; ++i) {
data->mcb_data_index[i].mcbmi_control = (uint32_t*)
(base + (i * (control_size + frag_size)));
data->mcb_data_index[i].mcbmi_data =
(((char*) data->mcb_data_index[i].mcbmi_control) +
control_size);
#if OPAL_HAVE_HWLOC
/* Memory affinity: control */
maffinity[j].mbs_len = c->sm_control_size;
maffinity[j].mbs_start_addr = (void *)
(((char*) data->mcb_data_index[i].mcbmi_control) +
(rank * c->sm_control_size));
++j;
/* Memory affinity: data */
maffinity[j].mbs_len = c->sm_fragment_size;
maffinity[j].mbs_start_addr =
((char*) data->mcb_data_index[i].mcbmi_data) +
(rank * c->sm_control_size);
++j;
#endif
}
#if OPAL_HAVE_HWLOC
/* Setup memory affinity so that the pages that belong to this
process are local to this process */
opal_hwloc_base_memory_set(maffinity, j);
free(maffinity);
#endif
/* Zero out the control structures that belong to this process */
memset(data->mcb_barrier_control_me, 0,
num_barrier_buffers * 2 * c->sm_control_size);
for (i = 0; i < c->sm_comm_num_segments; ++i) {
memset((void *) data->mcb_data_index[i].mcbmi_control, 0,
c->sm_control_size);
}
/* Save previous component's reduce information */
sm_module->previous_reduce = comm->c_coll.coll_reduce;
sm_module->previous_reduce_module = comm->c_coll.coll_reduce_module;
OBJ_RETAIN(sm_module->previous_reduce_module);
/* Indicate that we have successfully attached and setup */
opal_atomic_add(&(data->sm_bootstrap_meta->module_seg->seg_inited), 1);
/* Wait for everyone in this communicator to attach and setup */
opal_output_verbose(10, ompi_coll_base_framework.framework_output,
"coll:sm:enable (%d/%s): waiting for peers to attach",
comm->c_contextid, comm->c_name);
SPIN_CONDITION(size == data->sm_bootstrap_meta->module_seg->seg_inited, seg_init_exit);
/* Once we're all here, remove the mmap file; it's not needed anymore */
if (0 == rank) {
unlink(data->sm_bootstrap_meta->shmem_ds.seg_name);
opal_output_verbose(10, ompi_coll_base_framework.framework_output,
"coll:sm:enable (%d/%s): removed mmap file %s",
comm->c_contextid, comm->c_name,
data->sm_bootstrap_meta->shmem_ds.seg_name);
}
/* All done */
opal_output_verbose(10, ompi_coll_base_framework.framework_output,
"coll:sm:enable (%d/%s): success!",
comm->c_contextid, comm->c_name);
return OMPI_SUCCESS;
}
/*
* Progress the event library and any functions that have registered to
* be called. We don't propogate errors from the progress functions,
* so no action is taken if they return failures. The functions are
* expected to return the number of events progressed, to determine
* whether or not we should call sched_yield() during MPI progress.
* This is only losely tracked, as an error return can cause the number
* of progressed events to appear lower than it actually is. We don't
* care, as the cost of that happening is far outweighed by the cost
* of the if checks (they were resulting in bad pipe stalling behavior)
*/
void
opal_progress(void)
{
size_t i;
int events = 0;
#if OPAL_HAVE_THREAD_SUPPORT
opal_atomic_add(&opal_progress_recursion_depth_counter, 1);
#else
++opal_progress_recursion_depth_counter;
#endif
if( opal_progress_event_flag != 0 ) {
#if (OPAL_ENABLE_PROGRESS_THREADS == 0) && OPAL_HAVE_WORKING_EVENTOPS
#if OPAL_PROGRESS_USE_TIMERS
#if OPAL_TIMER_USEC_NATIVE
opal_timer_t now = opal_timer_base_get_usec();
#else
opal_timer_t now = opal_timer_base_get_cycles();
#endif /* OPAL_TIMER_USEC_NATIVE */
/* trip the event library if we've reached our tick rate and we are
enabled */
if (now - event_progress_last_time > event_progress_delta ) {
event_progress_last_time = (num_event_users > 0) ?
now - event_progress_delta : now;
events += opal_event_loop(opal_progress_event_flag);
}
#else /* OPAL_PROGRESS_USE_TIMERS */
/* trip the event library if we've reached our tick rate and we are
enabled */
if (OPAL_THREAD_ADD32(&event_progress_counter, -1) <= 0 ) {
event_progress_counter =
(num_event_users > 0) ? 0 : event_progress_delta;
events += opal_event_loop(opal_progress_event_flag);
}
#endif /* OPAL_PROGRESS_USE_TIMERS */
#endif /* OPAL_ENABLE_PROGRESS_THREADS == 0 && OPAL_HAVE_WORKING_EVENTOPS */
}
/* progress all registered callbacks */
for (i = 0 ; i < callbacks_len ; ++i) {
events += (callbacks[i])();
}
#if defined(__WINDOWS__) || defined(HAVE_SCHED_YIELD)
if (call_yield && events <= 0) {
/* If there is nothing to do - yield the processor - otherwise
* we could consume the processor for the entire time slice. If
* the processor is oversubscribed - this will result in a best-case
* latency equivalent to the time-slice.
*/
#if defined(__WINDOWS__)
SwitchToThread();
#else
sched_yield();
#endif /* defined(__WINDOWS__) */
}
#endif /* defined(__WINDOWS__) || defined(HAVE_SCHED_YIELD) */
#if OPAL_HAVE_THREAD_SUPPORT
opal_atomic_add(&opal_progress_recursion_depth_counter, -1);
#else
--opal_progress_recursion_depth_counter;
#endif
}
/* look up the remote pointer in the peer rcache and attach if
* necessary */
mca_mpool_base_registration_t *vader_get_registation (struct mca_btl_base_endpoint_t *ep, void *rem_ptr,
size_t size, int flags, void **local_ptr)
{
struct mca_rcache_base_module_t *rcache = ep->rcache;
mca_mpool_base_registration_t *regs[10], *reg = NULL;
xpmem_addr_t xpmem_addr;
uintptr_t base, bound;
int rc, i;
/* protect rcache access */
OPAL_THREAD_LOCK(&ep->lock);
/* use btl/self for self communication */
assert (ep->peer_smp_rank != MCA_BTL_VADER_LOCAL_RANK);
base = (uintptr_t) down_align_addr(rem_ptr, mca_btl_vader_component.log_attach_align);
bound = (uintptr_t) up_align_addr((void *)((uintptr_t) rem_ptr + size - 1),
mca_btl_vader_component.log_attach_align) + 1;
if (OPAL_UNLIKELY(bound > VADER_MAX_ADDRESS)) {
bound = VADER_MAX_ADDRESS;
}
/* several segments may match the base pointer */
rc = rcache->rcache_find_all (rcache, (void *) base, bound - base, regs, 10);
for (i = 0 ; i < rc ; ++i) {
if (bound <= (uintptr_t)regs[i]->bound && base >= (uintptr_t)regs[i]->base) {
opal_atomic_add (®s[i]->ref_count, 1);
reg = regs[i];
goto reg_found;
}
if (regs[i]->flags & MCA_MPOOL_FLAGS_PERSIST) {
continue;
}
/* remove this pointer from the rcache and decrement its reference count
(so it is detached later) */
rc = rcache->rcache_delete (rcache, regs[i]);
if (OPAL_UNLIKELY(0 != rc)) {
/* someone beat us to it? */
break;
}
/* start the new segment from the lower of the two bases */
base = (uintptr_t) regs[i]->base < base ? (uintptr_t) regs[i]->base : base;
opal_atomic_add (®s[i]->ref_count, -1);
if (OPAL_LIKELY(0 == regs[i]->ref_count)) {
/* this pointer is not in use */
(void) xpmem_detach (regs[i]->alloc_base);
OBJ_RELEASE(regs[i]);
}
break;
}
reg = OBJ_NEW(mca_mpool_base_registration_t);
if (OPAL_LIKELY(NULL != reg)) {
/* stick around for awhile */
reg->ref_count = 2;
reg->base = (unsigned char *) base;
reg->bound = (unsigned char *) bound;
reg->flags = flags;
#if defined(HAVE_SN_XPMEM_H)
xpmem_addr.id = ep->apid;
#else
xpmem_addr.apid = ep->apid;
#endif
xpmem_addr.offset = base;
reg->alloc_base = xpmem_attach (xpmem_addr, bound - base, NULL);
if (OPAL_UNLIKELY((void *)-1 == reg->alloc_base)) {
OPAL_THREAD_UNLOCK(&ep->lock);
OBJ_RELEASE(reg);
return NULL;
}
opal_memchecker_base_mem_defined (reg->alloc_base, bound - base);
rcache->rcache_insert (rcache, reg, 0);
}
reg_found:
opal_atomic_wmb ();
*local_ptr = (void *) ((uintptr_t) reg->alloc_base +
(ptrdiff_t)((uintptr_t) rem_ptr - (uintptr_t) reg->base));
OPAL_THREAD_UNLOCK(&ep->lock);
return reg;
}
static void* thr2_run(opal_object_t* obj)
{
opal_atomic_add(&count, 2);
return NULL;
}
