int
ompi_osc_pt2pt_module_test(ompi_win_t *win,
int *flag)
{
ompi_group_t *group;
ompi_osc_pt2pt_module_t *module = P2P_MODULE(win);
#if !OPAL_ENABLE_PROGRESS_THREADS
opal_progress();
#endif
if (0 != (module->p2p_num_pending_in) ||
0 != (module->p2p_num_complete_msgs)) {
*flag = 0;
return OMPI_SUCCESS;
}
*flag = 1;
ompi_win_remove_mode(win, OMPI_WIN_EXPOSE_EPOCH | OMPI_WIN_POSTED);
OPAL_THREAD_LOCK(&(module->p2p_lock));
group = module->p2p_pw_group;
module->p2p_pw_group = NULL;
OPAL_THREAD_UNLOCK(&(module->p2p_lock));
ompi_group_decrement_proc_count(group);
OBJ_RELEASE(group);
return OMPI_SUCCESS;
}
int ompi_group_free ( ompi_group_t **group )
{
ompi_group_t *l_group;
l_group = (ompi_group_t *) *group;
ompi_group_decrement_proc_count (l_group);
OBJ_RELEASE(l_group);
*group = MPI_GROUP_NULL;
return OMPI_SUCCESS;
}
static void
ompi_win_destruct(ompi_win_t *win)
{
if (NULL != win->w_keyhash) {
ompi_attr_delete_all(WIN_ATTR, win, win->w_keyhash);
OBJ_RELEASE(win->w_keyhash);
}
if (NULL != win->error_handler) {
OBJ_RELEASE(win->error_handler);
}
if (NULL != win->w_group) {
ompi_group_decrement_proc_count(win->w_group);
OBJ_RELEASE(win->w_group);
}
OBJ_DESTRUCT(&win->w_lock);
}
/*
* group destructor
*/
static void ompi_group_destruct(ompi_group_t *group)
{
/* Note that we do *NOT* decrease the refcount on all the included
procs here because that is handled at a different level (e.g.,
the proc counts are not increased during the constructor,
either). */
/* release thegrp_proc_pointers memory */
if (NULL != group->grp_proc_pointers) {
free(group->grp_proc_pointers);
}
if (OMPI_GROUP_IS_SPORADIC(group)) {
if (NULL != group->sparse_data.grp_sporadic.grp_sporadic_list) {
free(group->sparse_data.grp_sporadic.grp_sporadic_list);
}
}
if (OMPI_GROUP_IS_BITMAP(group)) {
if (NULL != group->sparse_data.grp_bitmap.grp_bitmap_array) {
free(group->sparse_data.grp_bitmap.grp_bitmap_array);
}
}
if (NULL != group->grp_parent_group_ptr){
ompi_group_decrement_proc_count(group->grp_parent_group_ptr);
OBJ_RELEASE(group->grp_parent_group_ptr);
}
/* reset the ompi_group_f_to_c_table entry - make sure that the
* entry is in the table */
if (NULL != opal_pointer_array_get_item(&ompi_group_f_to_c_table,
group->grp_f_to_c_index)) {
opal_pointer_array_set_item(&ompi_group_f_to_c_table,
group->grp_f_to_c_index, NULL);
}
/* return */
return;
}
int
ompi_osc_pt2pt_module_wait(ompi_win_t *win)
{
ompi_group_t *group;
ompi_osc_pt2pt_module_t *module = P2P_MODULE(win);
OPAL_THREAD_LOCK(&module->p2p_lock);
while (0 != (module->p2p_num_pending_in) ||
0 != (module->p2p_num_complete_msgs)) {
opal_condition_wait(&module->p2p_cond, &module->p2p_lock);
}
group = module->p2p_pw_group;
module->p2p_pw_group = NULL;
OPAL_THREAD_UNLOCK(&module->p2p_lock);
ompi_win_remove_mode(win, OMPI_WIN_EXPOSE_EPOCH | OMPI_WIN_POSTED);
ompi_group_decrement_proc_count(group);
OBJ_RELEASE(group);
return OMPI_SUCCESS;
}
int
ompi_osc_pt2pt_module_complete(ompi_win_t *win)
{
int i;
int ret = OMPI_SUCCESS;
ompi_group_t *group;
opal_list_item_t *item;
ompi_osc_pt2pt_module_t *module = P2P_MODULE(win);
/* wait for all the post messages */
OPAL_THREAD_LOCK(&module->p2p_lock);
while (0 != module->p2p_num_post_msgs) {
opal_condition_wait(&module->p2p_cond, &module->p2p_lock);
}
ompi_osc_pt2pt_flip_sendreqs(module);
/* for each process in group, send a control message with number
of updates coming, then start all the requests */
for (i = 0 ; i < ompi_group_size(module->p2p_sc_group) ; ++i) {
int comm_rank = module->p2p_sc_remote_ranks[i];
module->p2p_num_pending_out +=
module->p2p_copy_num_pending_sendreqs[comm_rank];
}
OPAL_THREAD_UNLOCK(&module->p2p_lock);
for (i = 0 ; i < ompi_group_size(module->p2p_sc_group) ; ++i) {
int comm_rank = module->p2p_sc_remote_ranks[i];
ret = ompi_osc_pt2pt_control_send(module,
ompi_group_peer_lookup(module->p2p_sc_group, i),
OMPI_OSC_PT2PT_HDR_COMPLETE,
module->p2p_copy_num_pending_sendreqs[comm_rank],
0);
assert(ret == OMPI_SUCCESS);
}
/* try to start all the requests. We've copied everything we
need out of pending_sendreqs, so don't need the lock
here */
while (NULL !=
(item = opal_list_remove_first(&(module->p2p_copy_pending_sendreqs)))) {
ompi_osc_pt2pt_sendreq_t *req =
(ompi_osc_pt2pt_sendreq_t*) item;
ret = ompi_osc_pt2pt_sendreq_send(module, req);
if (OMPI_ERR_TEMP_OUT_OF_RESOURCE == ret ) {
opal_output_verbose(5, ompi_osc_base_output,
"complete: failure in starting sendreq (%d). Will try later.",
ret);
opal_list_append(&(module->p2p_copy_pending_sendreqs), item);
} else if (OMPI_SUCCESS != ret) {
return ret;
}
}
/* wait for all the requests */
OPAL_THREAD_LOCK(&module->p2p_lock);
while (0 != module->p2p_num_pending_out) {
opal_condition_wait(&module->p2p_cond, &module->p2p_lock);
}
group = module->p2p_sc_group;
module->p2p_sc_group = NULL;
OPAL_THREAD_UNLOCK(&module->p2p_lock);
/* remove WIN_POSTED from our mode */
ompi_win_remove_mode(win, OMPI_WIN_ACCESS_EPOCH | OMPI_WIN_STARTED);
ompi_group_decrement_proc_count(group);
OBJ_RELEASE(group);
return ret;
}
int
ompi_osc_pt2pt_module_start(ompi_group_t *group,
int assert,
ompi_win_t *win)
{
int i, ret = OMPI_SUCCESS;
ompi_osc_pt2pt_module_t *module = P2P_MODULE(win);
OBJ_RETAIN(group);
ompi_group_increment_proc_count(group);
OPAL_THREAD_LOCK(&(module->p2p_lock));
if (NULL != module->p2p_sc_group) {
OPAL_THREAD_UNLOCK(&module->p2p_lock);
ret = MPI_ERR_RMA_SYNC;
goto cleanup;
}
module->p2p_sc_group = group;
/* possible we've already received a couple in messages, so
add however many we're going to wait for */
module->p2p_num_post_msgs += ompi_group_size(module->p2p_sc_group);
OPAL_THREAD_UNLOCK(&(module->p2p_lock));
memset(module->p2p_sc_remote_active_ranks, 0,
sizeof(bool) * ompi_comm_size(module->p2p_comm));
/* for each process in the specified group, find it's rank in our
communicator, store those indexes, and set the true / false in
the active ranks table */
for (i = 0 ; i < ompi_group_size(group) ; i++) {
int comm_rank = -1, j;
/* find the rank in the communicator associated with this windows */
for (j = 0 ; j < ompi_comm_size(module->p2p_comm) ; ++j) {
if (ompi_group_peer_lookup(module->p2p_sc_group, i) ==
ompi_comm_peer_lookup(module->p2p_comm, j)) {
comm_rank = j;
break;
}
}
if (comm_rank == -1) {
ret = MPI_ERR_RMA_SYNC;
goto cleanup;
}
module->p2p_sc_remote_active_ranks[comm_rank] = true;
module->p2p_sc_remote_ranks[i] = comm_rank;
}
/* Set our mode to access w/ start */
ompi_win_remove_mode(win, OMPI_WIN_FENCE);
ompi_win_append_mode(win, OMPI_WIN_ACCESS_EPOCH | OMPI_WIN_STARTED);
return OMPI_SUCCESS;
cleanup:
ompi_group_decrement_proc_count(group);
OBJ_RELEASE(group);
return ret;
}
static void ompi_comm_destruct(ompi_communicator_t* comm)
{
/* Note that the attributes were already released on this
communicator in ompi_comm_free() (i.e., from MPI_COMM_FREE /
MPI_COMM_DISCONNECT). See the lengthy comment in
communicator/comm.c in ompi_comm_free() for the reasons why. */
/* Release the collective module */
if ( MPI_COMM_NULL != comm ) {
mca_coll_base_comm_unselect(comm);
}
/* Tell the PML that this communicator is done.
MCA_PML_CALL(add_comm()) was called explicitly in
ompi_comm_init() when setting up COMM_WORLD and COMM_SELF; it's
called in ompi_comm_set() for all others. This means that all
communicators must be destroyed before the PML shuts down.
Also -- do not invoke the pml_del_comm if the corresponding
pml_add_comm was never invoked. This can happen in an error
situation where, for example, attributes do not copy properly
from one communicator to another and we end up destroying the
new communication while propagating the error up the stack. We
want to make it all the way up the stack to invoke the MPI
exception, not cause a seg fault in pml_del_comm because it was
never pml_add_com'ed. */
if ( MPI_COMM_NULL != comm && OMPI_COMM_IS_PML_ADDED(comm) ) {
MCA_PML_CALL(del_comm (comm));
}
/* Release topology module */
if (NULL != comm->c_topo) {
OBJ_RELEASE(comm->c_topo);
comm->c_topo = NULL;
}
if (NULL != comm->c_local_group) {
ompi_group_decrement_proc_count (comm->c_local_group);
OBJ_RELEASE ( comm->c_local_group );
comm->c_local_group = NULL;
if ( OMPI_COMM_IS_INTRA(comm) ) {
/* We have to decrement the ref count on the remote group
even if it is identical to the local one in case of
intra-comm */
OBJ_RELEASE ( comm->c_remote_group );
comm->c_remote_group = NULL;
}
}
if (NULL != comm->c_remote_group) {
ompi_group_decrement_proc_count (comm->c_remote_group);
OBJ_RELEASE ( comm->c_remote_group );
comm->c_remote_group = NULL;
}
if (NULL != comm->error_handler) {
OBJ_RELEASE ( comm->error_handler );
comm->error_handler = NULL;
}
/* reset the ompi_comm_f_to_c_table entry */
if ( MPI_UNDEFINED != comm->c_f_to_c_index &&
NULL != opal_pointer_array_get_item(&ompi_mpi_communicators,
comm->c_f_to_c_index )) {
opal_pointer_array_set_item ( &ompi_mpi_communicators,
comm->c_f_to_c_index, NULL);
}
return;
}
