int ompi_group_intersection(ompi_group_t* group1,ompi_group_t* group2,
ompi_group_t **new_group)
{
int proc1,proc2,k, result;
int *ranks_included=NULL;
ompi_group_t *group1_pointer, *group2_pointer;
ompi_proc_t *proc1_pointer, *proc2_pointer;
group1_pointer=(ompi_group_t *)group1;
group2_pointer=(ompi_group_t *)group2;
/* determine the number of included processes for the incl-method */
k = 0;
for (proc1 = 0; proc1 < group1_pointer->grp_proc_count; proc1++) {
proc1_pointer = ompi_group_peer_lookup (group1_pointer , proc1);
/* check to see if this proc is in group2 */
for (proc2 = 0; proc2 < group2_pointer->grp_proc_count; proc2++) {
proc2_pointer = ompi_group_peer_lookup (group2_pointer , proc2);
if( proc1_pointer == proc2_pointer ) {
k++;
break;
}
} /* end proc2 loop */
} /* end proc1 loop */
if (0 != k) {
ranks_included = (int *)malloc( k*(sizeof(int)));
}
/* determine the list of included processes for the incl-method */
k = 0;
for (proc1 = 0; proc1 < group1_pointer->grp_proc_count; proc1++) {
proc1_pointer = ompi_group_peer_lookup (group1_pointer , proc1);
/* check to see if this proc is in group2 */
for (proc2 = 0; proc2 < group2_pointer->grp_proc_count; proc2++) {
proc2_pointer = ompi_group_peer_lookup (group2_pointer ,proc2);
if( proc1_pointer == proc2_pointer ) {
ranks_included[k] = proc1;
k++;
break;
}
} /* end proc2 loop */
} /* end proc1 loop */
result = ompi_group_incl(group1, k, ranks_included, new_group);
if (NULL != ranks_included) {
free(ranks_included);
}
return result;
}
/* All we want to do in this function is determine if the number of
* jobids in the local and/or remote group is > 1. This tells us to
* set the disconnect flag. We don't actually care what the true
* number -is-, only that it is > 1
*/
void ompi_dpm_base_mark_dyncomm (ompi_communicator_t *comm)
{
int i;
int size, rsize;
bool found=false;
ompi_jobid_t thisjobid;
ompi_group_t *grp=NULL;
ompi_proc_t *proc = NULL;
/* special case for MPI_COMM_NULL */
if ( comm == MPI_COMM_NULL ) {
return;
}
size = ompi_comm_size (comm);
rsize = ompi_comm_remote_size(comm);
/* loop over all processes in local group and check for
* a different jobid
*/
grp = comm->c_local_group;
proc = ompi_group_peer_lookup(grp,0);
thisjobid = ((ompi_process_name_t*)&proc->super.proc_name)->jobid;
for (i=1; i< size; i++) {
proc = ompi_group_peer_lookup(grp,i);
if (thisjobid != ((ompi_process_name_t*)&proc->super.proc_name)->jobid) {
/* at least one is different */
found = true;
goto complete;
}
}
/* if inter-comm, loop over all processes in remote_group
* and see if any are different from thisjobid
*/
grp = comm->c_remote_group;
for (i=0; i< rsize; i++) {
proc = ompi_group_peer_lookup(grp,i);
if (thisjobid != ((ompi_process_name_t*)&proc->super.proc_name)->jobid) {
/* at least one is different */
found = true;
break;
}
}
complete:
/* if a different jobid was found, set the disconnect flag*/
if (found) {
ompi_comm_num_dyncomm++;
OMPI_COMM_SET_DYNAMIC(comm);
}
return;
}
int
ompi_osc_pt2pt_module_post(ompi_group_t *group,
int assert,
ompi_win_t *win)
{
int i;
ompi_osc_pt2pt_module_t *module = P2P_MODULE(win);
OBJ_RETAIN(group);
ompi_group_increment_proc_count(group);
OPAL_THREAD_LOCK(&(module->p2p_lock));
assert(NULL == module->p2p_pw_group);
module->p2p_pw_group = group;
/* Set our mode to expose w/ post */
ompi_win_remove_mode(win, OMPI_WIN_FENCE);
ompi_win_append_mode(win, OMPI_WIN_EXPOSE_EPOCH | OMPI_WIN_POSTED);
/* list how many complete counters we're still waiting on */
module->p2p_num_complete_msgs +=
ompi_group_size(module->p2p_pw_group);
OPAL_THREAD_UNLOCK(&(module->p2p_lock));
/* send a hello counter to everyone in group */
for (i = 0 ; i < ompi_group_size(module->p2p_pw_group) ; ++i) {
ompi_osc_pt2pt_control_send(module,
ompi_group_peer_lookup(group, i),
OMPI_OSC_PT2PT_HDR_POST, 1, 0);
}
return OMPI_SUCCESS;
}
struct mca_sharedfp_base_module_1_0_0_t * mca_sharedfp_sm_component_file_query(mca_io_ompio_file_t *fh, int *priority)
{
int i;
ompi_proc_t *proc;
ompi_communicator_t * comm = fh->f_comm;
int size = ompi_comm_size(comm);
*priority = 0;
/* test, and update priority. All processes have to be
** on a single node.
** original test copied from mca/coll/sm/coll_sm_module.c:
*/
ompi_group_t *group = comm->c_local_group;
for (i = 0; i < size; ++i) {
proc = ompi_group_peer_lookup(group,i);
if (!OPAL_PROC_ON_LOCAL_NODE(proc->proc_flags)){
opal_output(1,"mca_sharedfp_sm_component_file_query: Disqualifying myself: (%d/%s) "
"not all processes are on the same node.",
comm->c_contextid, comm->c_name);
return NULL;
}
}
/* This module can run */
*priority = mca_sharedfp_sm_priority;
return &sm;
}
int ompi_group_incl_plist(ompi_group_t* group, int n, const int *ranks,
ompi_group_t **new_group)
{
/* local variables */
int proc,my_group_rank;
ompi_group_t *group_pointer, *new_group_pointer;
ompi_proc_t *my_proc_pointer;
group_pointer = (ompi_group_t *)group;
if ( 0 == n ) {
*new_group = MPI_GROUP_EMPTY;
OBJ_RETAIN(MPI_GROUP_EMPTY);
return OMPI_SUCCESS;
}
/* get new group struct */
new_group_pointer=ompi_group_allocate(n);
if( NULL == new_group_pointer ) {
return MPI_ERR_GROUP;
}
/* put group elements in the list */
for (proc = 0; proc < n; proc++) {
new_group_pointer->grp_proc_pointers[proc] =
ompi_group_peer_lookup(group_pointer,ranks[proc]);
} /* end proc loop */
/* increment proc reference counters */
ompi_group_increment_proc_count(new_group_pointer);
/* find my rank */
my_group_rank=group_pointer->grp_my_rank;
if (MPI_UNDEFINED != my_group_rank) {
my_proc_pointer=ompi_group_peer_lookup (group_pointer,my_group_rank);
ompi_set_group_rank(new_group_pointer,my_proc_pointer);
}
else {
new_group_pointer->grp_my_rank = MPI_UNDEFINED;
}
*new_group = (MPI_Group)new_group_pointer;
return OMPI_SUCCESS;
}
void ompi_group_decrement_proc_count(ompi_group_t *group)
{
int proc;
ompi_proc_t * proc_pointer;
for (proc = 0; proc < group->grp_proc_count; proc++) {
proc_pointer = ompi_group_peer_lookup(group,proc);
OBJ_RELEASE(proc_pointer);
}
return;
}
static bool have_local_peers(ompi_group_t *group, size_t size)
{
size_t i;
ompi_proc_t *proc;
for (i = 0; i < size; ++i) {
proc = ompi_group_peer_lookup(group,i);
if (!OPAL_PROC_ON_LOCAL_NODE(proc->proc_flags)) {
return false;
}
}
return true;
}
int mca_pml_dr_add_comm(ompi_communicator_t* comm)
{
/* allocate pml specific comm data */
mca_pml_dr_comm_t* pml_comm = OBJ_NEW(mca_pml_dr_comm_t);
int i;
if (NULL == pml_comm) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
mca_pml_dr_comm_init(pml_comm, comm);
comm->c_pml_comm = pml_comm;
for( i = 0; i < comm->c_remote_group->grp_proc_count; i++ ) {
pml_comm->procs[i].ompi_proc = ompi_group_peer_lookup(comm->c_remote_group,i);
}
return OMPI_SUCCESS;
}
static int have_remote_peers(ompi_group_t *group, size_t size, int *local_peers)
{
ompi_proc_t *proc;
size_t i;
int ret;
*local_peers = 0;
ret = 0;
for (i = 0; i < size; ++i) {
proc = ompi_group_peer_lookup(group, i);
if (FCA_IS_LOCAL_PROCESS(proc->proc_flags)) {
++*local_peers;
} else {
ret = 1;
}
}
return ret;
}
bool ompi_group_have_remote_peers (ompi_group_t *group)
{
for (int i = 0 ; i < group->grp_proc_count ; ++i) {
ompi_proc_t *proc = NULL;
#if OMPI_GROUP_SPARSE
proc = ompi_group_peer_lookup (group, i);
#else
proc = ompi_group_get_proc_ptr_raw (group, i);
if (ompi_proc_is_sentinel (proc)) {
/* the proc must be stored in the group or cached in the proc
* hash table if the process resides in the local node
* (see ompi_proc_complete_init) */
return true;
}
#endif
if (!OPAL_PROC_ON_LOCAL_NODE(proc->super.proc_flags)) {
return true;
}
}
return false;
}
int mca_topo_base_graph_create(mca_topo_base_module_t *topo,
ompi_communicator_t* old_comm,
int nnodes,
int *index,
int *edges,
bool reorder,
ompi_communicator_t** comm_topo)
{
ompi_communicator_t *new_comm;
int new_rank, num_procs, ret, i;
ompi_proc_t **topo_procs = NULL;
mca_topo_base_comm_graph_2_2_0_t* graph;
num_procs = old_comm->c_local_group->grp_proc_count;
new_rank = old_comm->c_local_group->grp_my_rank;
assert(topo->type == OMPI_COMM_GRAPH);
if( num_procs < nnodes ) {
return MPI_ERR_DIMS;
}
if( num_procs > nnodes ) {
num_procs = nnodes;
}
if( new_rank > (nnodes - 1) ) {
new_rank = MPI_UNDEFINED;
num_procs = 0;
nnodes = 0;
}
graph = OBJ_NEW(mca_topo_base_comm_graph_2_2_0_t);
if( NULL == graph ) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
graph->nnodes = nnodes;
/* Don't do any of the other initialization if we're not supposed
to be part of the new communicator (because nnodes has been
reset to 0, making things like index[nnodes-1] be junk).
JMS: This should really be refactored to use
comm_create_group(), because ompi_comm_allocate() still
complains about 0-byte mallocs in debug builds for 0-member
groups. */
if (MPI_UNDEFINED != new_rank) {
graph->index = (int*)malloc(sizeof(int) * nnodes);
if (NULL == graph->index) {
OBJ_RELEASE(graph);
return OMPI_ERR_OUT_OF_RESOURCE;
}
memcpy(graph->index, index, nnodes * sizeof(int));
/* Graph communicator; copy the right data to the common information */
graph->edges = (int*)malloc(sizeof(int) * index[nnodes-1]);
if (NULL == graph->edges) {
OBJ_RELEASE(graph);
return OMPI_ERR_OUT_OF_RESOURCE;
}
memcpy(graph->edges, edges, index[nnodes-1] * sizeof(int));
topo_procs = (ompi_proc_t**)malloc(num_procs * sizeof(ompi_proc_t *));
if (NULL == topo_procs) {
OBJ_RELEASE(graph);
return OMPI_ERR_OUT_OF_RESOURCE;
}
if(OMPI_GROUP_IS_DENSE(old_comm->c_local_group)) {
memcpy(topo_procs,
old_comm->c_local_group->grp_proc_pointers,
num_procs * sizeof(ompi_proc_t *));
} else {
for(i = 0 ; i < num_procs; i++) {
topo_procs[i] = ompi_group_peer_lookup(old_comm->c_local_group,i);
}
}
}
/* allocate a new communicator */
new_comm = ompi_comm_allocate(nnodes, 0);
if (NULL == new_comm) {
free(topo_procs);
OBJ_RELEASE(graph);
return OMPI_ERR_OUT_OF_RESOURCE;
}
ret = ompi_comm_enable(old_comm, new_comm,
new_rank, num_procs, topo_procs);
if (OMPI_SUCCESS != ret) {
free(topo_procs);
OBJ_RELEASE(graph);
ompi_comm_free (&new_comm);
return ret;
}
new_comm->c_topo = topo;
new_comm->c_topo->mtc.graph = graph;
new_comm->c_flags |= OMPI_COMM_GRAPH;
new_comm->c_topo->reorder = reorder;
*comm_topo = new_comm;
if( MPI_UNDEFINED == new_rank ) {
ompi_comm_free(&new_comm);
*comm_topo = MPI_COMM_NULL;
}
return OMPI_SUCCESS;
}
int ompi_group_translate_ranks ( ompi_group_t *group1,
int n_ranks, const int *ranks1,
ompi_group_t *group2,
int *ranks2)
{
int rank, proc, proc2;
struct ompi_proc_t *proc1_pointer, *proc2_pointer;
if ( MPI_GROUP_EMPTY == group1 || MPI_GROUP_EMPTY == group2 ) {
for (proc = 0; proc < n_ranks ; proc++) {
ranks2[proc] = MPI_UNDEFINED;
}
return MPI_SUCCESS;
}
/*
* If we are translating from a parent to a child that uses the sparse format
* or vice versa, we use the translate ranks function corresponding to the
* format used. Generally, all these functions require less time than the
* original method that loops over the processes of both groups till we
* find a match.
*/
if( group1->grp_parent_group_ptr == group2 ) { /* from child to parent */
if(OMPI_GROUP_IS_SPORADIC(group1)) {
return ompi_group_translate_ranks_sporadic_reverse
(group1,n_ranks,ranks1,group2,ranks2);
}
else if(OMPI_GROUP_IS_STRIDED(group1)) {
return ompi_group_translate_ranks_strided_reverse
(group1,n_ranks,ranks1,group2,ranks2);
}
else if(OMPI_GROUP_IS_BITMAP(group1)) {
return ompi_group_translate_ranks_bmap_reverse
(group1,n_ranks,ranks1,group2,ranks2);
}
}
else if( group2->grp_parent_group_ptr == group1 ) { /* from parent to child*/
if(OMPI_GROUP_IS_SPORADIC(group2)) {
return ompi_group_translate_ranks_sporadic
(group1,n_ranks,ranks1,group2,ranks2);
}
else if(OMPI_GROUP_IS_STRIDED(group2)) {
return ompi_group_translate_ranks_strided
(group1,n_ranks,ranks1,group2,ranks2);
}
else if(OMPI_GROUP_IS_BITMAP(group2)) {
return ompi_group_translate_ranks_bmap
(group1,n_ranks,ranks1,group2,ranks2);
}
}
else {
/* loop over all ranks */
for (proc = 0; proc < n_ranks; proc++) {
rank=ranks1[proc];
if ( MPI_PROC_NULL == rank) {
ranks2[proc] = MPI_PROC_NULL;
}
else {
proc1_pointer = ompi_group_peer_lookup(group1 ,rank);
/* initialize to no "match" */
ranks2[proc] = MPI_UNDEFINED;
for (proc2 = 0; proc2 < group2->grp_proc_count; proc2++) {
proc2_pointer= ompi_group_peer_lookup(group2, proc2);
if ( proc1_pointer == proc2_pointer) {
ranks2[proc] = proc2;
break;
}
} /* end proc2 loop */
} /* end proc loop */
}
}
return MPI_SUCCESS;
}
int MPI_Group_compare(MPI_Group group1, MPI_Group group2, int *result) {
/* local variables */
int return_value, proc1, proc2, match;
bool similar, identical;
ompi_group_t *group1_pointer, *group2_pointer;
ompi_proc_t *proc1_pointer, *proc2_pointer;
OPAL_CR_NOOP_PROGRESS();
/* initialization */
return_value=MPI_SUCCESS;
/* check for errors */
if( MPI_PARAM_CHECK ) {
OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
if( ( MPI_GROUP_NULL == group1 ) || ( MPI_GROUP_NULL == group2 ) ||
(NULL == group1) || (NULL==group2) ){
return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_GROUP,
FUNC_NAME);
} else if (NULL == result) {
return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG,
FUNC_NAME);
}
}
/* check for same groups */
if( group1 == group2 ) {
*result=MPI_IDENT;
return return_value;
}
/* check to see if either is MPI_GROUP_NULL or MPI_GROUP_EMPTY */
if( ( MPI_GROUP_EMPTY == group1 ) || ( MPI_GROUP_EMPTY == group2 ) ) {
*result=MPI_UNEQUAL;
return return_value;
}
/* get group pointers */
group1_pointer = (ompi_group_t *)group1;
group2_pointer = (ompi_group_t *)group2;
/* compare sizes */
if( group1_pointer->grp_proc_count != group2_pointer->grp_proc_count ) {
/* if not same size - return */
*result=MPI_UNEQUAL;
return return_value;
}
/* check for similarity */
/* loop over group1 processes */
similar=true;
identical=true;
for(proc1=0 ; proc1 < group1_pointer->grp_proc_count ; proc1++ ) {
proc1_pointer= ompi_group_peer_lookup(group1_pointer,proc1);
/* loop over group2 processes to find "match" */
match=-1;
for(proc2=0 ; proc2 < group2_pointer->grp_proc_count ; proc2++ ) {
proc2_pointer=ompi_group_peer_lookup(group2_pointer,proc2);
if( proc1_pointer == proc2_pointer ) {
if(proc1 != proc2 ) {
identical=false;
}
match=proc2;
break;
}
} /* end proc2 loop */
if( match== -1 ) {
similar=false;
identical=false;
break;
}
} /* end proc1 loop */
/* set comparison result */
if( identical ) {
*result=MPI_IDENT;
} else if( similar ) {
*result=MPI_SIMILAR;
} else {
*result=MPI_UNEQUAL;
}
return return_value;
}
int mca_topo_base_cart_create(mca_topo_base_module_t *topo,
ompi_communicator_t* old_comm,
int ndims,
const int *dims,
const int *periods,
bool reorder,
ompi_communicator_t** comm_topo)
{
int nprocs = 1, i, new_rank, num_procs, ret;
ompi_communicator_t *new_comm;
ompi_proc_t **topo_procs = NULL;
mca_topo_base_comm_cart_2_2_0_t* cart;
num_procs = old_comm->c_local_group->grp_proc_count;
new_rank = old_comm->c_local_group->grp_my_rank;
assert(topo->type == OMPI_COMM_CART);
/* Calculate the number of processes in this grid */
for (i = 0; i < ndims; ++i) {
if(dims[i] <= 0) {
return OMPI_ERROR;
}
nprocs *= dims[i];
}
/* check for the error condition */
if (num_procs < nprocs) {
return MPI_ERR_DIMS;
}
/* check if we have to trim the list of processes */
if (nprocs < num_procs) {
num_procs = nprocs;
}
if (new_rank > (nprocs-1)) {
ndims = 0;
new_rank = MPI_UNDEFINED;
num_procs = 0;
}
cart = OBJ_NEW(mca_topo_base_comm_cart_2_2_0_t);
if( NULL == cart ) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
cart->ndims = ndims;
/* MPI-2.1 allows 0-dimension cartesian communicators, so prevent
a 0-byte malloc -- leave dims as NULL */
if( ndims > 0 ) {
cart->dims = (int*)malloc(sizeof(int) * ndims);
if (NULL == cart->dims) {
OBJ_RELEASE(cart);
return OMPI_ERROR;
}
memcpy(cart->dims, dims, ndims * sizeof(int));
/* Cartesian communicator; copy the right data to the common information */
cart->periods = (int*)malloc(sizeof(int) * ndims);
if (NULL == cart->periods) {
OBJ_RELEASE(cart);
return OMPI_ERR_OUT_OF_RESOURCE;
}
memcpy(cart->periods, periods, ndims * sizeof(int));
cart->coords = (int*)malloc(sizeof(int) * ndims);
if (NULL == cart->coords) {
OBJ_RELEASE(cart);
return OMPI_ERR_OUT_OF_RESOURCE;
}
{ /* setup the cartesian topology */
int nprocs = num_procs, rank = new_rank;
for (i = 0; i < ndims; ++i) {
nprocs /= cart->dims[i];
cart->coords[i] = rank / nprocs;
rank %= nprocs;
}
}
}
/* JMS: This should really be refactored to use
comm_create_group(), because ompi_comm_allocate() still
complains about 0-byte mallocs in debug builds for 0-member
groups. */
if (num_procs > 0) {
/* Copy the proc structure from the previous communicator over to
the new one. The topology module is then able to work on this
copy and rearrange it as it deems fit. */
topo_procs = (ompi_proc_t**)malloc(num_procs * sizeof(ompi_proc_t *));
if (NULL == topo_procs) {
OBJ_RELEASE(cart);
return OMPI_ERR_OUT_OF_RESOURCE;
}
if(OMPI_GROUP_IS_DENSE(old_comm->c_local_group)) {
memcpy(topo_procs,
old_comm->c_local_group->grp_proc_pointers,
num_procs * sizeof(ompi_proc_t *));
} else {
for(i = 0 ; i < num_procs; i++) {
topo_procs[i] = ompi_group_peer_lookup(old_comm->c_local_group,i);
}
}
}
/* allocate a new communicator */
new_comm = ompi_comm_allocate(num_procs, 0);
if (NULL == new_comm) {
free(topo_procs);
OBJ_RELEASE(cart);
return MPI_ERR_INTERN;
}
assert(NULL == new_comm->c_topo);
assert(!(new_comm->c_flags & OMPI_COMM_CART));
new_comm->c_topo = topo;
new_comm->c_topo->mtc.cart = cart;
new_comm->c_topo->reorder = reorder;
new_comm->c_flags |= OMPI_COMM_CART;
ret = ompi_comm_enable(old_comm, new_comm,
new_rank, num_procs, topo_procs);
if (OMPI_SUCCESS != ret) {
/* something wrong happened during setting the communicator */
free(topo_procs);
OBJ_RELEASE(cart);
if (MPI_COMM_NULL != new_comm) {
new_comm->c_topo = NULL;
new_comm->c_flags &= ~OMPI_COMM_CART;
ompi_comm_free (&new_comm);
}
return ret;
}
*comm_topo = new_comm;
if( MPI_UNDEFINED == new_rank ) {
ompi_comm_free(&new_comm);
*comm_topo = MPI_COMM_NULL;
}
/* end here */
return OMPI_SUCCESS;
}
int mca_pml_ob1_add_comm(ompi_communicator_t* comm)
{
/* allocate pml specific comm data */
mca_pml_ob1_comm_t* pml_comm = OBJ_NEW(mca_pml_ob1_comm_t);
opal_list_item_t *item, *next_item;
mca_pml_ob1_recv_frag_t* frag;
mca_pml_ob1_comm_proc_t* pml_proc;
mca_pml_ob1_match_hdr_t* hdr;
int i;
if (NULL == pml_comm) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* should never happen, but it was, so check */
if (comm->c_contextid > mca_pml_ob1.super.pml_max_contextid) {
OBJ_RELEASE(pml_comm);
return OMPI_ERR_OUT_OF_RESOURCE;
}
mca_pml_ob1_comm_init_size(pml_comm, comm->c_remote_group->grp_proc_count);
comm->c_pml_comm = pml_comm;
for( i = 0; i < comm->c_remote_group->grp_proc_count; i++ ) {
pml_comm->procs[i].ompi_proc = ompi_group_peer_lookup(comm->c_remote_group,i);
OBJ_RETAIN(pml_comm->procs[i].ompi_proc);
}
/* Grab all related messages from the non_existing_communicator pending queue */
for( item = opal_list_get_first(&mca_pml_ob1.non_existing_communicator_pending);
item != opal_list_get_end(&mca_pml_ob1.non_existing_communicator_pending);
item = next_item ) {
frag = (mca_pml_ob1_recv_frag_t*)item;
next_item = opal_list_get_next(item);
hdr = &frag->hdr.hdr_match;
/* Is this fragment for the current communicator ? */
if( frag->hdr.hdr_match.hdr_ctx != comm->c_contextid )
continue;
/* As we now know we work on a fragment for this communicator
* we should remove it from the
* non_existing_communicator_pending list. */
opal_list_remove_item( &mca_pml_ob1.non_existing_communicator_pending,
item );
add_fragment_to_unexpected:
/* We generate the MSG_ARRIVED event as soon as the PML is aware
* of a matching fragment arrival. Independing if it is received
* on the correct order or not. This will allow the tools to
* figure out if the messages are not received in the correct
* order (if multiple network interfaces).
*/
PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_MSG_ARRIVED, comm,
hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV);
/* There is no matching to be done, and no lock to be held on the communicator as
* we know at this point that the communicator has not yet been returned to the user.
* The only required protection is around the non_existing_communicator_pending queue.
* We just have to push the fragment into the unexpected list of the corresponding
* proc, or into the out-of-order (cant_match) list.
*/
pml_proc = &(pml_comm->procs[hdr->hdr_src]);
if( ((uint16_t)hdr->hdr_seq) == ((uint16_t)pml_proc->expected_sequence) ) {
/* We're now expecting the next sequence number. */
pml_proc->expected_sequence++;
opal_list_append( &pml_proc->unexpected_frags, (opal_list_item_t*)frag );
PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_MSG_INSERT_IN_UNEX_Q, comm,
hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV);
/* And now the ugly part. As some fragments can be inserted in the cant_match list,
* every time we succesfully add a fragment in the unexpected list we have to make
* sure the next one is not in the cant_match. Otherwise, we will endup in a deadlock
* situation as the cant_match is only checked when a new fragment is received from
* the network.
*/
for(frag = (mca_pml_ob1_recv_frag_t *)opal_list_get_first(&pml_proc->frags_cant_match);
frag != (mca_pml_ob1_recv_frag_t *)opal_list_get_end(&pml_proc->frags_cant_match);
frag = (mca_pml_ob1_recv_frag_t *)opal_list_get_next(frag)) {
hdr = &frag->hdr.hdr_match;
/* If the message has the next expected seq from that proc... */
if(hdr->hdr_seq != pml_proc->expected_sequence)
continue;
opal_list_remove_item(&pml_proc->frags_cant_match, (opal_list_item_t*)frag);
goto add_fragment_to_unexpected;
}
} else {
opal_list_append( &pml_proc->frags_cant_match, (opal_list_item_t*)frag );
}
}
return OMPI_SUCCESS;
}
/*
* Invoked when there's a new communicator that has been created.
* Look at the communicator and decide which set of functions and
* priority we want to return.
*/
mca_scoll_base_module_t *
mca_scoll_mpi_comm_query(oshmem_group_t *osh_group, int *priority)
{
mca_scoll_base_module_t *module;
mca_scoll_mpi_module_t *mpi_module;
int err, i;
int tag;
ompi_group_t* parent_group, *new_group;
ompi_communicator_t* newcomm = NULL;
*priority = 0;
mca_scoll_mpi_component_t *cm;
cm = &mca_scoll_mpi_component;
int* ranks;
if (!cm->mpi_enable){
return NULL;
}
if ((osh_group->proc_count < 2) || (osh_group->proc_count < cm->mpi_np)) {
return NULL;
}
/* Create OMPI_Comm object and store ptr to it in group obj*/
if (NULL == oshmem_group_all) {
osh_group->ompi_comm = &(ompi_mpi_comm_world.comm);
} else {
err = ompi_comm_group(&(ompi_mpi_comm_world.comm), &parent_group);
if (OPAL_UNLIKELY(OMPI_SUCCESS != err)) {
return NULL;
}
ranks = (int*) malloc(osh_group->proc_count * sizeof(int));
if (OPAL_UNLIKELY(NULL == ranks)) {
return NULL;
}
tag = 1;
for (i = 0; i < osh_group->proc_count; i++) {
ompi_proc_t* ompi_proc;
for( int j = 0; j < ompi_group_size(parent_group); j++ ) {
ompi_proc = ompi_group_peer_lookup(parent_group, j);
if( 0 == opal_compare_proc(ompi_proc->super.proc_name, osh_group->proc_array[i]->super.proc_name)) {
ranks[i] = j;
break;
}
}
}
err = ompi_group_incl(parent_group, osh_group->proc_count, ranks, &new_group);
if (OPAL_UNLIKELY(OMPI_SUCCESS != err)) {
free(ranks);
return NULL;
}
err = ompi_comm_create_group(&(ompi_mpi_comm_world.comm), new_group, tag, &newcomm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != err)) {
free(ranks);
return NULL;
}
err = ompi_group_free(&new_group);
if (OPAL_UNLIKELY(OMPI_SUCCESS != err)) {
free(ranks);
return NULL;
}
free(ranks);
osh_group->ompi_comm = newcomm;
}
mpi_module = OBJ_NEW(mca_scoll_mpi_module_t);
if (!mpi_module){
return NULL;
}
mpi_module->comm = osh_group->ompi_comm;
mpi_module->super.scoll_module_enable = mca_scoll_mpi_module_enable;
mpi_module->super.scoll_barrier = mca_scoll_mpi_barrier;
mpi_module->super.scoll_broadcast = mca_scoll_mpi_broadcast;
mpi_module->super.scoll_reduce = mca_scoll_mpi_reduce;
mpi_module->super.scoll_collect = mca_scoll_mpi_collect;
mpi_module->super.scoll_alltoall = NULL;
*priority = cm->mpi_priority;
module = &mpi_module->super;
return module;
}
int
ompi_osc_pt2pt_module_start(ompi_group_t *group,
int assert,
ompi_win_t *win)
{
int i, ret = OMPI_SUCCESS;
ompi_osc_pt2pt_module_t *module = P2P_MODULE(win);
OBJ_RETAIN(group);
ompi_group_increment_proc_count(group);
OPAL_THREAD_LOCK(&(module->p2p_lock));
if (NULL != module->p2p_sc_group) {
OPAL_THREAD_UNLOCK(&module->p2p_lock);
ret = MPI_ERR_RMA_SYNC;
goto cleanup;
}
module->p2p_sc_group = group;
/* possible we've already received a couple in messages, so
add however many we're going to wait for */
module->p2p_num_post_msgs += ompi_group_size(module->p2p_sc_group);
OPAL_THREAD_UNLOCK(&(module->p2p_lock));
memset(module->p2p_sc_remote_active_ranks, 0,
sizeof(bool) * ompi_comm_size(module->p2p_comm));
/* for each process in the specified group, find it's rank in our
communicator, store those indexes, and set the true / false in
the active ranks table */
for (i = 0 ; i < ompi_group_size(group) ; i++) {
int comm_rank = -1, j;
/* find the rank in the communicator associated with this windows */
for (j = 0 ; j < ompi_comm_size(module->p2p_comm) ; ++j) {
if (ompi_group_peer_lookup(module->p2p_sc_group, i) ==
ompi_comm_peer_lookup(module->p2p_comm, j)) {
comm_rank = j;
break;
}
}
if (comm_rank == -1) {
ret = MPI_ERR_RMA_SYNC;
goto cleanup;
}
module->p2p_sc_remote_active_ranks[comm_rank] = true;
module->p2p_sc_remote_ranks[i] = comm_rank;
}
/* Set our mode to access w/ start */
ompi_win_remove_mode(win, OMPI_WIN_FENCE);
ompi_win_append_mode(win, OMPI_WIN_ACCESS_EPOCH | OMPI_WIN_STARTED);
return OMPI_SUCCESS;
cleanup:
ompi_group_decrement_proc_count(group);
OBJ_RELEASE(group);
return ret;
}
/*
* Group Difference has to use the dense format since we don't support
* two parent groups in the group structure and maintain functions
*/
int ompi_group_difference(ompi_group_t* group1, ompi_group_t* group2,
ompi_group_t **new_group) {
/* local varibles */
int new_group_size, proc1, proc2, found_in_group2, cnt;
int my_group_rank;
ompi_group_t *group1_pointer, *group2_pointer, *new_group_pointer;
ompi_proc_t *proc1_pointer, *proc2_pointer, *my_proc_pointer = NULL;
group1_pointer=(ompi_group_t *)group1;
group2_pointer=(ompi_group_t *)group2;
/*
* form union
*/
/* get new group size */
new_group_size=0;
/* loop over group1 members */
for( proc1=0; proc1 < group1_pointer->grp_proc_count; proc1++ ) {
proc1_pointer = ompi_group_peer_lookup(group1_pointer,proc1);
/* check to see if this proc is in group2 */
found_in_group2=0;
for( proc2=0 ; proc2 < group2_pointer->grp_proc_count ; proc2++ ) {
proc2_pointer = ompi_group_peer_lookup(group2_pointer,proc2);
if( proc1_pointer == proc2_pointer ) {
found_in_group2=true;
break;
}
} /* end proc1 loop */
if(found_in_group2) {
continue;
}
new_group_size++;
} /* end proc loop */
if ( 0 == new_group_size ) {
*new_group = MPI_GROUP_EMPTY;
OBJ_RETAIN(MPI_GROUP_EMPTY);
return MPI_SUCCESS;
}
/* allocate a new ompi_group_t structure */
new_group_pointer=ompi_group_allocate(new_group_size);
if( NULL == new_group_pointer ) {
return MPI_ERR_GROUP;
}
/* fill in group list */
cnt=0;
/* loop over group1 members */
for( proc1=0; proc1 < group1_pointer->grp_proc_count; proc1++ ) {
proc1_pointer = ompi_group_peer_lookup(group1_pointer,proc1);
/* check to see if this proc is in group2 */
found_in_group2=0;
for( proc2=0 ; proc2 < group2_pointer->grp_proc_count ; proc2++ ) {
proc2_pointer = ompi_group_peer_lookup(group2_pointer,proc2);
if( proc1_pointer == proc2_pointer ) {
found_in_group2=true;
break;
}
} /* end proc1 loop */
if(found_in_group2) {
continue;
}
new_group_pointer->grp_proc_pointers[cnt] =
ompi_group_peer_lookup(group1_pointer,proc1);
cnt++;
} /* end proc loop */
/* increment proc reference counters */
ompi_group_increment_proc_count(new_group_pointer);
/* find my rank */
my_group_rank=group1_pointer->grp_my_rank;
if ( MPI_UNDEFINED != my_group_rank ) {
my_proc_pointer = ompi_group_peer_lookup(group1_pointer,my_group_rank);
}
else {
my_group_rank=group2_pointer->grp_my_rank;
if ( MPI_UNDEFINED != my_group_rank ) {
my_proc_pointer = ompi_group_peer_lookup(group2_pointer,my_group_rank);
}
}
if ( MPI_UNDEFINED == my_group_rank ) {
new_group_pointer->grp_my_rank = MPI_UNDEFINED;
}
else {
ompi_set_group_rank(new_group_pointer,my_proc_pointer);
}
*new_group = (MPI_Group)new_group_pointer;
return OMPI_SUCCESS;
}
static int bootstrap_comm(ompi_communicator_t *comm,
mca_coll_sm_module_t *module)
{
int i;
char *shortpath, *fullpath;
mca_coll_sm_component_t *c = &mca_coll_sm_component;
mca_coll_sm_comm_t *data = module->sm_comm_data;
int comm_size = ompi_comm_size(comm);
int num_segments = c->sm_comm_num_segments;
int num_in_use = c->sm_comm_num_in_use_flags;
int frag_size = c->sm_fragment_size;
int control_size = c->sm_control_size;
ompi_process_name_t *lowest_name = NULL;
size_t size;
ompi_proc_t *proc;
/* Make the rendezvous filename for this communicators shmem data
segment. The CID is not guaranteed to be unique among all
procs on this node, so also pair it with the PID of the proc
with the lowest ORTE name to form a unique filename. */
proc = ompi_group_peer_lookup(comm->c_local_group, 0);
lowest_name = &(proc->proc_name);
for (i = 1; i < comm_size; ++i) {
proc = ompi_group_peer_lookup(comm->c_local_group, i);
if (ompi_rte_compare_name_fields(OMPI_RTE_CMP_ALL,
&(proc->proc_name),
lowest_name) < 0) {
lowest_name = &(proc->proc_name);
}
}
asprintf(&shortpath, "coll-sm-cid-%d-name-%s.mmap", comm->c_contextid,
OMPI_NAME_PRINT(lowest_name));
if (NULL == shortpath) {
opal_output_verbose(10, ompi_coll_base_framework.framework_output,
"coll:sm:enable:bootstrap comm (%d/%s): asprintf failed",
comm->c_contextid, comm->c_name);
return OMPI_ERR_OUT_OF_RESOURCE;
}
fullpath = opal_os_path(false, ompi_process_info.job_session_dir,
shortpath, NULL);
free(shortpath);
if (NULL == fullpath) {
opal_output_verbose(10, ompi_coll_base_framework.framework_output,
"coll:sm:enable:bootstrap comm (%d/%s): opal_os_path failed",
comm->c_contextid, comm->c_name);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* Calculate how much space we need in the per-communicator shmem
data segment. There are several values to add:
- size of the barrier data (2 of these):
- fan-in data (num_procs * control_size)
- fan-out data (num_procs * control_size)
- size of the "in use" buffers:
- num_in_use_buffers * control_size
- size of the message fragment area (one for each segment):
- control (num_procs * control_size)
- fragment data (num_procs * (frag_size))
So it's:
barrier: 2 * control_size + 2 * control_size
in use: num_in_use * control_size
control: num_segments * (num_procs * control_size * 2 +
num_procs * control_size)
message: num_segments * (num_procs * frag_size)
*/
size = 4 * control_size +
(num_in_use * control_size) +
(num_segments * (comm_size * control_size * 2)) +
(num_segments * (comm_size * frag_size));
opal_output_verbose(10, ompi_coll_base_framework.framework_output,
"coll:sm:enable:bootstrap comm (%d/%s): attaching to %" PRIsize_t " byte mmap: %s",
comm->c_contextid, comm->c_name, size, fullpath);
data->sm_bootstrap_meta =
mca_common_sm_init_group(comm->c_local_group, size, fullpath,
sizeof(mca_common_sm_seg_header_t),
getpagesize());
if (NULL == data->sm_bootstrap_meta) {
opal_output_verbose(10, ompi_coll_base_framework.framework_output,
"coll:sm:enable:bootstrap comm (%d/%s): mca_common_sm_init_group failed",
comm->c_contextid, comm->c_name);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* All done */
return OMPI_SUCCESS;
}
int ompi_group_compare(ompi_group_t *group1,
ompi_group_t *group2,
int *result)
{
int return_value = OMPI_SUCCESS;
int proc1, proc2, match;
bool similar, identical;
ompi_group_t *group1_pointer, *group2_pointer;
ompi_proc_t *proc1_pointer, *proc2_pointer;
/* check for same groups */
if( group1 == group2 ) {
*result=MPI_IDENT;
return return_value;
}
/* check to see if either is MPI_GROUP_NULL or MPI_GROUP_EMPTY */
if( ( MPI_GROUP_EMPTY == group1 ) || ( MPI_GROUP_EMPTY == group2 ) ) {
*result=MPI_UNEQUAL;
return return_value;
}
/* get group pointers */
group1_pointer = (ompi_group_t *)group1;
group2_pointer = (ompi_group_t *)group2;
/* compare sizes */
if( group1_pointer->grp_proc_count != group2_pointer->grp_proc_count ) {
/* if not same size - return */
*result=MPI_UNEQUAL;
return return_value;
}
/* check for similarity */
/* loop over group1 processes */
similar=true;
identical=true;
for(proc1=0 ; proc1 < group1_pointer->grp_proc_count ; proc1++ ) {
proc1_pointer= ompi_group_peer_lookup(group1_pointer,proc1);
/* loop over group2 processes to find "match" */
match=-1;
for(proc2=0 ; proc2 < group2_pointer->grp_proc_count ; proc2++ ) {
proc2_pointer=ompi_group_peer_lookup(group2_pointer,proc2);
if( proc1_pointer == proc2_pointer ) {
if(proc1 != proc2 ) {
identical=false;
}
match=proc2;
break;
}
} /* end proc2 loop */
if( match== -1 ) {
similar=false;
identical=false;
break;
}
} /* end proc1 loop */
/* set comparison result */
if( identical ) {
*result=MPI_IDENT;
} else if( similar ) {
*result=MPI_SIMILAR;
} else {
*result=MPI_UNEQUAL;
}
return return_value;
}
int
ompi_osc_pt2pt_module_complete(ompi_win_t *win)
{
int i;
int ret = OMPI_SUCCESS;
ompi_group_t *group;
opal_list_item_t *item;
ompi_osc_pt2pt_module_t *module = P2P_MODULE(win);
/* wait for all the post messages */
OPAL_THREAD_LOCK(&module->p2p_lock);
while (0 != module->p2p_num_post_msgs) {
opal_condition_wait(&module->p2p_cond, &module->p2p_lock);
}
ompi_osc_pt2pt_flip_sendreqs(module);
/* for each process in group, send a control message with number
of updates coming, then start all the requests */
for (i = 0 ; i < ompi_group_size(module->p2p_sc_group) ; ++i) {
int comm_rank = module->p2p_sc_remote_ranks[i];
module->p2p_num_pending_out +=
module->p2p_copy_num_pending_sendreqs[comm_rank];
}
OPAL_THREAD_UNLOCK(&module->p2p_lock);
for (i = 0 ; i < ompi_group_size(module->p2p_sc_group) ; ++i) {
int comm_rank = module->p2p_sc_remote_ranks[i];
ret = ompi_osc_pt2pt_control_send(module,
ompi_group_peer_lookup(module->p2p_sc_group, i),
OMPI_OSC_PT2PT_HDR_COMPLETE,
module->p2p_copy_num_pending_sendreqs[comm_rank],
0);
assert(ret == OMPI_SUCCESS);
}
/* try to start all the requests. We've copied everything we
need out of pending_sendreqs, so don't need the lock
here */
while (NULL !=
(item = opal_list_remove_first(&(module->p2p_copy_pending_sendreqs)))) {
ompi_osc_pt2pt_sendreq_t *req =
(ompi_osc_pt2pt_sendreq_t*) item;
ret = ompi_osc_pt2pt_sendreq_send(module, req);
if (OMPI_ERR_TEMP_OUT_OF_RESOURCE == ret ) {
opal_output_verbose(5, ompi_osc_base_output,
"complete: failure in starting sendreq (%d). Will try later.",
ret);
opal_list_append(&(module->p2p_copy_pending_sendreqs), item);
} else if (OMPI_SUCCESS != ret) {
return ret;
}
}
/* wait for all the requests */
OPAL_THREAD_LOCK(&module->p2p_lock);
while (0 != module->p2p_num_pending_out) {
opal_condition_wait(&module->p2p_cond, &module->p2p_lock);
}
group = module->p2p_sc_group;
module->p2p_sc_group = NULL;
OPAL_THREAD_UNLOCK(&module->p2p_lock);
/* remove WIN_POSTED from our mode */
ompi_win_remove_mode(win, OMPI_WIN_ACCESS_EPOCH | OMPI_WIN_STARTED);
ompi_group_decrement_proc_count(group);
OBJ_RELEASE(group);
return ret;
}
static inline ompi_proc_t* __local_rank_lookup(ompi_communicator_t *comm, int rank)
{
return ompi_group_peer_lookup(comm->c_local_group, rank);
}
/*
* Invoked when there's a new communicator that has been created.
* Look at the communicator and decide which set of functions and
* priority we want to return.
*/
mca_scoll_base_module_t *
mca_scoll_mpi_comm_query(oshmem_group_t *osh_group, int *priority)
{
mca_scoll_base_module_t *module;
mca_scoll_mpi_module_t *mpi_module;
int err, i;
int tag;
ompi_group_t* parent_group, *new_group;
ompi_communicator_t* newcomm = NULL;
*priority = 0;
mca_scoll_mpi_component_t *cm;
cm = &mca_scoll_mpi_component;
int* ranks;
if (!cm->mpi_enable){
return NULL;
}
if ((osh_group->proc_count < 2) || (osh_group->proc_count < cm->mpi_np)) {
return NULL;
}
/* Create OMPI_Comm object and store ptr to it in group obj*/
if (NULL == oshmem_group_all) {
osh_group->ompi_comm = &(ompi_mpi_comm_world.comm);
} else {
int my_rank = MPI_UNDEFINED;
err = ompi_comm_group(&(ompi_mpi_comm_world.comm), &parent_group);
if (OPAL_UNLIKELY(OMPI_SUCCESS != err)) {
return NULL;
}
ranks = (int*) malloc(osh_group->proc_count * sizeof(int));
if (OPAL_UNLIKELY(NULL == ranks)) {
return NULL;
}
tag = 1;
for (i = 0; i < osh_group->proc_count; i++) {
ompi_proc_t* ompi_proc;
for( int j = 0; j < ompi_group_size(parent_group); j++ ) {
ompi_proc = ompi_group_peer_lookup(parent_group, j);
if( 0 == opal_compare_proc(ompi_proc->super.proc_name, osh_group->proc_array[i]->super.proc_name)) {
ranks[i] = j;
break;
}
}
/* NTH: keep track of my rank in the new group for the workaround below */
if (ranks[i] == ompi_comm_rank (&ompi_mpi_comm_world.comm)) {
my_rank = i;
}
}
err = ompi_group_incl(parent_group, osh_group->proc_count, ranks, &new_group);
if (OPAL_UNLIKELY(OMPI_SUCCESS != err)) {
free(ranks);
return NULL;
}
/* NTH: XXX -- WORKAROUND -- The oshmem code overwrites ompi_proc_local_proc with its
* own proc but does not update the proc list in comm world or comm self. This causes
* the code in ompi_group_incl that updates grp_my_rank to fail. This will cause failures
* here and when an application attempts to mix oshmem and mpi so it will really need to
* be fixed in oshmem/proc and not here. For now we need to work around a new jenkins
* failure so set my group ranking so we do not crash when running ompi_comm_create_group. */
new_group->grp_my_rank = my_rank;
err = ompi_comm_create_group(&(ompi_mpi_comm_world.comm), new_group, tag, &newcomm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != err)) {
free(ranks);
return NULL;
}
err = ompi_group_free(&new_group);
if (OPAL_UNLIKELY(OMPI_SUCCESS != err)) {
free(ranks);
return NULL;
}
free(ranks);
osh_group->ompi_comm = newcomm;
}
mpi_module = OBJ_NEW(mca_scoll_mpi_module_t);
if (!mpi_module){
return NULL;
}
mpi_module->comm = osh_group->ompi_comm;
mpi_module->super.scoll_module_enable = mca_scoll_mpi_module_enable;
mpi_module->super.scoll_barrier = mca_scoll_mpi_barrier;
mpi_module->super.scoll_broadcast = mca_scoll_mpi_broadcast;
mpi_module->super.scoll_reduce = mca_scoll_mpi_reduce;
mpi_module->super.scoll_collect = mca_scoll_mpi_collect;
*priority = cm->mpi_priority;
module = &mpi_module->super;
return module;
}
/*
* Group Union has to use the dense format since we don't support
* two parent groups in the group structure and maintain functions
*/
int ompi_group_union (ompi_group_t* group1, ompi_group_t* group2,
ompi_group_t **new_group)
{
/* local variables */
int new_group_size, proc1, proc2, found_in_group;
int my_group_rank, cnt;
ompi_group_t *group1_pointer, *group2_pointer, *new_group_pointer;
ompi_proc_t *proc1_pointer, *proc2_pointer, *my_proc_pointer = NULL;
group1_pointer = (ompi_group_t *) group1;
group2_pointer = (ompi_group_t *) group2;
/*
* form union
*/
/* get new group size */
new_group_size = group1_pointer->grp_proc_count;
/* check group2 elements to see if they need to be included in the list */
for (proc2 = 0; proc2 < group2_pointer->grp_proc_count; proc2++) {
proc2_pointer = ompi_group_peer_lookup(group2_pointer,proc2);
/* check to see if this proc2 is alread in the group */
found_in_group = 0;
for (proc1 = 0; proc1 < group1_pointer->grp_proc_count; proc1++) {
proc1_pointer = ompi_group_peer_lookup(group1_pointer,proc1);
if (proc1_pointer == proc2_pointer) {
/* proc2 is in group1 - don't double count */
found_in_group = 1;
break;
}
} /* end proc1 loop */
if (found_in_group) {
continue;
}
new_group_size++;
} /* end proc loop */
if ( 0 == new_group_size ) {
*new_group = MPI_GROUP_EMPTY;
OBJ_RETAIN(MPI_GROUP_EMPTY);
return MPI_SUCCESS;
}
/* get new group struct */
new_group_pointer = ompi_group_allocate(new_group_size);
if (NULL == new_group_pointer) {
return MPI_ERR_GROUP;
}
/* fill in the new group list */
/* put group1 elements in the list */
for (proc1 = 0; proc1 < group1_pointer->grp_proc_count; proc1++) {
new_group_pointer->grp_proc_pointers[proc1] =
ompi_group_peer_lookup(group1_pointer,proc1);
}
cnt = group1_pointer->grp_proc_count;
/* check group2 elements to see if they need to be included in the list */
for (proc2 = 0; proc2 < group2_pointer->grp_proc_count; proc2++) {
proc2_pointer = ompi_group_peer_lookup(group2_pointer,proc2);
/* check to see if this proc2 is alread in the group */
found_in_group = 0;
for (proc1 = 0; proc1 < group1_pointer->grp_proc_count; proc1++) {
proc1_pointer = ompi_group_peer_lookup(group1_pointer,proc1);
if (proc1_pointer == proc2_pointer) {
/* proc2 is in group1 - don't double count */
found_in_group = 1;
break;
}
} /* end proc1 loop */
if (found_in_group) {
continue;
}
new_group_pointer->grp_proc_pointers[cnt] =
ompi_group_peer_lookup(group2_pointer,proc2);
cnt++;
} /* end proc loop */
/* increment proc reference counters */
ompi_group_increment_proc_count(new_group_pointer);
/* find my rank */
my_group_rank = group1_pointer->grp_my_rank;
if (MPI_UNDEFINED == my_group_rank) {
my_group_rank = group2_pointer->grp_my_rank;
if ( MPI_UNDEFINED != my_group_rank) {
my_proc_pointer = ompi_group_peer_lookup(group2_pointer,my_group_rank);
}
} else {
my_proc_pointer = ompi_group_peer_lookup(group1_pointer,my_group_rank);
}
if ( MPI_UNDEFINED == my_group_rank ) {
new_group_pointer->grp_my_rank = MPI_UNDEFINED;
}
else {
ompi_set_group_rank(new_group_pointer, my_proc_pointer);
}
*new_group = (MPI_Group) new_group_pointer;
return OMPI_SUCCESS;
}
int mca_sharedfp_lockedfile_file_open (struct ompi_communicator_t *comm,
const char* filename,
int amode,
struct opal_info_t *info,
mca_io_ompio_file_t *fh)
{
int err = MPI_SUCCESS;
char * lockedfilename;
int handle, rank;
struct mca_sharedfp_lockedfile_data * module_data = NULL;
struct mca_sharedfp_base_data_t* sh;
mca_io_ompio_file_t * shfileHandle, *ompio_fh;
mca_io_ompio_data_t *data;
/*------------------------------------------------------------*/
/*Open the same file again without shared file pointer support*/
/*------------------------------------------------------------*/
shfileHandle = (mca_io_ompio_file_t *)malloc(sizeof(mca_io_ompio_file_t));
err = mca_common_ompio_file_open(comm,filename,amode,info,shfileHandle,false);
if ( OMPI_SUCCESS != err) {
opal_output(0, "mca_sharedfp_lockedfile_file_open: Error during file open\n");
return err;
}
shfileHandle->f_fh = fh->f_fh;
data = (mca_io_ompio_data_t *) fh->f_fh->f_io_selected_data;
ompio_fh = &data->ompio_fh;
err = mca_common_ompio_set_view (shfileHandle,
ompio_fh->f_disp,
ompio_fh->f_etype,
ompio_fh->f_orig_filetype,
ompio_fh->f_datarep,
&(MPI_INFO_NULL->super));
/*Memory is allocated here for the sh structure*/
sh = (struct mca_sharedfp_base_data_t*)malloc(sizeof(struct mca_sharedfp_base_data_t));
if ( NULL == sh){
opal_output(0, "mca_sharedfp_lockedfile_file_open: Error, unable to malloc f_sharedfp_ptr struct\n");
free ( shfileHandle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/*Populate the sh file structure based on the implementation*/
sh->sharedfh = shfileHandle; /* Shared file pointer*/
sh->global_offset = 0; /* Global Offset*/
sh->comm = comm; /* Communicator*/
sh->selected_module_data = NULL;
rank = ompi_comm_rank ( sh->comm);
/*Open a new file which will maintain the pointer for this file open*/
if ( mca_sharedfp_lockedfile_verbose ) {
opal_output(ompi_sharedfp_base_framework.framework_output,
"mca_sharedfp_lockedfile_file_open: open locked file.\n");
}
module_data = (struct mca_sharedfp_lockedfile_data*)malloc(sizeof(struct mca_sharedfp_lockedfile_data));
if ( NULL == module_data ) {
opal_output(ompi_sharedfp_base_framework.framework_output,
"mca_sharedfp_lockedfile_file_open: Error, unable to malloc lockedfile_data struct\n");
free (shfileHandle);
free (sh);
return OMPI_ERR_OUT_OF_RESOURCE;
}
opal_jobid_t masterjobid;
if ( 0 == comm->c_my_rank ) {
ompi_proc_t *masterproc = ompi_group_peer_lookup(comm->c_local_group, 0 );
masterjobid = OMPI_CAST_RTE_NAME(&masterproc->super.proc_name)->jobid;
}
comm->c_coll->coll_bcast ( &masterjobid, 1, MPI_UNSIGNED, 0, comm,
comm->c_coll->coll_bcast_module );
size_t filenamelen = strlen(filename) + 16;
lockedfilename = (char*)malloc(sizeof(char) * filenamelen);
if ( NULL == lockedfilename ) {
free (shfileHandle);
free (sh);
free (module_data);
return OMPI_ERR_OUT_OF_RESOURCE;
}
snprintf(lockedfilename, filenamelen, "%s-%u%s",filename,masterjobid,".lock");
module_data->filename = lockedfilename;
/*-------------------------------------------------*/
/*Open the lockedfile without shared file pointer */
/*-------------------------------------------------*/
if ( 0 == rank ) {
OMPI_MPI_OFFSET_TYPE position=0;
/*only let main process initialize file pointer,
*therefore there is no need to lock the file
*/
handle = open ( lockedfilename, O_RDWR | O_CREAT, 0644 );
write ( handle, &position, sizeof(OMPI_MPI_OFFSET_TYPE) );
close ( handle );
}
comm->c_coll->coll_barrier ( comm, comm->c_coll->coll_barrier_module );
handle = open ( lockedfilename, O_RDWR, 0644 );
if ( -1 == handle ) {
opal_output(0, "[%d]mca_sharedfp_lockedfile_file_open: Error during file open\n", rank);
free (shfileHandle);
free (sh);
free(module_data);
return OMPI_ERROR;
}
/*Store the new file handle*/
module_data->handle = handle;
/* Assign the lockedfile_data to sh->handle*/
sh->selected_module_data = module_data;
/*remember the shared file handle*/
fh->f_sharedfp_data = sh;
comm->c_coll->coll_barrier ( comm, comm->c_coll->coll_barrier_module );
return err;
}
