/*
* Invoked when there's a new communicator that has been created.
* Look at the communicator and decide which set of functions and
* priority we want to return.
*/
mca_coll_base_module_t *
mca_coll_fca_comm_query(struct ompi_communicator_t *comm, int *priority)
{
mca_coll_base_module_t *module;
int size = ompi_comm_size(comm);
int local_peers;
mca_coll_fca_module_t *fca_module;
*priority = 0;
module = NULL;
if (!mca_coll_fca_component.fca_enable)
goto exit;
if (size < mca_coll_fca_component.fca_np)
goto exit;
if (!have_remote_peers(comm->c_local_group, size, &local_peers) || OMPI_COMM_IS_INTER(comm))
goto exit;
fca_module = OBJ_NEW(mca_coll_fca_module_t);
if (!fca_module)
goto exit;
fca_module->super.coll_module_enable = mca_coll_fca_module_enable;
fca_module->super.ft_event = mca_coll_fca_ft_event;
fca_module->super.coll_allgather = mca_coll_fca_component.fca_enable_allgather? mca_coll_fca_allgather : NULL;
fca_module->super.coll_allgatherv = mca_coll_fca_component.fca_enable_allgatherv? mca_coll_fca_allgatherv : NULL;
fca_module->super.coll_allreduce = mca_coll_fca_component.fca_enable_allreduce? mca_coll_fca_allreduce : NULL;
fca_module->super.coll_alltoall = mca_coll_fca_component.fca_enable_alltoall? mca_coll_fca_alltoall : NULL;
fca_module->super.coll_alltoallv = mca_coll_fca_component.fca_enable_alltoallv? mca_coll_fca_alltoallv : NULL;
fca_module->super.coll_alltoallw = mca_coll_fca_component.fca_enable_alltoallw? mca_coll_fca_alltoallw : NULL;
fca_module->super.coll_barrier = mca_coll_fca_component.fca_enable_barrier? mca_coll_fca_barrier : NULL;
fca_module->super.coll_bcast = mca_coll_fca_component.fca_enable_bcast? mca_coll_fca_bcast : NULL;
fca_module->super.coll_exscan = NULL;
fca_module->super.coll_gather = mca_coll_fca_component.fca_enable_gather? mca_coll_fca_gather : NULL;
fca_module->super.coll_gatherv = mca_coll_fca_component.fca_enable_gatherv? mca_coll_fca_gatherv : NULL;
fca_module->super.coll_reduce = mca_coll_fca_component.fca_enable_reduce? mca_coll_fca_reduce : NULL;
fca_module->super.coll_reduce_scatter = mca_coll_fca_component.fca_enable_reduce_scatter? mca_coll_fca_reduce_scatter : NULL;
fca_module->super.coll_scan = NULL;
fca_module->super.coll_scatter = NULL;
fca_module->super.coll_scatterv = NULL;
*priority = mca_coll_fca_component.fca_priority;
module = &fca_module->super;
exit:
FCA_VERBOSE(4, "Query FCA module for comm %p size %d rank %d local_peers=%d: priority=%d %s",
(void *)comm, size, ompi_comm_rank(comm), local_peers,
*priority, module ? "enabled" : "disabled");
return module;
}
/*
* * Invoked when there's a new communicator that has been created.
* * Look at the communicator and decide which set of functions and
* * priority we want to return.
* */
mca_scoll_base_module_t *
mca_scoll_fca_comm_query(struct oshmem_group_t *comm, int *priority)
{
mca_scoll_base_module_t *module;
int size = comm->proc_count;
int local_peers = 0;
mca_scoll_fca_module_t *fca_module;
*priority = 0;
module = NULL;
if (!mca_scoll_fca_component.fca_enable) {
FCA_VERBOSE(20, "FCA is disable on user request => exiting");
goto exit;
}
if (mca_memheap.memheap_component == NULL ) {
FCA_VERBOSE(20, "No memheap => exiting");
goto exit;
}
if (NULL == mca_scoll_fca_component.ret) {
MCA_MEMHEAP_CALL(private_alloc(sizeof(int),(void **)&mca_scoll_fca_component.ret));
MCA_MEMHEAP_CALL(private_alloc(oshmem_group_all->proc_count*sizeof(*mca_scoll_fca_component.rcounts), (void **)&mca_scoll_fca_component.rcounts ));
MCA_MEMHEAP_CALL(private_alloc(/*info_size*/20,&mca_scoll_fca_component.my_info_exchangeable));
MCA_MEMHEAP_CALL(private_alloc(sizeof(fca_comm_desc_t), &mca_scoll_fca_component.fca_comm_desc_exchangeable));
}
if (size < mca_scoll_fca_component.fca_np) {
FCA_VERBOSE(20,
"size(%d) < fca_np(%d)",
size, mca_scoll_fca_component.fca_np);
goto exit;
}
if (size < 2) {
FCA_VERBOSE(20, "size(%d) < 2", size);
goto exit;
}
if (!have_remote_peers(comm,
size,
&local_peers) /* || OMPI_COMM_IS_INTER(comm)*/) {
FCA_VERBOSE(1,
"all peers in group are on the same node, fca disabled\n");
goto exit;
}
fca_module = OBJ_NEW(mca_scoll_fca_module_t);
if (!fca_module) {
goto exit_fatal;
}
fca_module->super.scoll_module_enable = mca_scoll_fca_module_enable;
fca_module->super.scoll_collect =
mca_scoll_fca_component.fca_enable_allgather ?
mca_scoll_fca_collect : NULL;
fca_module->super.scoll_reduce =
mca_scoll_fca_component.fca_enable_allreduce ?
mca_scoll_fca_reduce : NULL;
fca_module->super.scoll_barrier =
mca_scoll_fca_component.fca_enable_barrier ? mca_scoll_fca_barrier :
NULL;
fca_module->super.scoll_broadcast =
mca_scoll_fca_component.fca_enable_bcast ? mca_scoll_fca_broadcast :
NULL;
*priority = mca_scoll_fca_component.fca_priority;
module = &fca_module->super;
exit:
FCA_VERBOSE(4,
"Query FCA module for comm %p size %d rank %d local_peers=%d: priority=%d %s",
(void *)comm, size, comm->my_pe, local_peers, *priority, module ? "enabled" : "disabled");
return module;
exit_fatal:
/* it is possible that other pe(s) succesfully initialized fca.
* So differnt frameworks will be used for collective ops
*/
FCA_ERROR("FCA module query failed - aborting");
oshmem_shmem_abort(-1);
return NULL ;
}
