void test_bitmap_find_and_set(opal_bitmap_t *bm)
{
int bsize;
int result=0;
opal_bitmap_clear_all_bits(bm);
result = find_and_set(bm, 0);
TEST_AND_REPORT(result, 0, "opal_bitmap_find_and_set_first_unset_bit");
result = find_and_set(bm, 1);
TEST_AND_REPORT(result, 0, "opal_bitmap_find_and_set_first_unset_bit");
result = find_and_set(bm, 2);
TEST_AND_REPORT(result, 0, "opal_bitmap_find_and_set_first_unset_bit");
result = find_and_set(bm, 3);
TEST_AND_REPORT(result, 0, "opal_bitmap_find_and_set_first_unset_bit");
result = opal_bitmap_set_bit(bm, 5);
result = find_and_set(bm, 4);
TEST_AND_REPORT(result, 0, "opal_bitmap_find_and_set_first_unset_bit");
result = opal_bitmap_set_bit(bm, 6);
result = opal_bitmap_set_bit(bm, 7);
/* Setting beyond a char boundary */
result = find_and_set(bm, 8);
TEST_AND_REPORT(result, 0, "opal_bitmap_find_and_set_first_unset_bit");
opal_bitmap_set_bit(bm, 9);
result = find_and_set(bm, 10);
TEST_AND_REPORT(result, 0, "opal_bitmap_find_and_set_first_unset_bit");
/* Setting beyond the current size of bitmap */
opal_bitmap_set_all_bits(bm);
bsize = bm->array_size * SIZE_OF_CHAR;
result = find_and_set(bm, bsize);
TEST_AND_REPORT(result, 0, "opal_bitmap_find_and_set_first_unset_bit");
}
static int ompi_group_dense_overlap (ompi_group_t *group1, ompi_group_t *group2, opal_bitmap_t *bitmap)
{
ompi_proc_t *proc1_pointer, *proc2_pointer;
int rc, overlap_count;
overlap_count = 0;
for (int proc1 = 0 ; proc1 < group1->grp_proc_count ; ++proc1) {
proc1_pointer = ompi_group_get_proc_ptr_raw (group1, proc1);
/* check to see if this proc is in group2 */
for (int proc2 = 0 ; proc2 < group2->grp_proc_count ; ++proc2) {
proc2_pointer = ompi_group_get_proc_ptr_raw (group2, proc2);
if( proc1_pointer == proc2_pointer ) {
rc = opal_bitmap_set_bit (bitmap, proc2);
if (OPAL_SUCCESS != rc) {
return rc;
}
++overlap_count;
break;
}
} /* end proc1 loop */
} /* end proc loop */
return overlap_count;
}
void mca_oob_tcp_component_set_module(int fd, short args, void *cbdata)
{
mca_oob_tcp_peer_op_t *pop = (mca_oob_tcp_peer_op_t*)cbdata;
uint64_t ui64;
int rc;
orte_oob_base_peer_t *bpr;
opal_output_verbose(OOB_TCP_DEBUG_CONNECT, orte_oob_base_framework.framework_output,
"%s tcp:set_module called for peer %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&pop->peer));
/* make sure the OOB knows that we can reach this peer - we
* are in the same event base as the OOB base, so we can
* directly access its storage
*/
memcpy(&ui64, (char*)&pop->peer, sizeof(uint64_t));
if (OPAL_SUCCESS != opal_hash_table_get_value_uint64(&orte_oob_base.peers,
ui64, (void**)&bpr) || NULL == bpr) {
bpr = OBJ_NEW(orte_oob_base_peer_t);
}
opal_bitmap_set_bit(&bpr->addressable, mca_oob_tcp_component.super.idx);
bpr->component = &mca_oob_tcp_component.super;
if (OPAL_SUCCESS != (rc = opal_hash_table_set_value_uint64(&orte_oob_base.peers,
ui64, bpr))) {
ORTE_ERROR_LOG(rc);
}
OBJ_RELEASE(pop);
}
static int vader_add_procs (struct mca_btl_base_module_t* btl,
size_t nprocs, struct ompi_proc_t **procs,
struct mca_btl_base_endpoint_t **peers,
opal_bitmap_t *reachability)
{
mca_btl_vader_component_t *component = &mca_btl_vader_component;
mca_btl_vader_t *vader_btl = (mca_btl_vader_t *) btl;
int32_t proc, local_rank;
ompi_proc_t *my_proc;
int rc;
/* initializion */
/* get pointer to my proc structure */
if (NULL == (my_proc = ompi_proc_local())) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* jump out if there's not someone we can talk to */
if (1 > MCA_BTL_VADER_NUM_LOCAL_PEERS) {
return OMPI_SUCCESS;
}
/* make sure that my local rank has been defined */
if (ORTE_LOCAL_RANK_INVALID == MCA_BTL_VADER_LOCAL_RANK) {
return OMPI_ERROR;
}
if (!vader_btl->btl_inited) {
rc = vader_btl_first_time_init (vader_btl, 1 + MCA_BTL_VADER_NUM_LOCAL_PEERS);
if (rc != OMPI_SUCCESS) {
return rc;
}
}
for (proc = 0, local_rank = 0 ; proc < (int32_t) nprocs ; ++proc) {
/* check to see if this proc can be reached via shmem (i.e.,
if they're on my local host and in my job) */
if (procs[proc]->proc_name.jobid != my_proc->proc_name.jobid ||
!OPAL_PROC_ON_LOCAL_NODE(procs[proc]->proc_flags)) {
peers[proc] = NULL;
continue;
}
if (my_proc != procs[proc]) {
/* add this proc to shared memory accessibility list */
rc = opal_bitmap_set_bit (reachability, proc);
if(OMPI_SUCCESS != rc) {
return rc;
}
}
/* setup endpoint */
peers[proc] = component->endpoints + local_rank;
init_vader_endpoint (peers[proc], procs[proc], local_rank++);
}
return OMPI_SUCCESS;
}
int down_search(int rank, int parent, int me, int num_procs,
int *num_children, opal_list_t *children, opal_bitmap_t *relatives)
{
int i, bitmap, peer, hibit, mask, found;
orte_routed_tree_t *child;
opal_bitmap_t *relations;
/* is this me? */
if (me == rank) {
bitmap = opal_cube_dim(num_procs);
hibit = opal_hibit(rank, bitmap);
--bitmap;
for (i = hibit + 1, mask = 1 << i; i <= bitmap; ++i, mask <<= 1) {
peer = rank | mask;
if (peer < num_procs) {
child = OBJ_NEW(orte_routed_tree_t);
child->vpid = peer;
if (NULL != children) {
/* this is a direct child - add it to my list */
opal_list_append(children, &child->super);
(*num_children)++;
/* setup the relatives bitmap */
opal_bitmap_init(&child->relatives, num_procs);
/* point to the relatives */
relations = &child->relatives;
} else {
/* we are recording someone's relatives - set the bit */
opal_bitmap_set_bit(relatives, peer);
/* point to this relations */
relations = relatives;
}
/* search for this child's relatives */
down_search(0, 0, peer, num_procs, NULL, NULL, relations);
}
}
return parent;
}
/* find the children of this rank */
bitmap = opal_cube_dim(num_procs);
hibit = opal_hibit(rank, bitmap);
--bitmap;
for (i = hibit + 1, mask = 1 << i; i <= bitmap; ++i, mask <<= 1) {
peer = rank | mask;
if (peer < num_procs) {
/* execute compute on this child */
if (0 <= (found = down_search(peer, rank, me, num_procs, num_children, children, relatives))) {
return found;
}
}
}
return -1;
}
int set_bit(opal_bitmap_t *bm, int bit)
{
int err = opal_bitmap_set_bit(bm, bit);
if (err != 0 || !opal_bitmap_is_set_bit(bm, bit)) {
fprintf(error_out, "ERROR: set_bit for bit = %d\n\n", bit);
return ERR_CODE;
}
return 0;
}
int mca_btl_udapl_add_procs(
struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t **ompi_procs,
struct mca_btl_base_endpoint_t** peers,
opal_bitmap_t* reachable)
{
mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*)btl;
int i, rc;
for(i = 0; i < (int) nprocs; i++) {
struct ompi_proc_t* ompi_proc = ompi_procs[i];
mca_btl_udapl_proc_t* udapl_proc;
mca_btl_base_endpoint_t* udapl_endpoint;
if(ompi_proc == ompi_proc_local())
continue;
if(NULL == (udapl_proc = mca_btl_udapl_proc_create(ompi_proc))) {
continue;
}
OPAL_THREAD_LOCK(&udapl_proc->proc_lock);
/* The btl_proc datastructure is shared by all uDAPL BTL
* instances that are trying to reach this destination.
* Cache the peer instance on the btl_proc.
*/
udapl_endpoint = OBJ_NEW(mca_btl_udapl_endpoint_t);
if(NULL == udapl_endpoint) {
OPAL_THREAD_UNLOCK(&udapl_proc->proc_lock);
return OMPI_ERR_OUT_OF_RESOURCE;
}
udapl_endpoint->endpoint_btl = udapl_btl;
rc = mca_btl_udapl_proc_insert(udapl_proc, udapl_endpoint);
if(rc != OMPI_SUCCESS) {
OBJ_RELEASE(udapl_endpoint);
OPAL_THREAD_UNLOCK(&udapl_proc->proc_lock);
continue;
}
opal_bitmap_set_bit(reachable, i);
OPAL_THREAD_UNLOCK(&udapl_proc->proc_lock);
peers[i] = udapl_endpoint;
}
/* resize based on number of processes */
if (OMPI_SUCCESS !=
mca_btl_udapl_set_peer_parameters(udapl_btl, nprocs)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
return OMPI_SUCCESS;
}
int set_bit(opal_bitmap_t *bm, int bit)
{
int err = opal_bitmap_set_bit(bm, bit);
if (err != 0
|| !(bm->bitmap[bit/SIZE_OF_CHAR] & (1 << bit % SIZE_OF_CHAR))) {
fprintf(error_out, "ERROR: set_bit for bit = %d\n\n", bit);
return ERR_CODE;
}
return 0;
}
/**
* PML->BTL notification of change in the process list.
* PML->BTL Notification that a receive fragment has been matched.
* Called for message that is send from process with the virtual
* address of the shared memory segment being different than that of
* the receiver.
*
* @param btl (IN)
* @param proc (IN)
* @param peer (OUT)
* @return OPAL_SUCCESS or error status on failure.
*
*/
static int mca_btl_self_add_procs (struct mca_btl_base_module_t *btl, size_t nprocs,
struct opal_proc_t **procs,
struct mca_btl_base_endpoint_t **peers,
opal_bitmap_t* reachability)
{
for (int i = 0; i < (int)nprocs; i++ ) {
if( 0 == opal_compare_proc(procs[i]->proc_name, OPAL_PROC_MY_NAME) ) {
opal_bitmap_set_bit( reachability, i );
/* need to return something to keep the bml from ignoring us */
peers[i] = (struct mca_btl_base_endpoint_t *) 1;
break; /* there will always be only one ... */
}
}
return OPAL_SUCCESS;
}
int mca_btl_self_add_procs( struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_btl_base_endpoint_t **peers,
opal_bitmap_t* reachability )
{
int i;
for( i = 0; i < (int)nprocs; i++ ) {
if( procs[i] == ompi_proc_local_proc ) {
opal_bitmap_set_bit( reachability, i );
break; /* there will always be only one ... */
}
}
return OMPI_SUCCESS;
}
int mca_btl_self_add_procs( struct mca_btl_base_module_t* btl,
size_t nprocs,
struct opal_proc_t **procs,
struct mca_btl_base_endpoint_t **peers,
opal_bitmap_t* reachability )
{
int i;
for( i = 0; i < (int)nprocs; i++ ) {
if( 0 == opal_compare_proc(procs[i]->proc_name, OPAL_PROC_MY_NAME) ) {
opal_bitmap_set_bit( reachability, i );
break; /* there will always be only one ... */
}
}
return OPAL_SUCCESS;
}
static void radix_tree(int rank, int *num_children,
opal_list_t *children, opal_bitmap_t *relatives)
{
int i, peer, Sum, NInLevel;
orte_routed_tree_t *child;
opal_bitmap_t *relations;
/* compute how many procs are at my level */
Sum=1;
NInLevel=1;
while ( Sum < (rank+1) ) {
NInLevel *= mca_routed_radix_component.radix;
Sum += NInLevel;
}
/* our children start at our rank + num_in_level */
peer = rank + NInLevel;
for (i = 0; i < mca_routed_radix_component.radix; i++) {
if (peer < (int)orte_process_info.num_procs) {
child = OBJ_NEW(orte_routed_tree_t);
child->vpid = peer;
if (NULL != children) {
/* this is a direct child - add it to my list */
opal_list_append(children, &child->super);
(*num_children)++;
/* setup the relatives bitmap */
opal_bitmap_init(&child->relatives, orte_process_info.num_procs);
/* point to the relatives */
relations = &child->relatives;
} else {
/* we are recording someone's relatives - set the bit */
if (OPAL_SUCCESS != opal_bitmap_set_bit(relatives, peer)) {
opal_output(0, "%s Error: could not set relations bit!", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
}
/* point to this relations */
relations = relatives;
}
/* search for this child's relatives */
radix_tree(peer, NULL, NULL, relations);
}
peer += NInLevel;
}
}
int
opal_bitmap_find_and_set_first_unset_bit(opal_bitmap_t *bm, int *position)
{
int i = 0;
unsigned char temp;
unsigned char all_ones = 0xff;
if (NULL == bm) {
return OPAL_ERR_BAD_PARAM;
}
/* Neglect all which don't have an unset bit */
*position = 0;
while((i < bm->array_size) && (bm->bitmap[i] == all_ones)) {
++i;
}
if (i == bm->array_size) {
/* increase the bitmap size then */
*position = bm->array_size * SIZE_OF_CHAR;
return opal_bitmap_set_bit(bm, *position);
}
/* This one has an unset bit, find its bit number */
temp = bm->bitmap[i];
while (temp & 0x1) {
++(*position);
temp >>= 1;
}
/* Now set the bit number */
bm->bitmap[i] |= (bm->bitmap[i] + 1);
(*position) += i * SIZE_OF_CHAR;
return OPAL_SUCCESS;
}
static orte_vpid_t get_routing_tree(opal_list_t *children)
{
orte_routed_tree_t *nm;
orte_vpid_t v;
/* if I am anything other than a daemon or the HNP, this
* is a meaningless command as I am not allowed to route
*/
if (!ORTE_PROC_IS_DAEMON && !ORTE_PROC_IS_HNP) {
return ORTE_VPID_INVALID;
}
/* the linear routing tree consists of a chain of daemons
* extending from the HNP to orte_process_info.num_procs-1.
* Accordingly, my child is just the my_vpid+1 daemon
*/
if (NULL != children &&
ORTE_PROC_MY_NAME->vpid < orte_process_info.num_procs-1) {
/* my child is just the vpid+1 daemon */
nm = OBJ_NEW(orte_routed_tree_t);
opal_bitmap_init(&nm->relatives, orte_process_info.num_procs);
nm->vpid = ORTE_PROC_MY_NAME->vpid + 1;
/* my relatives are everyone above that point */
for (v=nm->vpid+1; v < orte_process_info.num_procs; v++) {
opal_bitmap_set_bit(&nm->relatives, v);
}
opal_list_append(children, &nm->super);
}
if (ORTE_PROC_IS_HNP) {
/* the parent of the HNP is invalid */
return ORTE_VPID_INVALID;
}
/* my parent is the my_vpid-1 daemon */
return (ORTE_PROC_MY_NAME->vpid - 1);
}
int mca_btl_tcp_add_procs( struct mca_btl_base_module_t* btl,
size_t nprocs,
struct opal_proc_t **procs,
struct mca_btl_base_endpoint_t** peers,
opal_bitmap_t* reachable )
{
mca_btl_tcp_module_t* tcp_btl = (mca_btl_tcp_module_t*)btl;
const opal_proc_t* my_proc; /* pointer to caller's proc structure */
int i, rc;
/* get pointer to my proc structure */
if( NULL == (my_proc = opal_proc_local_get()) )
return OPAL_ERR_OUT_OF_RESOURCE;
for(i = 0; i < (int) nprocs; i++) {
struct opal_proc_t* opal_proc = procs[i];
mca_btl_tcp_proc_t* tcp_proc;
mca_btl_base_endpoint_t* tcp_endpoint;
bool existing_found = false;
/* Do not create loopback TCP connections */
if( my_proc == opal_proc ) {
continue;
}
if(NULL == (tcp_proc = mca_btl_tcp_proc_create(opal_proc))) {
continue;
}
/*
* Check to make sure that the peer has at least as many interface
* addresses exported as we are trying to use. If not, then
* don't bind this BTL instance to the proc.
*/
OPAL_THREAD_LOCK(&tcp_proc->proc_lock);
for (uint32_t j = 0 ; j < (uint32_t)tcp_proc->proc_endpoint_count ; ++j) {
tcp_endpoint = tcp_proc->proc_endpoints[j];
if (tcp_endpoint->endpoint_btl == tcp_btl) {
existing_found = true;
break;
}
}
if (!existing_found) {
/* The btl_proc datastructure is shared by all TCP BTL
* instances that are trying to reach this destination.
* Cache the peer instance on the btl_proc.
*/
tcp_endpoint = OBJ_NEW(mca_btl_tcp_endpoint_t);
if(NULL == tcp_endpoint) {
OPAL_THREAD_UNLOCK(&tcp_proc->proc_lock);
return OPAL_ERR_OUT_OF_RESOURCE;
}
tcp_endpoint->endpoint_btl = tcp_btl;
rc = mca_btl_tcp_proc_insert(tcp_proc, tcp_endpoint);
if(rc != OPAL_SUCCESS) {
OPAL_THREAD_UNLOCK(&tcp_proc->proc_lock);
OBJ_RELEASE(tcp_endpoint);
continue;
}
OPAL_THREAD_LOCK(&tcp_btl->tcp_endpoints_mutex);
opal_list_append(&tcp_btl->tcp_endpoints, (opal_list_item_t*)tcp_endpoint);
OPAL_THREAD_UNLOCK(&tcp_btl->tcp_endpoints_mutex);
}
OPAL_THREAD_UNLOCK(&tcp_proc->proc_lock);
if (NULL != reachable) {
opal_bitmap_set_bit(reachable, i);
}
peers[i] = tcp_endpoint;
/* we increase the count of MPI users of the event library
once per peer, so that we are used until we aren't
connected to a peer */
opal_progress_event_users_increment();
}
return OPAL_SUCCESS;
}
int
mca_btl_portals4_add_procs(struct mca_btl_base_module_t* btl_base,
size_t nprocs,
struct opal_proc_t **procs,
struct mca_btl_base_endpoint_t** btl_peer_data,
opal_bitmap_t* reachable)
{
struct mca_btl_portals4_module_t* portals4_btl = (struct mca_btl_portals4_module_t*) btl_base;
int ret;
size_t i;
opal_output_verbose(50, opal_btl_base_framework.framework_output,
"mca_btl_portals4_add_procs: Adding %d procs (%d) for NI %d",
(int) nprocs,
(int) portals4_btl->portals_num_procs,
portals4_btl->interface_num);
/*
* The PML handed us a list of procs that need Portals4
* peer info. Complete those procs here.
*/
for (i = 0 ; i < nprocs ; ++i) {
struct opal_proc_t *curr_proc = procs[i];
/* portals doesn't support heterogeneous yet... */
if (opal_proc_local_get()->proc_arch != curr_proc->proc_arch) {
opal_output_verbose(1, opal_btl_base_framework.framework_output,
"Portals 4 BTL does not support heterogeneous operations.");
opal_output_verbose(1, opal_btl_base_framework.framework_output,
"Proc %s architecture %x, mine %x.",
OPAL_NAME_PRINT(curr_proc->proc_name),
curr_proc->proc_arch, opal_proc_local_get()->proc_arch);
return OPAL_ERR_NOT_SUPPORTED;
}
ret = create_endpoint(portals4_btl->interface_num,
curr_proc,
&btl_peer_data[i]);
OPAL_THREAD_ADD_FETCH32(&portals4_btl->portals_num_procs, 1);
/* and here we can reach */
opal_bitmap_set_bit(reachable, i);
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"add_procs: rank=%lx nid=%x pid=%x for NI %d",
i,
btl_peer_data[i]->ptl_proc.phys.nid,
btl_peer_data[i]->ptl_proc.phys.pid,
portals4_btl->interface_num));
}
if (mca_btl_portals4_component.need_init && portals4_btl->portals_num_procs > 0) {
if (mca_btl_portals4_component.use_logical) {
ret = create_maptable(portals4_btl, nprocs, procs, btl_peer_data);
if (OPAL_SUCCESS != ret) {
opal_output_verbose(1, opal_btl_base_framework.framework_output,
"%s:%d: mca_btl_portals4_add_procs::create_maptable() failed: %d\n",
__FILE__, __LINE__, ret);
return ret;
}
}
ret = btl_portals4_init_interface();
if (OPAL_SUCCESS != ret) {
opal_output_verbose(1, opal_btl_base_framework.framework_output,
"%s:%d: portals4 interface initialization failed: %d",
__FILE__, __LINE__, ret);
return ret;
}
mca_btl_portals4_component.need_init = 0;
}
return OPAL_SUCCESS;
}
static int binomial_tree(int rank, int parent, int me, int num_procs,
int *nchildren, opal_list_t *childrn,
opal_bitmap_t *relatives, bool mine)
{
int i, bitmap, peer, hibit, mask, found;
orte_routed_tree_t *child;
opal_bitmap_t *relations;
OPAL_OUTPUT_VERBOSE((3, orte_routed_base_framework.framework_output,
"%s routed:binomial rank %d parent %d me %d num_procs %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
rank, parent, me, num_procs));
/* is this me? */
if (me == rank) {
bitmap = opal_cube_dim(num_procs);
hibit = opal_hibit(rank, bitmap);
--bitmap;
for (i = hibit + 1, mask = 1 << i; i <= bitmap; ++i, mask <<= 1) {
peer = rank | mask;
if (peer < num_procs) {
child = OBJ_NEW(orte_routed_tree_t);
child->vpid = peer;
OPAL_OUTPUT_VERBOSE((3, orte_routed_base_framework.framework_output,
"%s routed:binomial %d found child %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
rank,
ORTE_VPID_PRINT(child->vpid)));
if (mine) {
/* this is a direct child - add it to my list */
opal_list_append(childrn, &child->super);
(*nchildren)++;
/* setup the relatives bitmap */
opal_bitmap_init(&child->relatives, num_procs);
/* point to the relatives */
relations = &child->relatives;
} else {
/* we are recording someone's relatives - set the bit */
opal_bitmap_set_bit(relatives, peer);
/* point to this relations */
relations = relatives;
}
/* search for this child's relatives */
binomial_tree(0, 0, peer, num_procs, nchildren, childrn, relations, false);
}
}
return parent;
}
/* find the children of this rank */
OPAL_OUTPUT_VERBOSE((5, orte_routed_base_framework.framework_output,
"%s routed:binomial find children of rank %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), rank));
bitmap = opal_cube_dim(num_procs);
hibit = opal_hibit(rank, bitmap);
--bitmap;
for (i = hibit + 1, mask = 1 << i; i <= bitmap; ++i, mask <<= 1) {
peer = rank | mask;
OPAL_OUTPUT_VERBOSE((5, orte_routed_base_framework.framework_output,
"%s routed:binomial find children checking peer %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), peer));
if (peer < num_procs) {
OPAL_OUTPUT_VERBOSE((5, orte_routed_base_framework.framework_output,
"%s routed:binomial find children computing tree",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* execute compute on this child */
if (0 <= (found = binomial_tree(peer, rank, me, num_procs, nchildren, childrn, relatives, mine))) {
OPAL_OUTPUT_VERBOSE((5, orte_routed_base_framework.framework_output,
"%s routed:binomial find children returning found value %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), found));
return found;
}
}
}
return -1;
}
int mca_btl_sm_add_procs(
struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_btl_base_endpoint_t **peers,
opal_bitmap_t* reachability)
{
int return_code = OMPI_SUCCESS;
int32_t n_local_procs = 0, proc, j, my_smp_rank = -1;
ompi_proc_t* my_proc; /* pointer to caller's proc structure */
mca_btl_sm_t *sm_btl;
bool have_connected_peer = false;
char **bases;
/* for easy access to the mpool_sm_module */
mca_mpool_sm_module_t *sm_mpool_modp = NULL;
/* initializion */
sm_btl = (mca_btl_sm_t *)btl;
/* get pointer to my proc structure */
if(NULL == (my_proc = ompi_proc_local()))
return OMPI_ERR_OUT_OF_RESOURCE;
/* Get unique host identifier for each process in the list,
* and idetify procs that are on this host. Add procs on this
* host to shared memory reachbility list. Also, get number
* of local procs in the procs list. */
for (proc = 0; proc < (int32_t)nprocs; proc++) {
/* check to see if this proc can be reached via shmem (i.e.,
if they're on my local host and in my job) */
if (procs[proc]->proc_name.jobid != my_proc->proc_name.jobid ||
!OPAL_PROC_ON_LOCAL_NODE(procs[proc]->proc_flags)) {
peers[proc] = NULL;
continue;
}
/* check to see if this is me */
if(my_proc == procs[proc]) {
my_smp_rank = mca_btl_sm_component.my_smp_rank = n_local_procs++;
continue;
}
/* sm doesn't support heterogeneous yet... */
if (procs[proc]->proc_arch != my_proc->proc_arch) {
continue;
}
/* we have someone to talk to */
have_connected_peer = true;
if(!(peers[proc] = create_sm_endpoint(n_local_procs, procs[proc]))) {
return_code = OMPI_ERROR;
goto CLEANUP;
}
n_local_procs++;
/* add this proc to shared memory accessibility list */
return_code = opal_bitmap_set_bit(reachability, proc);
if(OMPI_SUCCESS != return_code)
goto CLEANUP;
}
/* jump out if there's not someone we can talk to */
if (!have_connected_peer)
goto CLEANUP;
/* make sure that my_smp_rank has been defined */
if (-1 == my_smp_rank) {
return_code = OMPI_ERROR;
goto CLEANUP;
}
if (!sm_btl->btl_inited) {
return_code =
sm_btl_first_time_init(sm_btl, my_smp_rank,
mca_btl_sm_component.sm_max_procs);
if (return_code != OMPI_SUCCESS) {
goto CLEANUP;
}
}
/* set local proc's smp rank in the peers structure for
* rapid access and calculate reachability */
for(proc = 0; proc < (int32_t)nprocs; proc++) {
if(NULL == peers[proc])
continue;
mca_btl_sm_component.sm_peers[peers[proc]->peer_smp_rank] = peers[proc];
peers[proc]->my_smp_rank = my_smp_rank;
}
bases = mca_btl_sm_component.shm_bases;
sm_mpool_modp = (mca_mpool_sm_module_t *)mca_btl_sm_component.sm_mpool;
/* initialize own FIFOs */
/*
* The receiver initializes all its FIFOs. All components will
* be allocated near the receiver. Nothing will be local to
* "the sender" since there will be many senders.
*/
for(j = mca_btl_sm_component.num_smp_procs;
j < mca_btl_sm_component.num_smp_procs + FIFO_MAP_NUM(n_local_procs); j++) {
return_code = sm_fifo_init( mca_btl_sm_component.fifo_size,
mca_btl_sm_component.sm_mpool,
&mca_btl_sm_component.fifo[my_smp_rank][j],
mca_btl_sm_component.fifo_lazy_free);
if(return_code != OMPI_SUCCESS)
goto CLEANUP;
}
opal_atomic_wmb();
/* Sync with other local procs. Force the FIFO initialization to always
* happens before the readers access it.
*/
opal_atomic_add_32(&mca_btl_sm_component.sm_seg->module_seg->seg_inited, 1);
while( n_local_procs >
mca_btl_sm_component.sm_seg->module_seg->seg_inited) {
opal_progress();
opal_atomic_rmb();
}
/* it is now safe to unlink the shared memory segment. only one process
* needs to do this, so just let smp rank zero take care of it. */
if (0 == my_smp_rank) {
if (OMPI_SUCCESS !=
mca_common_sm_module_unlink(mca_btl_sm_component.sm_seg)) {
/* it is "okay" if this fails at this point. we have gone this far,
* so just warn about the failure and continue. this is probably
* only triggered by a programming error. */
opal_output(0, "WARNING: common_sm_module_unlink failed.\n");
}
/* SKG - another abstraction violation here, but I don't want to add
* extra code in the sm mpool for further synchronization. */
/* at this point, all processes have attached to the mpool segment. so
* it is safe to unlink it here. */
if (OMPI_SUCCESS !=
mca_common_sm_module_unlink(sm_mpool_modp->sm_common_module)) {
opal_output(0, "WARNING: common_sm_module_unlink failed.\n");
}
if (-1 == unlink(mca_btl_sm_component.sm_mpool_rndv_file_name)) {
opal_output(0, "WARNING: %s unlink failed.\n",
mca_btl_sm_component.sm_mpool_rndv_file_name);
}
if (-1 == unlink(mca_btl_sm_component.sm_rndv_file_name)) {
opal_output(0, "WARNING: %s unlink failed.\n",
mca_btl_sm_component.sm_rndv_file_name);
}
}
/* free up some space used by the name buffers */
free(mca_btl_sm_component.sm_mpool_ctl_file_name);
free(mca_btl_sm_component.sm_mpool_rndv_file_name);
free(mca_btl_sm_component.sm_ctl_file_name);
free(mca_btl_sm_component.sm_rndv_file_name);
/* coordinate with other processes */
for(j = mca_btl_sm_component.num_smp_procs;
j < mca_btl_sm_component.num_smp_procs + n_local_procs; j++) {
ptrdiff_t diff;
/* spin until this element is allocated */
/* doesn't really wait for that process... FIFO might be allocated, but not initialized */
opal_atomic_rmb();
while(NULL == mca_btl_sm_component.shm_fifo[j]) {
opal_progress();
opal_atomic_rmb();
}
/* Calculate the difference as (my_base - their_base) */
diff = ADDR2OFFSET(bases[my_smp_rank], bases[j]);
/* store local address of remote fifos */
mca_btl_sm_component.fifo[j] =
(sm_fifo_t*)OFFSET2ADDR(diff, mca_btl_sm_component.shm_fifo[j]);
/* cache local copy of peer memory node number */
mca_btl_sm_component.mem_nodes[j] = mca_btl_sm_component.shm_mem_nodes[j];
}
/* update the local smp process count */
mca_btl_sm_component.num_smp_procs += n_local_procs;
/* make sure we have enough eager fragmnents for each process */
return_code = ompi_free_list_resize_mt(&mca_btl_sm_component.sm_frags_eager,
mca_btl_sm_component.num_smp_procs * 2);
if (OMPI_SUCCESS != return_code)
goto CLEANUP;
CLEANUP:
return return_code;
}
int mca_btl_scif_add_procs(struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_btl_base_endpoint_t **peers,
opal_bitmap_t *reachable) {
mca_btl_scif_module_t *scif_module = (mca_btl_scif_module_t *) btl;
size_t procs_on_board, i, board_proc;
ompi_proc_t *my_proc = ompi_proc_local();
int rc;
/* determine how many procs are on this board */
for (i = 0, procs_on_board = 0 ; i < nprocs ; ++i) {
struct ompi_proc_t *ompi_proc = procs[i];
if (my_proc == ompi_proc) {
continue;
}
if (!OPAL_PROC_ON_LOCAL_HOST(ompi_proc->proc_flags) ||
my_proc == ompi_proc) {
/* scif can only be used with procs on this board */
continue;
}
procs_on_board++;
}
/* allocate space for the detected peers and setup the mpool */
if (NULL == scif_module->endpoints) {
scif_module->endpoints = calloc (procs_on_board, sizeof (mca_btl_base_endpoint_t));
if (OPAL_UNLIKELY(NULL == scif_module->endpoints)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
rc = mca_btl_scif_setup_mpools (scif_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
BTL_ERROR(("btl/scif error setting up mpools/free lists"));
return rc;
}
}
for (i = 0, board_proc = 0 ; i < nprocs ; ++i) {
struct ompi_proc_t *ompi_proc = procs[i];
if (my_proc == ompi_proc) {
continue;
}
if (!OPAL_PROC_ON_LOCAL_HOST(ompi_proc->proc_flags) ||
my_proc == ompi_proc) {
peers[i] = NULL;
/* scif can only be used with procs on this board */
continue;
}
/* Initialize endpoints */
rc = mca_btl_scif_ep_init (scif_module->endpoints + board_proc, (mca_btl_scif_module_t *) btl, ompi_proc);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
BTL_ERROR(("btl/scif error initializing endpoint"));
return rc;
}
scif_module->endpoints[board_proc].id = board_proc;
/* Set the reachable bit */
rc = opal_bitmap_set_bit (reachable, i);
/* Store a reference to this peer */
peers[i] = scif_module->endpoints + board_proc;
board_proc++;
}
BTL_VERBOSE(("%lu procs on board\n", (unsigned long) procs_on_board));
scif_module->endpoint_count = procs_on_board;
/* start listening thread */
rc = pthread_create (&mca_btl_scif_module.listen_thread, NULL, mca_btl_scif_connect_accept, NULL);
if (0 > rc) {
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
int mca_btl_sctp_add_procs(
struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t **ompi_procs,
struct mca_btl_base_endpoint_t** peers,
opal_bitmap_t* reachable)
{
mca_btl_sctp_module_t* sctp_btl = (mca_btl_sctp_module_t*)btl;
ompi_proc_t* my_proc; /* pointer to caller's proc structure */
int i, rc;
/* get pointer to my proc structure */
my_proc = ompi_proc_local();
if( NULL == my_proc ) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
for(i = 0; i < (int) nprocs; i++) {
struct ompi_proc_t* ompi_proc = ompi_procs[i];
mca_btl_sctp_proc_t* sctp_proc;
mca_btl_base_endpoint_t* sctp_endpoint;
/* Do not create loopback SCTP connections */
if( my_proc == ompi_proc ) {
continue;
}
if(NULL == (sctp_proc = mca_btl_sctp_proc_create(ompi_proc))) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/*
* Check to make sure that the peer has at least as many interface
* addresses exported as we are trying to use. If not, then
* don't bind this BTL instance to the proc.
*/
OPAL_THREAD_LOCK(&sctp_proc->proc_lock);
/* The btl_proc datastructure is shared by all SCTP BTL
* instances that are trying to reach this destination.
* Cache the peer instance on the btl_proc.
*/
sctp_endpoint = OBJ_NEW(mca_btl_sctp_endpoint_t);
if(NULL == sctp_endpoint) {
OPAL_THREAD_UNLOCK(&sctp_proc->proc_lock);
return OMPI_ERR_OUT_OF_RESOURCE;
}
sctp_endpoint->endpoint_btl = sctp_btl;
rc = mca_btl_sctp_proc_insert(sctp_proc, sctp_endpoint);
if(rc != OMPI_SUCCESS) {
OBJ_RELEASE(sctp_endpoint);
OPAL_THREAD_UNLOCK(&sctp_proc->proc_lock);
continue;
}
opal_bitmap_set_bit(reachable, i);
OPAL_THREAD_UNLOCK(&sctp_proc->proc_lock);
peers[i] = sctp_endpoint;
opal_list_append(&sctp_btl->sctp_endpoints, (opal_list_item_t*)sctp_endpoint);
/* we increase the count of MPI users of the event library
once per peer, so that we are used until we aren't
connected to a peer */
opal_progress_event_users_increment();
}
return OMPI_SUCCESS;
}
