void test_bitmap_find_and_set(opal_bitmap_t *bm) { int bsize; int result=0; opal_bitmap_clear_all_bits(bm); result = find_and_set(bm, 0); TEST_AND_REPORT(result, 0, "opal_bitmap_find_and_set_first_unset_bit"); result = find_and_set(bm, 1); TEST_AND_REPORT(result, 0, "opal_bitmap_find_and_set_first_unset_bit"); result = find_and_set(bm, 2); TEST_AND_REPORT(result, 0, "opal_bitmap_find_and_set_first_unset_bit"); result = find_and_set(bm, 3); TEST_AND_REPORT(result, 0, "opal_bitmap_find_and_set_first_unset_bit"); result = opal_bitmap_set_bit(bm, 5); result = find_and_set(bm, 4); TEST_AND_REPORT(result, 0, "opal_bitmap_find_and_set_first_unset_bit"); result = opal_bitmap_set_bit(bm, 6); result = opal_bitmap_set_bit(bm, 7); /* Setting beyond a char boundary */ result = find_and_set(bm, 8); TEST_AND_REPORT(result, 0, "opal_bitmap_find_and_set_first_unset_bit"); opal_bitmap_set_bit(bm, 9); result = find_and_set(bm, 10); TEST_AND_REPORT(result, 0, "opal_bitmap_find_and_set_first_unset_bit"); /* Setting beyond the current size of bitmap */ opal_bitmap_set_all_bits(bm); bsize = bm->array_size * SIZE_OF_CHAR; result = find_and_set(bm, bsize); TEST_AND_REPORT(result, 0, "opal_bitmap_find_and_set_first_unset_bit"); }
static int ompi_group_dense_overlap (ompi_group_t *group1, ompi_group_t *group2, opal_bitmap_t *bitmap) { ompi_proc_t *proc1_pointer, *proc2_pointer; int rc, overlap_count; overlap_count = 0; for (int proc1 = 0 ; proc1 < group1->grp_proc_count ; ++proc1) { proc1_pointer = ompi_group_get_proc_ptr_raw (group1, proc1); /* check to see if this proc is in group2 */ for (int proc2 = 0 ; proc2 < group2->grp_proc_count ; ++proc2) { proc2_pointer = ompi_group_get_proc_ptr_raw (group2, proc2); if( proc1_pointer == proc2_pointer ) { rc = opal_bitmap_set_bit (bitmap, proc2); if (OPAL_SUCCESS != rc) { return rc; } ++overlap_count; break; } } /* end proc1 loop */ } /* end proc loop */ return overlap_count; }
void mca_oob_tcp_component_set_module(int fd, short args, void *cbdata) { mca_oob_tcp_peer_op_t *pop = (mca_oob_tcp_peer_op_t*)cbdata; uint64_t ui64; int rc; orte_oob_base_peer_t *bpr; opal_output_verbose(OOB_TCP_DEBUG_CONNECT, orte_oob_base_framework.framework_output, "%s tcp:set_module called for peer %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_NAME_PRINT(&pop->peer)); /* make sure the OOB knows that we can reach this peer - we * are in the same event base as the OOB base, so we can * directly access its storage */ memcpy(&ui64, (char*)&pop->peer, sizeof(uint64_t)); if (OPAL_SUCCESS != opal_hash_table_get_value_uint64(&orte_oob_base.peers, ui64, (void**)&bpr) || NULL == bpr) { bpr = OBJ_NEW(orte_oob_base_peer_t); } opal_bitmap_set_bit(&bpr->addressable, mca_oob_tcp_component.super.idx); bpr->component = &mca_oob_tcp_component.super; if (OPAL_SUCCESS != (rc = opal_hash_table_set_value_uint64(&orte_oob_base.peers, ui64, bpr))) { ORTE_ERROR_LOG(rc); } OBJ_RELEASE(pop); }
static int vader_add_procs (struct mca_btl_base_module_t* btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers, opal_bitmap_t *reachability) { mca_btl_vader_component_t *component = &mca_btl_vader_component; mca_btl_vader_t *vader_btl = (mca_btl_vader_t *) btl; int32_t proc, local_rank; ompi_proc_t *my_proc; int rc; /* initializion */ /* get pointer to my proc structure */ if (NULL == (my_proc = ompi_proc_local())) { return OMPI_ERR_OUT_OF_RESOURCE; } /* jump out if there's not someone we can talk to */ if (1 > MCA_BTL_VADER_NUM_LOCAL_PEERS) { return OMPI_SUCCESS; } /* make sure that my local rank has been defined */ if (ORTE_LOCAL_RANK_INVALID == MCA_BTL_VADER_LOCAL_RANK) { return OMPI_ERROR; } if (!vader_btl->btl_inited) { rc = vader_btl_first_time_init (vader_btl, 1 + MCA_BTL_VADER_NUM_LOCAL_PEERS); if (rc != OMPI_SUCCESS) { return rc; } } for (proc = 0, local_rank = 0 ; proc < (int32_t) nprocs ; ++proc) { /* check to see if this proc can be reached via shmem (i.e., if they're on my local host and in my job) */ if (procs[proc]->proc_name.jobid != my_proc->proc_name.jobid || !OPAL_PROC_ON_LOCAL_NODE(procs[proc]->proc_flags)) { peers[proc] = NULL; continue; } if (my_proc != procs[proc]) { /* add this proc to shared memory accessibility list */ rc = opal_bitmap_set_bit (reachability, proc); if(OMPI_SUCCESS != rc) { return rc; } } /* setup endpoint */ peers[proc] = component->endpoints + local_rank; init_vader_endpoint (peers[proc], procs[proc], local_rank++); } return OMPI_SUCCESS; }
int down_search(int rank, int parent, int me, int num_procs, int *num_children, opal_list_t *children, opal_bitmap_t *relatives) { int i, bitmap, peer, hibit, mask, found; orte_routed_tree_t *child; opal_bitmap_t *relations; /* is this me? */ if (me == rank) { bitmap = opal_cube_dim(num_procs); hibit = opal_hibit(rank, bitmap); --bitmap; for (i = hibit + 1, mask = 1 << i; i <= bitmap; ++i, mask <<= 1) { peer = rank | mask; if (peer < num_procs) { child = OBJ_NEW(orte_routed_tree_t); child->vpid = peer; if (NULL != children) { /* this is a direct child - add it to my list */ opal_list_append(children, &child->super); (*num_children)++; /* setup the relatives bitmap */ opal_bitmap_init(&child->relatives, num_procs); /* point to the relatives */ relations = &child->relatives; } else { /* we are recording someone's relatives - set the bit */ opal_bitmap_set_bit(relatives, peer); /* point to this relations */ relations = relatives; } /* search for this child's relatives */ down_search(0, 0, peer, num_procs, NULL, NULL, relations); } } return parent; } /* find the children of this rank */ bitmap = opal_cube_dim(num_procs); hibit = opal_hibit(rank, bitmap); --bitmap; for (i = hibit + 1, mask = 1 << i; i <= bitmap; ++i, mask <<= 1) { peer = rank | mask; if (peer < num_procs) { /* execute compute on this child */ if (0 <= (found = down_search(peer, rank, me, num_procs, num_children, children, relatives))) { return found; } } } return -1; }
int set_bit(opal_bitmap_t *bm, int bit) { int err = opal_bitmap_set_bit(bm, bit); if (err != 0 || !opal_bitmap_is_set_bit(bm, bit)) { fprintf(error_out, "ERROR: set_bit for bit = %d\n\n", bit); return ERR_CODE; } return 0; }
int mca_btl_udapl_add_procs( struct mca_btl_base_module_t* btl, size_t nprocs, struct ompi_proc_t **ompi_procs, struct mca_btl_base_endpoint_t** peers, opal_bitmap_t* reachable) { mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*)btl; int i, rc; for(i = 0; i < (int) nprocs; i++) { struct ompi_proc_t* ompi_proc = ompi_procs[i]; mca_btl_udapl_proc_t* udapl_proc; mca_btl_base_endpoint_t* udapl_endpoint; if(ompi_proc == ompi_proc_local()) continue; if(NULL == (udapl_proc = mca_btl_udapl_proc_create(ompi_proc))) { continue; } OPAL_THREAD_LOCK(&udapl_proc->proc_lock); /* The btl_proc datastructure is shared by all uDAPL BTL * instances that are trying to reach this destination. * Cache the peer instance on the btl_proc. */ udapl_endpoint = OBJ_NEW(mca_btl_udapl_endpoint_t); if(NULL == udapl_endpoint) { OPAL_THREAD_UNLOCK(&udapl_proc->proc_lock); return OMPI_ERR_OUT_OF_RESOURCE; } udapl_endpoint->endpoint_btl = udapl_btl; rc = mca_btl_udapl_proc_insert(udapl_proc, udapl_endpoint); if(rc != OMPI_SUCCESS) { OBJ_RELEASE(udapl_endpoint); OPAL_THREAD_UNLOCK(&udapl_proc->proc_lock); continue; } opal_bitmap_set_bit(reachable, i); OPAL_THREAD_UNLOCK(&udapl_proc->proc_lock); peers[i] = udapl_endpoint; } /* resize based on number of processes */ if (OMPI_SUCCESS != mca_btl_udapl_set_peer_parameters(udapl_btl, nprocs)) { return OMPI_ERR_OUT_OF_RESOURCE; } return OMPI_SUCCESS; }
int set_bit(opal_bitmap_t *bm, int bit) { int err = opal_bitmap_set_bit(bm, bit); if (err != 0 || !(bm->bitmap[bit/SIZE_OF_CHAR] & (1 << bit % SIZE_OF_CHAR))) { fprintf(error_out, "ERROR: set_bit for bit = %d\n\n", bit); return ERR_CODE; } return 0; }
/** * PML->BTL notification of change in the process list. * PML->BTL Notification that a receive fragment has been matched. * Called for message that is send from process with the virtual * address of the shared memory segment being different than that of * the receiver. * * @param btl (IN) * @param proc (IN) * @param peer (OUT) * @return OPAL_SUCCESS or error status on failure. * */ static int mca_btl_self_add_procs (struct mca_btl_base_module_t *btl, size_t nprocs, struct opal_proc_t **procs, struct mca_btl_base_endpoint_t **peers, opal_bitmap_t* reachability) { for (int i = 0; i < (int)nprocs; i++ ) { if( 0 == opal_compare_proc(procs[i]->proc_name, OPAL_PROC_MY_NAME) ) { opal_bitmap_set_bit( reachability, i ); /* need to return something to keep the bml from ignoring us */ peers[i] = (struct mca_btl_base_endpoint_t *) 1; break; /* there will always be only one ... */ } } return OPAL_SUCCESS; }
int mca_btl_self_add_procs( struct mca_btl_base_module_t* btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers, opal_bitmap_t* reachability ) { int i; for( i = 0; i < (int)nprocs; i++ ) { if( procs[i] == ompi_proc_local_proc ) { opal_bitmap_set_bit( reachability, i ); break; /* there will always be only one ... */ } } return OMPI_SUCCESS; }
int mca_btl_self_add_procs( struct mca_btl_base_module_t* btl, size_t nprocs, struct opal_proc_t **procs, struct mca_btl_base_endpoint_t **peers, opal_bitmap_t* reachability ) { int i; for( i = 0; i < (int)nprocs; i++ ) { if( 0 == opal_compare_proc(procs[i]->proc_name, OPAL_PROC_MY_NAME) ) { opal_bitmap_set_bit( reachability, i ); break; /* there will always be only one ... */ } } return OPAL_SUCCESS; }
static void radix_tree(int rank, int *num_children, opal_list_t *children, opal_bitmap_t *relatives) { int i, peer, Sum, NInLevel; orte_routed_tree_t *child; opal_bitmap_t *relations; /* compute how many procs are at my level */ Sum=1; NInLevel=1; while ( Sum < (rank+1) ) { NInLevel *= mca_routed_radix_component.radix; Sum += NInLevel; } /* our children start at our rank + num_in_level */ peer = rank + NInLevel; for (i = 0; i < mca_routed_radix_component.radix; i++) { if (peer < (int)orte_process_info.num_procs) { child = OBJ_NEW(orte_routed_tree_t); child->vpid = peer; if (NULL != children) { /* this is a direct child - add it to my list */ opal_list_append(children, &child->super); (*num_children)++; /* setup the relatives bitmap */ opal_bitmap_init(&child->relatives, orte_process_info.num_procs); /* point to the relatives */ relations = &child->relatives; } else { /* we are recording someone's relatives - set the bit */ if (OPAL_SUCCESS != opal_bitmap_set_bit(relatives, peer)) { opal_output(0, "%s Error: could not set relations bit!", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)); } /* point to this relations */ relations = relatives; } /* search for this child's relatives */ radix_tree(peer, NULL, NULL, relations); } peer += NInLevel; } }
int opal_bitmap_find_and_set_first_unset_bit(opal_bitmap_t *bm, int *position) { int i = 0; unsigned char temp; unsigned char all_ones = 0xff; if (NULL == bm) { return OPAL_ERR_BAD_PARAM; } /* Neglect all which don't have an unset bit */ *position = 0; while((i < bm->array_size) && (bm->bitmap[i] == all_ones)) { ++i; } if (i == bm->array_size) { /* increase the bitmap size then */ *position = bm->array_size * SIZE_OF_CHAR; return opal_bitmap_set_bit(bm, *position); } /* This one has an unset bit, find its bit number */ temp = bm->bitmap[i]; while (temp & 0x1) { ++(*position); temp >>= 1; } /* Now set the bit number */ bm->bitmap[i] |= (bm->bitmap[i] + 1); (*position) += i * SIZE_OF_CHAR; return OPAL_SUCCESS; }
static orte_vpid_t get_routing_tree(opal_list_t *children) { orte_routed_tree_t *nm; orte_vpid_t v; /* if I am anything other than a daemon or the HNP, this * is a meaningless command as I am not allowed to route */ if (!ORTE_PROC_IS_DAEMON && !ORTE_PROC_IS_HNP) { return ORTE_VPID_INVALID; } /* the linear routing tree consists of a chain of daemons * extending from the HNP to orte_process_info.num_procs-1. * Accordingly, my child is just the my_vpid+1 daemon */ if (NULL != children && ORTE_PROC_MY_NAME->vpid < orte_process_info.num_procs-1) { /* my child is just the vpid+1 daemon */ nm = OBJ_NEW(orte_routed_tree_t); opal_bitmap_init(&nm->relatives, orte_process_info.num_procs); nm->vpid = ORTE_PROC_MY_NAME->vpid + 1; /* my relatives are everyone above that point */ for (v=nm->vpid+1; v < orte_process_info.num_procs; v++) { opal_bitmap_set_bit(&nm->relatives, v); } opal_list_append(children, &nm->super); } if (ORTE_PROC_IS_HNP) { /* the parent of the HNP is invalid */ return ORTE_VPID_INVALID; } /* my parent is the my_vpid-1 daemon */ return (ORTE_PROC_MY_NAME->vpid - 1); }
int mca_btl_tcp_add_procs( struct mca_btl_base_module_t* btl, size_t nprocs, struct opal_proc_t **procs, struct mca_btl_base_endpoint_t** peers, opal_bitmap_t* reachable ) { mca_btl_tcp_module_t* tcp_btl = (mca_btl_tcp_module_t*)btl; const opal_proc_t* my_proc; /* pointer to caller's proc structure */ int i, rc; /* get pointer to my proc structure */ if( NULL == (my_proc = opal_proc_local_get()) ) return OPAL_ERR_OUT_OF_RESOURCE; for(i = 0; i < (int) nprocs; i++) { struct opal_proc_t* opal_proc = procs[i]; mca_btl_tcp_proc_t* tcp_proc; mca_btl_base_endpoint_t* tcp_endpoint; bool existing_found = false; /* Do not create loopback TCP connections */ if( my_proc == opal_proc ) { continue; } if(NULL == (tcp_proc = mca_btl_tcp_proc_create(opal_proc))) { continue; } /* * Check to make sure that the peer has at least as many interface * addresses exported as we are trying to use. If not, then * don't bind this BTL instance to the proc. */ OPAL_THREAD_LOCK(&tcp_proc->proc_lock); for (uint32_t j = 0 ; j < (uint32_t)tcp_proc->proc_endpoint_count ; ++j) { tcp_endpoint = tcp_proc->proc_endpoints[j]; if (tcp_endpoint->endpoint_btl == tcp_btl) { existing_found = true; break; } } if (!existing_found) { /* The btl_proc datastructure is shared by all TCP BTL * instances that are trying to reach this destination. * Cache the peer instance on the btl_proc. */ tcp_endpoint = OBJ_NEW(mca_btl_tcp_endpoint_t); if(NULL == tcp_endpoint) { OPAL_THREAD_UNLOCK(&tcp_proc->proc_lock); return OPAL_ERR_OUT_OF_RESOURCE; } tcp_endpoint->endpoint_btl = tcp_btl; rc = mca_btl_tcp_proc_insert(tcp_proc, tcp_endpoint); if(rc != OPAL_SUCCESS) { OPAL_THREAD_UNLOCK(&tcp_proc->proc_lock); OBJ_RELEASE(tcp_endpoint); continue; } OPAL_THREAD_LOCK(&tcp_btl->tcp_endpoints_mutex); opal_list_append(&tcp_btl->tcp_endpoints, (opal_list_item_t*)tcp_endpoint); OPAL_THREAD_UNLOCK(&tcp_btl->tcp_endpoints_mutex); } OPAL_THREAD_UNLOCK(&tcp_proc->proc_lock); if (NULL != reachable) { opal_bitmap_set_bit(reachable, i); } peers[i] = tcp_endpoint; /* we increase the count of MPI users of the event library once per peer, so that we are used until we aren't connected to a peer */ opal_progress_event_users_increment(); } return OPAL_SUCCESS; }
int mca_btl_portals4_add_procs(struct mca_btl_base_module_t* btl_base, size_t nprocs, struct opal_proc_t **procs, struct mca_btl_base_endpoint_t** btl_peer_data, opal_bitmap_t* reachable) { struct mca_btl_portals4_module_t* portals4_btl = (struct mca_btl_portals4_module_t*) btl_base; int ret; size_t i; opal_output_verbose(50, opal_btl_base_framework.framework_output, "mca_btl_portals4_add_procs: Adding %d procs (%d) for NI %d", (int) nprocs, (int) portals4_btl->portals_num_procs, portals4_btl->interface_num); /* * The PML handed us a list of procs that need Portals4 * peer info. Complete those procs here. */ for (i = 0 ; i < nprocs ; ++i) { struct opal_proc_t *curr_proc = procs[i]; /* portals doesn't support heterogeneous yet... */ if (opal_proc_local_get()->proc_arch != curr_proc->proc_arch) { opal_output_verbose(1, opal_btl_base_framework.framework_output, "Portals 4 BTL does not support heterogeneous operations."); opal_output_verbose(1, opal_btl_base_framework.framework_output, "Proc %s architecture %x, mine %x.", OPAL_NAME_PRINT(curr_proc->proc_name), curr_proc->proc_arch, opal_proc_local_get()->proc_arch); return OPAL_ERR_NOT_SUPPORTED; } ret = create_endpoint(portals4_btl->interface_num, curr_proc, &btl_peer_data[i]); OPAL_THREAD_ADD_FETCH32(&portals4_btl->portals_num_procs, 1); /* and here we can reach */ opal_bitmap_set_bit(reachable, i); OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "add_procs: rank=%lx nid=%x pid=%x for NI %d", i, btl_peer_data[i]->ptl_proc.phys.nid, btl_peer_data[i]->ptl_proc.phys.pid, portals4_btl->interface_num)); } if (mca_btl_portals4_component.need_init && portals4_btl->portals_num_procs > 0) { if (mca_btl_portals4_component.use_logical) { ret = create_maptable(portals4_btl, nprocs, procs, btl_peer_data); if (OPAL_SUCCESS != ret) { opal_output_verbose(1, opal_btl_base_framework.framework_output, "%s:%d: mca_btl_portals4_add_procs::create_maptable() failed: %d\n", __FILE__, __LINE__, ret); return ret; } } ret = btl_portals4_init_interface(); if (OPAL_SUCCESS != ret) { opal_output_verbose(1, opal_btl_base_framework.framework_output, "%s:%d: portals4 interface initialization failed: %d", __FILE__, __LINE__, ret); return ret; } mca_btl_portals4_component.need_init = 0; } return OPAL_SUCCESS; }
static int binomial_tree(int rank, int parent, int me, int num_procs, int *nchildren, opal_list_t *childrn, opal_bitmap_t *relatives, bool mine) { int i, bitmap, peer, hibit, mask, found; orte_routed_tree_t *child; opal_bitmap_t *relations; OPAL_OUTPUT_VERBOSE((3, orte_routed_base_framework.framework_output, "%s routed:binomial rank %d parent %d me %d num_procs %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), rank, parent, me, num_procs)); /* is this me? */ if (me == rank) { bitmap = opal_cube_dim(num_procs); hibit = opal_hibit(rank, bitmap); --bitmap; for (i = hibit + 1, mask = 1 << i; i <= bitmap; ++i, mask <<= 1) { peer = rank | mask; if (peer < num_procs) { child = OBJ_NEW(orte_routed_tree_t); child->vpid = peer; OPAL_OUTPUT_VERBOSE((3, orte_routed_base_framework.framework_output, "%s routed:binomial %d found child %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), rank, ORTE_VPID_PRINT(child->vpid))); if (mine) { /* this is a direct child - add it to my list */ opal_list_append(childrn, &child->super); (*nchildren)++; /* setup the relatives bitmap */ opal_bitmap_init(&child->relatives, num_procs); /* point to the relatives */ relations = &child->relatives; } else { /* we are recording someone's relatives - set the bit */ opal_bitmap_set_bit(relatives, peer); /* point to this relations */ relations = relatives; } /* search for this child's relatives */ binomial_tree(0, 0, peer, num_procs, nchildren, childrn, relations, false); } } return parent; } /* find the children of this rank */ OPAL_OUTPUT_VERBOSE((5, orte_routed_base_framework.framework_output, "%s routed:binomial find children of rank %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), rank)); bitmap = opal_cube_dim(num_procs); hibit = opal_hibit(rank, bitmap); --bitmap; for (i = hibit + 1, mask = 1 << i; i <= bitmap; ++i, mask <<= 1) { peer = rank | mask; OPAL_OUTPUT_VERBOSE((5, orte_routed_base_framework.framework_output, "%s routed:binomial find children checking peer %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), peer)); if (peer < num_procs) { OPAL_OUTPUT_VERBOSE((5, orte_routed_base_framework.framework_output, "%s routed:binomial find children computing tree", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); /* execute compute on this child */ if (0 <= (found = binomial_tree(peer, rank, me, num_procs, nchildren, childrn, relatives, mine))) { OPAL_OUTPUT_VERBOSE((5, orte_routed_base_framework.framework_output, "%s routed:binomial find children returning found value %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), found)); return found; } } } return -1; }
int mca_btl_sm_add_procs( struct mca_btl_base_module_t* btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers, opal_bitmap_t* reachability) { int return_code = OMPI_SUCCESS; int32_t n_local_procs = 0, proc, j, my_smp_rank = -1; ompi_proc_t* my_proc; /* pointer to caller's proc structure */ mca_btl_sm_t *sm_btl; bool have_connected_peer = false; char **bases; /* for easy access to the mpool_sm_module */ mca_mpool_sm_module_t *sm_mpool_modp = NULL; /* initializion */ sm_btl = (mca_btl_sm_t *)btl; /* get pointer to my proc structure */ if(NULL == (my_proc = ompi_proc_local())) return OMPI_ERR_OUT_OF_RESOURCE; /* Get unique host identifier for each process in the list, * and idetify procs that are on this host. Add procs on this * host to shared memory reachbility list. Also, get number * of local procs in the procs list. */ for (proc = 0; proc < (int32_t)nprocs; proc++) { /* check to see if this proc can be reached via shmem (i.e., if they're on my local host and in my job) */ if (procs[proc]->proc_name.jobid != my_proc->proc_name.jobid || !OPAL_PROC_ON_LOCAL_NODE(procs[proc]->proc_flags)) { peers[proc] = NULL; continue; } /* check to see if this is me */ if(my_proc == procs[proc]) { my_smp_rank = mca_btl_sm_component.my_smp_rank = n_local_procs++; continue; } /* sm doesn't support heterogeneous yet... */ if (procs[proc]->proc_arch != my_proc->proc_arch) { continue; } /* we have someone to talk to */ have_connected_peer = true; if(!(peers[proc] = create_sm_endpoint(n_local_procs, procs[proc]))) { return_code = OMPI_ERROR; goto CLEANUP; } n_local_procs++; /* add this proc to shared memory accessibility list */ return_code = opal_bitmap_set_bit(reachability, proc); if(OMPI_SUCCESS != return_code) goto CLEANUP; } /* jump out if there's not someone we can talk to */ if (!have_connected_peer) goto CLEANUP; /* make sure that my_smp_rank has been defined */ if (-1 == my_smp_rank) { return_code = OMPI_ERROR; goto CLEANUP; } if (!sm_btl->btl_inited) { return_code = sm_btl_first_time_init(sm_btl, my_smp_rank, mca_btl_sm_component.sm_max_procs); if (return_code != OMPI_SUCCESS) { goto CLEANUP; } } /* set local proc's smp rank in the peers structure for * rapid access and calculate reachability */ for(proc = 0; proc < (int32_t)nprocs; proc++) { if(NULL == peers[proc]) continue; mca_btl_sm_component.sm_peers[peers[proc]->peer_smp_rank] = peers[proc]; peers[proc]->my_smp_rank = my_smp_rank; } bases = mca_btl_sm_component.shm_bases; sm_mpool_modp = (mca_mpool_sm_module_t *)mca_btl_sm_component.sm_mpool; /* initialize own FIFOs */ /* * The receiver initializes all its FIFOs. All components will * be allocated near the receiver. Nothing will be local to * "the sender" since there will be many senders. */ for(j = mca_btl_sm_component.num_smp_procs; j < mca_btl_sm_component.num_smp_procs + FIFO_MAP_NUM(n_local_procs); j++) { return_code = sm_fifo_init( mca_btl_sm_component.fifo_size, mca_btl_sm_component.sm_mpool, &mca_btl_sm_component.fifo[my_smp_rank][j], mca_btl_sm_component.fifo_lazy_free); if(return_code != OMPI_SUCCESS) goto CLEANUP; } opal_atomic_wmb(); /* Sync with other local procs. Force the FIFO initialization to always * happens before the readers access it. */ opal_atomic_add_32(&mca_btl_sm_component.sm_seg->module_seg->seg_inited, 1); while( n_local_procs > mca_btl_sm_component.sm_seg->module_seg->seg_inited) { opal_progress(); opal_atomic_rmb(); } /* it is now safe to unlink the shared memory segment. only one process * needs to do this, so just let smp rank zero take care of it. */ if (0 == my_smp_rank) { if (OMPI_SUCCESS != mca_common_sm_module_unlink(mca_btl_sm_component.sm_seg)) { /* it is "okay" if this fails at this point. we have gone this far, * so just warn about the failure and continue. this is probably * only triggered by a programming error. */ opal_output(0, "WARNING: common_sm_module_unlink failed.\n"); } /* SKG - another abstraction violation here, but I don't want to add * extra code in the sm mpool for further synchronization. */ /* at this point, all processes have attached to the mpool segment. so * it is safe to unlink it here. */ if (OMPI_SUCCESS != mca_common_sm_module_unlink(sm_mpool_modp->sm_common_module)) { opal_output(0, "WARNING: common_sm_module_unlink failed.\n"); } if (-1 == unlink(mca_btl_sm_component.sm_mpool_rndv_file_name)) { opal_output(0, "WARNING: %s unlink failed.\n", mca_btl_sm_component.sm_mpool_rndv_file_name); } if (-1 == unlink(mca_btl_sm_component.sm_rndv_file_name)) { opal_output(0, "WARNING: %s unlink failed.\n", mca_btl_sm_component.sm_rndv_file_name); } } /* free up some space used by the name buffers */ free(mca_btl_sm_component.sm_mpool_ctl_file_name); free(mca_btl_sm_component.sm_mpool_rndv_file_name); free(mca_btl_sm_component.sm_ctl_file_name); free(mca_btl_sm_component.sm_rndv_file_name); /* coordinate with other processes */ for(j = mca_btl_sm_component.num_smp_procs; j < mca_btl_sm_component.num_smp_procs + n_local_procs; j++) { ptrdiff_t diff; /* spin until this element is allocated */ /* doesn't really wait for that process... FIFO might be allocated, but not initialized */ opal_atomic_rmb(); while(NULL == mca_btl_sm_component.shm_fifo[j]) { opal_progress(); opal_atomic_rmb(); } /* Calculate the difference as (my_base - their_base) */ diff = ADDR2OFFSET(bases[my_smp_rank], bases[j]); /* store local address of remote fifos */ mca_btl_sm_component.fifo[j] = (sm_fifo_t*)OFFSET2ADDR(diff, mca_btl_sm_component.shm_fifo[j]); /* cache local copy of peer memory node number */ mca_btl_sm_component.mem_nodes[j] = mca_btl_sm_component.shm_mem_nodes[j]; } /* update the local smp process count */ mca_btl_sm_component.num_smp_procs += n_local_procs; /* make sure we have enough eager fragmnents for each process */ return_code = ompi_free_list_resize_mt(&mca_btl_sm_component.sm_frags_eager, mca_btl_sm_component.num_smp_procs * 2); if (OMPI_SUCCESS != return_code) goto CLEANUP; CLEANUP: return return_code; }
int mca_btl_scif_add_procs(struct mca_btl_base_module_t* btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers, opal_bitmap_t *reachable) { mca_btl_scif_module_t *scif_module = (mca_btl_scif_module_t *) btl; size_t procs_on_board, i, board_proc; ompi_proc_t *my_proc = ompi_proc_local(); int rc; /* determine how many procs are on this board */ for (i = 0, procs_on_board = 0 ; i < nprocs ; ++i) { struct ompi_proc_t *ompi_proc = procs[i]; if (my_proc == ompi_proc) { continue; } if (!OPAL_PROC_ON_LOCAL_HOST(ompi_proc->proc_flags) || my_proc == ompi_proc) { /* scif can only be used with procs on this board */ continue; } procs_on_board++; } /* allocate space for the detected peers and setup the mpool */ if (NULL == scif_module->endpoints) { scif_module->endpoints = calloc (procs_on_board, sizeof (mca_btl_base_endpoint_t)); if (OPAL_UNLIKELY(NULL == scif_module->endpoints)) { return OMPI_ERR_OUT_OF_RESOURCE; } rc = mca_btl_scif_setup_mpools (scif_module); if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) { BTL_ERROR(("btl/scif error setting up mpools/free lists")); return rc; } } for (i = 0, board_proc = 0 ; i < nprocs ; ++i) { struct ompi_proc_t *ompi_proc = procs[i]; if (my_proc == ompi_proc) { continue; } if (!OPAL_PROC_ON_LOCAL_HOST(ompi_proc->proc_flags) || my_proc == ompi_proc) { peers[i] = NULL; /* scif can only be used with procs on this board */ continue; } /* Initialize endpoints */ rc = mca_btl_scif_ep_init (scif_module->endpoints + board_proc, (mca_btl_scif_module_t *) btl, ompi_proc); if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) { BTL_ERROR(("btl/scif error initializing endpoint")); return rc; } scif_module->endpoints[board_proc].id = board_proc; /* Set the reachable bit */ rc = opal_bitmap_set_bit (reachable, i); /* Store a reference to this peer */ peers[i] = scif_module->endpoints + board_proc; board_proc++; } BTL_VERBOSE(("%lu procs on board\n", (unsigned long) procs_on_board)); scif_module->endpoint_count = procs_on_board; /* start listening thread */ rc = pthread_create (&mca_btl_scif_module.listen_thread, NULL, mca_btl_scif_connect_accept, NULL); if (0 > rc) { return OMPI_ERROR; } return OMPI_SUCCESS; }
int mca_btl_sctp_add_procs( struct mca_btl_base_module_t* btl, size_t nprocs, struct ompi_proc_t **ompi_procs, struct mca_btl_base_endpoint_t** peers, opal_bitmap_t* reachable) { mca_btl_sctp_module_t* sctp_btl = (mca_btl_sctp_module_t*)btl; ompi_proc_t* my_proc; /* pointer to caller's proc structure */ int i, rc; /* get pointer to my proc structure */ my_proc = ompi_proc_local(); if( NULL == my_proc ) { return OMPI_ERR_OUT_OF_RESOURCE; } for(i = 0; i < (int) nprocs; i++) { struct ompi_proc_t* ompi_proc = ompi_procs[i]; mca_btl_sctp_proc_t* sctp_proc; mca_btl_base_endpoint_t* sctp_endpoint; /* Do not create loopback SCTP connections */ if( my_proc == ompi_proc ) { continue; } if(NULL == (sctp_proc = mca_btl_sctp_proc_create(ompi_proc))) { return OMPI_ERR_OUT_OF_RESOURCE; } /* * Check to make sure that the peer has at least as many interface * addresses exported as we are trying to use. If not, then * don't bind this BTL instance to the proc. */ OPAL_THREAD_LOCK(&sctp_proc->proc_lock); /* The btl_proc datastructure is shared by all SCTP BTL * instances that are trying to reach this destination. * Cache the peer instance on the btl_proc. */ sctp_endpoint = OBJ_NEW(mca_btl_sctp_endpoint_t); if(NULL == sctp_endpoint) { OPAL_THREAD_UNLOCK(&sctp_proc->proc_lock); return OMPI_ERR_OUT_OF_RESOURCE; } sctp_endpoint->endpoint_btl = sctp_btl; rc = mca_btl_sctp_proc_insert(sctp_proc, sctp_endpoint); if(rc != OMPI_SUCCESS) { OBJ_RELEASE(sctp_endpoint); OPAL_THREAD_UNLOCK(&sctp_proc->proc_lock); continue; } opal_bitmap_set_bit(reachable, i); OPAL_THREAD_UNLOCK(&sctp_proc->proc_lock); peers[i] = sctp_endpoint; opal_list_append(&sctp_btl->sctp_endpoints, (opal_list_item_t*)sctp_endpoint); /* we increase the count of MPI users of the event library once per peer, so that we are used until we aren't connected to a peer */ opal_progress_event_users_increment(); } return OMPI_SUCCESS; }