int mca_btl_mx_add_procs( struct mca_btl_base_module_t* btl, size_t nprocs, struct ompi_proc_t** ompi_procs, struct mca_btl_base_endpoint_t** peers, ompi_bitmap_t* reachable ) { mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*)btl; int i, rc; for( i = 0; i < (int) nprocs; i++ ) { struct ompi_proc_t* ompi_proc = ompi_procs[i]; mca_btl_mx_proc_t* mx_proc; mca_btl_base_endpoint_t* mx_endpoint; /** * By default don't allow communications with self nor with any * other processes on the same node. The BTL self and sm are * supposed to take care of such communications. */ if( ompi_procs[i]->proc_flags & OMPI_PROC_FLAG_LOCAL ) { if( ompi_procs[i] == ompi_proc_local_proc ) { if( 0 == mca_btl_mx_component.mx_support_self ) continue; } else { if( 0 == mca_btl_mx_component.mx_support_sharedmem ) continue; } } if( NULL == (mx_proc = mca_btl_mx_proc_create(ompi_proc)) ) { continue; } OPAL_THREAD_LOCK(&mx_proc->proc_lock); /* The btl_proc datastructure is shared by all MX BTL * instances that are trying to reach this destination. * Cache the peer instance on the btl_proc. */ mx_endpoint = OBJ_NEW(mca_btl_mx_endpoint_t); if(NULL == mx_endpoint) { OPAL_THREAD_UNLOCK(&mx_proc->proc_lock); return OMPI_ERR_OUT_OF_RESOURCE; } mx_endpoint->endpoint_btl = mx_btl; rc = mca_btl_mx_proc_insert( mx_proc, mx_endpoint ); if( rc != OMPI_SUCCESS ) { OBJ_RELEASE(mx_endpoint); OBJ_RELEASE(mx_proc); OPAL_THREAD_UNLOCK(&mx_proc->proc_lock); continue; } ompi_bitmap_set_bit(reachable, i); OPAL_THREAD_UNLOCK(&mx_proc->proc_lock); peers[i] = mx_endpoint; } return OMPI_SUCCESS; }
int mca_btl_self_add_procs( struct mca_btl_base_module_t* btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers, ompi_bitmap_t* reachability ) { size_t i; for( i = 0; i < nprocs; i++ ) { if( procs[i] == ompi_proc_local_proc ) { ompi_bitmap_set_bit( reachability, i ); break; /* there will always be only one ... */ } } return OMPI_SUCCESS; }
int ompi_bitmap_find_and_set_first_unset_bit(ompi_bitmap_t *bm, int *position) { int i = 0; unsigned char temp; unsigned char all_ones = 0xff; if (NULL == bm) { return OMPI_ERR_BAD_PARAM; } /* Neglect all which dont have an unset bit */ *position = 0; while((i < bm->array_size) && (bm->bitmap[i] == all_ones)) { ++i; } if (i == bm->array_size) { /* increase the bitmap size then */ *position = bm->array_size * SIZE_OF_CHAR; return ompi_bitmap_set_bit(bm, *position); } /* This one has an unset bit, find its bit number */ temp = bm->bitmap[i]; while (temp & 0x1) { ++(*position); temp >>= 1; } /* Now set the bit number */ bm->bitmap[i] |= (bm->bitmap[i] + 1); (*position) += i * SIZE_OF_CHAR; return OMPI_SUCCESS; }
int mca_btl_portals_add_procs(struct mca_btl_base_module_t* btl_base, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t** peers, ompi_bitmap_t* reachable) { int ret; struct ompi_proc_t *curr_proc = NULL; ptl_process_id_t *portals_procs = NULL; size_t i; unsigned long distance; bool need_activate = false; assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base); opal_output_verbose(50, mca_btl_portals_component.portals_output, "Adding %d procs (%d)", nprocs, mca_btl_portals_module.portals_num_procs); /* if we havne't already, get our network handle */ if (mca_btl_portals_module.portals_ni_h == PTL_INVALID_HANDLE) { ret = ompi_common_portals_ni_initialize(&mca_btl_portals_module.portals_ni_h); if (OMPI_SUCCESS != ret) return ret; } portals_procs = malloc(nprocs * sizeof(ptl_process_id_t)); ret = ompi_common_portals_get_procs(nprocs, procs, portals_procs); if (OMPI_SUCCESS != ret) return ret; if (0 == mca_btl_portals_module.portals_num_procs) { need_activate = true; } /* loop through all procs, setting our reachable flag */ for (i= 0; i < nprocs ; ++i) { curr_proc = procs[i]; /* portals doesn't support heterogeneous yet... */ if (ompi_proc_local()->proc_arch != curr_proc->proc_arch) { continue; } peers[i] = malloc(sizeof(mca_btl_base_endpoint_t)); if (NULL == peers[i]) return OMPI_ERROR; *((mca_btl_base_endpoint_t*) peers[i]) = portals_procs[i]; /* make sure we can reach the process - this is supposed to be a cheap-ish operation */ ret = PtlNIDist(mca_btl_portals_module.portals_ni_h, portals_procs[i], &distance); if (ret != PTL_OK) { opal_output_verbose(10, mca_btl_portals_component.portals_output, "Could not find distance to process %d", i); continue; } OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_num_procs, 1); /* and here we can reach */ ompi_bitmap_set_bit(reachable, i); } if (NULL != portals_procs) free(portals_procs); if (need_activate && mca_btl_portals_module.portals_num_procs > 0) { /* create eqs */ int i; opal_output_verbose(50, mca_btl_portals_component.portals_output, "Enabling progress"); for (i = 0 ; i < OMPI_BTL_PORTALS_EQ_SIZE ; ++i) { int ptl_ret = PtlEQAlloc(mca_btl_portals_module.portals_ni_h, mca_btl_portals_module.portals_eq_sizes[i], PTL_EQ_HANDLER_NONE, &(mca_btl_portals_module.portals_eq_handles[i])); if (PTL_OK != ptl_ret) { opal_output(mca_btl_portals_component.portals_output, "Error creating EQ %d: %d", i, ptl_ret); /* BWB - better error code? */ return OMPI_ERROR; } } ret = mca_btl_portals_recv_enable(&mca_btl_portals_module); /* fill in send memory descriptor */ mca_btl_portals_module.md_send.start = NULL; mca_btl_portals_module.md_send.length = 0; mca_btl_portals_module.md_send.threshold = PTL_MD_THRESH_INF; mca_btl_portals_module.md_send.max_size = 0; mca_btl_portals_module.md_send.options = PTL_MD_EVENT_START_DISABLE; mca_btl_portals_module.md_send.user_ptr = NULL; mca_btl_portals_module.md_send.eq_handle = mca_btl_portals_module.portals_eq_handles[OMPI_BTL_PORTALS_EQ_SEND]; } else { ret = OMPI_SUCCESS; } return ret; }
/* * add a proc to this btl module * creates an endpoint that is setup on the * first send to the endpoint */ int mca_btl_openib_add_procs( struct mca_btl_base_module_t* btl, size_t nprocs, struct ompi_proc_t **ompi_procs, struct mca_btl_base_endpoint_t** peers, ompi_bitmap_t* reachable) { mca_btl_openib_module_t* openib_btl = (mca_btl_openib_module_t*)btl; int i,j, rc; int rem_subnet_id_port_cnt; int lcl_subnet_id_port_cnt = 0; int btl_rank; for(j=0; j < mca_btl_openib_component.ib_num_btls; j++){ if(mca_btl_openib_component.openib_btls[j].port_info.subnet_id == openib_btl->port_info.subnet_id) { lcl_subnet_id_port_cnt++; } if(openib_btl == &(mca_btl_openib_component.openib_btls[j])) { btl_rank = j; } } for(i = 0; i < (int) nprocs; i++) { struct ompi_proc_t* ompi_proc = ompi_procs[i]; mca_btl_openib_proc_t* ib_proc; mca_btl_base_endpoint_t* endpoint; if(NULL == (ib_proc = mca_btl_openib_proc_create(ompi_proc))) { return OMPI_ERR_OUT_OF_RESOURCE; } rem_subnet_id_port_cnt = 0; /* check if the remote proc has a reachable subnet first */ BTL_VERBOSE(("got %d port_infos \n", ib_proc->proc_port_count)); for(j = 0; j < (int) ib_proc->proc_port_count; j++){ BTL_VERBOSE(("got a subnet %016x\n", ib_proc->proc_ports[j].subnet_id)); if(ib_proc->proc_ports[j].subnet_id == openib_btl->port_info.subnet_id) { BTL_VERBOSE(("Got a matching subnet!\n")); rem_subnet_id_port_cnt ++; } } if(!rem_subnet_id_port_cnt ) { /* no use trying to communicate with this endpointlater */ BTL_VERBOSE(("No matching subnet id was found, moving on.. \n")); continue; } #if 0 num_endpoints = rem_subnet_id_port_cnt / lcl_subnet_id_port_cnt + (btl_rank < (rem_subnet_id_port_cnt / lcl_subnet_id_port_cnt)) ? 1:0; #endif if(rem_subnet_id_port_cnt < lcl_subnet_id_port_cnt && btl_rank >= rem_subnet_id_port_cnt ) { BTL_VERBOSE(("Not enough remote ports on this subnet id, moving on.. \n")); continue; } OPAL_THREAD_LOCK(&ib_proc->proc_lock); /* The btl_proc datastructure is shared by all IB PTL * instances that are trying to reach this destination. * Cache the peer instance on the btl_proc. */ endpoint = OBJ_NEW(mca_btl_openib_endpoint_t); if(NULL == endpoint) { OPAL_THREAD_UNLOCK(&ib_proc->proc_lock); return OMPI_ERR_OUT_OF_RESOURCE; } endpoint->endpoint_btl = openib_btl; endpoint->use_eager_rdma = openib_btl->hca->use_eager_rdma & mca_btl_openib_component.use_eager_rdma; endpoint->subnet_id = openib_btl->port_info.subnet_id; rc = mca_btl_openib_proc_insert(ib_proc, endpoint); if(rc != OMPI_SUCCESS) { OBJ_RELEASE(endpoint); OPAL_THREAD_UNLOCK(&ib_proc->proc_lock); continue; } orte_pointer_array_add((orte_std_cntr_t*)&endpoint->index, openib_btl->endpoints, (void*)endpoint); ompi_bitmap_set_bit(reachable, i); OPAL_THREAD_UNLOCK(&ib_proc->proc_lock); peers[i] = endpoint; } return mca_btl_openib_size_queues(openib_btl, nprocs); }