int ompi_common_mx_initialize(void) { mx_return_t mx_return; struct mca_mpool_base_resources_t mpool_resources; int index, value; ompi_common_mx_initialize_ref_cnt++; if(ompi_common_mx_initialize_ref_cnt == 1) { /* set the MX error handle to always return. This function is the * only MX function allowed to be called before mx_init in order * to make sure that if the MX is not up and running the MX * library does not exit the application. */ mx_set_error_handler(MX_ERRORS_RETURN); /* If we have a memory manager available, and mpi_leave_pinned == -1, then set mpi_leave_pinned to 1. We have a memory manager if: - we have both FREE and MUNMAP support - we have MUNMAP support and the linux mallopt */ value = opal_mem_hooks_support_level(); if ((value & (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT)) == (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT)) { index = mca_base_param_find("mpi", NULL, "leave_pinned"); if (index >= 0) if ((mca_base_param_lookup_int(index, &value) == OPAL_SUCCESS) && (value == -1)) { ompi_mpi_leave_pinned = 1; setenv("MX_RCACHE", "2", 1); mpool_resources.regcache_clean = mx__regcache_clean; ompi_common_mx_fake_mpool = mca_mpool_base_module_create("fake", NULL, &mpool_resources); if (!ompi_common_mx_fake_mpool) { ompi_mpi_leave_pinned = 0; setenv("MX_RCACHE", "0", 1); opal_output(0, "Error creating fake mpool (error %s)\n", strerror(errno)); } } } /* initialize the mx library */ mx_return = mx_init(); if(MX_SUCCESS != mx_return) { opal_output(0, "Error in mx_init (error %s)\n", mx_strerror(mx_return)); return OMPI_ERR_NOT_AVAILABLE; } } return OMPI_SUCCESS; }
static int mca_btl_scif_setup_mpools (mca_btl_scif_module_t *scif_module) { struct mca_mpool_base_resources_t mpool_resources; int rc; /* initialize the grdma mpool */ mpool_resources.pool_name = "scif"; mpool_resources.reg_data = (void *) scif_module; mpool_resources.sizeof_reg = sizeof (mca_btl_scif_reg_t); mpool_resources.register_mem = scif_reg_mem; mpool_resources.deregister_mem = scif_dereg_mem; scif_module->super.btl_mpool = mca_mpool_base_module_create("grdma", scif_module, &mpool_resources); if (NULL == scif_module->super.btl_mpool) { BTL_ERROR(("error creating grdma mpool")); return OMPI_ERROR; } /* setup free lists for fragments. dma fragments will be used for * rma operations and in-place sends. eager frags will be used for * buffered sends. */ rc = ompi_free_list_init_new (&scif_module->dma_frags, sizeof (mca_btl_scif_dma_frag_t), 64, OBJ_CLASS(mca_btl_scif_dma_frag_t), 128, getpagesize (), mca_btl_scif_component.scif_free_list_num, mca_btl_scif_component.scif_free_list_max, mca_btl_scif_component.scif_free_list_inc, NULL); if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) { return rc; } rc = ompi_free_list_init_new (&scif_module->eager_frags, sizeof (mca_btl_scif_eager_frag_t), 8, OBJ_CLASS(mca_btl_scif_eager_frag_t), 128 + scif_module->super.btl_eager_limit, 64, mca_btl_scif_component.scif_free_list_num, mca_btl_scif_component.scif_free_list_max, mca_btl_scif_component.scif_free_list_inc, NULL); if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) { BTL_ERROR(("error creating eager receive fragment free list")); return rc; } return OMPI_SUCCESS; }
static struct mca_btl_base_endpoint_t * create_sm_endpoint(int local_proc, struct opal_proc_t *proc) { struct mca_btl_base_endpoint_t *ep; #if OPAL_ENABLE_PROGRESS_THREADS == 1 char path[PATH_MAX]; #endif ep = (struct mca_btl_base_endpoint_t*) malloc(sizeof(struct mca_btl_base_endpoint_t)); if(NULL == ep) return NULL; ep->peer_smp_rank = local_proc + mca_btl_smcuda_component.num_smp_procs; OBJ_CONSTRUCT(&ep->pending_sends, opal_list_t); OBJ_CONSTRUCT(&ep->endpoint_lock, opal_mutex_t); #if OPAL_ENABLE_PROGRESS_THREADS == 1 sprintf(path, "%s"OPAL_PATH_SEP"sm_fifo.%lu", opal_process_info.job_session_dir, (unsigned long)proc->proc_name); ep->fifo_fd = open(path, O_WRONLY); if(ep->fifo_fd < 0) { opal_output(0, "mca_btl_smcuda_add_procs: open(%s) failed with errno=%d\n", path, errno); free(ep); return NULL; } #endif #if OPAL_CUDA_SUPPORT { mca_mpool_base_resources_t resources; /* unused, but needed */ /* Create a remote memory pool on the endpoint. Note that the resources * argument is just to satisfy the function signature. The rcuda mpool * actually takes care of filling in the resources. */ ep->mpool = mca_mpool_base_module_create("rgpusm", NULL, &resources); } #endif /* OPAL_CUDA_SUPPORT */ return ep; }
static int sm_btl_first_time_init(mca_btl_sm_t *sm_btl, int32_t my_smp_rank, int n) { size_t length, length_payload; sm_fifo_t *my_fifos; int my_mem_node, num_mem_nodes, i, rc; mca_mpool_base_resources_t *res = NULL; mca_btl_sm_component_t* m = &mca_btl_sm_component; /* Assume we don't have hwloc support and fill in dummy info */ mca_btl_sm_component.mem_node = my_mem_node = 0; mca_btl_sm_component.num_mem_nodes = num_mem_nodes = 1; #if OPAL_HAVE_HWLOC /* If we have hwloc support, then get accurate information */ if (NULL != opal_hwloc_topology) { i = opal_hwloc_base_get_nbobjs_by_type(opal_hwloc_topology, HWLOC_OBJ_NODE, 0, OPAL_HWLOC_AVAILABLE); /* If we find >0 NUMA nodes, then investigate further */ if (i > 0) { int numa=0, w; unsigned n_bound=0; hwloc_cpuset_t avail; hwloc_obj_t obj; /* JMS This tells me how many numa nodes are *available*, but it's not how many are being used *by this job*. Note that this is the value we've previously used (from the previous carto-based implementation), but it really should be improved to be how many NUMA nodes are being used *in this job*. */ mca_btl_sm_component.num_mem_nodes = num_mem_nodes = i; /* if we are not bound, then there is nothing further to do */ if (NULL != ompi_process_info.cpuset) { /* count the number of NUMA nodes to which we are bound */ for (w=0; w < i; w++) { if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_NODE, 0, w, OPAL_HWLOC_AVAILABLE))) { continue; } /* get that NUMA node's available cpus */ avail = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj); /* see if we intersect */ if (hwloc_bitmap_intersects(avail, opal_hwloc_my_cpuset)) { n_bound++; numa = w; } } /* if we are located on more than one NUMA, or we didn't find * a NUMA we are on, then not much we can do */ if (1 == n_bound) { mca_btl_sm_component.mem_node = my_mem_node = numa; } else { mca_btl_sm_component.mem_node = my_mem_node = -1; } } } } #endif if (NULL == (res = calloc(1, sizeof(*res)))) { return OMPI_ERR_OUT_OF_RESOURCE; } /* lookup shared memory pool */ mca_btl_sm_component.sm_mpools = (mca_mpool_base_module_t **)calloc(num_mem_nodes, sizeof(mca_mpool_base_module_t *)); /* Disable memory binding, because each MPI process will claim pages in the * mpool for their local NUMA node */ res->mem_node = -1; if (OMPI_SUCCESS != (rc = setup_mpool_base_resources(m, res))) { free(res); return rc; } /* now that res is fully populated, create the thing */ mca_btl_sm_component.sm_mpools[0] = mca_mpool_base_module_create(mca_btl_sm_component.sm_mpool_name, sm_btl, res); /* Sanity check to ensure that we found it */ if (NULL == mca_btl_sm_component.sm_mpools[0]) { free(res); return OMPI_ERR_OUT_OF_RESOURCE; } mca_btl_sm_component.sm_mpool = mca_btl_sm_component.sm_mpools[0]; mca_btl_sm_component.sm_mpool_base = mca_btl_sm_component.sm_mpools[0]->mpool_base(mca_btl_sm_component.sm_mpools[0]); /* create a list of peers */ mca_btl_sm_component.sm_peers = (struct mca_btl_base_endpoint_t**) calloc(n, sizeof(struct mca_btl_base_endpoint_t*)); if (NULL == mca_btl_sm_component.sm_peers) { free(res); return OMPI_ERR_OUT_OF_RESOURCE; } /* remember that node rank zero is already attached */ if (0 != my_smp_rank) { if (OMPI_SUCCESS != (rc = sm_segment_attach(m))) { free(res); return rc; } } /* it is now safe to free the mpool resources */ free(res); /* check to make sure number of local procs is within the * specified limits */ if(mca_btl_sm_component.sm_max_procs > 0 && mca_btl_sm_component.num_smp_procs + n > mca_btl_sm_component.sm_max_procs) { return OMPI_ERROR; } mca_btl_sm_component.shm_fifo = (volatile sm_fifo_t **)mca_btl_sm_component.sm_seg->module_data_addr; mca_btl_sm_component.shm_bases = (char**)(mca_btl_sm_component.shm_fifo + n); mca_btl_sm_component.shm_mem_nodes = (uint16_t*)(mca_btl_sm_component.shm_bases + n); /* set the base of the shared memory segment */ mca_btl_sm_component.shm_bases[mca_btl_sm_component.my_smp_rank] = (char*)mca_btl_sm_component.sm_mpool_base; mca_btl_sm_component.shm_mem_nodes[mca_btl_sm_component.my_smp_rank] = (uint16_t)my_mem_node; /* initialize the array of fifo's "owned" by this process */ if(NULL == (my_fifos = (sm_fifo_t*)mpool_calloc(FIFO_MAP_NUM(n), sizeof(sm_fifo_t)))) return OMPI_ERR_OUT_OF_RESOURCE; mca_btl_sm_component.shm_fifo[mca_btl_sm_component.my_smp_rank] = my_fifos; /* cache the pointer to the 2d fifo array. These addresses * are valid in the current process space */ mca_btl_sm_component.fifo = (sm_fifo_t**)malloc(sizeof(sm_fifo_t*) * n); if(NULL == mca_btl_sm_component.fifo) return OMPI_ERR_OUT_OF_RESOURCE; mca_btl_sm_component.fifo[mca_btl_sm_component.my_smp_rank] = my_fifos; mca_btl_sm_component.mem_nodes = (uint16_t *) malloc(sizeof(uint16_t) * n); if(NULL == mca_btl_sm_component.mem_nodes) return OMPI_ERR_OUT_OF_RESOURCE; /* initialize fragment descriptor free lists */ /* allocation will be for the fragment descriptor and payload buffer */ length = sizeof(mca_btl_sm_frag1_t); length_payload = sizeof(mca_btl_sm_hdr_t) + mca_btl_sm_component.eager_limit; i = ompi_free_list_init_new(&mca_btl_sm_component.sm_frags_eager, length, opal_cache_line_size, OBJ_CLASS(mca_btl_sm_frag1_t), length_payload, opal_cache_line_size, mca_btl_sm_component.sm_free_list_num, mca_btl_sm_component.sm_free_list_max, mca_btl_sm_component.sm_free_list_inc, mca_btl_sm_component.sm_mpool); if ( OMPI_SUCCESS != i ) return i; length = sizeof(mca_btl_sm_frag2_t); length_payload = sizeof(mca_btl_sm_hdr_t) + mca_btl_sm_component.max_frag_size; i = ompi_free_list_init_new(&mca_btl_sm_component.sm_frags_max, length, opal_cache_line_size, OBJ_CLASS(mca_btl_sm_frag2_t), length_payload, opal_cache_line_size, mca_btl_sm_component.sm_free_list_num, mca_btl_sm_component.sm_free_list_max, mca_btl_sm_component.sm_free_list_inc, mca_btl_sm_component.sm_mpool); if ( OMPI_SUCCESS != i ) return i; i = ompi_free_list_init_new(&mca_btl_sm_component.sm_frags_user, sizeof(mca_btl_sm_user_t), opal_cache_line_size, OBJ_CLASS(mca_btl_sm_user_t), sizeof(mca_btl_sm_hdr_t), opal_cache_line_size, mca_btl_sm_component.sm_free_list_num, mca_btl_sm_component.sm_free_list_max, mca_btl_sm_component.sm_free_list_inc, mca_btl_sm_component.sm_mpool); if ( OMPI_SUCCESS != i ) return i; mca_btl_sm_component.num_outstanding_frags = 0; mca_btl_sm_component.num_pending_sends = 0; i = opal_free_list_init(&mca_btl_sm_component.pending_send_fl, sizeof(btl_sm_pending_send_item_t), OBJ_CLASS(opal_free_list_item_t), 16, -1, 32); if ( OMPI_SUCCESS != i ) return i; /* set flag indicating btl has been inited */ sm_btl->btl_inited = true; return OMPI_SUCCESS; }
int mca_btl_udapl_init(DAT_NAME_PTR ia_name, mca_btl_udapl_module_t* btl) { mca_mpool_base_resources_t res; DAT_CONN_QUAL port; DAT_RETURN rc; /* open the uDAPL interface */ btl->udapl_evd_async = DAT_HANDLE_NULL; rc = dat_ia_open(ia_name, btl->udapl_async_evd_qlen, &btl->udapl_evd_async, &btl->udapl_ia); if(DAT_SUCCESS != rc) { char* major; char* minor; dat_strerror(rc, (const char**)&major, (const char**)&minor); #if defined(__SVR4) && defined(__sun) if (strcmp(major, "DAT_INVALID_PARAMETER") == 0 && strcmp(minor, "DAT_INVALID_RO_COOKIE") == 0) { /* Some platforms that Solaris runs on implement the PCI * standard for relaxed ordering(RO). Using RDMA with * polling on a memory location as the uDAPL (and openib * by the way) BTL does for short messages with * relaxed ordering could potentially produce silent data * corruption. For this reason we need to take extra * steps and this is accomplished by setting * "ro_aware_system = 1" and handling as required. * * The uDAPL standard does not provide an interface to * inform users of this scenario so Sun has implemented the * following: If a platform supports relaxed ordering * when the interface name is passed into the * dat_ia_open() call, the call will return * DAT_INVALID_PARAMETER and DAT_INVALID_RO_COOKIE. * DAT_INVALID_RO_COOKIE is not part of the uDAPL standard * at this time. The only way to open this interface is * to prefix the following cookie "RO_AWARE_" to the ia * name that was retreived from the dat registry. * * Example: ia_name = "ib0", new expected name will be * "RO_AWARE_ib0". * * Here, since our first ia open attempt failed in the * standard way, add the cookie and try to open again. */ DAT_NAME_PTR ro_ia_name; /* prefix relaxed order cookie to ia_name */ asprintf(&ro_ia_name, "RO_AWARE_%s", ia_name); if (NULL == ro_ia_name) { return OMPI_ERR_OUT_OF_RESOURCE; } /* because this is not standard inform user in some way */ BTL_UDAPL_VERBOSE_HELP(VERBOSE_INFORM, ("help-mpi-btl-udapl.txt", "relaxed order support", true, ia_name, ro_ia_name)); /* try and open again */ btl->udapl_evd_async = DAT_HANDLE_NULL; rc = dat_ia_open(ro_ia_name, btl->udapl_async_evd_qlen, &btl->udapl_evd_async, &btl->udapl_ia); dat_strerror(rc, (const char**)&major, (const char**)&minor); if (DAT_SUCCESS == rc) { mca_btl_udapl_component.ro_aware_system = 1; free(ro_ia_name); } else { BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt", "dat_ia_open fail RO", true, ro_ia_name, major, minor, ia_name)); free(ro_ia_name); return OMPI_ERROR; } } else { #endif BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt", "dat_ia_open fail", true, ia_name, major, minor)); return OMPI_ERROR; #if defined(__SVR4) && defined(__sun) } #endif } /* create a protection zone */ rc = dat_pz_create(btl->udapl_ia, &btl->udapl_pz); if(DAT_SUCCESS != rc) { char* major; char* minor; dat_strerror(rc, (const char**)&major, (const char**)&minor); BTL_ERROR(("ERROR: %s %s %s\n", "dat_pz_create", major, minor)); goto failure; } /* query to get address information */ rc = dat_ia_query(btl->udapl_ia, &btl->udapl_evd_async, DAT_IA_ALL, &(btl->udapl_ia_attr), 0, NULL); if(DAT_SUCCESS != rc) { char* major; char* minor; dat_strerror(rc, (const char**)&major, (const char**)&minor); BTL_ERROR(("ERROR: %s %s %s\n", "dat_ia_query", major, minor)); goto failure; } memcpy(&btl->udapl_addr.addr, (btl->udapl_ia_attr).ia_address_ptr, sizeof(DAT_SOCK_ADDR)); /* determine netmask */ mca_btl_udapl_assign_netmask(btl); /* check evd qlen against adapter max */ if (btl->udapl_dto_evd_qlen > (btl->udapl_ia_attr).max_evd_qlen) { BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt", "evd_qlen adapter max", true, "btl_udapl_dto_evd_qlen", btl->udapl_dto_evd_qlen, (btl->udapl_ia_attr).max_evd_qlen)); btl->udapl_dto_evd_qlen = btl->udapl_ia_attr.max_evd_qlen; } if (btl->udapl_conn_evd_qlen > (btl->udapl_ia_attr).max_evd_qlen) { BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt", "evd_qlen adapter max", true, "btl_udapl_conn_evd_qlen", btl->udapl_conn_evd_qlen, (btl->udapl_ia_attr).max_evd_qlen)); btl->udapl_conn_evd_qlen = btl->udapl_ia_attr.max_evd_qlen; } /* set up evd's */ rc = dat_evd_create(btl->udapl_ia, btl->udapl_dto_evd_qlen, DAT_HANDLE_NULL, DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG, &btl->udapl_evd_dto); if(DAT_SUCCESS != rc) { char* major; char* minor; dat_strerror(rc, (const char**)&major, (const char**)&minor); BTL_ERROR(("ERROR: %s %s %s\n", "dat_evd_create (dto)", major, minor)); goto failure; } rc = dat_evd_create(btl->udapl_ia, btl->udapl_conn_evd_qlen, DAT_HANDLE_NULL, DAT_EVD_CR_FLAG | DAT_EVD_CONNECTION_FLAG, &btl->udapl_evd_conn); if(DAT_SUCCESS != rc) { char* major; char* minor; dat_strerror(rc, (const char**)&major, (const char**)&minor); BTL_ERROR(("ERROR: %s %s %s\n", "dat_evd_create (conn)", major, minor)); goto failure; } /* create our public service point */ rc = dat_psp_create_any(btl->udapl_ia, &port, btl->udapl_evd_conn, DAT_PSP_CONSUMER_FLAG, &btl->udapl_psp); if(DAT_SUCCESS != rc) { char* major; char* minor; dat_strerror(rc, (const char**)&major, (const char**)&minor); BTL_ERROR(("ERROR: %s %s %s\n", "dat_psp_create_any", major, minor)); goto failure; } /* establish endpoint parameters */ rc = mca_btl_udapl_endpoint_get_params(btl, &(btl->udapl_ep_param)); if(OMPI_SUCCESS != rc) { /* by not erroring out here we can try to continue with * the default endpoint parameter values */ BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt", "use default endpoint params", true)); } /* Save the port with the address information */ /* TODO - since we're doing the hack below, do we need our own port? */ btl->udapl_addr.port = port; /* Using dat_ep_query to obtain the remote port would be ideal but * since the current udapl implementations don't seem to support * this we store the port in udapl_addr and explictly exchange the * information later. */ ((struct sockaddr_in*)&btl->udapl_addr.addr)->sin_port = htons(port); /* initialize the memory pool */ res.pool_name = "udapl"; res.reg_data = btl; res.sizeof_reg = sizeof(mca_btl_udapl_reg_t); res.register_mem = udapl_reg_mr; res.deregister_mem = udapl_dereg_mr; btl->super.btl_mpool = mca_mpool_base_module_create( mca_btl_udapl_component.udapl_mpool_name, &btl->super, &res); if (NULL == btl->super.btl_mpool) { BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_INFORM, ("WARNING: Failed to create mpool.")); goto failure; } /* initialize objects */ OBJ_CONSTRUCT(&btl->udapl_frag_eager, ompi_free_list_t); OBJ_CONSTRUCT(&btl->udapl_frag_eager_recv, ompi_free_list_t); OBJ_CONSTRUCT(&btl->udapl_frag_max, ompi_free_list_t); OBJ_CONSTRUCT(&btl->udapl_frag_max_recv, ompi_free_list_t); OBJ_CONSTRUCT(&btl->udapl_frag_user, ompi_free_list_t); OBJ_CONSTRUCT(&btl->udapl_frag_control, ompi_free_list_t); OBJ_CONSTRUCT(&btl->udapl_lock, opal_mutex_t); /* check buffer alignment against dat library */ if (mca_btl_udapl_component.udapl_buffer_alignment != DAT_OPTIMAL_ALIGNMENT) { BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt", "optimal buffer alignment mismatch", true, DAT_OPTIMAL_ALIGNMENT, mca_btl_udapl_component.udapl_buffer_alignment, DAT_OPTIMAL_ALIGNMENT)); } /* initialize free lists */ ompi_free_list_init_ex_new(&btl->udapl_frag_eager, sizeof(mca_btl_udapl_frag_eager_t) + mca_btl_udapl_component.udapl_eager_frag_size, mca_btl_udapl_component.udapl_buffer_alignment, OBJ_CLASS(mca_btl_udapl_frag_eager_t), mca_btl_udapl_component.udapl_eager_frag_size, mca_btl_udapl_component.udapl_buffer_alignment, mca_btl_udapl_component.udapl_free_list_num, mca_btl_udapl_component.udapl_free_list_max, mca_btl_udapl_component.udapl_free_list_inc, btl->super.btl_mpool, NULL, NULL); ompi_free_list_init_ex_new(&btl->udapl_frag_eager_recv, sizeof(mca_btl_udapl_frag_eager_t) + mca_btl_udapl_component.udapl_eager_frag_size, mca_btl_udapl_component.udapl_buffer_alignment, OBJ_CLASS(mca_btl_udapl_frag_eager_t), mca_btl_udapl_component.udapl_eager_frag_size, mca_btl_udapl_component.udapl_buffer_alignment, mca_btl_udapl_component.udapl_free_list_num, mca_btl_udapl_component.udapl_free_list_max, mca_btl_udapl_component.udapl_free_list_inc, btl->super.btl_mpool, NULL, NULL); ompi_free_list_init_ex_new(&btl->udapl_frag_max, sizeof(mca_btl_udapl_frag_max_t) + mca_btl_udapl_component.udapl_max_frag_size, mca_btl_udapl_component.udapl_buffer_alignment, OBJ_CLASS(mca_btl_udapl_frag_max_t), mca_btl_udapl_component.udapl_max_frag_size, mca_btl_udapl_component.udapl_buffer_alignment, mca_btl_udapl_component.udapl_free_list_num, mca_btl_udapl_component.udapl_free_list_max, mca_btl_udapl_component.udapl_free_list_inc, btl->super.btl_mpool, NULL, NULL); ompi_free_list_init_ex_new(&btl->udapl_frag_max_recv, sizeof(mca_btl_udapl_frag_max_t) + mca_btl_udapl_component.udapl_max_frag_size, mca_btl_udapl_component.udapl_buffer_alignment, OBJ_CLASS(mca_btl_udapl_frag_max_t), mca_btl_udapl_component.udapl_max_frag_size, mca_btl_udapl_component.udapl_buffer_alignment, mca_btl_udapl_component.udapl_free_list_num, mca_btl_udapl_component.udapl_free_list_max, mca_btl_udapl_component.udapl_free_list_inc, btl->super.btl_mpool, NULL, NULL); ompi_free_list_init_ex_new(&btl->udapl_frag_user, sizeof(mca_btl_udapl_frag_user_t), mca_btl_udapl_component.udapl_buffer_alignment, OBJ_CLASS(mca_btl_udapl_frag_user_t), 0,0, mca_btl_udapl_component.udapl_free_list_num, mca_btl_udapl_component.udapl_free_list_max, mca_btl_udapl_component.udapl_free_list_inc, NULL, NULL, NULL); ompi_free_list_init_ex_new(&btl->udapl_frag_control, sizeof(mca_btl_udapl_frag_eager_t) + mca_btl_udapl_component.udapl_eager_frag_size, mca_btl_udapl_component.udapl_buffer_alignment, OBJ_CLASS(mca_btl_udapl_frag_eager_t), mca_btl_udapl_component.udapl_eager_frag_size, mca_btl_udapl_component.udapl_buffer_alignment, mca_btl_udapl_component.udapl_free_list_num, -1, mca_btl_udapl_component.udapl_free_list_inc, btl->super.btl_mpool, NULL, NULL); /* initialize eager rdma buffer info */ btl->udapl_eager_rdma_endpoints = OBJ_NEW(opal_pointer_array_t); opal_pointer_array_init(btl->udapl_eager_rdma_endpoints, mca_btl_udapl_component.udapl_max_eager_rdma_peers, mca_btl_udapl_component.udapl_max_eager_rdma_peers, 0); btl->udapl_eager_rdma_endpoint_count = 0; OBJ_CONSTRUCT(&btl->udapl_eager_rdma_lock, opal_mutex_t); /* initialize miscellaneous variables */ btl->udapl_async_events = 0; btl->udapl_connect_inprogress = 0; btl->udapl_num_peers = 0; /* TODO - Set up SRQ when it is supported */ return OMPI_SUCCESS; failure: dat_ia_close(btl->udapl_ia, DAT_CLOSE_ABRUPT_FLAG); return OMPI_ERROR; }