int
ompi_common_mx_initialize(void)
{
mx_return_t mx_return;
struct mca_mpool_base_resources_t mpool_resources;
int index, value;
ompi_common_mx_initialize_ref_cnt++;
if(ompi_common_mx_initialize_ref_cnt == 1) {
/* set the MX error handle to always return. This function is the
* only MX function allowed to be called before mx_init in order
* to make sure that if the MX is not up and running the MX
* library does not exit the application.
*/
mx_set_error_handler(MX_ERRORS_RETURN);
/* If we have a memory manager available, and
mpi_leave_pinned == -1, then set mpi_leave_pinned to 1.
We have a memory manager if:
- we have both FREE and MUNMAP support
- we have MUNMAP support and the linux mallopt */
value = opal_mem_hooks_support_level();
if ((value & (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT))
== (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT)) {
index = mca_base_param_find("mpi", NULL, "leave_pinned");
if (index >= 0)
if ((mca_base_param_lookup_int(index, &value) == OPAL_SUCCESS)
&& (value == -1)) {
ompi_mpi_leave_pinned = 1;
setenv("MX_RCACHE", "2", 1);
mpool_resources.regcache_clean = mx__regcache_clean;
ompi_common_mx_fake_mpool =
mca_mpool_base_module_create("fake", NULL, &mpool_resources);
if (!ompi_common_mx_fake_mpool) {
ompi_mpi_leave_pinned = 0;
setenv("MX_RCACHE", "0", 1);
opal_output(0, "Error creating fake mpool (error %s)\n",
strerror(errno));
}
}
}
/* initialize the mx library */
mx_return = mx_init();
if(MX_SUCCESS != mx_return) {
opal_output(0,
"Error in mx_init (error %s)\n",
mx_strerror(mx_return));
return OMPI_ERR_NOT_AVAILABLE;
}
}
return OMPI_SUCCESS;
}
static int
mca_btl_scif_setup_mpools (mca_btl_scif_module_t *scif_module)
{
struct mca_mpool_base_resources_t mpool_resources;
int rc;
/* initialize the grdma mpool */
mpool_resources.pool_name = "scif";
mpool_resources.reg_data = (void *) scif_module;
mpool_resources.sizeof_reg = sizeof (mca_btl_scif_reg_t);
mpool_resources.register_mem = scif_reg_mem;
mpool_resources.deregister_mem = scif_dereg_mem;
scif_module->super.btl_mpool =
mca_mpool_base_module_create("grdma", scif_module, &mpool_resources);
if (NULL == scif_module->super.btl_mpool) {
BTL_ERROR(("error creating grdma mpool"));
return OMPI_ERROR;
}
/* setup free lists for fragments. dma fragments will be used for
* rma operations and in-place sends. eager frags will be used for
* buffered sends. */
rc = ompi_free_list_init_new (&scif_module->dma_frags,
sizeof (mca_btl_scif_dma_frag_t), 64,
OBJ_CLASS(mca_btl_scif_dma_frag_t),
128, getpagesize (),
mca_btl_scif_component.scif_free_list_num,
mca_btl_scif_component.scif_free_list_max,
mca_btl_scif_component.scif_free_list_inc,
NULL);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
return rc;
}
rc = ompi_free_list_init_new (&scif_module->eager_frags,
sizeof (mca_btl_scif_eager_frag_t), 8,
OBJ_CLASS(mca_btl_scif_eager_frag_t),
128 + scif_module->super.btl_eager_limit, 64,
mca_btl_scif_component.scif_free_list_num,
mca_btl_scif_component.scif_free_list_max,
mca_btl_scif_component.scif_free_list_inc,
NULL);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
BTL_ERROR(("error creating eager receive fragment free list"));
return rc;
}
return OMPI_SUCCESS;
}
static struct mca_btl_base_endpoint_t *
create_sm_endpoint(int local_proc, struct opal_proc_t *proc)
{
struct mca_btl_base_endpoint_t *ep;
#if OPAL_ENABLE_PROGRESS_THREADS == 1
char path[PATH_MAX];
#endif
ep = (struct mca_btl_base_endpoint_t*)
malloc(sizeof(struct mca_btl_base_endpoint_t));
if(NULL == ep)
return NULL;
ep->peer_smp_rank = local_proc + mca_btl_smcuda_component.num_smp_procs;
OBJ_CONSTRUCT(&ep->pending_sends, opal_list_t);
OBJ_CONSTRUCT(&ep->endpoint_lock, opal_mutex_t);
#if OPAL_ENABLE_PROGRESS_THREADS == 1
sprintf(path, "%s"OPAL_PATH_SEP"sm_fifo.%lu",
opal_process_info.job_session_dir,
(unsigned long)proc->proc_name);
ep->fifo_fd = open(path, O_WRONLY);
if(ep->fifo_fd < 0) {
opal_output(0, "mca_btl_smcuda_add_procs: open(%s) failed with errno=%d\n",
path, errno);
free(ep);
return NULL;
}
#endif
#if OPAL_CUDA_SUPPORT
{
mca_mpool_base_resources_t resources; /* unused, but needed */
/* Create a remote memory pool on the endpoint. Note that the resources
* argument is just to satisfy the function signature. The rcuda mpool
* actually takes care of filling in the resources. */
ep->mpool = mca_mpool_base_module_create("rgpusm",
NULL,
&resources);
}
#endif /* OPAL_CUDA_SUPPORT */
return ep;
}
static int
sm_btl_first_time_init(mca_btl_sm_t *sm_btl,
int32_t my_smp_rank,
int n)
{
size_t length, length_payload;
sm_fifo_t *my_fifos;
int my_mem_node, num_mem_nodes, i, rc;
mca_mpool_base_resources_t *res = NULL;
mca_btl_sm_component_t* m = &mca_btl_sm_component;
/* Assume we don't have hwloc support and fill in dummy info */
mca_btl_sm_component.mem_node = my_mem_node = 0;
mca_btl_sm_component.num_mem_nodes = num_mem_nodes = 1;
#if OPAL_HAVE_HWLOC
/* If we have hwloc support, then get accurate information */
if (NULL != opal_hwloc_topology) {
i = opal_hwloc_base_get_nbobjs_by_type(opal_hwloc_topology,
HWLOC_OBJ_NODE, 0,
OPAL_HWLOC_AVAILABLE);
/* If we find >0 NUMA nodes, then investigate further */
if (i > 0) {
int numa=0, w;
unsigned n_bound=0;
hwloc_cpuset_t avail;
hwloc_obj_t obj;
/* JMS This tells me how many numa nodes are *available*,
but it's not how many are being used *by this job*.
Note that this is the value we've previously used (from
the previous carto-based implementation), but it really
should be improved to be how many NUMA nodes are being
used *in this job*. */
mca_btl_sm_component.num_mem_nodes = num_mem_nodes = i;
/* if we are not bound, then there is nothing further to do */
if (NULL != ompi_process_info.cpuset) {
/* count the number of NUMA nodes to which we are bound */
for (w=0; w < i; w++) {
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology,
HWLOC_OBJ_NODE, 0, w,
OPAL_HWLOC_AVAILABLE))) {
continue;
}
/* get that NUMA node's available cpus */
avail = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
/* see if we intersect */
if (hwloc_bitmap_intersects(avail, opal_hwloc_my_cpuset)) {
n_bound++;
numa = w;
}
}
/* if we are located on more than one NUMA, or we didn't find
* a NUMA we are on, then not much we can do
*/
if (1 == n_bound) {
mca_btl_sm_component.mem_node = my_mem_node = numa;
} else {
mca_btl_sm_component.mem_node = my_mem_node = -1;
}
}
}
}
#endif
if (NULL == (res = calloc(1, sizeof(*res)))) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* lookup shared memory pool */
mca_btl_sm_component.sm_mpools =
(mca_mpool_base_module_t **)calloc(num_mem_nodes,
sizeof(mca_mpool_base_module_t *));
/* Disable memory binding, because each MPI process will claim pages in the
* mpool for their local NUMA node */
res->mem_node = -1;
if (OMPI_SUCCESS != (rc = setup_mpool_base_resources(m, res))) {
free(res);
return rc;
}
/* now that res is fully populated, create the thing */
mca_btl_sm_component.sm_mpools[0] =
mca_mpool_base_module_create(mca_btl_sm_component.sm_mpool_name,
sm_btl, res);
/* Sanity check to ensure that we found it */
if (NULL == mca_btl_sm_component.sm_mpools[0]) {
free(res);
return OMPI_ERR_OUT_OF_RESOURCE;
}
mca_btl_sm_component.sm_mpool = mca_btl_sm_component.sm_mpools[0];
mca_btl_sm_component.sm_mpool_base =
mca_btl_sm_component.sm_mpools[0]->mpool_base(mca_btl_sm_component.sm_mpools[0]);
/* create a list of peers */
mca_btl_sm_component.sm_peers = (struct mca_btl_base_endpoint_t**)
calloc(n, sizeof(struct mca_btl_base_endpoint_t*));
if (NULL == mca_btl_sm_component.sm_peers) {
free(res);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* remember that node rank zero is already attached */
if (0 != my_smp_rank) {
if (OMPI_SUCCESS != (rc = sm_segment_attach(m))) {
free(res);
return rc;
}
}
/* it is now safe to free the mpool resources */
free(res);
/* check to make sure number of local procs is within the
* specified limits */
if(mca_btl_sm_component.sm_max_procs > 0 &&
mca_btl_sm_component.num_smp_procs + n >
mca_btl_sm_component.sm_max_procs) {
return OMPI_ERROR;
}
mca_btl_sm_component.shm_fifo = (volatile sm_fifo_t **)mca_btl_sm_component.sm_seg->module_data_addr;
mca_btl_sm_component.shm_bases = (char**)(mca_btl_sm_component.shm_fifo + n);
mca_btl_sm_component.shm_mem_nodes = (uint16_t*)(mca_btl_sm_component.shm_bases + n);
/* set the base of the shared memory segment */
mca_btl_sm_component.shm_bases[mca_btl_sm_component.my_smp_rank] =
(char*)mca_btl_sm_component.sm_mpool_base;
mca_btl_sm_component.shm_mem_nodes[mca_btl_sm_component.my_smp_rank] =
(uint16_t)my_mem_node;
/* initialize the array of fifo's "owned" by this process */
if(NULL == (my_fifos = (sm_fifo_t*)mpool_calloc(FIFO_MAP_NUM(n), sizeof(sm_fifo_t))))
return OMPI_ERR_OUT_OF_RESOURCE;
mca_btl_sm_component.shm_fifo[mca_btl_sm_component.my_smp_rank] = my_fifos;
/* cache the pointer to the 2d fifo array. These addresses
* are valid in the current process space */
mca_btl_sm_component.fifo = (sm_fifo_t**)malloc(sizeof(sm_fifo_t*) * n);
if(NULL == mca_btl_sm_component.fifo)
return OMPI_ERR_OUT_OF_RESOURCE;
mca_btl_sm_component.fifo[mca_btl_sm_component.my_smp_rank] = my_fifos;
mca_btl_sm_component.mem_nodes = (uint16_t *) malloc(sizeof(uint16_t) * n);
if(NULL == mca_btl_sm_component.mem_nodes)
return OMPI_ERR_OUT_OF_RESOURCE;
/* initialize fragment descriptor free lists */
/* allocation will be for the fragment descriptor and payload buffer */
length = sizeof(mca_btl_sm_frag1_t);
length_payload =
sizeof(mca_btl_sm_hdr_t) + mca_btl_sm_component.eager_limit;
i = ompi_free_list_init_new(&mca_btl_sm_component.sm_frags_eager, length,
opal_cache_line_size, OBJ_CLASS(mca_btl_sm_frag1_t),
length_payload, opal_cache_line_size,
mca_btl_sm_component.sm_free_list_num,
mca_btl_sm_component.sm_free_list_max,
mca_btl_sm_component.sm_free_list_inc,
mca_btl_sm_component.sm_mpool);
if ( OMPI_SUCCESS != i )
return i;
length = sizeof(mca_btl_sm_frag2_t);
length_payload =
sizeof(mca_btl_sm_hdr_t) + mca_btl_sm_component.max_frag_size;
i = ompi_free_list_init_new(&mca_btl_sm_component.sm_frags_max, length,
opal_cache_line_size, OBJ_CLASS(mca_btl_sm_frag2_t),
length_payload, opal_cache_line_size,
mca_btl_sm_component.sm_free_list_num,
mca_btl_sm_component.sm_free_list_max,
mca_btl_sm_component.sm_free_list_inc,
mca_btl_sm_component.sm_mpool);
if ( OMPI_SUCCESS != i )
return i;
i = ompi_free_list_init_new(&mca_btl_sm_component.sm_frags_user,
sizeof(mca_btl_sm_user_t),
opal_cache_line_size, OBJ_CLASS(mca_btl_sm_user_t),
sizeof(mca_btl_sm_hdr_t), opal_cache_line_size,
mca_btl_sm_component.sm_free_list_num,
mca_btl_sm_component.sm_free_list_max,
mca_btl_sm_component.sm_free_list_inc,
mca_btl_sm_component.sm_mpool);
if ( OMPI_SUCCESS != i )
return i;
mca_btl_sm_component.num_outstanding_frags = 0;
mca_btl_sm_component.num_pending_sends = 0;
i = opal_free_list_init(&mca_btl_sm_component.pending_send_fl,
sizeof(btl_sm_pending_send_item_t),
OBJ_CLASS(opal_free_list_item_t),
16, -1, 32);
if ( OMPI_SUCCESS != i )
return i;
/* set flag indicating btl has been inited */
sm_btl->btl_inited = true;
return OMPI_SUCCESS;
}
int
mca_btl_udapl_init(DAT_NAME_PTR ia_name, mca_btl_udapl_module_t* btl)
{
mca_mpool_base_resources_t res;
DAT_CONN_QUAL port;
DAT_RETURN rc;
/* open the uDAPL interface */
btl->udapl_evd_async = DAT_HANDLE_NULL;
rc = dat_ia_open(ia_name, btl->udapl_async_evd_qlen,
&btl->udapl_evd_async, &btl->udapl_ia);
if(DAT_SUCCESS != rc) {
char* major;
char* minor;
dat_strerror(rc, (const char**)&major,
(const char**)&minor);
#if defined(__SVR4) && defined(__sun)
if (strcmp(major, "DAT_INVALID_PARAMETER") == 0 &&
strcmp(minor, "DAT_INVALID_RO_COOKIE") == 0) {
/* Some platforms that Solaris runs on implement the PCI
* standard for relaxed ordering(RO). Using RDMA with
* polling on a memory location as the uDAPL (and openib
* by the way) BTL does for short messages with
* relaxed ordering could potentially produce silent data
* corruption. For this reason we need to take extra
* steps and this is accomplished by setting
* "ro_aware_system = 1" and handling as required.
*
* The uDAPL standard does not provide an interface to
* inform users of this scenario so Sun has implemented the
* following: If a platform supports relaxed ordering
* when the interface name is passed into the
* dat_ia_open() call, the call will return
* DAT_INVALID_PARAMETER and DAT_INVALID_RO_COOKIE.
* DAT_INVALID_RO_COOKIE is not part of the uDAPL standard
* at this time. The only way to open this interface is
* to prefix the following cookie "RO_AWARE_" to the ia
* name that was retreived from the dat registry.
*
* Example: ia_name = "ib0", new expected name will be
* "RO_AWARE_ib0".
*
* Here, since our first ia open attempt failed in the
* standard way, add the cookie and try to open again.
*/
DAT_NAME_PTR ro_ia_name;
/* prefix relaxed order cookie to ia_name */
asprintf(&ro_ia_name, "RO_AWARE_%s", ia_name);
if (NULL == ro_ia_name) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* because this is not standard inform user in some way */
BTL_UDAPL_VERBOSE_HELP(VERBOSE_INFORM,
("help-mpi-btl-udapl.txt", "relaxed order support",
true, ia_name, ro_ia_name));
/* try and open again */
btl->udapl_evd_async = DAT_HANDLE_NULL;
rc = dat_ia_open(ro_ia_name, btl->udapl_async_evd_qlen,
&btl->udapl_evd_async, &btl->udapl_ia);
dat_strerror(rc, (const char**)&major,
(const char**)&minor);
if (DAT_SUCCESS == rc) {
mca_btl_udapl_component.ro_aware_system = 1;
free(ro_ia_name);
} else {
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
("help-mpi-btl-udapl.txt",
"dat_ia_open fail RO", true, ro_ia_name,
major, minor, ia_name));
free(ro_ia_name);
return OMPI_ERROR;
}
} else {
#endif
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt",
"dat_ia_open fail", true, ia_name, major, minor));
return OMPI_ERROR;
#if defined(__SVR4) && defined(__sun)
}
#endif
}
/* create a protection zone */
rc = dat_pz_create(btl->udapl_ia, &btl->udapl_pz);
if(DAT_SUCCESS != rc) {
char* major;
char* minor;
dat_strerror(rc, (const char**)&major,
(const char**)&minor);
BTL_ERROR(("ERROR: %s %s %s\n", "dat_pz_create",
major, minor));
goto failure;
}
/* query to get address information */
rc = dat_ia_query(btl->udapl_ia, &btl->udapl_evd_async,
DAT_IA_ALL, &(btl->udapl_ia_attr), 0, NULL);
if(DAT_SUCCESS != rc) {
char* major;
char* minor;
dat_strerror(rc, (const char**)&major,
(const char**)&minor);
BTL_ERROR(("ERROR: %s %s %s\n", "dat_ia_query",
major, minor));
goto failure;
}
memcpy(&btl->udapl_addr.addr, (btl->udapl_ia_attr).ia_address_ptr,
sizeof(DAT_SOCK_ADDR));
/* determine netmask */
mca_btl_udapl_assign_netmask(btl);
/* check evd qlen against adapter max */
if (btl->udapl_dto_evd_qlen > (btl->udapl_ia_attr).max_evd_qlen) {
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt",
"evd_qlen adapter max",
true,
"btl_udapl_dto_evd_qlen",
btl->udapl_dto_evd_qlen,
(btl->udapl_ia_attr).max_evd_qlen));
btl->udapl_dto_evd_qlen = btl->udapl_ia_attr.max_evd_qlen;
}
if (btl->udapl_conn_evd_qlen > (btl->udapl_ia_attr).max_evd_qlen) {
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt",
"evd_qlen adapter max",
true,
"btl_udapl_conn_evd_qlen",
btl->udapl_conn_evd_qlen,
(btl->udapl_ia_attr).max_evd_qlen));
btl->udapl_conn_evd_qlen = btl->udapl_ia_attr.max_evd_qlen;
}
/* set up evd's */
rc = dat_evd_create(btl->udapl_ia,
btl->udapl_dto_evd_qlen, DAT_HANDLE_NULL,
DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG, &btl->udapl_evd_dto);
if(DAT_SUCCESS != rc) {
char* major;
char* minor;
dat_strerror(rc, (const char**)&major,
(const char**)&minor);
BTL_ERROR(("ERROR: %s %s %s\n", "dat_evd_create (dto)",
major, minor));
goto failure;
}
rc = dat_evd_create(btl->udapl_ia,
btl->udapl_conn_evd_qlen, DAT_HANDLE_NULL,
DAT_EVD_CR_FLAG | DAT_EVD_CONNECTION_FLAG, &btl->udapl_evd_conn);
if(DAT_SUCCESS != rc) {
char* major;
char* minor;
dat_strerror(rc, (const char**)&major,
(const char**)&minor);
BTL_ERROR(("ERROR: %s %s %s\n", "dat_evd_create (conn)",
major, minor));
goto failure;
}
/* create our public service point */
rc = dat_psp_create_any(btl->udapl_ia, &port, btl->udapl_evd_conn,
DAT_PSP_CONSUMER_FLAG, &btl->udapl_psp);
if(DAT_SUCCESS != rc) {
char* major;
char* minor;
dat_strerror(rc, (const char**)&major,
(const char**)&minor);
BTL_ERROR(("ERROR: %s %s %s\n", "dat_psp_create_any",
major, minor));
goto failure;
}
/* establish endpoint parameters */
rc = mca_btl_udapl_endpoint_get_params(btl, &(btl->udapl_ep_param));
if(OMPI_SUCCESS != rc) {
/* by not erroring out here we can try to continue with
* the default endpoint parameter values
*/
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt",
"use default endpoint params",
true));
}
/* Save the port with the address information */
/* TODO - since we're doing the hack below, do we need our own port? */
btl->udapl_addr.port = port;
/* Using dat_ep_query to obtain the remote port would be ideal but
* since the current udapl implementations don't seem to support
* this we store the port in udapl_addr and explictly exchange the
* information later.
*/
((struct sockaddr_in*)&btl->udapl_addr.addr)->sin_port = htons(port);
/* initialize the memory pool */
res.pool_name = "udapl";
res.reg_data = btl;
res.sizeof_reg = sizeof(mca_btl_udapl_reg_t);
res.register_mem = udapl_reg_mr;
res.deregister_mem = udapl_dereg_mr;
btl->super.btl_mpool = mca_mpool_base_module_create(
mca_btl_udapl_component.udapl_mpool_name, &btl->super, &res);
if (NULL == btl->super.btl_mpool) {
BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_INFORM,
("WARNING: Failed to create mpool."));
goto failure;
}
/* initialize objects */
OBJ_CONSTRUCT(&btl->udapl_frag_eager, ompi_free_list_t);
OBJ_CONSTRUCT(&btl->udapl_frag_eager_recv, ompi_free_list_t);
OBJ_CONSTRUCT(&btl->udapl_frag_max, ompi_free_list_t);
OBJ_CONSTRUCT(&btl->udapl_frag_max_recv, ompi_free_list_t);
OBJ_CONSTRUCT(&btl->udapl_frag_user, ompi_free_list_t);
OBJ_CONSTRUCT(&btl->udapl_frag_control, ompi_free_list_t);
OBJ_CONSTRUCT(&btl->udapl_lock, opal_mutex_t);
/* check buffer alignment against dat library */
if (mca_btl_udapl_component.udapl_buffer_alignment !=
DAT_OPTIMAL_ALIGNMENT) {
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt",
"optimal buffer alignment mismatch",
true,
DAT_OPTIMAL_ALIGNMENT,
mca_btl_udapl_component.udapl_buffer_alignment,
DAT_OPTIMAL_ALIGNMENT));
}
/* initialize free lists */
ompi_free_list_init_ex_new(&btl->udapl_frag_eager,
sizeof(mca_btl_udapl_frag_eager_t) +
mca_btl_udapl_component.udapl_eager_frag_size,
mca_btl_udapl_component.udapl_buffer_alignment,
OBJ_CLASS(mca_btl_udapl_frag_eager_t),
mca_btl_udapl_component.udapl_eager_frag_size,
mca_btl_udapl_component.udapl_buffer_alignment,
mca_btl_udapl_component.udapl_free_list_num,
mca_btl_udapl_component.udapl_free_list_max,
mca_btl_udapl_component.udapl_free_list_inc,
btl->super.btl_mpool,
NULL,
NULL);
ompi_free_list_init_ex_new(&btl->udapl_frag_eager_recv,
sizeof(mca_btl_udapl_frag_eager_t) +
mca_btl_udapl_component.udapl_eager_frag_size,
mca_btl_udapl_component.udapl_buffer_alignment,
OBJ_CLASS(mca_btl_udapl_frag_eager_t),
mca_btl_udapl_component.udapl_eager_frag_size,
mca_btl_udapl_component.udapl_buffer_alignment,
mca_btl_udapl_component.udapl_free_list_num,
mca_btl_udapl_component.udapl_free_list_max,
mca_btl_udapl_component.udapl_free_list_inc,
btl->super.btl_mpool,
NULL,
NULL);
ompi_free_list_init_ex_new(&btl->udapl_frag_max,
sizeof(mca_btl_udapl_frag_max_t) +
mca_btl_udapl_component.udapl_max_frag_size,
mca_btl_udapl_component.udapl_buffer_alignment,
OBJ_CLASS(mca_btl_udapl_frag_max_t),
mca_btl_udapl_component.udapl_max_frag_size,
mca_btl_udapl_component.udapl_buffer_alignment,
mca_btl_udapl_component.udapl_free_list_num,
mca_btl_udapl_component.udapl_free_list_max,
mca_btl_udapl_component.udapl_free_list_inc,
btl->super.btl_mpool,
NULL,
NULL);
ompi_free_list_init_ex_new(&btl->udapl_frag_max_recv,
sizeof(mca_btl_udapl_frag_max_t) +
mca_btl_udapl_component.udapl_max_frag_size,
mca_btl_udapl_component.udapl_buffer_alignment,
OBJ_CLASS(mca_btl_udapl_frag_max_t),
mca_btl_udapl_component.udapl_max_frag_size,
mca_btl_udapl_component.udapl_buffer_alignment,
mca_btl_udapl_component.udapl_free_list_num,
mca_btl_udapl_component.udapl_free_list_max,
mca_btl_udapl_component.udapl_free_list_inc,
btl->super.btl_mpool,
NULL,
NULL);
ompi_free_list_init_ex_new(&btl->udapl_frag_user,
sizeof(mca_btl_udapl_frag_user_t),
mca_btl_udapl_component.udapl_buffer_alignment,
OBJ_CLASS(mca_btl_udapl_frag_user_t),
0,0,
mca_btl_udapl_component.udapl_free_list_num,
mca_btl_udapl_component.udapl_free_list_max,
mca_btl_udapl_component.udapl_free_list_inc,
NULL,
NULL,
NULL);
ompi_free_list_init_ex_new(&btl->udapl_frag_control,
sizeof(mca_btl_udapl_frag_eager_t) +
mca_btl_udapl_component.udapl_eager_frag_size,
mca_btl_udapl_component.udapl_buffer_alignment,
OBJ_CLASS(mca_btl_udapl_frag_eager_t),
mca_btl_udapl_component.udapl_eager_frag_size,
mca_btl_udapl_component.udapl_buffer_alignment,
mca_btl_udapl_component.udapl_free_list_num,
-1,
mca_btl_udapl_component.udapl_free_list_inc,
btl->super.btl_mpool,
NULL,
NULL);
/* initialize eager rdma buffer info */
btl->udapl_eager_rdma_endpoints = OBJ_NEW(opal_pointer_array_t);
opal_pointer_array_init(btl->udapl_eager_rdma_endpoints,
mca_btl_udapl_component.udapl_max_eager_rdma_peers,
mca_btl_udapl_component.udapl_max_eager_rdma_peers,
0);
btl->udapl_eager_rdma_endpoint_count = 0;
OBJ_CONSTRUCT(&btl->udapl_eager_rdma_lock, opal_mutex_t);
/* initialize miscellaneous variables */
btl->udapl_async_events = 0;
btl->udapl_connect_inprogress = 0;
btl->udapl_num_peers = 0;
/* TODO - Set up SRQ when it is supported */
return OMPI_SUCCESS;
failure:
dat_ia_close(btl->udapl_ia, DAT_CLOSE_ABRUPT_FLAG);
return OMPI_ERROR;
}
