static int udapl_reg_mr(void *reg_data, void *base, size_t size,
mca_mpool_base_registration_t *reg)
{
mca_btl_udapl_module_t *btl = (mca_btl_udapl_module_t*)reg_data;
mca_btl_udapl_reg_t *udapl_reg = (mca_btl_udapl_reg_t*)reg;
DAT_REGION_DESCRIPTION region;
DAT_VLEN dat_size;
DAT_VADDR dat_addr;
int rc;
DAT_MEM_TYPE lmr_mem_type = DAT_MEM_TYPE_VIRTUAL;
region.for_va = base;
udapl_reg->lmr_triplet.virtual_address = (DAT_VADDR)(uintptr_t)base;
udapl_reg->lmr_triplet.segment_length = size;
udapl_reg->lmr = NULL;
#if HAVE_DAT_MEM_TYPE_SO_VIRTUAL
if (reg->flags & MCA_MPOOL_FLAGS_SO_MEM) {
lmr_mem_type = DAT_MEM_TYPE_SO_VIRTUAL;
}
#endif
rc = dat_lmr_create(btl->udapl_ia, lmr_mem_type, region, size,
btl->udapl_pz, DAT_MEM_PRIV_ALL_FLAG, &udapl_reg->lmr,
&udapl_reg->lmr_triplet.lmr_context, &udapl_reg->rmr_context,
&dat_size, &dat_addr);
if(rc != DAT_SUCCESS) {
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt",
"dat_lmr_create DAT_INSUFFICIENT_RESOURCES", true));
return OMPI_ERR_OUT_OF_RESOURCE;
}
return OMPI_SUCCESS;
}
/*
* Find and assign system netmask for the address of the uDAPL BTL
* module, but only if udapl_if_mask has not been set by the "--mca
* btl_udapl_if_mask" parameter. This routine will either find
* the system netmask or set the value to 0.
*
* @param udapl_btl (IN) BTL module
*
* @return OMPI_SUCCESS or OMPI_ERROR
*/
static int mca_btl_udapl_assign_netmask(mca_btl_udapl_module_t* udapl_btl)
{
struct sockaddr *saddr;
struct sockaddr_in *btl_addr;
char btl_addr_string[INET_ADDRSTRLEN];
char btl_ifname[INET_ADDRSTRLEN];
/* Setting if_mask to 0 informs future steps to assume all
* addresses are reachable.
*/
udapl_btl->udapl_if_mask = 0;
if (mca_btl_udapl_component.udapl_compare_subnet) {
/* go get system netmask value */
/* use generic address to find address family */
saddr = (struct sockaddr *)&(udapl_btl->udapl_addr.addr);
if (saddr->sa_family == AF_INET) {
btl_addr = (struct sockaddr_in *)saddr;
/*
* Retrieve the netmask of the udapl btl address. To
* accomplish this requires 4 steps and the use of an opal
* utility. This same utility is used by the tcp oob.
* Steps:
* 1. Get string value of known udapl btl module address.
* 2. Use string value to find the interface name of address.
* 3. Use interface name to find its index.
* 4. From the index get the netmask.
*/
/* retrieve string value of udapl btl address */
inet_ntop(AF_INET, (void *) &btl_addr->sin_addr,
btl_addr_string, INET_ADDRSTRLEN);
/* use address string to retrieve associated interface name */
if (OPAL_SUCCESS !=
opal_ifaddrtoname(btl_addr_string,
btl_ifname, INET_ADDRSTRLEN)) {
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
("help-mpi-btl-udapl.txt", "interface not found",
true, ompi_process_info.nodename, btl_addr_string));
return OMPI_ERROR;
}
/* use interface name to retrieve index; then
* use index to retrieve udapl btl address netmask
*/
if (OPAL_SUCCESS !=
opal_ifindextomask(opal_ifnametoindex(btl_ifname),
&(udapl_btl->udapl_if_mask), sizeof(udapl_btl->udapl_if_mask))) {
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
("help-mpi-btl-udapl.txt", "netmask not found",
true, ompi_process_info.nodename, btl_addr_string));
return OMPI_ERROR;
}
/* report if_mask used by address */
BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_INFORM,
("uDAPL BTL address %s : if_mask = %d",
btl_addr_string, udapl_btl->udapl_if_mask));
} else {
/* current uDAPL BTL does not support IPv6 */
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
("help-mpi-btl-udapl.txt", "IPv4 only",
true, ompi_process_info.nodename));
return OMPI_ERROR;
}
}
return OMPI_SUCCESS;
}
static int mca_btl_udapl_set_peer_parameters(
struct mca_btl_udapl_module_t* udapl_btl,
size_t nprocs)
{
int rc = OMPI_SUCCESS;
DAT_RETURN dat_rc = DAT_SUCCESS;
uint potential_udapl_timeout;
int first_time_sizing = (udapl_btl->udapl_num_peers == 0 ? 1 : 0);
DAT_EVD_PARAM evd_param;
/* nprocs includes self so subtract 1 */
udapl_btl->udapl_num_peers += nprocs - 1;
/* resize dto_evd_qlen if not already at its max */
if (udapl_btl->udapl_dto_evd_qlen !=
udapl_btl->udapl_ia_attr.max_evd_qlen) {
int potential_dto_evd_qlen;
int max_connection_dto_events;
int eager_connection_dto_events;
/* eager connection dto events already factored into
* max_recv/request_dtos but need to calculate max connection dtos;
* see mca_btl_udapl_get_params() for max_recv/request_dtos
*/
eager_connection_dto_events = udapl_btl->udapl_max_recv_dtos +
udapl_btl->udapl_max_request_dtos;
max_connection_dto_events = mca_btl_udapl_component.udapl_num_recvs +
mca_btl_udapl_component.udapl_num_sends +
(mca_btl_udapl_component.udapl_num_recvs /
mca_btl_udapl_component.udapl_sr_win) + 1;
potential_dto_evd_qlen = udapl_btl->udapl_num_peers *
(eager_connection_dto_events + max_connection_dto_events);
/* here we use what the library calculates as the
* potential_dto_evd_qlen unless the user has set
*/
if (first_time_sizing) {
if (udapl_btl->udapl_dto_evd_qlen < potential_dto_evd_qlen) {
if (MCA_BTL_UDAPL_DTO_EVD_QLEN_DEFAULT !=
udapl_btl->udapl_dto_evd_qlen) {
/* user modified so warn */
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
("help-mpi-btl-udapl.txt",
"evd_qlen too low",
true,
"btl_udapl_dto_evd_qlen",
udapl_btl->udapl_dto_evd_qlen,
"btl_udapl_dto_evd_qlen",
potential_dto_evd_qlen));
} else {
udapl_btl->udapl_dto_evd_qlen = potential_dto_evd_qlen;
}
}
} else {
/* since this is not the first time attempting to resize the
* evd queue length just use the potential value; this may not
* be the best solution
*/
udapl_btl->udapl_dto_evd_qlen = potential_dto_evd_qlen;
}
udapl_btl->udapl_dto_evd_qlen = ((udapl_btl->udapl_dto_evd_qlen >
udapl_btl->udapl_ia_attr.max_evd_qlen) ?
udapl_btl->udapl_ia_attr.max_evd_qlen :
udapl_btl->udapl_dto_evd_qlen);
/* OFED stack does not return DAT_INVALID_STATE when
* the new qlen is less than current value so here we find
* current value and if greater than what we intend to set
* it to skip the resize.
*/
dat_rc = dat_evd_query(udapl_btl->udapl_evd_dto,
DAT_EVD_FIELD_EVD_QLEN, &evd_param);
if(DAT_SUCCESS != dat_rc) {
char* major;
char* minor;
dat_strerror(dat_rc, (const char**)&major,
(const char**)&minor);
BTL_ERROR(("ERROR: %s %s %s\n", "dat_evd_query",
major, minor));
}
if (udapl_btl->udapl_dto_evd_qlen > evd_param.evd_qlen) {
/* resize dto event dispatcher queue length */
dat_rc = dat_evd_resize(udapl_btl->udapl_evd_dto,
udapl_btl->udapl_dto_evd_qlen);
if(DAT_SUCCESS != dat_rc) {
char* major;
char* minor;
dat_strerror(dat_rc, (const char**)&major,
(const char**)&minor);
BTL_ERROR(("ERROR: %s %s %s\n", "dat_evd_resize",
major, minor));
rc = OMPI_ERR_OUT_OF_RESOURCE;
}
}
}
/* resize connection evd qlen */
if (udapl_btl->udapl_conn_evd_qlen !=
udapl_btl->udapl_ia_attr.max_evd_qlen) {
int potential_conn_evd_qlen = 2 * udapl_btl->udapl_num_peers;
if (first_time_sizing) {
if (udapl_btl->udapl_conn_evd_qlen < potential_conn_evd_qlen) {
if (MCA_BTL_UDAPL_CONN_EVD_QLEN_DEFAULT !=
udapl_btl->udapl_conn_evd_qlen) {
/* user modified so warn */
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
("help-mpi-btl-udapl.txt",
"evd_qlen too low",
true,
"btl_udapl_conn_evd_qlen",
udapl_btl->udapl_conn_evd_qlen,
"btl_udapl_conn_evd_qlen",
potential_conn_evd_qlen));
} else {
udapl_btl->udapl_conn_evd_qlen = potential_conn_evd_qlen;
}
}
} else {
/* since this is not the first time attempting to resize the
* evd queue length just use the potential value; this may not
* be the best solution
*/
udapl_btl->udapl_conn_evd_qlen = potential_conn_evd_qlen;
}
udapl_btl->udapl_conn_evd_qlen = ((udapl_btl->udapl_conn_evd_qlen >
udapl_btl->udapl_ia_attr.max_evd_qlen) ?
udapl_btl->udapl_ia_attr.max_evd_qlen :
udapl_btl->udapl_conn_evd_qlen);
/* OFED stack does not return DAT_INVALID_STATE when
* the new qlen is less than current value so here we find
* current value and if greater than what we intend to set
* it to skip the resize.
*/
dat_rc = dat_evd_query(udapl_btl->udapl_evd_conn,
DAT_EVD_FIELD_EVD_QLEN, &evd_param);
if(DAT_SUCCESS != dat_rc) {
char* major;
char* minor;
dat_strerror(dat_rc, (const char**)&major,
(const char**)&minor);
BTL_ERROR(("ERROR: %s %s %s\n", "dat_evd_query",
major, minor));
}
if (udapl_btl->udapl_conn_evd_qlen > evd_param.evd_qlen) {
/* resize conn evd queue length */
dat_rc = dat_evd_resize(udapl_btl->udapl_evd_conn,
udapl_btl->udapl_conn_evd_qlen);
if(DAT_SUCCESS != dat_rc) {
char* major;
char* minor;
dat_strerror(dat_rc, (const char**)&major,
(const char**)&minor);
BTL_ERROR(("ERROR: %s %s %s\n", "dat_evd_resize",
major, minor));
rc = OMPI_ERR_OUT_OF_RESOURCE;
}
}
}
/* adjust connection timeout value, calculated in microseconds */
potential_udapl_timeout = MCA_BTL_UDAPL_CONN_TIMEOUT_INC *
udapl_btl->udapl_num_peers;
if (mca_btl_udapl_component.udapl_timeout <
potential_udapl_timeout) {
if (MCA_BTL_UDAPL_CONN_TIMEOUT_DEFAULT !=
mca_btl_udapl_component.udapl_timeout) {
/* user modified so warn */
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
("help-mpi-btl-udapl.txt",
"connection timeout low",
true,
"btl_udapl_timeout",
mca_btl_udapl_component.udapl_timeout,
"btl_udapl_timeout",
potential_udapl_timeout));
} else {
mca_btl_udapl_component.udapl_timeout =
potential_udapl_timeout;
}
}
mca_btl_udapl_component.udapl_timeout =
((mca_btl_udapl_component.udapl_timeout >
MCA_BTL_UDAPL_CONN_TIMEOUT_MAX) ?
MCA_BTL_UDAPL_CONN_TIMEOUT_MAX :
mca_btl_udapl_component.udapl_timeout);
return rc;
}
int
mca_btl_udapl_init(DAT_NAME_PTR ia_name, mca_btl_udapl_module_t* btl)
{
mca_mpool_base_resources_t res;
DAT_CONN_QUAL port;
DAT_RETURN rc;
/* open the uDAPL interface */
btl->udapl_evd_async = DAT_HANDLE_NULL;
rc = dat_ia_open(ia_name, btl->udapl_async_evd_qlen,
&btl->udapl_evd_async, &btl->udapl_ia);
if(DAT_SUCCESS != rc) {
char* major;
char* minor;
dat_strerror(rc, (const char**)&major,
(const char**)&minor);
#if defined(__SVR4) && defined(__sun)
if (strcmp(major, "DAT_INVALID_PARAMETER") == 0 &&
strcmp(minor, "DAT_INVALID_RO_COOKIE") == 0) {
/* Some platforms that Solaris runs on implement the PCI
* standard for relaxed ordering(RO). Using RDMA with
* polling on a memory location as the uDAPL (and openib
* by the way) BTL does for short messages with
* relaxed ordering could potentially produce silent data
* corruption. For this reason we need to take extra
* steps and this is accomplished by setting
* "ro_aware_system = 1" and handling as required.
*
* The uDAPL standard does not provide an interface to
* inform users of this scenario so Sun has implemented the
* following: If a platform supports relaxed ordering
* when the interface name is passed into the
* dat_ia_open() call, the call will return
* DAT_INVALID_PARAMETER and DAT_INVALID_RO_COOKIE.
* DAT_INVALID_RO_COOKIE is not part of the uDAPL standard
* at this time. The only way to open this interface is
* to prefix the following cookie "RO_AWARE_" to the ia
* name that was retreived from the dat registry.
*
* Example: ia_name = "ib0", new expected name will be
* "RO_AWARE_ib0".
*
* Here, since our first ia open attempt failed in the
* standard way, add the cookie and try to open again.
*/
DAT_NAME_PTR ro_ia_name;
/* prefix relaxed order cookie to ia_name */
asprintf(&ro_ia_name, "RO_AWARE_%s", ia_name);
if (NULL == ro_ia_name) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* because this is not standard inform user in some way */
BTL_UDAPL_VERBOSE_HELP(VERBOSE_INFORM,
("help-mpi-btl-udapl.txt", "relaxed order support",
true, ia_name, ro_ia_name));
/* try and open again */
btl->udapl_evd_async = DAT_HANDLE_NULL;
rc = dat_ia_open(ro_ia_name, btl->udapl_async_evd_qlen,
&btl->udapl_evd_async, &btl->udapl_ia);
dat_strerror(rc, (const char**)&major,
(const char**)&minor);
if (DAT_SUCCESS == rc) {
mca_btl_udapl_component.ro_aware_system = 1;
free(ro_ia_name);
} else {
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
("help-mpi-btl-udapl.txt",
"dat_ia_open fail RO", true, ro_ia_name,
major, minor, ia_name));
free(ro_ia_name);
return OMPI_ERROR;
}
} else {
#endif
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt",
"dat_ia_open fail", true, ia_name, major, minor));
return OMPI_ERROR;
#if defined(__SVR4) && defined(__sun)
}
#endif
}
/* create a protection zone */
rc = dat_pz_create(btl->udapl_ia, &btl->udapl_pz);
if(DAT_SUCCESS != rc) {
char* major;
char* minor;
dat_strerror(rc, (const char**)&major,
(const char**)&minor);
BTL_ERROR(("ERROR: %s %s %s\n", "dat_pz_create",
major, minor));
goto failure;
}
/* query to get address information */
rc = dat_ia_query(btl->udapl_ia, &btl->udapl_evd_async,
DAT_IA_ALL, &(btl->udapl_ia_attr), 0, NULL);
if(DAT_SUCCESS != rc) {
char* major;
char* minor;
dat_strerror(rc, (const char**)&major,
(const char**)&minor);
BTL_ERROR(("ERROR: %s %s %s\n", "dat_ia_query",
major, minor));
goto failure;
}
memcpy(&btl->udapl_addr.addr, (btl->udapl_ia_attr).ia_address_ptr,
sizeof(DAT_SOCK_ADDR));
/* determine netmask */
mca_btl_udapl_assign_netmask(btl);
/* check evd qlen against adapter max */
if (btl->udapl_dto_evd_qlen > (btl->udapl_ia_attr).max_evd_qlen) {
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt",
"evd_qlen adapter max",
true,
"btl_udapl_dto_evd_qlen",
btl->udapl_dto_evd_qlen,
(btl->udapl_ia_attr).max_evd_qlen));
btl->udapl_dto_evd_qlen = btl->udapl_ia_attr.max_evd_qlen;
}
if (btl->udapl_conn_evd_qlen > (btl->udapl_ia_attr).max_evd_qlen) {
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt",
"evd_qlen adapter max",
true,
"btl_udapl_conn_evd_qlen",
btl->udapl_conn_evd_qlen,
(btl->udapl_ia_attr).max_evd_qlen));
btl->udapl_conn_evd_qlen = btl->udapl_ia_attr.max_evd_qlen;
}
/* set up evd's */
rc = dat_evd_create(btl->udapl_ia,
btl->udapl_dto_evd_qlen, DAT_HANDLE_NULL,
DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG, &btl->udapl_evd_dto);
if(DAT_SUCCESS != rc) {
char* major;
char* minor;
dat_strerror(rc, (const char**)&major,
(const char**)&minor);
BTL_ERROR(("ERROR: %s %s %s\n", "dat_evd_create (dto)",
major, minor));
goto failure;
}
rc = dat_evd_create(btl->udapl_ia,
btl->udapl_conn_evd_qlen, DAT_HANDLE_NULL,
DAT_EVD_CR_FLAG | DAT_EVD_CONNECTION_FLAG, &btl->udapl_evd_conn);
if(DAT_SUCCESS != rc) {
char* major;
char* minor;
dat_strerror(rc, (const char**)&major,
(const char**)&minor);
BTL_ERROR(("ERROR: %s %s %s\n", "dat_evd_create (conn)",
major, minor));
goto failure;
}
/* create our public service point */
rc = dat_psp_create_any(btl->udapl_ia, &port, btl->udapl_evd_conn,
DAT_PSP_CONSUMER_FLAG, &btl->udapl_psp);
if(DAT_SUCCESS != rc) {
char* major;
char* minor;
dat_strerror(rc, (const char**)&major,
(const char**)&minor);
BTL_ERROR(("ERROR: %s %s %s\n", "dat_psp_create_any",
major, minor));
goto failure;
}
/* establish endpoint parameters */
rc = mca_btl_udapl_endpoint_get_params(btl, &(btl->udapl_ep_param));
if(OMPI_SUCCESS != rc) {
/* by not erroring out here we can try to continue with
* the default endpoint parameter values
*/
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt",
"use default endpoint params",
true));
}
/* Save the port with the address information */
/* TODO - since we're doing the hack below, do we need our own port? */
btl->udapl_addr.port = port;
/* Using dat_ep_query to obtain the remote port would be ideal but
* since the current udapl implementations don't seem to support
* this we store the port in udapl_addr and explictly exchange the
* information later.
*/
((struct sockaddr_in*)&btl->udapl_addr.addr)->sin_port = htons(port);
/* initialize the memory pool */
res.pool_name = "udapl";
res.reg_data = btl;
res.sizeof_reg = sizeof(mca_btl_udapl_reg_t);
res.register_mem = udapl_reg_mr;
res.deregister_mem = udapl_dereg_mr;
btl->super.btl_mpool = mca_mpool_base_module_create(
mca_btl_udapl_component.udapl_mpool_name, &btl->super, &res);
if (NULL == btl->super.btl_mpool) {
BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_INFORM,
("WARNING: Failed to create mpool."));
goto failure;
}
/* initialize objects */
OBJ_CONSTRUCT(&btl->udapl_frag_eager, ompi_free_list_t);
OBJ_CONSTRUCT(&btl->udapl_frag_eager_recv, ompi_free_list_t);
OBJ_CONSTRUCT(&btl->udapl_frag_max, ompi_free_list_t);
OBJ_CONSTRUCT(&btl->udapl_frag_max_recv, ompi_free_list_t);
OBJ_CONSTRUCT(&btl->udapl_frag_user, ompi_free_list_t);
OBJ_CONSTRUCT(&btl->udapl_frag_control, ompi_free_list_t);
OBJ_CONSTRUCT(&btl->udapl_lock, opal_mutex_t);
/* check buffer alignment against dat library */
if (mca_btl_udapl_component.udapl_buffer_alignment !=
DAT_OPTIMAL_ALIGNMENT) {
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt",
"optimal buffer alignment mismatch",
true,
DAT_OPTIMAL_ALIGNMENT,
mca_btl_udapl_component.udapl_buffer_alignment,
DAT_OPTIMAL_ALIGNMENT));
}
/* initialize free lists */
ompi_free_list_init_ex_new(&btl->udapl_frag_eager,
sizeof(mca_btl_udapl_frag_eager_t) +
mca_btl_udapl_component.udapl_eager_frag_size,
mca_btl_udapl_component.udapl_buffer_alignment,
OBJ_CLASS(mca_btl_udapl_frag_eager_t),
mca_btl_udapl_component.udapl_eager_frag_size,
mca_btl_udapl_component.udapl_buffer_alignment,
mca_btl_udapl_component.udapl_free_list_num,
mca_btl_udapl_component.udapl_free_list_max,
mca_btl_udapl_component.udapl_free_list_inc,
btl->super.btl_mpool,
NULL,
NULL);
ompi_free_list_init_ex_new(&btl->udapl_frag_eager_recv,
sizeof(mca_btl_udapl_frag_eager_t) +
mca_btl_udapl_component.udapl_eager_frag_size,
mca_btl_udapl_component.udapl_buffer_alignment,
OBJ_CLASS(mca_btl_udapl_frag_eager_t),
mca_btl_udapl_component.udapl_eager_frag_size,
mca_btl_udapl_component.udapl_buffer_alignment,
mca_btl_udapl_component.udapl_free_list_num,
mca_btl_udapl_component.udapl_free_list_max,
mca_btl_udapl_component.udapl_free_list_inc,
btl->super.btl_mpool,
NULL,
NULL);
ompi_free_list_init_ex_new(&btl->udapl_frag_max,
sizeof(mca_btl_udapl_frag_max_t) +
mca_btl_udapl_component.udapl_max_frag_size,
mca_btl_udapl_component.udapl_buffer_alignment,
OBJ_CLASS(mca_btl_udapl_frag_max_t),
mca_btl_udapl_component.udapl_max_frag_size,
mca_btl_udapl_component.udapl_buffer_alignment,
mca_btl_udapl_component.udapl_free_list_num,
mca_btl_udapl_component.udapl_free_list_max,
mca_btl_udapl_component.udapl_free_list_inc,
btl->super.btl_mpool,
NULL,
NULL);
ompi_free_list_init_ex_new(&btl->udapl_frag_max_recv,
sizeof(mca_btl_udapl_frag_max_t) +
mca_btl_udapl_component.udapl_max_frag_size,
mca_btl_udapl_component.udapl_buffer_alignment,
OBJ_CLASS(mca_btl_udapl_frag_max_t),
mca_btl_udapl_component.udapl_max_frag_size,
mca_btl_udapl_component.udapl_buffer_alignment,
mca_btl_udapl_component.udapl_free_list_num,
mca_btl_udapl_component.udapl_free_list_max,
mca_btl_udapl_component.udapl_free_list_inc,
btl->super.btl_mpool,
NULL,
NULL);
ompi_free_list_init_ex_new(&btl->udapl_frag_user,
sizeof(mca_btl_udapl_frag_user_t),
mca_btl_udapl_component.udapl_buffer_alignment,
OBJ_CLASS(mca_btl_udapl_frag_user_t),
0,0,
mca_btl_udapl_component.udapl_free_list_num,
mca_btl_udapl_component.udapl_free_list_max,
mca_btl_udapl_component.udapl_free_list_inc,
NULL,
NULL,
NULL);
ompi_free_list_init_ex_new(&btl->udapl_frag_control,
sizeof(mca_btl_udapl_frag_eager_t) +
mca_btl_udapl_component.udapl_eager_frag_size,
mca_btl_udapl_component.udapl_buffer_alignment,
OBJ_CLASS(mca_btl_udapl_frag_eager_t),
mca_btl_udapl_component.udapl_eager_frag_size,
mca_btl_udapl_component.udapl_buffer_alignment,
mca_btl_udapl_component.udapl_free_list_num,
-1,
mca_btl_udapl_component.udapl_free_list_inc,
btl->super.btl_mpool,
NULL,
NULL);
/* initialize eager rdma buffer info */
btl->udapl_eager_rdma_endpoints = OBJ_NEW(opal_pointer_array_t);
opal_pointer_array_init(btl->udapl_eager_rdma_endpoints,
mca_btl_udapl_component.udapl_max_eager_rdma_peers,
mca_btl_udapl_component.udapl_max_eager_rdma_peers,
0);
btl->udapl_eager_rdma_endpoint_count = 0;
OBJ_CONSTRUCT(&btl->udapl_eager_rdma_lock, opal_mutex_t);
/* initialize miscellaneous variables */
btl->udapl_async_events = 0;
btl->udapl_connect_inprogress = 0;
btl->udapl_num_peers = 0;
/* TODO - Set up SRQ when it is supported */
return OMPI_SUCCESS;
failure:
dat_ia_close(btl->udapl_ia, DAT_CLOSE_ABRUPT_FLAG);
return OMPI_ERROR;
}
/*
* Find an address on the peer_process which matches stated criteria
* to the udapl btl module address information. Return in peer_addr_idx
* the index to the peer_process address that matches the btl module
* address. Where match criteria is:
* - the address in not already in use
* - compare addresses using netmask, the netmask value can be modified with
* "--mca btl_udapl_if_mask"
*
* Note: since this is called from mca_btl_udapl_proc_insert() it
* is assumed that the process lock is locked when entered.
*
* @param udapl_btl (IN) BTL module
* @param peer_process (IN) BTL peer process
* @param peer_addr_idx(IN/OUT) Index of address on peer_process
* which matches the udapl_btl address data.
* On success should be >= 0.
* @return OMPI_SUCCESS or error status on failure
*/
static int mca_btl_udapl_proc_address_match(
mca_btl_udapl_module_t* udapl_btl,
mca_btl_udapl_proc_t* peer_proc,
int* peer_addr_idx)
{
int i;
struct sockaddr *saddr;
struct sockaddr_in *btl_addr;
struct sockaddr_in *peer_addr;
char btl_addr_string[INET_ADDRSTRLEN];
char peer_addr_string[INET_ADDRSTRLEN];
*peer_addr_idx = MCA_BTL_UDAPL_INVALID_PEER_ADDR_IDX;
/* use generic address to find address family */
saddr = (struct sockaddr *)&(udapl_btl->udapl_addr.addr);
if (saddr->sa_family == AF_INET) {
btl_addr = (struct sockaddr_in *)saddr;
/* Loop thru peer process addresses looking for match.
* Match criteria:
* - address should not be "inuse"
* - both udapl btl module and peer address should be on
* the same subnet (compare with if_mask value)
*/
for(i = 0; i < (int) peer_proc->proc_addr_count; i++) {
peer_addr =
(struct sockaddr_in *)&(peer_proc->proc_addrs[i].addr);
if (VERBOSE_INFORM <=
mca_btl_udapl_component.udapl_verbosity) {
/* retrieve udapl btl and peer address string for reporting */
inet_ntop(AF_INET, (void *) &btl_addr->sin_addr,
btl_addr_string, INET_ADDRSTRLEN);
inet_ntop(AF_INET, (void *) &peer_addr->sin_addr,
peer_addr_string, INET_ADDRSTRLEN);
}
if ((false == peer_proc->proc_addrs[i].inuse) &&
(opal_net_samenetwork((struct sockaddr *)btl_addr,
(struct sockaddr *)peer_addr, udapl_btl->udapl_if_mask))) {
/* capture index of remote address where match found */
*peer_addr_idx = i;
/* mark this address as now being used */
peer_proc->proc_addrs[i].inuse = true;
/* report what address was found to match */
BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_INFORM,
("uDAPL BTL module(%s) matched %s",
btl_addr_string, peer_addr_string));
break;
} else {
/* peer address already used by another udapl btl
* module or netmask check not successful so skip
*/
BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_INFORM,
("uDAPL BTL module(%s) either skipped because it "
"is already in use or match criteria not successful "
"for peer address %s",
btl_addr_string, peer_addr_string));
}
}
} else {
/* current uDAPL BTL only supports IPv4 */
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
("help-mpi-btl-udapl.txt", "IPv4 only",
true, orte_process_info.nodename));
return OMPI_ERROR;
}
if (MCA_BTL_UDAPL_INVALID_PEER_ADDR_IDX == *peer_addr_idx) {
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
("help-mpi-btl-udapl.txt", "no network match",
true, btl_addr_string, orte_process_info.nodename,
peer_proc->proc_ompi->proc_hostname));
return OMPI_ERR_OUT_OF_RESOURCE;
}
return OMPI_SUCCESS;
}
static int mca_btl_udapl_modify_ia_list(DAT_COUNT *num_info_entries,
DAT_PROVIDER_INFO* datinfo)
{
int i,j,k,found;
DAT_PROVIDER_INFO* tmp_datinfo = NULL;
DAT_COUNT tmp_num_entries = 0;
tmp_datinfo = malloc((*num_info_entries) * sizeof(DAT_PROVIDER_INFO));
if(NULL == tmp_datinfo) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
for (i = 0; i < *num_info_entries; i++) {
j = 0;
found = 0;
/* search for datinfo entry on the if_list list */
while (mca_btl_udapl_component.if_list[j]) {
if (0 == strcmp(datinfo[i].ia_name,
mca_btl_udapl_component.if_list[j])) {
found = 1;
/* remove from if_list */
k = opal_argv_count(mca_btl_udapl_component.if_list);
opal_argv_delete(&k, &(mca_btl_udapl_component.if_list),
j, 1);
break;
}
j++;
}
if (found) {
if (NULL != mca_btl_udapl_component.if_include_list) {
/* explicitly include */
tmp_datinfo[tmp_num_entries] = datinfo[i];
tmp_num_entries++;
}
/* if this is if_exclude case and match found do nothing */
} else {
/* if this is if_include case and match not found do nothing */
if (NULL != mca_btl_udapl_component.if_exclude_list) {
/* not found for exclude case so actually include here */
tmp_datinfo[tmp_num_entries] = datinfo[i];
tmp_num_entries++;
}
}
}
/* set new values */
*num_info_entries = tmp_num_entries;
for (j = 0; j < *num_info_entries; j++) {
datinfo[j] = tmp_datinfo[j];
}
/* if if_list not NULL, either not found or user error */
if (opal_argv_count(mca_btl_udapl_component.if_list)) {
char *str = opal_argv_join(mca_btl_udapl_component.if_list, ',');
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
("help-mpi-btl-udapl.txt", "nonexistent entry",
true, orte_process_info.nodename,
((NULL != mca_btl_udapl_component.if_include) ?
"in" : "ex"), str));
free(str);
}
free(tmp_datinfo);
return OMPI_SUCCESS;
}
