/*
* Attempt to resolve the address (given as either IPv4/IPv6 string
* or hostname) and return the kernel index of the interface
* on the same network as the specified address
*/
int16_t opal_ifaddrtokindex(const char* if_addr)
{
opal_if_t* intf;
int error;
struct addrinfo hints, *res = NULL, *r;
size_t len;
if (OPAL_SUCCESS != mca_base_framework_open(&opal_if_base_framework, 0)) {
return OPAL_ERROR;
}
memset(&hints, 0, sizeof(hints));
hints.ai_family = PF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
error = getaddrinfo(if_addr, NULL, &hints, &res);
if (error) {
if (NULL != res) {
freeaddrinfo (res);
}
return OPAL_ERR_NOT_FOUND;
}
for (r = res; r != NULL; r = r->ai_next) {
for (intf = (opal_if_t*)opal_list_get_first(&opal_if_list);
intf != (opal_if_t*)opal_list_get_end(&opal_if_list);
intf = (opal_if_t*)opal_list_get_next(intf)) {
if (AF_INET == r->ai_family && AF_INET == intf->af_family) {
struct sockaddr_in ipv4;
len = (r->ai_addrlen < sizeof(struct sockaddr_in)) ? r->ai_addrlen : sizeof(struct sockaddr_in);
memcpy(&ipv4, r->ai_addr, len);
if (opal_net_samenetwork((struct sockaddr*)&ipv4, (struct sockaddr*)&intf->if_addr, intf->if_mask)) {
return intf->if_kernel_index;
}
}
#if OPAL_ENABLE_IPV6
else if (AF_INET6 == r->ai_family && AF_INET6 == intf->af_family) {
struct sockaddr_in6 ipv6;
len = (r->ai_addrlen < sizeof(struct sockaddr_in6)) ? r->ai_addrlen : sizeof(struct sockaddr_in6);
memcpy(&ipv6, r->ai_addr, len);
if (opal_net_samenetwork((struct sockaddr*)((struct sockaddr_in6*)&intf->if_addr),
(struct sockaddr*)&ipv6, intf->if_mask)) {
return intf->if_kernel_index;
}
}
#endif
}
}
if (NULL != res) {
freeaddrinfo (res);
}
return OPAL_ERR_NOT_FOUND;
}
bool
opal_net_addr_isipv6linklocal(const struct sockaddr *addr)
{
#if OPAL_ENABLE_IPV6
struct sockaddr_in6 if_addr;
#endif
switch (addr->sa_family) {
#if OPAL_ENABLE_IPV6
case AF_INET6:
if_addr.sin6_family = AF_INET6;
if (1 != inet_pton(AF_INET6, "fe80::0000", &if_addr.sin6_addr)) {
return false;
}
return opal_net_samenetwork(addr, (struct sockaddr*)&if_addr, 64);
#endif
case AF_INET:
return false;
default:
opal_output (0,
"unhandled sa_family %d passed to opal_net_addr_isipv6linklocal\n",
addr->sa_family);
}
return false;
}
/*
* Find an address on the peer_process which matches stated criteria
* to the udapl btl module address information. Return in peer_addr_idx
* the index to the peer_process address that matches the btl module
* address. Where match criteria is:
* - the address in not already in use
* - compare addresses using netmask, the netmask value can be modified with
* "--mca btl_udapl_if_mask"
*
* Note: since this is called from mca_btl_udapl_proc_insert() it
* is assumed that the process lock is locked when entered.
*
* @param udapl_btl (IN) BTL module
* @param peer_process (IN) BTL peer process
* @param peer_addr_idx(IN/OUT) Index of address on peer_process
* which matches the udapl_btl address data.
* On success should be >= 0.
* @return OMPI_SUCCESS or error status on failure
*/
static int mca_btl_udapl_proc_address_match(
mca_btl_udapl_module_t* udapl_btl,
mca_btl_udapl_proc_t* peer_proc,
int* peer_addr_idx)
{
int i;
struct sockaddr *saddr;
struct sockaddr_in *btl_addr;
struct sockaddr_in *peer_addr;
char btl_addr_string[INET_ADDRSTRLEN];
char peer_addr_string[INET_ADDRSTRLEN];
*peer_addr_idx = MCA_BTL_UDAPL_INVALID_PEER_ADDR_IDX;
/* use generic address to find address family */
saddr = (struct sockaddr *)&(udapl_btl->udapl_addr.addr);
if (saddr->sa_family == AF_INET) {
btl_addr = (struct sockaddr_in *)saddr;
/* Loop thru peer process addresses looking for match.
* Match criteria:
* - address should not be "inuse"
* - both udapl btl module and peer address should be on
* the same subnet (compare with if_mask value)
*/
for(i = 0; i < (int) peer_proc->proc_addr_count; i++) {
peer_addr =
(struct sockaddr_in *)&(peer_proc->proc_addrs[i].addr);
if (VERBOSE_INFORM <=
mca_btl_udapl_component.udapl_verbosity) {
/* retrieve udapl btl and peer address string for reporting */
inet_ntop(AF_INET, (void *) &btl_addr->sin_addr,
btl_addr_string, INET_ADDRSTRLEN);
inet_ntop(AF_INET, (void *) &peer_addr->sin_addr,
peer_addr_string, INET_ADDRSTRLEN);
}
if ((false == peer_proc->proc_addrs[i].inuse) &&
(opal_net_samenetwork((struct sockaddr *)btl_addr,
(struct sockaddr *)peer_addr, udapl_btl->udapl_if_mask))) {
/* capture index of remote address where match found */
*peer_addr_idx = i;
/* mark this address as now being used */
peer_proc->proc_addrs[i].inuse = true;
/* report what address was found to match */
BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_INFORM,
("uDAPL BTL module(%s) matched %s",
btl_addr_string, peer_addr_string));
break;
} else {
/* peer address already used by another udapl btl
* module or netmask check not successful so skip
*/
BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_INFORM,
("uDAPL BTL module(%s) either skipped because it "
"is already in use or match criteria not successful "
"for peer address %s",
btl_addr_string, peer_addr_string));
}
}
} else {
/* current uDAPL BTL only supports IPv4 */
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
("help-mpi-btl-udapl.txt", "IPv4 only",
true, orte_process_info.nodename));
return OMPI_ERROR;
}
if (MCA_BTL_UDAPL_INVALID_PEER_ADDR_IDX == *peer_addr_idx) {
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
("help-mpi-btl-udapl.txt", "no network match",
true, btl_addr_string, orte_process_info.nodename,
peer_proc->proc_ompi->proc_hostname));
return OMPI_ERR_OUT_OF_RESOURCE;
}
return OMPI_SUCCESS;
}
int mca_oob_tcp_addr_get_next(mca_oob_tcp_addr_t* addr, struct sockaddr* retval)
{
static uint32_t i_have = MCA_OOB_TCP_ADDR_UNCLASSIFIED; /* my own capabilities */
if((NULL == addr) || (0 == addr->addr_count)) {
return ORTE_ERROR;
}
if(MCA_OOB_TCP_ADDR_UNCLASSIFIED == addr->addr_matched) {
orte_std_cntr_t i=0;
for(i=addr->addr_next; i<addr->addr_count; i++) {
opal_list_item_t *item;
for (item = opal_list_get_first(&mca_oob_tcp_component.tcp_available_devices) ;
item != opal_list_get_end(&mca_oob_tcp_component.tcp_available_devices) ;
item = opal_list_get_next(item)) {
mca_oob_tcp_device_t *dev = (mca_oob_tcp_device_t*) item;
uint32_t inmask;
opal_ifindextomask(dev->if_index, &inmask, sizeof(inmask));
/* Decide which address to try first; note that we're
called multiple times and each time we need to
present a different address
Precedence rules:
- IPv4public has the highest priority
- when IPv4private + IPv6, use IPv6 (this should
be changed when there is something like a CellID)
*/
if (true == opal_net_addr_isipv4public ((struct sockaddr*) &dev->if_addr)) {
i_have |= MCA_OOB_TCP_ADDR_IPV4public;
}
if (true == opal_net_addr_isipv4public ((struct sockaddr*)&addr->addr_inet[i])) {
addr->addr_matched |= MCA_OOB_TCP_ADDR_IPV4public;
}
if ((MCA_OOB_TCP_ADDR_IPV4public ==
(i_have & MCA_OOB_TCP_ADDR_IPV4public)) &&
(MCA_OOB_TCP_ADDR_IPV4public ==
(addr->addr_matched & MCA_OOB_TCP_ADDR_IPV4public))) {
addr->addr_next = i;
goto done;
}
if (AF_INET6 == dev->if_addr.ss_family) {
i_have |= MCA_OOB_TCP_ADDR_IPV6;
}
if (AF_INET6 ==
((struct sockaddr_in6*)&addr->addr_inet[i])->sin6_family) {
addr->addr_matched |= MCA_OOB_TCP_ADDR_IPV6;
addr->addr_next = i;
goto done;
}
/* if match on network prefix - start here */
/* Bug, FIXME: This code is dangerous, it will prefer
local addresses even if they point to wrong hosts
(the multicluster problem).
We need more magic to select the best address
adi@2006-09-30
*/
if(opal_net_samenetwork((struct sockaddr*) &dev->if_addr,
(struct sockaddr*)&addr->addr_inet[i],
inmask)) {
addr->addr_matched |= MCA_OOB_TCP_ADDR_MATCHED;
addr->addr_next = i;
goto done;
}
}
}
done:
; /* NOP */
}
if (addr->addr_inet[addr->addr_next].ss_family == AF_INET) {
memcpy(retval, &addr->addr_inet[addr->addr_next],
sizeof(struct sockaddr_in));
} else {
memcpy(retval, &addr->addr_inet[addr->addr_next],
sizeof(struct sockaddr_in6));
}
if(++addr->addr_next >= addr->addr_count)
addr->addr_next = 0;
return ORTE_SUCCESS;
}
/*
* Note that this routine must be called with the lock on the process
* already held. Insert a btl instance into the proc array and assign
* it an address.
*/
int mca_btl_tcp2_proc_insert( mca_btl_tcp2_proc_t* btl_proc,
mca_btl_base_endpoint_t* btl_endpoint )
{
struct sockaddr_storage endpoint_addr_ss;
unsigned int perm_size;
int rc, *a = NULL;
size_t i, j;
#ifndef WORDS_BIGENDIAN
/* if we are little endian and our peer is not so lucky, then we
need to put all information sent to him in big endian (aka
Network Byte Order) and expect all information received to
be in NBO. Since big endian machines always send and receive
in NBO, we don't care so much about that case. */
if (btl_proc->proc_ompi->proc_arch & OPAL_ARCH_ISBIGENDIAN) {
btl_endpoint->endpoint_nbo = true;
}
#endif
/* insert into endpoint array */
btl_endpoint->endpoint_proc = btl_proc;
btl_proc->proc_endpoints[btl_proc->proc_endpoint_count++] = btl_endpoint;
/* sanity checks */
if( NULL == local_interfaces ) {
if( NULL == mca_btl_tcp2_retrieve_local_interfaces() )
return OMPI_ERR_OUT_OF_RESOURCE;
}
if( 0 == num_local_interfaces ) {
return OMPI_ERR_UNREACH;
}
if( NULL == peer_interfaces ) {
max_peer_interfaces = max_local_interfaces;
peer_interfaces = (mca_btl_tcp2_interface_t**)malloc( max_peer_interfaces * sizeof(mca_btl_tcp2_interface_t*) );
}
num_peer_interfaces = 0;
memset(peer_kindex_to_index, -1, sizeof(int)*MAX_KERNEL_INTERFACE_INDEX);
memset(peer_interfaces, 0, max_peer_interfaces * sizeof(mca_btl_tcp2_interface_t*));
/*
* identify all kernel interfaces and the associated addresses of
* the peer
*/
for( i = 0; i < btl_proc->proc_addr_count; i++ ) {
int index;
mca_btl_tcp2_addr_t* endpoint_addr = btl_proc->proc_addrs + i;
mca_btl_tcp2_proc_tosocks (endpoint_addr, &endpoint_addr_ss);
index = peer_kindex_to_index[endpoint_addr->addr_ifkindex];
if(-1 == index) {
index = num_peer_interfaces++;
peer_kindex_to_index[endpoint_addr->addr_ifkindex] = index;
if( num_peer_interfaces == max_peer_interfaces ) {
max_peer_interfaces <<= 1;
peer_interfaces = (mca_btl_tcp2_interface_t**)realloc( peer_interfaces,
max_peer_interfaces * sizeof(mca_btl_tcp2_interface_t*) );
if( NULL == peer_interfaces )
return OMPI_ERR_OUT_OF_RESOURCE;
}
peer_interfaces[index] = (mca_btl_tcp2_interface_t *) malloc(sizeof(mca_btl_tcp2_interface_t));
mca_btl_tcp2_initialise_interface(peer_interfaces[index],
endpoint_addr->addr_ifkindex, index);
}
/*
* in case one of the peer addresses is already in use,
* mark the complete peer interface as 'not available'
*/
if(endpoint_addr->addr_inuse) {
peer_interfaces[index]->inuse = 1;
}
switch(endpoint_addr_ss.ss_family) {
case AF_INET:
peer_interfaces[index]->ipv4_address = (struct sockaddr_storage*) malloc(sizeof(endpoint_addr_ss));
peer_interfaces[index]->ipv4_endpoint_addr = endpoint_addr;
memcpy(peer_interfaces[index]->ipv4_address,
&endpoint_addr_ss, sizeof(endpoint_addr_ss));
break;
case AF_INET6:
peer_interfaces[index]->ipv6_address = (struct sockaddr_storage*) malloc(sizeof(endpoint_addr_ss));
peer_interfaces[index]->ipv6_endpoint_addr = endpoint_addr;
memcpy(peer_interfaces[index]->ipv6_address,
&endpoint_addr_ss, sizeof(endpoint_addr_ss));
break;
default:
opal_output(0, "unknown address family for tcp: %d\n",
endpoint_addr_ss.ss_family);
/*
* return OMPI_UNREACH or some error, as this is not
* good
*/
}
}
/*
* assign weights to each possible pair of interfaces
*/
perm_size = num_local_interfaces;
if(num_peer_interfaces > perm_size) {
perm_size = num_peer_interfaces;
}
weights = (enum mca_btl_tcp2_connection_quality**) malloc(perm_size
* sizeof(enum mca_btl_tcp2_connection_quality*));
best_addr = (mca_btl_tcp2_addr_t ***) malloc(perm_size
* sizeof(mca_btl_tcp2_addr_t **));
for(i = 0; i < perm_size; ++i) {
weights[i] = (enum mca_btl_tcp2_connection_quality*) malloc(perm_size *
sizeof(enum mca_btl_tcp2_connection_quality));
memset(weights[i], 0, perm_size * sizeof(enum mca_btl_tcp2_connection_quality));
best_addr[i] = (mca_btl_tcp2_addr_t **) malloc(perm_size *
sizeof(mca_btl_tcp2_addr_t *));
memset(best_addr[i], 0, perm_size * sizeof(mca_btl_tcp2_addr_t *));
}
for(i=0; i<num_local_interfaces; ++i) {
for(j=0; j<num_peer_interfaces; ++j) {
/* initially, assume no connection is possible */
weights[i][j] = CQ_NO_CONNECTION;
/* check state of ipv4 address pair */
if(NULL != local_interfaces[i]->ipv4_address &&
NULL != peer_interfaces[j]->ipv4_address) {
/* check for RFC1918 */
if(opal_net_addr_isipv4public((struct sockaddr*) local_interfaces[i]->ipv4_address)
&& opal_net_addr_isipv4public((struct sockaddr*)
peer_interfaces[j]->ipv4_address)) {
if(opal_net_samenetwork((struct sockaddr*) local_interfaces[i]->ipv4_address,
(struct sockaddr*) peer_interfaces[j]->ipv4_address,
local_interfaces[i]->ipv4_netmask)) {
weights[i][j] = CQ_PUBLIC_SAME_NETWORK;
} else {
weights[i][j] = CQ_PUBLIC_DIFFERENT_NETWORK;
}
best_addr[i][j] = peer_interfaces[j]->ipv4_endpoint_addr;
continue;
} else {
if(opal_net_samenetwork((struct sockaddr*) local_interfaces[i]->ipv4_address,
(struct sockaddr*) peer_interfaces[j]->ipv4_address,
local_interfaces[i]->ipv4_netmask)) {
weights[i][j] = CQ_PRIVATE_SAME_NETWORK;
} else {
weights[i][j] = CQ_PRIVATE_DIFFERENT_NETWORK;
}
best_addr[i][j] = peer_interfaces[j]->ipv4_endpoint_addr;
}
}
/* check state of ipv6 address pair - ipv6 is always public,
* since link-local addresses are skipped in opal_ifinit()
*/
if(NULL != local_interfaces[i]->ipv6_address &&
NULL != peer_interfaces[j]->ipv6_address) {
if(opal_net_samenetwork((struct sockaddr*) local_interfaces[i]->ipv6_address,
(struct sockaddr*) peer_interfaces[j]->ipv6_address,
local_interfaces[i]->ipv6_netmask)) {
weights[i][j] = CQ_PUBLIC_SAME_NETWORK;
} else {
weights[i][j] = CQ_PUBLIC_DIFFERENT_NETWORK;
}
best_addr[i][j] = peer_interfaces[j]->ipv6_endpoint_addr;
}
} /* for each peer interface */
} /* for each local interface */
/*
* determine the size of the set to permute (max number of
* interfaces
*/
best_assignment = (unsigned int *) malloc (perm_size * sizeof(int));
a = (int *) malloc(perm_size * sizeof(int));
if (NULL == a) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* Can only find the best set of connections when the number of
* interfaces is not too big. When it gets larger, we fall back
* to a simpler and faster (and not as optimal) algorithm. */
if (perm_size <= MAX_PERMUTATION_INTERFACES) {
memset(a, 0, perm_size * sizeof(int));
max_assignment_cardinality = -1;
max_assignment_weight = -1;
visit(0, -1, perm_size, a);
rc = OMPI_ERR_UNREACH;
for(i = 0; i < perm_size; ++i) {
if(best_assignment[i] > num_peer_interfaces
|| weights[i][best_assignment[i]] == CQ_NO_CONNECTION
|| peer_interfaces[best_assignment[i]]->inuse
|| NULL == peer_interfaces[best_assignment[i]]) {
continue;
}
peer_interfaces[best_assignment[i]]->inuse++;
btl_endpoint->endpoint_addr = best_addr[i][best_assignment[i]];
btl_endpoint->endpoint_addr->addr_inuse++;
rc = OMPI_SUCCESS;
break;
}
} else {
enum mca_btl_tcp2_connection_quality max;
int i_max, j_max;
/* Find the best connection that is not in use. Save away
* the indices of the best location. */
max = CQ_NO_CONNECTION;
for(i=0; i<num_local_interfaces; ++i) {
for(j=0; j<num_peer_interfaces; ++j) {
if (!peer_interfaces[j]->inuse) {
if (weights[i][j] > max) {
max = weights[i][j];
i_max = i;
j_max = j;
}
}
}
}
/* Now see if there is a some type of connection available. */
rc = OMPI_ERR_UNREACH;
if (CQ_NO_CONNECTION != max) {
peer_interfaces[j_max]->inuse++;
btl_endpoint->endpoint_addr = best_addr[i_max][j_max];
btl_endpoint->endpoint_addr->addr_inuse++;
rc = OMPI_SUCCESS;
}
}
for(i = 0; i < perm_size; ++i) {
free(weights[i]);
free(best_addr[i]);
}
for(i = 0; i < num_peer_interfaces; ++i) {
if(NULL != peer_interfaces[i]->ipv4_address) {
free(peer_interfaces[i]->ipv4_address);
}
if(NULL != peer_interfaces[i]->ipv6_address) {
free(peer_interfaces[i]->ipv6_address);
}
free(peer_interfaces[i]);
}
free(peer_interfaces);
peer_interfaces = NULL;
max_peer_interfaces = 0;
for(i = 0; i < num_local_interfaces; ++i) {
if(NULL != local_interfaces[i]->ipv4_address) {
free(local_interfaces[i]->ipv4_address);
}
if(NULL != local_interfaces[i]->ipv6_address) {
free(local_interfaces[i]->ipv6_address);
}
free(local_interfaces[i]);
}
free(local_interfaces);
local_interfaces = NULL;
max_local_interfaces = 0;
free(weights);
free(best_addr);
free(best_assignment);
free(a);
return rc;
}
