int ompi_btl_usnic_find_ip(ompi_btl_usnic_module_t *module, uint8_t mac[6])
{
int i;
uint8_t localmac[6];
char addr_string[32], mac_string[32];
struct sockaddr sa;
struct sockaddr_in *sai;
/* Loop through all IP interfaces looking for the one with the
right MAC */
for (i = opal_ifbegin(); i != -1; i = opal_ifnext(i)) {
if (OPAL_SUCCESS == opal_ifindextomac(i, localmac)) {
/* Is this the MAC I'm looking for? */
if (0 != memcmp(mac, localmac, 6)) {
continue;
}
/* Yes, it is! */
if (OPAL_SUCCESS != opal_ifindextoname(i, module->if_name,
sizeof(module->if_name)) ||
OPAL_SUCCESS != opal_ifindextoaddr(i, &sa, sizeof(sa)) ||
OPAL_SUCCESS != opal_ifindextomask(i, &module->if_cidrmask,
sizeof(module->if_cidrmask)) ||
OPAL_SUCCESS != opal_ifindextomac(i, module->if_mac) ||
OPAL_SUCCESS != opal_ifindextomtu(i, &module->if_mtu)) {
continue;
}
sai = (struct sockaddr_in *) &sa;
memcpy(&module->if_ipv4_addr, &sai->sin_addr, 4);
/* Save this information to my local address field on the
module so that it gets sent in the modex */
module->local_addr.ipv4_addr = module->if_ipv4_addr;
module->local_addr.cidrmask = module->if_cidrmask;
/* Since verbs doesn't offer a way to get standard
Ethernet MTUs (as of libibverbs 1.1.5, the MTUs are
enums, and don't inlcude values for 1500 or 9000), look
up the MTU in the corresponding enic interface. */
module->local_addr.mtu = module->if_mtu;
inet_ntop(AF_INET, &(module->if_ipv4_addr),
addr_string, sizeof(addr_string));
ompi_btl_usnic_sprintf_mac(mac_string, mac);
opal_output_verbose(5, mca_btl_base_verbose,
"btl:usnic: found usNIC device corresponds to IP device %s, %s/%d, MAC %s",
module->if_name, addr_string, module->if_cidrmask,
mac_string);
return OMPI_SUCCESS;
}
}
return OMPI_ERR_NOT_FOUND;
}
int main(int argc, char* argv[])
{
int rc, idx;
uint32_t addr, netmask, netaddr;
struct sockaddr_in inaddr;
char **aliases=NULL;
if (0 > (rc = opal_init(&argc, &argv))) {
fprintf(stderr, "orte_interface: couldn't init opal - error code %d\n", rc);
return rc;
}
if (2 == argc) {
rc = opal_iftupletoaddr(argv[1], &netaddr, &netmask);
fprintf(stderr, "netaddr %03d.%03d.%03d.%03d netmask %03d.%03d.%03d.%03d rc %d\n",
OPAL_IF_FORMAT_ADDR(netaddr), OPAL_IF_FORMAT_ADDR(netmask), rc);
/* search for a matching interface - take the first one within the returned scope */
idx = opal_ifbegin();
while (0 < idx) {
/* ignore the loopback interface */
if (opal_ifisloopback(idx)) {
fprintf(stderr, "LOOPBACK IGNORED\n");
idx = opal_ifnext(idx);
continue;
}
if (0 != (rc = opal_ifindextoaddr(idx, (struct sockaddr*)&inaddr, sizeof(inaddr)))) {
break;
}
addr = ntohl(inaddr.sin_addr.s_addr);
fprintf(stderr, "checking netaddr %03d.%03d.%03d.%03d addr %03d.%03d.%03d.%03d netmask %03d.%03d.%03d.%03d rc %d\n",
OPAL_IF_FORMAT_ADDR(netaddr), OPAL_IF_FORMAT_ADDR(addr), OPAL_IF_FORMAT_ADDR(netmask), rc);
if (netaddr == (addr & netmask)) {
fprintf(stderr, "MATCH FOUND\n");
}
idx = opal_ifnext(idx);
}
}
/* check the aliases */
opal_ifgetaliases(&aliases);
idx = 0;
while (NULL != aliases[idx]) {
fprintf(stderr, "alias: %s\n", aliases[idx]);
idx++;
}
opal_finalize();
return 0;
}
static bool component_available(void)
{
int i, rc;
char **interfaces = NULL;
bool including = false, excluding = false;
char name[32];
struct sockaddr_storage my_ss;
int kindex;
opal_output_verbose(5, orte_oob_base_framework.framework_output,
"oob:tcp: component_available called");
/* if interface include was given, construct a list
* of those interfaces which match the specifications - remember,
* the includes could be given as named interfaces, IP addrs, or
* subnet+mask
*/
if (NULL != mca_oob_tcp_component.if_include) {
interfaces = split_and_resolve(&mca_oob_tcp_component.if_include,
"include");
including = true;
excluding = false;
} else if (NULL != mca_oob_tcp_component.if_exclude) {
interfaces = split_and_resolve(&mca_oob_tcp_component.if_exclude,
"exclude");
including = false;
excluding = true;
}
/* look at all available interfaces */
for (i = opal_ifbegin(); i >= 0; i = opal_ifnext(i)) {
if (OPAL_SUCCESS != opal_ifindextoaddr(i, (struct sockaddr*) &my_ss,
sizeof (my_ss))) {
opal_output (0, "oob_tcp: problems getting address for index %i (kernel index %i)\n",
i, opal_ifindextokindex(i));
continue;
}
/* ignore non-ip4/6 interfaces */
if (AF_INET != my_ss.ss_family
#if OPAL_ENABLE_IPV6
&& AF_INET6 != my_ss.ss_family
#endif
) {
continue;
}
kindex = opal_ifindextokindex(i);
if (kindex <= 0) {
continue;
}
opal_output_verbose(10, orte_oob_base_framework.framework_output,
"WORKING INTERFACE %d KERNEL INDEX %d FAMILY: %s", i, kindex,
(AF_INET == my_ss.ss_family) ? "V4" : "V6");
/* get the name for diagnostic purposes */
opal_ifindextoname(i, name, sizeof(name));
/* handle include/exclude directives */
if (NULL != interfaces) {
/* check for match */
rc = opal_ifmatches(kindex, interfaces);
/* if one of the network specifications isn't parseable, then
* error out as we can't do what was requested
*/
if (OPAL_ERR_NETWORK_NOT_PARSEABLE == rc) {
orte_show_help("help-oob-tcp.txt", "not-parseable", true);
opal_argv_free(interfaces);
return false;
}
/* if we are including, then ignore this if not present */
if (including) {
if (OPAL_SUCCESS != rc) {
opal_output_verbose(20, orte_oob_base_framework.framework_output,
"%s oob:tcp:init rejecting interface %s (not in include list)",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), name);
continue;
}
} else {
/* we are excluding, so ignore if present */
if (OPAL_SUCCESS == rc) {
opal_output_verbose(20, orte_oob_base_framework.framework_output,
"%s oob:tcp:init rejecting interface %s (in exclude list)",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), name);
continue;
}
}
} else {
/* if no specific interfaces were provided, we ignore the loopback
* interface unless nothing else is available
*/
if (1 < opal_ifcount() && opal_ifisloopback(i)) {
opal_output_verbose(20, orte_oob_base_framework.framework_output,
"%s oob:tcp:init rejecting loopback interface %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), name);
continue;
}
}
/* Refs ticket #3019
* it would probably be worthwhile to print out a warning if OMPI detects multiple
* IP interfaces that are "up" on the same subnet (because that's a Bad Idea). Note
* that we should only check for this after applying the relevant include/exclude
* list MCA params. If we detect redundant ports, we can also automatically ignore
* them so that applications won't hang.
*/
/* add this address to our connections */
if (AF_INET == my_ss.ss_family) {
opal_output_verbose(10, orte_oob_base_framework.framework_output,
"%s oob:tcp:init adding %s to our list of %s connections",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
opal_net_get_hostname((struct sockaddr*) &my_ss),
(AF_INET == my_ss.ss_family) ? "V4" : "V6");
opal_argv_append_nosize(&mca_oob_tcp_component.ipv4conns, opal_net_get_hostname((struct sockaddr*) &my_ss));
} else if (AF_INET6 == my_ss.ss_family) {
#if OPAL_ENABLE_IPV6
opal_output_verbose(10, orte_oob_base_framework.framework_output,
"%s oob:tcp:init adding %s to our list of %s connections",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
opal_net_get_hostname((struct sockaddr*) &my_ss),
(AF_INET == my_ss.ss_family) ? "V4" : "V6");
opal_argv_append_nosize(&mca_oob_tcp_component.ipv6conns, opal_net_get_hostname((struct sockaddr*) &my_ss));
#endif
} else {
opal_output_verbose(10, orte_oob_base_framework.framework_output,
"%s oob:tcp:init ignoring %s from out list of connections",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
opal_net_get_hostname((struct sockaddr*) &my_ss));
}
}
/* cleanup */
if (NULL != interfaces) {
opal_argv_free(interfaces);
}
if (0 == opal_argv_count(mca_oob_tcp_component.ipv4conns)
#if OPAL_ENABLE_IPV6
&& 0 == opal_argv_count(mca_oob_tcp_component.ipv6conns)
#endif
) {
if (including) {
orte_show_help("help-oob-tcp.txt", "no-included-found", true, mca_oob_tcp_component.if_include);
} else if (excluding) {
orte_show_help("help-oob-tcp.txt", "excluded-all", true, mca_oob_tcp_component.if_exclude);
}
return false;
}
/* set the module event base - this is where we would spin off a separate
* progress thread if so desired */
mca_oob_tcp_module.ev_base = orte_event_base;
return true;
}
static int mca_btl_sctp_create(int if_index, const char* if_name)
{
if(mca_btl_sctp_component.sctp_if_11) {
char param[256];
struct mca_btl_sctp_module_t* btl = (struct mca_btl_sctp_module_t *)malloc(sizeof(mca_btl_sctp_module_t));
if(NULL == btl) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
memcpy(btl, &mca_btl_sctp_module, sizeof(mca_btl_sctp_module));
OBJ_CONSTRUCT(&btl->sctp_endpoints, opal_list_t);
mca_btl_sctp_component.sctp_btls[mca_btl_sctp_component.sctp_num_btls++] = btl;
/* initialize the btl */
btl->sctp_ifindex = if_index;
#if MCA_BTL_SCTP_STATISTICS
btl->sctp_bytes_recv = 0;
btl->sctp_bytes_sent = 0;
btl->sctp_send_handler = 0;
#endif
opal_ifindextoaddr(if_index, (struct sockaddr*)&btl->sctp_ifaddr, sizeof(btl->sctp_ifaddr));
/* prepare for bind call later before connect */
btl->sctp_ifaddr.sin_family = AF_INET;
#ifdef FREEBSD
btl->sctp_ifaddr.sin_len = sizeof(struct sockaddr);
#endif
btl->sctp_ifaddr.sin_port = 0;
opal_ifindextomask(if_index, (uint32_t *)&btl->sctp_ifmask, sizeof(btl->sctp_ifmask));
/* allow user to specify interface bandwidth */
sprintf(param, "bandwidth_%s", if_name);
btl->super.btl_bandwidth = mca_btl_sctp_param_register_int(param, 0);
/* allow user to override/specify latency ranking */
sprintf(param, "latency_%s", if_name);
btl->super.btl_latency = mca_btl_sctp_param_register_int(param, 0);
#if 0 && OPAL_ENABLE_DEBUG
BTL_OUTPUT(("interface: %s bandwidth %d latency %d",
if_name, btl->super.btl_bandwidth, btl->super.btl_latency));
#endif
return OMPI_SUCCESS;
}
else {
/* 1 to many */
struct mca_btl_sctp_module_t* btl;
char param[256];
struct sockaddr_in next_ifaddr;
socklen_t len = sizeof(struct sockaddr_in);
opal_socklen_t addrlen;
/* check if this is the first time this function is being called */
if(0 == mca_btl_sctp_component.sctp_num_btls) {
/* fill in btl struct with first interface's information (arbitary) */
btl = (struct mca_btl_sctp_module_t *)malloc(sizeof(mca_btl_sctp_module_t));
if(NULL == btl) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
memcpy(btl, &mca_btl_sctp_module, sizeof(mca_btl_sctp_module));
OBJ_CONSTRUCT(&btl->sctp_endpoints, opal_list_t);
mca_btl_sctp_component.sctp_btls[mca_btl_sctp_component.sctp_num_btls++] = btl;
/* initialize the btl */
btl->sctp_ifindex = if_index;
#if MCA_BTL_SCTP_STATISTICS
btl->sctp_bytes_recv = 0;
btl->sctp_bytes_sent = 0;
btl->sctp_send_handler = 0;
#endif
opal_ifindextoaddr(if_index, (struct sockaddr*)&btl->sctp_ifaddr, sizeof(btl->sctp_ifaddr));
opal_ifindextomask(if_index, (uint32_t *)&btl->sctp_ifmask, sizeof(btl->sctp_ifmask));
/* allow user to specify interface bandwidth */
sprintf(param, "bandwidth_%s", if_name);
btl->super.btl_bandwidth = mca_btl_sctp_param_register_int(param, 0);
/* allow user to override/specify latency ranking */
sprintf(param, "latency_%s", if_name);
btl->super.btl_latency = mca_btl_sctp_param_register_int(param, 0);
#if 0 && OPAL_ENABLE_DEBUG
BTL_OUTPUT(("interface: %s bandwidth %d latency %d",
if_name, btl->super.btl_bandwidth, btl->super.btl_latency));
#endif
/* call bind to this (initial) addr */
opal_ifindextoaddr(if_index, (struct sockaddr*)&next_ifaddr, sizeof(next_ifaddr));
next_ifaddr.sin_family = AF_INET;
#ifdef FREEBSD
next_ifaddr.sin_len = sizeof(struct sockaddr);
#endif
next_ifaddr.sin_port = 0;
if(bind(mca_btl_sctp_component.sctp_listen_sd, (struct sockaddr *) &next_ifaddr, len) < 0) {
return OMPI_ERR_FATAL;
}
/* resolve system assignend port */
addrlen = sizeof(struct sockaddr_in);
if(getsockname(mca_btl_sctp_component.sctp_listen_sd, (struct sockaddr*)&next_ifaddr, &addrlen) < 0) {
BTL_ERROR(("getsockname() failed with errno=%d", opal_socket_errno));
return OMPI_ERROR;
}
/* need to get the port after the first bind call for subsequent
* sctp_bindx calls.
*/
mca_btl_sctp_component.sctp_listen_port = next_ifaddr.sin_port;
}
else {
next_ifaddr.sin_port = htons((unsigned short) mca_btl_sctp_component.sctp_listen_port);
/* add this addr to bindx */
opal_ifindextoaddr(if_index, (struct sockaddr*)&next_ifaddr, sizeof(next_ifaddr));
next_ifaddr.sin_family = AF_INET;
#ifdef FREEBSD
next_ifaddr.sin_len = sizeof(struct sockaddr);
#endif
if(sctp_bindx(mca_btl_sctp_component.sctp_listen_sd, (struct sockaddr *) &next_ifaddr,
1, SCTP_BINDX_ADD_ADDR) < 0) {
return OMPI_ERR_FATAL;
}
}
return OMPI_SUCCESS;
}
}
int main(int argc, char* argv[])
{
int rc, idx;
uint32_t addr, netmask, netaddr;
struct sockaddr_in inaddr, *paddr;
char **aliases=NULL;
opal_if_t *intf;
if (0 > (rc = opal_init(&argc, &argv))) {
fprintf(stderr, "opal_interface: couldn't init opal - error code %d\n", rc);
return rc;
}
if (OPAL_SUCCESS != mca_base_framework_open(&opal_if_base_framework, 0)) {
fprintf(stderr, "opal_interface: couldn't get interfaces\n");
return 1;
}
for (intf = (opal_if_t*)opal_list_get_first(&opal_if_list);
intf != (opal_if_t*)opal_list_get_end(&opal_if_list);
intf = (opal_if_t*)opal_list_get_next(intf)) {
paddr = (struct sockaddr_in*) &intf->if_addr;
fprintf(stderr, "intf: %s AF: %s indx: %d kindx: %d\n", intf->if_name,
(AF_INET == paddr->sin_family) ? "v4" : "v6",
intf->if_index, intf->if_kernel_index);
}
if (2 == argc) {
rc = opal_iftupletoaddr(argv[1], &netaddr, &netmask);
fprintf(stderr, "netaddr %03d.%03d.%03d.%03d netmask %03d.%03d.%03d.%03d rc %d\n",
OPAL_IF_FORMAT_ADDR(netaddr), OPAL_IF_FORMAT_ADDR(netmask), rc);
/* search for a matching interface - take the first one within the returned scope */
idx = opal_ifbegin();
while (0 < idx) {
/* ignore the loopback interface */
if (opal_ifisloopback(idx)) {
fprintf(stderr, "LOOPBACK IGNORED\n");
idx = opal_ifnext(idx);
continue;
}
if (0 != (rc = opal_ifindextoaddr(idx, (struct sockaddr*)&inaddr, sizeof(inaddr)))) {
break;
}
addr = ntohl(inaddr.sin_addr.s_addr);
fprintf(stderr, "checking netaddr %03d.%03d.%03d.%03d addr %03d.%03d.%03d.%03d netmask %03d.%03d.%03d.%03d rc %d\n",
OPAL_IF_FORMAT_ADDR(netaddr), OPAL_IF_FORMAT_ADDR(addr), OPAL_IF_FORMAT_ADDR(netmask), rc);
if (netaddr == (addr & netmask)) {
fprintf(stderr, "MATCH FOUND\n");
}
idx = opal_ifnext(idx);
}
}
/* check the aliases */
opal_ifgetaliases(&aliases);
idx = 0;
while (NULL != aliases[idx]) {
fprintf(stderr, "alias: %s\n", aliases[idx]);
idx++;
}
opal_finalize();
return 0;
}
