static unsigned int
opal_common_ugni_get_nic_address(int device_id)
{
unsigned int address, cpu_id;
gni_return_t status;
int i, alps_dev_id = -1;
char *token,*p_ptr;
p_ptr = getenv("PMI_GNI_DEV_ID");
if (!p_ptr) {
status = GNI_CdmGetNicAddress(device_id, &address, &cpu_id);
if(status != GNI_RC_SUCCESS) {
opal_output (0, "FAILED:GNI_CdmGetNicAddress returned error %d", status);
return (unsigned int)-1;
}
return address;
}
while (NULL != (token = strtok(p_ptr, ":"))) {
alps_dev_id = atoi(token);
if (alps_dev_id == device_id) {
break;
}
p_ptr = NULL;
}
if (OPAL_UNLIKELY(-1 == alps_dev_id)) {
return (unsigned int)-1;
}
p_ptr = getenv("PMI_GNI_LOC_ADDR");
if (OPAL_UNLIKELY(NULL == p_ptr)) {
return (unsigned int)-1;
}
i = 0;
while (NULL != (token = strtok(p_ptr, ":"))) {
if (i == alps_dev_id) {
return strtoul (token, NULL, 10);
}
p_ptr = NULL;
++i;
}
return (unsigned int)-1;
}
static ucs_status_t get_nic_address(uct_ugni_device_t *dev_p)
{
int alps_addr = -1;
int alps_dev_id = -1;
int i;
char *token, *pmi_env;
pmi_env = getenv("PMI_GNI_DEV_ID");
if (NULL == pmi_env) {
gni_return_t ugni_rc;
ugni_rc = GNI_CdmGetNicAddress(dev_p->device_id, &dev_p->address,
&dev_p->cpu_id);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_error("GNI_CdmGetNicAddress failed, device %d, Error status: %s %d",
dev_p->device_id, gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_NO_DEVICE;
}
CPU_SET(dev_p->cpu_id, &(dev_p->cpu_mask));
ucs_debug("(GNI) NIC address: %d", dev_p->address);
} else {
while ((token = strtok(pmi_env, ":")) != NULL) {
alps_dev_id = atoi(token);
if (alps_dev_id == dev_p->device_id) {
break;
}
pmi_env = NULL;
}
ucs_assert(alps_dev_id != -1);
pmi_env = getenv("PMI_GNI_LOC_ADDR");
ucs_assert(NULL != pmi_env);
i = 0;
while ((token = strtok(pmi_env, ":")) != NULL) {
if (i == alps_dev_id) {
alps_addr = atoi(token);
break;
}
pmi_env = NULL;
++i;
}
ucs_assert(alps_addr != -1);
dev_p->address = alps_addr;
ucs_debug("(PMI) NIC address: %d", dev_p->address);
}
return UCS_OK;
}
int _gnix_src_addr(struct gnix_ep_name *resolved_addr)
{
gni_return_t status;
uint32_t pe = -1;
uint32_t cpu_id = -1;
assert(resolved_addr);
memset(resolved_addr, 0, sizeof(*resolved_addr));
status = GNI_CdmGetNicAddress(0, &pe, &cpu_id);
if (status != GNI_RC_SUCCESS) {
GNIX_WARN(FI_LOG_FABRIC,
"Unable to get NIC address.");
return -FI_ENODATA;
}
resolved_addr->gnix_addr.device_addr = pe;
resolved_addr->name_type = GNIX_EPN_TYPE_UNBOUND;
return FI_SUCCESS;
}
/*
* gnix_resolve_name: given a node hint and a valid pointer to a gnix_ep_name
* will resolve the gnix specific address of node and fill the provided
* gnix_ep_name pointer with the information.
*
* node (IN) : Node name being resolved to gnix specific address
* service (IN) : Port number being resolved to gnix specific address
* resolved_addr (IN/OUT) : Pointer that must be provided to contain the
* resolved address.
*/
int _gnix_resolve_name(IN const char *node, IN const char *service,
IN uint64_t flags,
INOUT struct gnix_ep_name *resolved_addr)
{
uint32_t pe = -1;
uint32_t cpu_id = -1;
struct addrinfo *result = NULL;
struct addrinfo *rp = NULL;
struct sockaddr_in *sa = NULL;
struct sockaddr_in sin;
int ret = FI_SUCCESS;
gni_return_t status = GNI_RC_SUCCESS;
struct addrinfo hints = {
.ai_family = AF_INET,
.ai_socktype = SOCK_DGRAM,
};
GNIX_TRACE(FI_LOG_FABRIC, "\n");
if (flags & FI_SOURCE)
hints.ai_flags |= AI_PASSIVE;
if (flags & FI_NUMERICHOST)
hints.ai_flags |= AI_NUMERICHOST;
if (!resolved_addr) {
GNIX_WARN(FI_LOG_FABRIC,
"Resolved_addr must be a valid pointer.\n");
ret = -FI_EINVAL;
goto err;
}
ret = _gnix_local_ipaddr(&sin);
if (ret != FI_SUCCESS)
goto err;
ret = getaddrinfo(node, service, &hints, &result);
if (ret != 0) {
GNIX_WARN(FI_LOG_FABRIC,
"Failed to get address for node provided: %s\n",
gai_strerror(ret));
ret = -FI_EINVAL;
goto err;
}
for (rp = result; rp != NULL; rp = rp->ai_next) {
assert(rp->ai_addr->sa_family == AF_INET);
sa = (struct sockaddr_in *) rp->ai_addr;
/*
* If we are trying to resolve localhost then use
* CdmGetNicAddress.
*/
if (sa->sin_addr.s_addr == sin.sin_addr.s_addr) {
status = GNI_CdmGetNicAddress(0, &pe, &cpu_id);
if(status == GNI_RC_SUCCESS) {
break;
} else {
GNIX_WARN(FI_LOG_FABRIC,
"Unable to get NIC address.");
ret = gnixu_to_fi_errno(status);
goto err;
}
} else {
ret = __gnix_get_pe_from_ip("ipogif0",
inet_ntoa(sa->sin_addr), &pe);
if (ret == 0) {
break;
} else {
ret = __gnix_get_pe_from_ip("br0",
inet_ntoa(sa->sin_addr), &pe);
}
if (ret == 0)
break;
}
}
/*
* Make sure address is valid.
*/
if (pe == -1) {
GNIX_WARN(FI_LOG_FABRIC,
"Unable to acquire valid address for node %s\n",
node);
ret = -FI_EADDRNOTAVAIL;
goto err;
}
/*
* Fill the INOUT parameter resolved_addr with the address information
* acquired for the provided node parameter.
*/
memset(resolved_addr, 0, sizeof(struct gnix_ep_name));
resolved_addr->gnix_addr.device_addr = pe;
if (service) {
/* use resolved service/port */
resolved_addr->gnix_addr.cdm_id = ntohs(sa->sin_port);
resolved_addr->name_type = GNIX_EPN_TYPE_BOUND;
resolved_addr->cm_nic_cdm_id = resolved_addr->gnix_addr.cdm_id;
} else {
/* generate port internally */
resolved_addr->name_type = GNIX_EPN_TYPE_UNBOUND;
}
GNIX_INFO(FI_LOG_FABRIC, "Resolved: %s:%s to gnix_addr: 0x%lx\n",
node ?: "", service ?: "", resolved_addr->gnix_addr);
err:
if (result != NULL) {
freeaddrinfo(result);
}
return ret;
}
int _gnix_pe_to_ip(const struct gnix_ep_name *ep_name,
struct sockaddr_in *saddr)
{
int ret = -FI_EIO;
FILE *arp_table;
char buf[1024];
char ip_str[128], mac_str[128];
union mac_addr mac;
union mac_addr tmp_mac = {0};
gni_return_t status;
uint32_t pe, cpu_id;
status = GNI_CdmGetNicAddress(0, &pe, &cpu_id);
if (status == GNI_RC_SUCCESS &&
ep_name->gnix_addr.device_addr == pe) {
ret = _gnix_local_ipaddr(saddr);
saddr->sin_port = ep_name->gnix_addr.cdm_id;
return ret;
}
arp_table = fopen(ARP_TABLE_FILE, "r");
if (!arp_table) {
GNIX_WARN(FI_LOG_FABRIC, "Failed to fopen(): %s\n",
ARP_TABLE_FILE);
return -FI_EIO;
}
/* Eat header line. */
if (!fgets(buf, sizeof(buf), arp_table)) {
GNIX_WARN(FI_LOG_FABRIC, "Failed to fgets(): %s\n",
ARP_TABLE_FILE);
return -FI_EIO;
}
mac.u64 = __gnix_pe_to_mac(ep_name->gnix_addr.device_addr);
while (fscanf(arp_table, ARP_TABLE_FORMAT, ip_str, mac_str) == 2) {
ret = sscanf(mac_str, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx",
&tmp_mac.octets[5], &tmp_mac.octets[4],
&tmp_mac.octets[3], &tmp_mac.octets[2],
&tmp_mac.octets[1], &tmp_mac.octets[0]);
if (ret == 6) {
GNIX_DEBUG(FI_LOG_FABRIC,
"Comparing 0x%llx, 0x%llx\n",
mac.u64, tmp_mac.u64);
if (mac.u64 == tmp_mac.u64) {
saddr->sin_family = AF_INET;
saddr->sin_port = ep_name->gnix_addr.cdm_id;
saddr->sin_addr.s_addr = inet_addr(ip_str);
ret = FI_SUCCESS;
GNIX_DEBUG(FI_LOG_FABRIC,
"Translated %s->%s\n",
ip_str, mac_str);
break;
}
} else {
GNIX_WARN(FI_LOG_FABRIC, "Parse error: %d : %s\n",
ret, mac_str);
break;
}
}
fclose(arp_table);
return ret;
}
/*
* gnix_resolve_name: given a node hint and a valid pointer to a gnix_ep_name
* will resolve the gnix specific address of node and fill the provided
* gnix_ep_name pointer with the information.
*
* node (IN) : Node name being resolved to gnix specific address
* resolved_addr (IN/OUT) : Pointer that must be provided to contain the
* resolved address.
*
* TODO: consider a use for service.
*/
int gnix_resolve_name(IN const char *node, IN const char *service,
INOUT struct gnix_ep_name *resolved_addr)
{
int sock = -1;
uint32_t pe = -1;
uint32_t cpu_id = -1;
struct addrinfo *result = NULL;
struct addrinfo *rp = NULL;
struct ifreq ifr = {{{0}}};
struct sockaddr_in *sa = NULL;
struct sockaddr_in *sin = NULL;
int ret = FI_SUCCESS;
gni_return_t status = GNI_RC_SUCCESS;
struct addrinfo hints = {
.ai_family = AF_INET,
.ai_socktype = SOCK_DGRAM,
.ai_flags = AI_CANONNAME
};
if (!resolved_addr) {
GNIX_ERR(FI_LOG_FABRIC,
"Resolved_addr must be a valid pointer.\n");
ret = -FI_EINVAL;
goto err;
}
sock = socket(AF_INET, SOCK_DGRAM, 0);
if (sock == -1) {
GNIX_ERR(FI_LOG_FABRIC, "Socket creation failed: %s\n",
strerror(errno));
ret = -FI_EIO;
goto err;
}
/* Get the address for the ipogif0 interface */
ifr.ifr_addr.sa_family = AF_INET;
snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s", "ipogif0");
ret = ioctl(sock, SIOCGIFADDR, &ifr);
if (ret == -1) {
GNIX_ERR(FI_LOG_FABRIC,
"Failed to get address for ipogif0: %s\n",
strerror(errno));
ret = -FI_EIO;
goto sock_cleanup;
}
sin = (struct sockaddr_in *) &ifr.ifr_addr;
ret = getaddrinfo(node, "domain", &hints, &result);
if (ret != 0) {
GNIX_ERR(FI_LOG_FABRIC,
"Failed to get address for node provided: %s\n",
strerror(errno));
ret = -FI_EINVAL;
goto sock_cleanup;
}
for (rp = result; rp != NULL; rp = rp->ai_next) {
assert(rp->ai_addr->sa_family == AF_INET);
sa = (struct sockaddr_in *) rp->ai_addr;
/*
* If we are trying to resolve localhost then use
* CdmGetNicAddress.
*/
if (sa->sin_addr.s_addr == sin->sin_addr.s_addr) {
status = GNI_CdmGetNicAddress(0, &pe, &cpu_id);
if(status == GNI_RC_SUCCESS) {
break;
} else {
GNIX_ERR(FI_LOG_FABRIC,
"Unable to get NIC address.");
ret = gnixu_to_fi_errno(status);
goto sock_cleanup;
}
} else {
ret =
gnixu_get_pe_from_ip(inet_ntoa(sa->sin_addr), &pe);
if (ret == 0) {
break;
}
}
}
/*
* Make sure address is valid.
*/
if (pe == -1) {
GNIX_ERR(FI_LOG_FABRIC,
"Unable to acquire valid address for node %s\n",
node);
ret = -FI_EADDRNOTAVAIL;
goto sock_cleanup;
}
/*
* Fill the INOUT parameter resolved_addr with the address information
* acquired for the provided node parameter.
*/
memset(resolved_addr, 0, sizeof(struct gnix_ep_name));
resolved_addr->gnix_addr.device_addr = pe;
/* TODO: have to write a nameserver to get this info */
resolved_addr->gnix_addr.cdm_id = 0;
/* TODO: likely depend on service? */
resolved_addr->name_type = 0;
sock_cleanup:
if(close(sock) == -1) {
GNIX_ERR(FI_LOG_FABRIC, "Unable to close socket: %s\n",
strerror(errno));
}
err:
if (result != NULL) {
freeaddrinfo(result);
}
return ret;
}
