int ofi_nic_control(struct fid *fid, int command, void *arg)
{
struct fid_nic **nic = (struct fid_nic **) arg;
switch(command) {
case FI_DUP:
*nic = ofi_nic_dup(*nic);
return *nic ? FI_SUCCESS : -FI_ENOMEM;
default:
return -FI_ENOSYS;
}
}
int rxm_info_to_rxm(uint32_t version, const struct fi_info *core_info,
struct fi_info *info)
{
info->caps = rxm_info.caps;
// TODO find which other modes should be filtered
info->mode = (core_info->mode & ~FI_RX_CQ_DATA) | rxm_info.mode;
info->tx_attr->caps = rxm_info.tx_attr->caps;
info->tx_attr->mode = info->mode;
info->tx_attr->msg_order = core_info->tx_attr->msg_order;
info->tx_attr->comp_order = rxm_info.tx_attr->comp_order;
info->tx_attr->inject_size = rxm_info.tx_attr->inject_size;
info->tx_attr->size = rxm_info.tx_attr->size;
info->tx_attr->iov_limit = MIN(rxm_info.tx_attr->iov_limit,
core_info->tx_attr->iov_limit);
info->tx_attr->rma_iov_limit = MIN(rxm_info.tx_attr->rma_iov_limit,
core_info->tx_attr->rma_iov_limit);
info->rx_attr->caps = rxm_info.rx_attr->caps;
info->rx_attr->mode = info->rx_attr->mode & ~FI_RX_CQ_DATA;
info->rx_attr->msg_order = core_info->rx_attr->msg_order;
info->rx_attr->comp_order = rxm_info.rx_attr->comp_order;
info->rx_attr->size = rxm_info.rx_attr->size;
info->rx_attr->iov_limit = MIN(rxm_info.rx_attr->iov_limit,
core_info->rx_attr->iov_limit);
*info->ep_attr = *rxm_info.ep_attr;
info->ep_attr->max_msg_size = core_info->ep_attr->max_msg_size;
info->ep_attr->max_order_raw_size = core_info->ep_attr->max_order_raw_size;
info->ep_attr->max_order_war_size = core_info->ep_attr->max_order_war_size;
info->ep_attr->max_order_waw_size = core_info->ep_attr->max_order_waw_size;
*info->domain_attr = *rxm_info.domain_attr;
info->domain_attr->mr_mode |= core_info->domain_attr->mr_mode;
info->domain_attr->cq_data_size = MIN(core_info->domain_attr->cq_data_size,
rxm_info.domain_attr->cq_data_size);
info->domain_attr->mr_key_size = core_info->domain_attr->mr_key_size;
if (core_info->nic) {
info->nic = ofi_nic_dup(core_info->nic);
if (!info->nic)
return -FI_ENOMEM;
}
return 0;
}
int rxd_info_to_rxd(uint32_t version, const struct fi_info *core_info,
struct fi_info *info)
{
info->caps = rxd_info.caps;
info->mode = rxd_info.mode;
*info->tx_attr = *rxd_info.tx_attr;
info->tx_attr->inject_size = MIN(core_info->ep_attr->max_msg_size,
RXD_MAX_MTU_SIZE) - (sizeof(struct rxd_base_hdr) +
core_info->ep_attr->msg_prefix_size +
sizeof(struct rxd_rma_hdr) + (RXD_IOV_LIMIT *
sizeof(struct ofi_rma_iov)) + sizeof(struct rxd_atom_hdr));
*info->rx_attr = *rxd_info.rx_attr;
*info->ep_attr = *rxd_info.ep_attr;
*info->domain_attr = *rxd_info.domain_attr;
if (core_info->nic) {
info->nic = ofi_nic_dup(core_info->nic);
if (!info->nic)
return -FI_ENOMEM;
}
return 0;
}
static int efa_alloc_fid_nic(struct fi_info *fi, struct efa_context *ctx,
struct efa_device_attr *efa_device_attr,
struct ibv_port_attr *port_attr)
{
struct fi_device_attr *device_attr;
char driver_real_path[PATH_MAX];
struct fi_link_attr *link_attr;
char dbdf_real_path[PATH_MAX];
struct fi_bus_attr *bus_attr;
struct fi_pci_attr *pci_attr;
char *driver_sym_path;
char *dbdf_sym_path;
char *sysfs_path;
void *src_addr;
char *driver;
int name_len;
char *dbdf;
int ret;
/* Sets nic ops and allocates basic structure */
fi->nic = ofi_nic_dup(NULL);
if (!fi->nic)
return -FI_ENOMEM;
device_attr = fi->nic->device_attr;
bus_attr = fi->nic->bus_attr;
pci_attr = &bus_attr->attr.pci;
link_attr = fi->nic->link_attr;
/* fi_device_attr */
device_attr->name = strdup(ctx->ibv_ctx.device->name);
if (!device_attr->name) {
ret = -FI_ENOMEM;
goto err_free_nic;
}
ret = asprintf(&device_attr->device_id, "0x%x",
efa_device_attr->ibv_attr.vendor_part_id);
/* ofi_nic_close will free all attributes of the fi_nic struct */
if (ret < 0) {
ret = -FI_ENOMEM;
goto err_free_nic;
}
device_attr->device_version = calloc(1, EFA_ABI_VER_MAX_LEN + 1);
if (!device_attr->device_version) {
ret = -FI_ENOMEM;
goto err_free_nic;
}
sysfs_path = get_sysfs_path();
if (!sysfs_path) {
ret = -FI_ENOMEM;
goto err_free_nic;
}
ret = fi_read_file(sysfs_path, "class/infiniband_verbs/abi_version",
device_attr->device_version,
sizeof(device_attr->device_version));
if (ret < 0)
goto err_free_sysfs;
ret = asprintf(&device_attr->vendor_id, "0x%x",
efa_device_attr->ibv_attr.vendor_id);
if (ret < 0) {
ret = -FI_ENOMEM;
goto err_free_sysfs;
}
ret = asprintf(&driver_sym_path, "%s%s",
ctx->ibv_ctx.device->ibdev_path, "/device/driver");
if (ret < 0) {
ret = -FI_ENOMEM;
goto err_free_sysfs;
}
if (!realpath(driver_sym_path, driver_real_path)) {
ret = -errno;
goto err_free_driver_sym;
}
driver = strrchr(driver_real_path, '/');
if (!driver) {
ret = -FI_EINVAL;
goto err_free_driver_sym;
}
driver++;
device_attr->driver = strdup(driver);
if (!device_attr->driver) {
ret = -FI_ENOMEM;
goto err_free_driver_sym;
}
device_attr->firmware = strdup(efa_device_attr->ibv_attr.fw_ver);
if (!device_attr->firmware) {
ret = -FI_ENOMEM;
goto err_free_driver_sym;
}
/* fi_bus_attr */
bus_attr->bus_type = FI_BUS_PCI;
/* fi_pci_attr */
ret = asprintf(&dbdf_sym_path, "%s%s",
ctx->ibv_ctx.device->ibdev_path, "/device");
if (ret < 0) {
ret = -FI_ENOMEM;
goto err_free_driver_sym;
}
if (!realpath(dbdf_sym_path, dbdf_real_path)) {
ret = -errno;
goto err_free_dbdf_sym;
}
dbdf = strrchr(dbdf_real_path, '/');
if (!dbdf) {
ret = -FI_EINVAL;
goto err_free_dbdf_sym;
}
dbdf++;
ret = sscanf(dbdf, "%hx:%hhx:%hhx.%hhx", &pci_attr->domain_id,
&pci_attr->bus_id, &pci_attr->device_id,
&pci_attr->function_id);
if (ret != 4) {
ret = -FI_EINVAL;
goto err_free_dbdf_sym;
}
/* fi_link_attr */
src_addr = calloc(1, EFA_EP_ADDR_LEN);
if (!src_addr) {
ret = -FI_ENOMEM;
goto err_free_dbdf_sym;
}
ret = efa_get_addr(ctx, src_addr);
if (ret)
goto err_free_src_addr;
name_len = strlen(EFA_FABRIC_PREFIX) + INET6_ADDRSTRLEN;
link_attr->address = calloc(1, name_len + 1);
if (!link_attr->address) {
ret = -FI_ENOMEM;
goto err_free_src_addr;
}
efa_addr_to_str(src_addr, link_attr->address);
link_attr->mtu = port_attr->max_msg_sz;
link_attr->speed = 0;
switch (port_attr->state) {
case IBV_PORT_DOWN:
link_attr->state = FI_LINK_DOWN;
break;
case IBV_PORT_ACTIVE:
link_attr->state = FI_LINK_UP;
break;
default:
link_attr->state = FI_LINK_UNKNOWN;
break;
}
link_attr->network_type = strdup("Ethernet");
if (!link_attr->network_type) {
ret = -FI_ENOMEM;
goto err_free_src_addr;
}
free(src_addr);
free(dbdf_sym_path);
free(driver_sym_path);
free(sysfs_path);
return FI_SUCCESS;
err_free_src_addr:
free(src_addr);
err_free_dbdf_sym:
free(dbdf_sym_path);
err_free_driver_sym:
free(driver_sym_path);
err_free_sysfs:
free(sysfs_path);
err_free_nic:
fi_close(&fi->nic->fid);
fi->nic = NULL;
return ret;
}
static int usdf_alloc_fid_nic(struct fi_info *fi,
struct usd_device_attrs *dap)
{
int ret;
struct fid_nic *nic = NULL;
struct fi_device_attr *da = NULL;
struct fi_link_attr *la = NULL;
nic = ofi_nic_dup(NULL);
if (!nic)
goto nomem;
da = nic->device_attr;
da->name = strdup(dap->uda_devname);
if (!da->name)
goto nomem;
ret = asprintf(&da->device_id, "%s (%s)",
usd_devid_to_pid(dap->uda_vendor_id,
dap->uda_device_id),
usd_devid_to_nicname(dap->uda_vendor_id,
dap->uda_device_id));
if (ret < 0)
goto nomem;
ret = asprintf(&da->device_version, "0x%x", dap->uda_vendor_part_id);
if (ret < 0)
goto nomem;
ret = asprintf(&da->vendor_id, "0x%x", dap->uda_vendor_id);
if (ret < 0)
goto nomem;
da->driver = strdup("usnic_verbs");
if (!da->driver)
goto nomem;
da->firmware = strdup(dap->uda_firmware);
if (!da->firmware)
goto nomem;
// usnic does not currently expose PCI bus information, so we
// set the bus type to unknown.
nic->bus_attr->bus_type = FI_BUS_UNKNOWN;
la = nic->link_attr;
socklen_t size = INET_ADDRSTRLEN;
la->address = calloc(1, size);
if (!la->address)
goto nomem;
inet_ntop(AF_INET, &dap->uda_ipaddr_be, la->address, size);
la->mtu = dap->uda_mtu;
la->speed = dap->uda_bandwidth;
switch (dap->uda_link_state) {
case USD_LINK_UP:
la->state = FI_LINK_UP;
break;
case USD_LINK_DOWN:
la->state = FI_LINK_DOWN;
break;
default:
la->state = FI_LINK_UNKNOWN;
break;
}
la->network_type = strdup("Ethernet");
if (!la->network_type)
goto nomem;
fi->nic = nic;
return FI_SUCCESS;
nomem:
if (nic)
fi_close(&nic->fid);
return -FI_ENOMEM;
}