static int efa_alloc_fid_nic(struct fi_info *fi, struct efa_context *ctx,
struct efa_device_attr *efa_device_attr,
struct ibv_port_attr *port_attr)
{
struct fi_device_attr *device_attr;
char driver_real_path[PATH_MAX];
struct fi_link_attr *link_attr;
char dbdf_real_path[PATH_MAX];
struct fi_bus_attr *bus_attr;
struct fi_pci_attr *pci_attr;
char *driver_sym_path;
char *dbdf_sym_path;
char *sysfs_path;
void *src_addr;
char *driver;
int name_len;
char *dbdf;
int ret;
/* Sets nic ops and allocates basic structure */
fi->nic = ofi_nic_dup(NULL);
if (!fi->nic)
return -FI_ENOMEM;
device_attr = fi->nic->device_attr;
bus_attr = fi->nic->bus_attr;
pci_attr = &bus_attr->attr.pci;
link_attr = fi->nic->link_attr;
/* fi_device_attr */
device_attr->name = strdup(ctx->ibv_ctx.device->name);
if (!device_attr->name) {
ret = -FI_ENOMEM;
goto err_free_nic;
}
ret = asprintf(&device_attr->device_id, "0x%x",
efa_device_attr->ibv_attr.vendor_part_id);
/* ofi_nic_close will free all attributes of the fi_nic struct */
if (ret < 0) {
ret = -FI_ENOMEM;
goto err_free_nic;
}
device_attr->device_version = calloc(1, EFA_ABI_VER_MAX_LEN + 1);
if (!device_attr->device_version) {
ret = -FI_ENOMEM;
goto err_free_nic;
}
sysfs_path = get_sysfs_path();
if (!sysfs_path) {
ret = -FI_ENOMEM;
goto err_free_nic;
}
ret = fi_read_file(sysfs_path, "class/infiniband_verbs/abi_version",
device_attr->device_version,
sizeof(device_attr->device_version));
if (ret < 0)
goto err_free_sysfs;
ret = asprintf(&device_attr->vendor_id, "0x%x",
efa_device_attr->ibv_attr.vendor_id);
if (ret < 0) {
ret = -FI_ENOMEM;
goto err_free_sysfs;
}
ret = asprintf(&driver_sym_path, "%s%s",
ctx->ibv_ctx.device->ibdev_path, "/device/driver");
if (ret < 0) {
ret = -FI_ENOMEM;
goto err_free_sysfs;
}
if (!realpath(driver_sym_path, driver_real_path)) {
ret = -errno;
goto err_free_driver_sym;
}
driver = strrchr(driver_real_path, '/');
if (!driver) {
ret = -FI_EINVAL;
goto err_free_driver_sym;
}
driver++;
device_attr->driver = strdup(driver);
if (!device_attr->driver) {
ret = -FI_ENOMEM;
goto err_free_driver_sym;
}
device_attr->firmware = strdup(efa_device_attr->ibv_attr.fw_ver);
if (!device_attr->firmware) {
ret = -FI_ENOMEM;
goto err_free_driver_sym;
}
/* fi_bus_attr */
bus_attr->bus_type = FI_BUS_PCI;
/* fi_pci_attr */
ret = asprintf(&dbdf_sym_path, "%s%s",
ctx->ibv_ctx.device->ibdev_path, "/device");
if (ret < 0) {
ret = -FI_ENOMEM;
goto err_free_driver_sym;
}
if (!realpath(dbdf_sym_path, dbdf_real_path)) {
ret = -errno;
goto err_free_dbdf_sym;
}
dbdf = strrchr(dbdf_real_path, '/');
if (!dbdf) {
ret = -FI_EINVAL;
goto err_free_dbdf_sym;
}
dbdf++;
ret = sscanf(dbdf, "%hx:%hhx:%hhx.%hhx", &pci_attr->domain_id,
&pci_attr->bus_id, &pci_attr->device_id,
&pci_attr->function_id);
if (ret != 4) {
ret = -FI_EINVAL;
goto err_free_dbdf_sym;
}
/* fi_link_attr */
src_addr = calloc(1, EFA_EP_ADDR_LEN);
if (!src_addr) {
ret = -FI_ENOMEM;
goto err_free_dbdf_sym;
}
ret = efa_get_addr(ctx, src_addr);
if (ret)
goto err_free_src_addr;
name_len = strlen(EFA_FABRIC_PREFIX) + INET6_ADDRSTRLEN;
link_attr->address = calloc(1, name_len + 1);
if (!link_attr->address) {
ret = -FI_ENOMEM;
goto err_free_src_addr;
}
efa_addr_to_str(src_addr, link_attr->address);
link_attr->mtu = port_attr->max_msg_sz;
link_attr->speed = 0;
switch (port_attr->state) {
case IBV_PORT_DOWN:
link_attr->state = FI_LINK_DOWN;
break;
case IBV_PORT_ACTIVE:
link_attr->state = FI_LINK_UP;
break;
default:
link_attr->state = FI_LINK_UNKNOWN;
break;
}
link_attr->network_type = strdup("Ethernet");
if (!link_attr->network_type) {
ret = -FI_ENOMEM;
goto err_free_src_addr;
}
free(src_addr);
free(dbdf_sym_path);
free(driver_sym_path);
free(sysfs_path);
return FI_SUCCESS;
err_free_src_addr:
free(src_addr);
err_free_dbdf_sym:
free(dbdf_sym_path);
err_free_driver_sym:
free(driver_sym_path);
err_free_sysfs:
free(sysfs_path);
err_free_nic:
fi_close(&fi->nic->fid);
fi->nic = NULL;
return ret;
}
static inline int fi_ibv_get_qp_cap(struct ibv_context *ctx,
struct ibv_device_attr *device_attr,
struct fi_info *info)
{
struct ibv_pd *pd;
struct ibv_cq *cq;
struct ibv_qp *qp;
struct ibv_qp_init_attr init_attr;
int ret = 0;
pd = ibv_alloc_pd(ctx);
if (!pd) {
VERBS_INFO_ERRNO(FI_LOG_FABRIC, "ibv_alloc_pd", errno);
return -errno;
}
cq = ibv_create_cq(ctx, 1, NULL, NULL, 0);
if (!cq) {
VERBS_INFO_ERRNO(FI_LOG_FABRIC, "ibv_create_cq", errno);
ret = -errno;
goto err1;
}
/* TODO: serialize access to string buffers */
fi_read_file(FI_CONF_DIR, "def_tx_ctx_size",
def_tx_ctx_size, sizeof def_tx_ctx_size);
fi_read_file(FI_CONF_DIR, "def_rx_ctx_size",
def_rx_ctx_size, sizeof def_rx_ctx_size);
fi_read_file(FI_CONF_DIR, "def_tx_iov_limit",
def_tx_iov_limit, sizeof def_tx_iov_limit);
fi_read_file(FI_CONF_DIR, "def_rx_iov_limit",
def_rx_iov_limit, sizeof def_rx_iov_limit);
fi_read_file(FI_CONF_DIR, "def_inject_size",
def_inject_size, sizeof def_inject_size);
memset(&init_attr, 0, sizeof init_attr);
init_attr.send_cq = cq;
init_attr.recv_cq = cq;
init_attr.cap.max_send_wr = MIN(atoi(def_tx_ctx_size), device_attr->max_qp_wr);
init_attr.cap.max_recv_wr = MIN(atoi(def_rx_ctx_size), device_attr->max_qp_wr);
init_attr.cap.max_send_sge = MIN(atoi(def_tx_iov_limit), device_attr->max_sge);
init_attr.cap.max_recv_sge = MIN(atoi(def_rx_iov_limit), device_attr->max_sge);
init_attr.cap.max_inline_data = atoi(def_inject_size);
init_attr.qp_type = IBV_QPT_RC;
qp = ibv_create_qp(pd, &init_attr);
if (!qp) {
VERBS_INFO_ERRNO(FI_LOG_FABRIC, "ibv_create_qp", errno);
ret = -errno;
goto err2;
}
info->tx_attr->inject_size = init_attr.cap.max_inline_data;
info->tx_attr->iov_limit = init_attr.cap.max_send_sge;
info->tx_attr->size = init_attr.cap.max_send_wr;
info->rx_attr->iov_limit = init_attr.cap.max_recv_sge;
/*
* On some HW ibv_create_qp can increase max_recv_wr value more than
* it really supports. So, alignment with device capability is needed.
*/
info->rx_attr->size = MIN(init_attr.cap.max_recv_wr,
device_attr->max_qp_wr);
ibv_destroy_qp(qp);
err2:
ibv_destroy_cq(cq);
err1:
ibv_dealloc_pd(pd);
return ret;
}