static struct verbs_context *qelr_alloc_context(struct ibv_device *ibdev,
int cmd_fd,
void *private_data)
{
struct qelr_devctx *ctx;
struct qelr_get_context cmd;
struct qelr_get_context_resp resp;
ctx = verbs_init_and_alloc_context(ibdev, cmd_fd, ctx, ibv_ctx,
RDMA_DRIVER_QEDR);
if (!ctx)
return NULL;
memset(&resp, 0, sizeof(resp));
qelr_open_debug_file(ctx);
qelr_set_debug_mask();
if (ibv_cmd_get_context(&ctx->ibv_ctx, &cmd.ibv_cmd, sizeof(cmd),
&resp.ibv_resp, sizeof(resp)))
goto cmd_err;
verbs_set_ops(&ctx->ibv_ctx, &qelr_ctx_ops);
ctx->kernel_page_size = sysconf(_SC_PAGESIZE);
ctx->db_pa = resp.db_pa;
ctx->db_size = resp.db_size;
ctx->max_send_wr = resp.max_send_wr;
ctx->max_recv_wr = resp.max_recv_wr;
ctx->max_srq_wr = resp.max_srq_wr;
ctx->sges_per_send_wr = resp.sges_per_send_wr;
ctx->sges_per_recv_wr = resp.sges_per_recv_wr;
ctx->sges_per_srq_wr = resp.sges_per_recv_wr;
ctx->max_cqes = resp.max_cqes;
ctx->db_addr = mmap(NULL, ctx->db_size, PROT_WRITE, MAP_SHARED,
cmd_fd, ctx->db_pa);
if (ctx->db_addr == MAP_FAILED) {
int errsv = errno;
DP_ERR(ctx->dbg_fp,
"alloc context: doorbell mapping failed resp.db_pa = %llx resp.db_size=%d context->cmd_fd=%d errno=%d\n",
resp.db_pa, resp.db_size, cmd_fd, errsv);
goto cmd_err;
}
return &ctx->ibv_ctx;
cmd_err:
qelr_err("%s: Failed to allocate context for device.\n", __func__);
qelr_close_debug_file(ctx);
verbs_uninit_context(&ctx->ibv_ctx);
free(ctx);
return NULL;
}
static struct ibv_context *mthca_alloc_context(struct ibv_device *ibdev, int cmd_fd)
{
struct mthca_context *context;
struct ibv_get_context cmd;
struct mthca_alloc_ucontext_resp resp;
int i;
context = calloc(1, sizeof *context);
if (!context)
return NULL;
context->ibv_ctx.cmd_fd = cmd_fd;
if (ibv_cmd_get_context(&context->ibv_ctx, &cmd, sizeof cmd,
&resp.ibv_resp, sizeof resp))
goto err_free;
context->num_qps = resp.qp_tab_size;
context->qp_table_shift = ffs(context->num_qps) - 1 - MTHCA_QP_TABLE_BITS;
context->qp_table_mask = (1 << context->qp_table_shift) - 1;
/*
* Need to set ibv_ctx.device because mthca_is_memfree() will
* look at it to figure out the HCA type.
*/
context->ibv_ctx.device = ibdev;
if (mthca_is_memfree(&context->ibv_ctx)) {
context->db_tab = mthca_alloc_db_tab(resp.uarc_size);
if (!context->db_tab)
goto err_free;
} else
context->db_tab = NULL;
pthread_mutex_init(&context->qp_table_mutex, NULL);
for (i = 0; i < MTHCA_QP_TABLE_SIZE; ++i)
context->qp_table[i].refcnt = 0;
context->uar = mmap(NULL, to_mdev(ibdev)->page_size, PROT_WRITE,
MAP_SHARED, cmd_fd, 0);
if (context->uar == MAP_FAILED)
goto err_db_tab;
pthread_spin_init(&context->uar_lock, PTHREAD_PROCESS_PRIVATE);
context->pd = mthca_alloc_pd(&context->ibv_ctx);
if (!context->pd)
goto err_unmap;
context->pd->context = &context->ibv_ctx;
context->ibv_ctx.ops = mthca_ctx_ops;
if (mthca_is_memfree(&context->ibv_ctx)) {
context->ibv_ctx.ops.req_notify_cq = mthca_arbel_arm_cq;
context->ibv_ctx.ops.cq_event = mthca_arbel_cq_event;
context->ibv_ctx.ops.post_send = mthca_arbel_post_send;
context->ibv_ctx.ops.post_recv = mthca_arbel_post_recv;
context->ibv_ctx.ops.post_srq_recv = mthca_arbel_post_srq_recv;
} else {
context->ibv_ctx.ops.req_notify_cq = mthca_tavor_arm_cq;
context->ibv_ctx.ops.cq_event = NULL;
context->ibv_ctx.ops.post_send = mthca_tavor_post_send;
context->ibv_ctx.ops.post_recv = mthca_tavor_post_recv;
context->ibv_ctx.ops.post_srq_recv = mthca_tavor_post_srq_recv;
}
return &context->ibv_ctx;
err_unmap:
munmap(context->uar, to_mdev(ibdev)->page_size);
err_db_tab:
mthca_free_db_tab(context->db_tab);
err_free:
free(context);
return NULL;
}
static int mlx4_init_context(struct verbs_device *v_device,
struct ibv_context *ibv_ctx, int cmd_fd)
{
struct mlx4_context *context;
struct mlx4_alloc_ucontext_req req;
struct mlx4_alloc_ucontext_resp resp;
struct mlx4_alloc_ucontext_resp_v3 resp_v3;
int i;
struct ibv_exp_device_attr dev_attrs;
struct ibv_device_attr dev_legacy_attrs;
struct mlx4_device *dev = to_mdev(&v_device->device);
unsigned int qp_tab_size;
unsigned int bf_reg_size;
unsigned int cqe_size;
int hca_clock_offset;
void *hca_clock_page = NULL;
/* verbs_context should be used for new verbs.
* memory footprint of mlx4_context and verbs_context share
* struct ibv_context.
*/
struct verbs_context *verbs_ctx = verbs_get_ctx(ibv_ctx);
struct verbs_context_exp *verbs_exp_ctx = verbs_get_exp_ctx(ibv_ctx);
memset(&req, 0, sizeof(req));
context = to_mctx(ibv_ctx);
ibv_ctx->cmd_fd = cmd_fd;
ibv_ctx->device = &v_device->device;
if (pthread_mutex_init(&context->env_mtx, NULL))
return EIO;
if (dev->driver_abi_ver > 3) {
#ifdef MLX4_WQE_FORMAT
req.lib_caps = MLX4_USER_DEV_CAP_WQE_FORMAT;
#endif
if (ibv_cmd_get_context(ibv_ctx, &req.cmd, sizeof(req),
&resp.ibv_resp, sizeof(resp)))
return errno;
VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof(resp));
qp_tab_size = resp.qp_tab_size;
bf_reg_size = resp.bf_reg_size;
context->bf_regs_per_page = resp.bf_regs_per_page;
cqe_size = resp.cqe_size;
} else {
if (ibv_cmd_get_context(ibv_ctx, &req.cmd, sizeof(req.cmd),
&resp_v3.ibv_resp, sizeof(resp_v3)))
return errno;
VALGRIND_MAKE_MEM_DEFINED(&resp_v3, sizeof(resp_v3));
qp_tab_size = resp_v3.qp_tab_size;
bf_reg_size = resp_v3.bf_reg_size;
context->bf_regs_per_page = resp_v3.bf_regs_per_page;
cqe_size = 32;
}
context->num_qps = qp_tab_size;
context->qp_table_shift = ffs(context->num_qps) - 1 - MLX4_QP_TABLE_BITS;
context->qp_table_mask = (1 << context->qp_table_shift) - 1;
context->cqe_size = cqe_size;
for (i = 0; i < MLX4_PORTS_NUM; ++i)
context->port_query_cache[i].valid = 0;
pthread_mutex_init(&context->qp_table_mutex, NULL);
for (i = 0; i < MLX4_QP_TABLE_SIZE; ++i)
context->qp_table[i].refcnt = 0;
for (i = 0; i < MLX4_NUM_DB_TYPE; ++i)
context->db_list[i] = NULL;
mlx4_init_xsrq_table(&context->xsrq_table, qp_tab_size);
pthread_mutex_init(&context->db_list_mutex, NULL);
context->uar = mmap(NULL, dev->page_size, PROT_WRITE,
MAP_SHARED, cmd_fd, 0);
if (context->uar == MAP_FAILED)
return errno;
if (bf_reg_size) {
context->bfs.page = mmap(NULL, dev->page_size,
PROT_WRITE, MAP_SHARED, cmd_fd,
dev->page_size);
if (context->bfs.page == MAP_FAILED) {
fprintf(stderr, PFX "Warning: BlueFlame available, "
"but failed to mmap() BlueFlame page.\n");
context->bfs.page = NULL;
context->bfs.buf_size = 0;
context->bfs.num_dedic_bfs = 0;
} else {
context->bfs.num_dedic_bfs = min(context->bf_regs_per_page - 1,
MLX4_MAX_BFS_IN_PAGE - 1);
context->bfs.buf_size = bf_reg_size / 2;
mlx4_spinlock_init(&context->bfs.dedic_bf_lock, !mlx4_single_threaded);
context->bfs.cmn_bf.address = context->bfs.page;
mlx4_lock_init(&context->bfs.cmn_bf.lock,
!mlx4_single_threaded,
mlx4_get_locktype());
context->bfs.dedic_bf_free = context->bfs.num_dedic_bfs;
for (i = 0; i < context->bfs.num_dedic_bfs; i++) {
context->bfs.dedic_bf[i].address = context->bfs.page + (i + 1) * MLX4_BFS_STRIDE;
context->bfs.dedic_bf_used[i] = 0;
}
}
} else {
context->bfs.page = NULL;
context->bfs.buf_size = 0;
context->bfs.num_dedic_bfs = 0;
}
mlx4_spinlock_init(&context->uar_lock, !mlx4_single_threaded);
mlx4_spinlock_init(&context->send_db_lock, !mlx4_single_threaded);
INIT_LIST_HEAD(&context->send_db_list);
mlx4_spinlock_init(&context->hugetlb_lock, !mlx4_single_threaded);
INIT_LIST_HEAD(&context->hugetlb_list);
pthread_mutex_init(&context->task_mutex, NULL);
memset(&dev_attrs, 0, sizeof(dev_attrs));
dev_attrs.comp_mask = IBV_EXP_DEVICE_ATTR_WITH_TIMESTAMP_MASK |
IBV_EXP_DEVICE_ATTR_WITH_HCA_CORE_CLOCK |
IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS |
IBV_EXP_DEVICE_ATTR_MAX_CTX_RES_DOMAIN;
if (mlx4_exp_query_device(ibv_ctx, &dev_attrs)) {
if (mlx4_query_device(ibv_ctx, &dev_legacy_attrs))
goto query_free;
memcpy(&dev_attrs, &dev_legacy_attrs, sizeof(dev_legacy_attrs));
}
context->max_qp_wr = dev_attrs.max_qp_wr;
context->max_sge = dev_attrs.max_sge;
context->max_cqe = dev_attrs.max_cqe;
context->exp_device_cap_flags = dev_attrs.exp_device_cap_flags;
if (dev_attrs.comp_mask & IBV_EXP_DEVICE_ATTR_MAX_CTX_RES_DOMAIN)
context->max_ctx_res_domain = dev_attrs.max_ctx_res_domain;
VALGRIND_MAKE_MEM_DEFINED(&context->hca_core_clock, sizeof(context->hca_core_clock));
if (dev_attrs.comp_mask & IBV_EXP_DEVICE_ATTR_WITH_HCA_CORE_CLOCK) {
if (dev_attrs.hca_core_clock)
context->core_clk.mult = ((1ull * 1000) << 29) /
dev_attrs.hca_core_clock;
else
context->core_clk.mult = 0;
context->core_clk.shift = 29;
context->core_clk.mask = dev_attrs.timestamp_mask;
if (ioctl(cmd_fd, MLX4_IOCHWCLOCKOFFSET,
&hca_clock_offset) >= 0) {
VALGRIND_MAKE_MEM_DEFINED(&hca_clock_offset, sizeof(hca_clock_offset));
context->core_clk.offset = hca_clock_offset;
hca_clock_page = mmap(NULL, hca_clock_offset +
sizeof(context->core_clk.mask),
PROT_READ, MAP_SHARED, cmd_fd,
dev->page_size *
(MLX4_IB_MMAP_GET_HW_CLOCK));
if (hca_clock_page == MAP_FAILED) {
fprintf(stderr, PFX
"Warning: Timestamp available,\n"
"but failed to mmap() hca core "
"clock page.\n");
} else {
context->hca_core_clock = hca_clock_page +
context->core_clk.offset;
}
}
}
ibv_ctx->ops = mlx4_ctx_ops;
verbs_ctx->has_comp_mask |= VERBS_CONTEXT_XRCD | VERBS_CONTEXT_SRQ |
VERBS_CONTEXT_QP;
verbs_set_ctx_op(verbs_ctx, close_xrcd, mlx4_close_xrcd);
verbs_set_ctx_op(verbs_ctx, open_xrcd, mlx4_open_xrcd);
verbs_set_ctx_op(verbs_ctx, create_srq_ex, mlx4_create_srq_ex);
verbs_set_ctx_op(verbs_ctx, get_srq_num, verbs_get_srq_num);
verbs_set_ctx_op(verbs_ctx, create_qp_ex, mlx4_create_qp_ex);
verbs_set_ctx_op(verbs_ctx, open_qp, mlx4_open_qp);
verbs_set_ctx_op(verbs_ctx, create_flow, ibv_cmd_create_flow);
verbs_set_ctx_op(verbs_ctx, destroy_flow, ibv_cmd_destroy_flow);
/*
* Set experimental verbs
*/
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_ibv_reg_shared_mr, mlx4_reg_shared_mr);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_ibv_create_flow, ibv_exp_cmd_create_flow);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_ibv_destroy_flow, ibv_exp_cmd_destroy_flow);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_ibv_create_ah, mlx4_exp_create_ah);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_query_device, mlx4_exp_query_device);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_create_qp, mlx4_exp_create_qp);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_modify_qp, mlx4_exp_modify_qp);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_query_port, mlx4_exp_query_port);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_modify_cq, mlx4_modify_cq);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_post_task, mlx4_post_task);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_set_legacy_xrc, mlx4_set_legacy_xrc);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_get_legacy_xrc, mlx4_get_legacy_xrc);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_ibv_poll_cq, mlx4_exp_poll_cq);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_create_cq, mlx4_create_cq_ex);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_query_values, mlx4_query_values);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_reg_mr, mlx4_exp_reg_mr);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_post_send, mlx4_exp_post_send);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_bind_mw, mlx4_exp_bind_mw);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_rereg_mr, mlx4_exp_rereg_mr);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_dereg_mr, mlx4_exp_dereg_mr);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_create_res_domain, mlx4_exp_create_res_domain);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_destroy_res_domain, mlx4_exp_destroy_res_domain);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_query_intf, mlx4_exp_query_intf);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_release_intf, mlx4_exp_release_intf);
return 0;
query_free:
munmap(context->uar, dev->page_size);
if (context->bfs.page)
munmap(context->bfs.page, dev->page_size);
if (hca_clock_page)
munmap(hca_clock_page, hca_clock_offset +
sizeof(context->core_clk.mask));
return errno;
}
static int mlx5_alloc_context(struct verbs_device *vdev,
struct ibv_context *ctx, int cmd_fd)
{
struct mlx5_context *context;
struct mlx5_alloc_ucontext req;
struct mlx5_exp_alloc_ucontext_resp resp;
struct ibv_device *ibdev = &vdev->device;
struct verbs_context *verbs_ctx = verbs_get_ctx(ctx);
struct ibv_exp_device_attr attr;
int i;
int page_size = to_mdev(ibdev)->page_size;
int tot_uuars;
int low_lat_uuars;
int gross_uuars;
int j;
int uar_mapped;
off_t offset;
int err;
context = to_mctx(ctx);
if (pthread_mutex_init(&context->env_mtx, NULL))
return -1;
context->ibv_ctx.cmd_fd = cmd_fd;
memset(&resp, 0, sizeof(resp));
if (gethostname(context->hostname, sizeof(context->hostname)))
strcpy(context->hostname, "host_unknown");
tot_uuars = get_total_uuars();
gross_uuars = tot_uuars / MLX5_NUM_UUARS_PER_PAGE * 4;
context->bfs = calloc(gross_uuars, sizeof *context->bfs);
if (!context->bfs) {
errno = ENOMEM;
goto err_free;
}
low_lat_uuars = get_num_low_lat_uuars();
if (low_lat_uuars > tot_uuars - 1) {
errno = ENOMEM;
goto err_free_bf;
}
memset(&req, 0, sizeof(req));
req.total_num_uuars = tot_uuars;
req.num_low_latency_uuars = low_lat_uuars;
if (ibv_cmd_get_context(&context->ibv_ctx, &req.ibv_req, sizeof req,
&resp.ibv_resp, sizeof resp))
goto err_free_bf;
context->max_num_qps = resp.qp_tab_size;
context->bf_reg_size = resp.bf_reg_size;
context->tot_uuars = resp.tot_uuars;
context->low_lat_uuars = low_lat_uuars;
context->cache_line_size = resp.cache_line_size;
context->max_sq_desc_sz = resp.max_sq_desc_sz;
context->max_rq_desc_sz = resp.max_rq_desc_sz;
context->max_send_wqebb = resp.max_send_wqebb;
context->num_ports = resp.num_ports;
context->max_recv_wr = resp.max_recv_wr;
context->max_srq_recv_wr = resp.max_srq_recv_wr;
context->max_desc_sz_sq_dc = resp.max_desc_sz_sq_dc;
context->atomic_sizes_dc = resp.atomic_sizes_dc;
context->compact_av = resp.flags & MLX5_CAP_COMPACT_AV;
if (resp.exp_data.comp_mask & MLX5_EXP_ALLOC_CTX_RESP_MASK_CQE_COMP_MAX_NUM)
context->cqe_comp_max_num = resp.exp_data.cqe_comp_max_num;
if (resp.exp_data.comp_mask & MLX5_EXP_ALLOC_CTX_RESP_MASK_CQE_VERSION)
context->cqe_version = resp.exp_data.cqe_version;
if (resp.exp_data.comp_mask & MLX5_EXP_ALLOC_CTX_RESP_MASK_RROCE_UDP_SPORT_MIN)
context->rroce_udp_sport_min = resp.exp_data.rroce_udp_sport_min;
if (resp.exp_data.comp_mask & MLX5_EXP_ALLOC_CTX_RESP_MASK_RROCE_UDP_SPORT_MAX)
context->rroce_udp_sport_max = resp.exp_data.rroce_udp_sport_max;
ctx->ops = mlx5_ctx_ops;
if (context->cqe_version) {
if (context->cqe_version == 1) {
ctx->ops.poll_cq = mlx5_poll_cq_1;
} else {
printf("Unsupported cqe_vesion = %d, stay on cqe version 0\n",
context->cqe_version);
context->cqe_version = 0;
}
}
attr.comp_mask = IBV_EXP_DEVICE_ATTR_RESERVED - 1;
err = mlx5_query_device_ex(ctx, &attr);
if (!err) {
if (attr.comp_mask & IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS)
context->exp_device_cap_flags = attr.exp_device_cap_flags;
if (attr.comp_mask & IBV_EXP_DEVICE_ATTR_MAX_CTX_RES_DOMAIN) {
context->max_ctx_res_domain = attr.max_ctx_res_domain;
mlx5_spinlock_init(&context->send_db_lock, !mlx5_single_threaded);
INIT_LIST_HEAD(&context->send_wc_db_list);
INIT_LIST_HEAD(&context->wc_uar_list);
}
if (resp.exp_data.comp_mask & MLX5_EXP_ALLOC_CTX_RESP_MASK_HCA_CORE_CLOCK_OFFSET) {
context->core_clock.offset =
resp.exp_data.hca_core_clock_offset &
(to_mdev(ibdev)->page_size - 1);
mlx5_map_internal_clock(to_mdev(ibdev), ctx);
if (attr.hca_core_clock)
context->core_clock.mult = ((1ull * 1000) << 21) /
attr.hca_core_clock;
else
context->core_clock.mult = 0;
/* ConnectX-4 supports 64bit timestamp. We choose these numbers
* in order to make sure that after arithmetic operations,
* we don't overflow a 64bit variable.
*/
context->core_clock.shift = 21;
context->core_clock.mask = (1ULL << 49) - 1;
}
}
pthread_mutex_init(&context->rsc_table_mutex, NULL);
pthread_mutex_init(&context->srq_table_mutex, NULL);
for (i = 0; i < MLX5_QP_TABLE_SIZE; ++i)
context->rsc_table[i].refcnt = 0;
for (i = 0; i < MLX5_QP_TABLE_SIZE; ++i)
context->uidx_table[i].refcnt = 0;
context->db_list = NULL;
pthread_mutex_init(&context->db_list_mutex, NULL);
context->prefer_bf = get_always_bf(&context->ibv_ctx);
context->shut_up_bf = get_shut_up_bf(&context->ibv_ctx);
context->enable_cqe_comp = get_cqe_comp(&context->ibv_ctx);
mlx5_use_mutex = get_use_mutex(&context->ibv_ctx);
offset = 0;
set_command(MLX5_MMAP_MAP_DC_INFO_PAGE, &offset);
context->cc.buf = mmap(NULL, 4096 * context->num_ports, PROT_READ,
MAP_PRIVATE, cmd_fd, page_size * offset);
if (context->cc.buf == MAP_FAILED)
context->cc.buf = NULL;
mlx5_single_threaded = single_threaded_app(&context->ibv_ctx);
for (i = 0; i < resp.tot_uuars / MLX5_NUM_UUARS_PER_PAGE; ++i) {
uar_mapped = 0;
/* Don't map UAR to WC if BF is not used */
if (!context->shut_up_bf) {
context->uar[i].regs = mlx5_uar_mmap(i, MLX5_MMAP_GET_WC_PAGES_CMD, page_size, cmd_fd);
if (context->uar[i].regs != MAP_FAILED) {
context->uar[i].map_type = MLX5_UAR_MAP_WC;
uar_mapped = 1;
}
}
if (!uar_mapped) {
context->uar[i].regs = mlx5_uar_mmap(i, MLX5_MMAP_GET_NC_PAGES_CMD, page_size, cmd_fd);
if (context->uar[i].regs != MAP_FAILED) {
context->uar[i].map_type = MLX5_UAR_MAP_NC;
uar_mapped = 1;
}
}
if (!uar_mapped) {
/* for backward compatibility with old kernel driver */
context->uar[i].regs = mlx5_uar_mmap(i, MLX5_MMAP_GET_REGULAR_PAGES_CMD, page_size, cmd_fd);
if (context->uar[i].regs != MAP_FAILED) {
context->uar[i].map_type = MLX5_UAR_MAP_WC;
uar_mapped = 1;
}
}
if (!uar_mapped) {
context->uar[i].regs = NULL;
goto err_free_cc;
}
}
for (j = 0; j < gross_uuars; ++j) {
context->bfs[j].reg = context->uar[j / 4].regs +
MLX5_BF_OFFSET + (j % 4) * context->bf_reg_size;
context->bfs[j].need_lock = need_uuar_lock(context, j) &&
context->uar[j / 4].map_type == MLX5_UAR_MAP_WC;
mlx5_lock_init(&context->bfs[j].lock,
!mlx5_single_threaded,
mlx5_get_locktype());
context->bfs[j].offset = 0;
if (context->uar[j / 4].map_type == MLX5_UAR_MAP_WC) {
context->bfs[j].buf_size = context->bf_reg_size / 2;
context->bfs[j].db_method = (context->bfs[j].need_lock && !mlx5_single_threaded) ?
MLX5_DB_METHOD_BF :
(mlx5_single_threaded && wc_auto_evict_size() == 64 ?
MLX5_DB_METHOD_DEDIC_BF_1_THREAD :
MLX5_DB_METHOD_DEDIC_BF);
} else {
context->bfs[j].db_method = MLX5_DB_METHOD_DB;
}
context->bfs[j].uuarn = j;
}
mlx5_lock_init(&context->lock32,
!mlx5_single_threaded,
mlx5_get_locktype());
mlx5_spinlock_init(&context->hugetlb_lock, !mlx5_single_threaded);
INIT_LIST_HEAD(&context->hugetlb_list);
pthread_mutex_init(&context->task_mutex, NULL);
set_extended(verbs_ctx);
set_experimental(ctx);
for (i = 0; i < MLX5_MAX_PORTS_NUM; ++i)
context->port_query_cache[i].valid = 0;
return 0;
err_free_cc:
if (context->cc.buf)
munmap(context->cc.buf, 4096 * context->num_ports);
if (context->hca_core_clock)
munmap(context->hca_core_clock - context->core_clock.offset,
to_mdev(ibdev)->page_size);
err_free_bf:
free(context->bfs);
err_free:
for (i = 0; i < MLX5_MAX_UAR_PAGES; ++i) {
if (context->uar[i].regs)
munmap(context->uar[i].regs, page_size);
}
close_debug_file(context);
return errno;
}
static struct verbs_context *mthca_alloc_context(struct ibv_device *ibdev,
int cmd_fd,
void *private_data)
{
struct mthca_context *context;
struct ibv_get_context cmd;
struct umthca_alloc_ucontext_resp resp;
int i;
context = verbs_init_and_alloc_context(ibdev, cmd_fd, context, ibv_ctx,
RDMA_DRIVER_MTHCA);
if (!context)
return NULL;
if (ibv_cmd_get_context(&context->ibv_ctx, &cmd, sizeof cmd,
&resp.ibv_resp, sizeof resp))
goto err_free;
context->num_qps = resp.qp_tab_size;
context->qp_table_shift = ffs(context->num_qps) - 1 - MTHCA_QP_TABLE_BITS;
context->qp_table_mask = (1 << context->qp_table_shift) - 1;
if (mthca_is_memfree(&context->ibv_ctx.context)) {
context->db_tab = mthca_alloc_db_tab(resp.uarc_size);
if (!context->db_tab)
goto err_free;
} else
context->db_tab = NULL;
pthread_mutex_init(&context->qp_table_mutex, NULL);
for (i = 0; i < MTHCA_QP_TABLE_SIZE; ++i)
context->qp_table[i].refcnt = 0;
context->uar = mmap(NULL, to_mdev(ibdev)->page_size, PROT_WRITE,
MAP_SHARED, cmd_fd, 0);
if (context->uar == MAP_FAILED)
goto err_db_tab;
pthread_spin_init(&context->uar_lock, PTHREAD_PROCESS_PRIVATE);
context->pd = mthca_alloc_pd(&context->ibv_ctx.context);
if (!context->pd)
goto err_unmap;
context->pd->context = &context->ibv_ctx.context;
verbs_set_ops(&context->ibv_ctx, &mthca_ctx_common_ops);
if (mthca_is_memfree(&context->ibv_ctx.context))
verbs_set_ops(&context->ibv_ctx, &mthca_ctx_arbel_ops);
else
verbs_set_ops(&context->ibv_ctx, &mthca_ctx_tavor_ops);
return &context->ibv_ctx;
err_unmap:
munmap(context->uar, to_mdev(ibdev)->page_size);
err_db_tab:
mthca_free_db_tab(context->db_tab);
err_free:
verbs_uninit_context(&context->ibv_ctx);
free(context);
return NULL;
}
static int mlx5_alloc_context(struct verbs_device *vdev,
struct ibv_context *ctx, int cmd_fd)
{
struct mlx5_context *context;
struct mlx5_alloc_ucontext req;
struct mlx5_alloc_ucontext_resp resp;
struct ibv_device *ibdev = &vdev->device;
struct verbs_context *verbs_ctx = verbs_get_ctx(ctx);
int i;
int page_size = to_mdev(ibdev)->page_size;
int tot_uuars;
int low_lat_uuars;
int gross_uuars;
int j;
off_t offset;
mlx5_single_threaded = single_threaded_app();
context = to_mctx(ctx);
context->ibv_ctx.cmd_fd = cmd_fd;
memset(&resp, 0, sizeof(resp));
open_debug_file(context);
set_debug_mask();
set_freeze_on_error();
if (gethostname(context->hostname, sizeof(context->hostname)))
strcpy(context->hostname, "host_unknown");
tot_uuars = get_total_uuars();
if (tot_uuars <= 0) {
if (tot_uuars == 0)
errno = EINVAL;
else
errno = -tot_uuars;
goto err_free;
}
gross_uuars = tot_uuars / MLX5_NUM_UUARS_PER_PAGE * 4;
context->bfs = calloc(gross_uuars, sizeof *context->bfs);
if (!context->bfs) {
errno = ENOMEM;
goto err_free;
}
low_lat_uuars = get_num_low_lat_uuars();
if (low_lat_uuars < 0) {
errno = ENOMEM;
goto err_free_bf;
}
if (low_lat_uuars > tot_uuars - 1) {
errno = ENOMEM;
goto err_free_bf;
}
memset(&req, 0, sizeof(req));
req.total_num_uuars = tot_uuars;
req.num_low_latency_uuars = low_lat_uuars;
if (ibv_cmd_get_context(&context->ibv_ctx, &req.ibv_req, sizeof req,
&resp.ibv_resp, sizeof resp))
goto err_free_bf;
context->max_num_qps = resp.qp_tab_size;
context->bf_reg_size = resp.bf_reg_size;
context->tot_uuars = resp.tot_uuars;
context->low_lat_uuars = low_lat_uuars;
context->cache_line_size = resp.cache_line_size;
context->max_sq_desc_sz = resp.max_sq_desc_sz;
context->max_rq_desc_sz = resp.max_rq_desc_sz;
context->max_send_wqebb = resp.max_send_wqebb;
context->num_ports = resp.num_ports;
context->max_recv_wr = resp.max_recv_wr;
context->max_srq_recv_wr = resp.max_srq_recv_wr;
context->max_desc_sz_sq_dc = resp.max_desc_sz_sq_dc;
context->atomic_sizes_dc = resp.atomic_sizes_dc;
pthread_mutex_init(&context->rsc_table_mutex, NULL);
pthread_mutex_init(&context->srq_table_mutex, NULL);
for (i = 0; i < MLX5_QP_TABLE_SIZE; ++i)
context->rsc_table[i].refcnt = 0;
context->db_list = NULL;
pthread_mutex_init(&context->db_list_mutex, NULL);
for (i = 0; i < resp.tot_uuars / MLX5_NUM_UUARS_PER_PAGE; ++i) {
offset = 0;
set_command(MLX5_MMAP_GET_REGULAR_PAGES_CMD, &offset);
set_index(i, &offset);
context->uar[i] = mmap(NULL, to_mdev(ibdev)->page_size, PROT_WRITE,
MAP_SHARED, cmd_fd,
page_size * offset);
if (context->uar[i] == MAP_FAILED) {
context->uar[i] = NULL;
goto err_free_bf;
}
}
for (j = 0; j < gross_uuars; ++j) {
context->bfs[j].reg = context->uar[j / 4] +
MLX5_BF_OFFSET + (j % 4) * context->bf_reg_size;
context->bfs[j].need_lock = need_uuar_lock(context, j);
mlx5_spinlock_init(&context->bfs[j].lock);
context->bfs[j].offset = 0;
if (j)
context->bfs[j].buf_size = context->bf_reg_size / 2;
context->bfs[j].uuarn = j;
}
mlx5_spinlock_init(&context->lock32);
context->prefer_bf = get_always_bf();
context->shut_up_bf = get_shut_up_bf();
mlx5_read_env(ibdev, context);
mlx5_spinlock_init(&context->hugetlb_lock);
INIT_LIST_HEAD(&context->hugetlb_list);
pthread_mutex_init(&context->task_mutex, NULL);
ctx->ops = mlx5_ctx_ops;
set_extended(verbs_ctx);
set_experimental(ctx);
return 0;
err_free_bf:
free(context->bfs);
err_free:
for (i = 0; i < MLX5_MAX_UAR_PAGES; ++i) {
if (context->uar[i])
munmap(context->uar[i], page_size);
}
close_debug_file(context);
return errno;
}
static struct ibv_context *mlx4_alloc_context(struct ibv_device *ibdev, int cmd_fd)
{
struct mlx4_context *context;
struct ibv_get_context cmd;
struct mlx4_alloc_ucontext_resp resp;
int i;
struct ibv_device_attr dev_attrs;
context = calloc(1, sizeof *context);
if (!context)
return NULL;
context->ibv_ctx.cmd_fd = cmd_fd;
if (ibv_cmd_get_context(&context->ibv_ctx, &cmd, sizeof cmd,
&resp.ibv_resp, sizeof resp))
goto err_free;
context->num_qps = resp.qp_tab_size;
context->qp_table_shift = ffs(context->num_qps) - 1 - MLX4_QP_TABLE_BITS;
context->qp_table_mask = (1 << context->qp_table_shift) - 1;
pthread_mutex_init(&context->qp_table_mutex, NULL);
for (i = 0; i < MLX4_QP_TABLE_SIZE; ++i)
context->qp_table[i].refcnt = 0;
context->num_xrc_srqs = resp.qp_tab_size;
context->xrc_srq_table_shift = ffs(context->num_xrc_srqs) - 1
- MLX4_XRC_SRQ_TABLE_BITS;
context->xrc_srq_table_mask = (1 << context->xrc_srq_table_shift) - 1;
pthread_mutex_init(&context->xrc_srq_table_mutex, NULL);
for (i = 0; i < MLX4_XRC_SRQ_TABLE_SIZE; ++i)
context->xrc_srq_table[i].refcnt = 0;
for (i = 0; i < MLX4_NUM_DB_TYPE; ++i)
context->db_list[i] = NULL;
pthread_mutex_init(&context->db_list_mutex, NULL);
context->uar = mmap(NULL, to_mdev(ibdev)->page_size, PROT_WRITE,
MAP_SHARED, cmd_fd, 0);
if (context->uar == MAP_FAILED)
goto err_free;
if (resp.bf_reg_size) {
context->bf_page = mmap(NULL, to_mdev(ibdev)->page_size,
PROT_WRITE, MAP_SHARED, cmd_fd,
to_mdev(ibdev)->page_size);
if (context->bf_page == MAP_FAILED) {
fprintf(stderr, PFX "Warning: BlueFlame available, "
"but failed to mmap() BlueFlame page.\n");
context->bf_page = NULL;
context->bf_buf_size = 0;
} else {
context->bf_buf_size = resp.bf_reg_size / 2;
context->bf_offset = 0;
pthread_spin_init(&context->bf_lock, PTHREAD_PROCESS_PRIVATE);
}
} else {
context->bf_page = NULL;
context->bf_buf_size = 0;
}
context->cqe_size = resp.cqe_size;
pthread_spin_init(&context->uar_lock, PTHREAD_PROCESS_PRIVATE);
context->ibv_ctx.ops = mlx4_ctx_ops;
#ifdef HAVE_IBV_XRC_OPS
context->ibv_ctx.more_ops = &mlx4_more_ops;
#endif
if (mlx4_query_device(&context->ibv_ctx, &dev_attrs))
goto query_free;
context->max_qp_wr = dev_attrs.max_qp_wr;
context->max_sge = dev_attrs.max_sge;
context->max_cqe = dev_attrs.max_cqe;
if (!(dev_attrs.device_cap_flags & IBV_DEVICE_XRC)) {
fprintf(stderr, PFX "There is a mismatch between "
"the kernel and the userspace libraries: "
"Kernel does not support XRC. Exiting.\n");
goto query_free;
}
return &context->ibv_ctx;
query_free:
munmap(context->uar, to_mdev(ibdev)->page_size);
if (context->bf_page)
munmap(context->bf_page, to_mdev(ibdev)->page_size);
err_free:
free(context);
return NULL;
}
