static void set_experimental(struct ibv_context *ctx)
{
struct verbs_context_exp *verbs_exp_ctx = verbs_get_exp_ctx(ctx);
struct mlx5_context *mctx = to_mctx(ctx);
verbs_set_exp_ctx_op(verbs_exp_ctx, create_dct, mlx5_create_dct);
verbs_set_exp_ctx_op(verbs_exp_ctx, destroy_dct, mlx5_destroy_dct);
verbs_set_exp_ctx_op(verbs_exp_ctx, query_dct, mlx5_query_dct);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_arm_dct, mlx5_arm_dct);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_query_device, mlx5_query_device_ex);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_create_qp, mlx5_exp_create_qp);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_modify_qp, mlx5_modify_qp_ex);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_get_legacy_xrc, mlx5_get_legacy_xrc);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_set_legacy_xrc, mlx5_set_legacy_xrc);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_modify_cq, mlx5_modify_cq);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_create_cq, mlx5_create_cq_ex);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_ibv_poll_cq, mlx5_poll_cq_ex);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_post_task, mlx5_post_task);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_reg_mr, mlx5_exp_reg_mr);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_post_send, mlx5_exp_post_send);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_alloc_mkey_list_memory, mlx5_alloc_mkey_mem);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_dealloc_mkey_list_memory, mlx5_free_mkey_mem);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_query_mkey, mlx5_query_mkey);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_create_mr, mlx5_create_mr);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_prefetch_mr,
mlx5_prefetch_mr);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_dereg_mr, mlx5_exp_dereg_mr);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_poll_dc_info, mlx5_poll_dc_info);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_create_wq, mlx5_exp_create_wq);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_modify_wq, mlx5_exp_modify_wq);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_destroy_wq, mlx5_exp_destroy_wq);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_ibv_create_flow, ibv_exp_cmd_create_flow);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_ibv_destroy_flow, ibv_exp_cmd_destroy_flow);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_create_rwq_ind_table, mlx5_exp_create_rwq_ind_table);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_destroy_rwq_ind_table, mlx5_exp_destroy_rwq_ind_table);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_create_res_domain, mlx5_exp_create_res_domain);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_destroy_res_domain, mlx5_exp_destroy_res_domain);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_query_intf, mlx5_exp_query_intf);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_release_intf, mlx5_exp_release_intf);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_query_port, mlx5_exp_query_port);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_ibv_create_ah, mlx5_exp_create_ah);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_query_values, mlx5_exp_query_values);
verbs_set_exp_ctx_op(verbs_exp_ctx, alloc_ec_calc, mlx5_alloc_ec_calc);
verbs_set_exp_ctx_op(verbs_exp_ctx, dealloc_ec_calc, mlx5_dealloc_ec_calc);
verbs_set_exp_ctx_op(verbs_exp_ctx, ec_encode_async, mlx5_ec_encode_async);
verbs_set_exp_ctx_op(verbs_exp_ctx, ec_encode_sync, mlx5_ec_encode_sync);
verbs_set_exp_ctx_op(verbs_exp_ctx, ec_decode_async, mlx5_ec_decode_async);
verbs_set_exp_ctx_op(verbs_exp_ctx, ec_decode_sync, mlx5_ec_decode_sync);
verbs_set_exp_ctx_op(verbs_exp_ctx, ec_poll, mlx5_ec_poll);
verbs_set_exp_ctx_op(verbs_exp_ctx, ec_encode_send, mlx5_ec_encode_send);
verbs_set_exp_ctx_op(verbs_exp_ctx, ec_update_async, mlx5_ec_update_async);
verbs_set_exp_ctx_op(verbs_exp_ctx, ec_update_sync, mlx5_ec_update_sync);
if (mctx->cqe_version == 1)
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_ibv_poll_cq,
mlx5_poll_cq_ex_1);
else
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_ibv_poll_cq,
mlx5_poll_cq_ex);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_peer_commit_qp, mlx5_exp_peer_commit_qp);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_rollback_send, mlx5_exp_rollback_send);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_peer_peek_cq, mlx5_exp_peer_peek_cq);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_peer_abort_peek_cq, mlx5_exp_peer_abort_peek_cq);
}
static int mlx4_init_context(struct verbs_device *v_device,
struct ibv_context *ibv_ctx, int cmd_fd)
{
struct mlx4_context *context;
struct mlx4_alloc_ucontext_req req;
struct mlx4_alloc_ucontext_resp resp;
struct mlx4_alloc_ucontext_resp_v3 resp_v3;
int i;
struct ibv_exp_device_attr dev_attrs;
struct ibv_device_attr dev_legacy_attrs;
struct mlx4_device *dev = to_mdev(&v_device->device);
unsigned int qp_tab_size;
unsigned int bf_reg_size;
unsigned int cqe_size;
int hca_clock_offset;
void *hca_clock_page = NULL;
/* verbs_context should be used for new verbs.
* memory footprint of mlx4_context and verbs_context share
* struct ibv_context.
*/
struct verbs_context *verbs_ctx = verbs_get_ctx(ibv_ctx);
struct verbs_context_exp *verbs_exp_ctx = verbs_get_exp_ctx(ibv_ctx);
memset(&req, 0, sizeof(req));
context = to_mctx(ibv_ctx);
ibv_ctx->cmd_fd = cmd_fd;
ibv_ctx->device = &v_device->device;
if (pthread_mutex_init(&context->env_mtx, NULL))
return EIO;
if (dev->driver_abi_ver > 3) {
#ifdef MLX4_WQE_FORMAT
req.lib_caps = MLX4_USER_DEV_CAP_WQE_FORMAT;
#endif
if (ibv_cmd_get_context(ibv_ctx, &req.cmd, sizeof(req),
&resp.ibv_resp, sizeof(resp)))
return errno;
VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof(resp));
qp_tab_size = resp.qp_tab_size;
bf_reg_size = resp.bf_reg_size;
context->bf_regs_per_page = resp.bf_regs_per_page;
cqe_size = resp.cqe_size;
} else {
if (ibv_cmd_get_context(ibv_ctx, &req.cmd, sizeof(req.cmd),
&resp_v3.ibv_resp, sizeof(resp_v3)))
return errno;
VALGRIND_MAKE_MEM_DEFINED(&resp_v3, sizeof(resp_v3));
qp_tab_size = resp_v3.qp_tab_size;
bf_reg_size = resp_v3.bf_reg_size;
context->bf_regs_per_page = resp_v3.bf_regs_per_page;
cqe_size = 32;
}
context->num_qps = qp_tab_size;
context->qp_table_shift = ffs(context->num_qps) - 1 - MLX4_QP_TABLE_BITS;
context->qp_table_mask = (1 << context->qp_table_shift) - 1;
context->cqe_size = cqe_size;
for (i = 0; i < MLX4_PORTS_NUM; ++i)
context->port_query_cache[i].valid = 0;
pthread_mutex_init(&context->qp_table_mutex, NULL);
for (i = 0; i < MLX4_QP_TABLE_SIZE; ++i)
context->qp_table[i].refcnt = 0;
for (i = 0; i < MLX4_NUM_DB_TYPE; ++i)
context->db_list[i] = NULL;
mlx4_init_xsrq_table(&context->xsrq_table, qp_tab_size);
pthread_mutex_init(&context->db_list_mutex, NULL);
context->uar = mmap(NULL, dev->page_size, PROT_WRITE,
MAP_SHARED, cmd_fd, 0);
if (context->uar == MAP_FAILED)
return errno;
if (bf_reg_size) {
context->bfs.page = mmap(NULL, dev->page_size,
PROT_WRITE, MAP_SHARED, cmd_fd,
dev->page_size);
if (context->bfs.page == MAP_FAILED) {
fprintf(stderr, PFX "Warning: BlueFlame available, "
"but failed to mmap() BlueFlame page.\n");
context->bfs.page = NULL;
context->bfs.buf_size = 0;
context->bfs.num_dedic_bfs = 0;
} else {
context->bfs.num_dedic_bfs = min(context->bf_regs_per_page - 1,
MLX4_MAX_BFS_IN_PAGE - 1);
context->bfs.buf_size = bf_reg_size / 2;
mlx4_spinlock_init(&context->bfs.dedic_bf_lock, !mlx4_single_threaded);
context->bfs.cmn_bf.address = context->bfs.page;
mlx4_lock_init(&context->bfs.cmn_bf.lock,
!mlx4_single_threaded,
mlx4_get_locktype());
context->bfs.dedic_bf_free = context->bfs.num_dedic_bfs;
for (i = 0; i < context->bfs.num_dedic_bfs; i++) {
context->bfs.dedic_bf[i].address = context->bfs.page + (i + 1) * MLX4_BFS_STRIDE;
context->bfs.dedic_bf_used[i] = 0;
}
}
} else {
context->bfs.page = NULL;
context->bfs.buf_size = 0;
context->bfs.num_dedic_bfs = 0;
}
mlx4_spinlock_init(&context->uar_lock, !mlx4_single_threaded);
mlx4_spinlock_init(&context->send_db_lock, !mlx4_single_threaded);
INIT_LIST_HEAD(&context->send_db_list);
mlx4_spinlock_init(&context->hugetlb_lock, !mlx4_single_threaded);
INIT_LIST_HEAD(&context->hugetlb_list);
pthread_mutex_init(&context->task_mutex, NULL);
memset(&dev_attrs, 0, sizeof(dev_attrs));
dev_attrs.comp_mask = IBV_EXP_DEVICE_ATTR_WITH_TIMESTAMP_MASK |
IBV_EXP_DEVICE_ATTR_WITH_HCA_CORE_CLOCK |
IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS |
IBV_EXP_DEVICE_ATTR_MAX_CTX_RES_DOMAIN;
if (mlx4_exp_query_device(ibv_ctx, &dev_attrs)) {
if (mlx4_query_device(ibv_ctx, &dev_legacy_attrs))
goto query_free;
memcpy(&dev_attrs, &dev_legacy_attrs, sizeof(dev_legacy_attrs));
}
context->max_qp_wr = dev_attrs.max_qp_wr;
context->max_sge = dev_attrs.max_sge;
context->max_cqe = dev_attrs.max_cqe;
context->exp_device_cap_flags = dev_attrs.exp_device_cap_flags;
if (dev_attrs.comp_mask & IBV_EXP_DEVICE_ATTR_MAX_CTX_RES_DOMAIN)
context->max_ctx_res_domain = dev_attrs.max_ctx_res_domain;
VALGRIND_MAKE_MEM_DEFINED(&context->hca_core_clock, sizeof(context->hca_core_clock));
if (dev_attrs.comp_mask & IBV_EXP_DEVICE_ATTR_WITH_HCA_CORE_CLOCK) {
if (dev_attrs.hca_core_clock)
context->core_clk.mult = ((1ull * 1000) << 29) /
dev_attrs.hca_core_clock;
else
context->core_clk.mult = 0;
context->core_clk.shift = 29;
context->core_clk.mask = dev_attrs.timestamp_mask;
if (ioctl(cmd_fd, MLX4_IOCHWCLOCKOFFSET,
&hca_clock_offset) >= 0) {
VALGRIND_MAKE_MEM_DEFINED(&hca_clock_offset, sizeof(hca_clock_offset));
context->core_clk.offset = hca_clock_offset;
hca_clock_page = mmap(NULL, hca_clock_offset +
sizeof(context->core_clk.mask),
PROT_READ, MAP_SHARED, cmd_fd,
dev->page_size *
(MLX4_IB_MMAP_GET_HW_CLOCK));
if (hca_clock_page == MAP_FAILED) {
fprintf(stderr, PFX
"Warning: Timestamp available,\n"
"but failed to mmap() hca core "
"clock page.\n");
} else {
context->hca_core_clock = hca_clock_page +
context->core_clk.offset;
}
}
}
ibv_ctx->ops = mlx4_ctx_ops;
verbs_ctx->has_comp_mask |= VERBS_CONTEXT_XRCD | VERBS_CONTEXT_SRQ |
VERBS_CONTEXT_QP;
verbs_set_ctx_op(verbs_ctx, close_xrcd, mlx4_close_xrcd);
verbs_set_ctx_op(verbs_ctx, open_xrcd, mlx4_open_xrcd);
verbs_set_ctx_op(verbs_ctx, create_srq_ex, mlx4_create_srq_ex);
verbs_set_ctx_op(verbs_ctx, get_srq_num, verbs_get_srq_num);
verbs_set_ctx_op(verbs_ctx, create_qp_ex, mlx4_create_qp_ex);
verbs_set_ctx_op(verbs_ctx, open_qp, mlx4_open_qp);
verbs_set_ctx_op(verbs_ctx, create_flow, ibv_cmd_create_flow);
verbs_set_ctx_op(verbs_ctx, destroy_flow, ibv_cmd_destroy_flow);
/*
* Set experimental verbs
*/
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_ibv_reg_shared_mr, mlx4_reg_shared_mr);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_ibv_create_flow, ibv_exp_cmd_create_flow);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_ibv_destroy_flow, ibv_exp_cmd_destroy_flow);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_ibv_create_ah, mlx4_exp_create_ah);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_query_device, mlx4_exp_query_device);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_create_qp, mlx4_exp_create_qp);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_modify_qp, mlx4_exp_modify_qp);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_query_port, mlx4_exp_query_port);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_modify_cq, mlx4_modify_cq);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_post_task, mlx4_post_task);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_set_legacy_xrc, mlx4_set_legacy_xrc);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_get_legacy_xrc, mlx4_get_legacy_xrc);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_ibv_poll_cq, mlx4_exp_poll_cq);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_create_cq, mlx4_create_cq_ex);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_query_values, mlx4_query_values);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_reg_mr, mlx4_exp_reg_mr);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_post_send, mlx4_exp_post_send);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_bind_mw, mlx4_exp_bind_mw);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_rereg_mr, mlx4_exp_rereg_mr);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_dereg_mr, mlx4_exp_dereg_mr);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_create_res_domain, mlx4_exp_create_res_domain);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_destroy_res_domain, mlx4_exp_destroy_res_domain);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_query_intf, mlx4_exp_query_intf);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_release_intf, mlx4_exp_release_intf);
return 0;
query_free:
munmap(context->uar, dev->page_size);
if (context->bfs.page)
munmap(context->bfs.page, dev->page_size);
if (hca_clock_page)
munmap(hca_clock_page, hca_clock_offset +
sizeof(context->core_clk.mask));
return errno;
}
static void set_experimental(struct ibv_context *ctx)
{
struct verbs_context_exp *verbs_exp_ctx = verbs_get_exp_ctx(ctx);
verbs_set_exp_ctx_op(verbs_exp_ctx, create_dct, mlx5_create_dct);
verbs_set_exp_ctx_op(verbs_exp_ctx, destroy_dct, mlx5_destroy_dct);
verbs_set_exp_ctx_op(verbs_exp_ctx, query_dct, mlx5_query_dct);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_arm_dct, mlx5_arm_dct);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_query_device, mlx5_query_device_ex);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_create_qp, mlx5_exp_create_qp);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_modify_qp, mlx5_modify_qp_ex);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_get_legacy_xrc, mlx5_get_legacy_xrc);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_set_legacy_xrc, mlx5_set_legacy_xrc);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_modify_cq, mlx5_modify_cq);
verbs_set_exp_ctx_op(verbs_exp_ctx, exp_create_cq, mlx5_create_cq_ex);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_ibv_poll_cq, mlx5_poll_cq_ex);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_post_task, mlx5_post_task);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_reg_mr, mlx5_exp_reg_mr);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_post_send, mlx5_exp_post_send);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_alloc_mkey_list_memory, mlx5_alloc_mkey_mem);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_dealloc_mkey_list_memory, mlx5_free_mkey_mem);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_query_mkey, mlx5_query_mkey);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_create_mr, mlx5_create_mr);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_prefetch_mr,
mlx5_prefetch_mr);
verbs_set_exp_ctx_op(verbs_exp_ctx, drv_exp_dereg_mr, mlx5_exp_dereg_mr);
}
