void read_init_vars(struct mlx5_context *ctx)
{
pthread_mutex_lock(&ctx->env_mtx);
if (!ctx->env_initialized) {
mlx5_single_threaded = single_threaded_app(&ctx->ibv_ctx);
mlx5_use_mutex = get_use_mutex(&ctx->ibv_ctx);
open_debug_file(ctx);
set_debug_mask(&ctx->ibv_ctx);
set_freeze_on_error(&ctx->ibv_ctx);
ctx->prefer_bf = get_always_bf(&ctx->ibv_ctx);
ctx->shut_up_bf = get_shut_up_bf(&ctx->ibv_ctx);
mlx5_read_env(ctx);
ctx->env_initialized = 1;
}
pthread_mutex_unlock(&ctx->env_mtx);
}
static int mlx5_alloc_context(struct verbs_device *vdev,
struct ibv_context *ctx, int cmd_fd)
{
struct mlx5_context *context;
struct mlx5_alloc_ucontext req;
struct mlx5_exp_alloc_ucontext_resp resp;
struct ibv_device *ibdev = &vdev->device;
struct verbs_context *verbs_ctx = verbs_get_ctx(ctx);
struct ibv_exp_device_attr attr;
int i;
int page_size = to_mdev(ibdev)->page_size;
int tot_uuars;
int low_lat_uuars;
int gross_uuars;
int j;
int uar_mapped;
off_t offset;
int err;
context = to_mctx(ctx);
if (pthread_mutex_init(&context->env_mtx, NULL))
return -1;
context->ibv_ctx.cmd_fd = cmd_fd;
memset(&resp, 0, sizeof(resp));
if (gethostname(context->hostname, sizeof(context->hostname)))
strcpy(context->hostname, "host_unknown");
tot_uuars = get_total_uuars();
gross_uuars = tot_uuars / MLX5_NUM_UUARS_PER_PAGE * 4;
context->bfs = calloc(gross_uuars, sizeof *context->bfs);
if (!context->bfs) {
errno = ENOMEM;
goto err_free;
}
low_lat_uuars = get_num_low_lat_uuars();
if (low_lat_uuars > tot_uuars - 1) {
errno = ENOMEM;
goto err_free_bf;
}
memset(&req, 0, sizeof(req));
req.total_num_uuars = tot_uuars;
req.num_low_latency_uuars = low_lat_uuars;
if (ibv_cmd_get_context(&context->ibv_ctx, &req.ibv_req, sizeof req,
&resp.ibv_resp, sizeof resp))
goto err_free_bf;
context->max_num_qps = resp.qp_tab_size;
context->bf_reg_size = resp.bf_reg_size;
context->tot_uuars = resp.tot_uuars;
context->low_lat_uuars = low_lat_uuars;
context->cache_line_size = resp.cache_line_size;
context->max_sq_desc_sz = resp.max_sq_desc_sz;
context->max_rq_desc_sz = resp.max_rq_desc_sz;
context->max_send_wqebb = resp.max_send_wqebb;
context->num_ports = resp.num_ports;
context->max_recv_wr = resp.max_recv_wr;
context->max_srq_recv_wr = resp.max_srq_recv_wr;
context->max_desc_sz_sq_dc = resp.max_desc_sz_sq_dc;
context->atomic_sizes_dc = resp.atomic_sizes_dc;
context->compact_av = resp.flags & MLX5_CAP_COMPACT_AV;
if (resp.exp_data.comp_mask & MLX5_EXP_ALLOC_CTX_RESP_MASK_CQE_COMP_MAX_NUM)
context->cqe_comp_max_num = resp.exp_data.cqe_comp_max_num;
if (resp.exp_data.comp_mask & MLX5_EXP_ALLOC_CTX_RESP_MASK_CQE_VERSION)
context->cqe_version = resp.exp_data.cqe_version;
if (resp.exp_data.comp_mask & MLX5_EXP_ALLOC_CTX_RESP_MASK_RROCE_UDP_SPORT_MIN)
context->rroce_udp_sport_min = resp.exp_data.rroce_udp_sport_min;
if (resp.exp_data.comp_mask & MLX5_EXP_ALLOC_CTX_RESP_MASK_RROCE_UDP_SPORT_MAX)
context->rroce_udp_sport_max = resp.exp_data.rroce_udp_sport_max;
ctx->ops = mlx5_ctx_ops;
if (context->cqe_version) {
if (context->cqe_version == 1) {
ctx->ops.poll_cq = mlx5_poll_cq_1;
} else {
printf("Unsupported cqe_vesion = %d, stay on cqe version 0\n",
context->cqe_version);
context->cqe_version = 0;
}
}
attr.comp_mask = IBV_EXP_DEVICE_ATTR_RESERVED - 1;
err = mlx5_query_device_ex(ctx, &attr);
if (!err) {
if (attr.comp_mask & IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS)
context->exp_device_cap_flags = attr.exp_device_cap_flags;
if (attr.comp_mask & IBV_EXP_DEVICE_ATTR_MAX_CTX_RES_DOMAIN) {
context->max_ctx_res_domain = attr.max_ctx_res_domain;
mlx5_spinlock_init(&context->send_db_lock, !mlx5_single_threaded);
INIT_LIST_HEAD(&context->send_wc_db_list);
INIT_LIST_HEAD(&context->wc_uar_list);
}
if (resp.exp_data.comp_mask & MLX5_EXP_ALLOC_CTX_RESP_MASK_HCA_CORE_CLOCK_OFFSET) {
context->core_clock.offset =
resp.exp_data.hca_core_clock_offset &
(to_mdev(ibdev)->page_size - 1);
mlx5_map_internal_clock(to_mdev(ibdev), ctx);
if (attr.hca_core_clock)
context->core_clock.mult = ((1ull * 1000) << 21) /
attr.hca_core_clock;
else
context->core_clock.mult = 0;
/* ConnectX-4 supports 64bit timestamp. We choose these numbers
* in order to make sure that after arithmetic operations,
* we don't overflow a 64bit variable.
*/
context->core_clock.shift = 21;
context->core_clock.mask = (1ULL << 49) - 1;
}
}
pthread_mutex_init(&context->rsc_table_mutex, NULL);
pthread_mutex_init(&context->srq_table_mutex, NULL);
for (i = 0; i < MLX5_QP_TABLE_SIZE; ++i)
context->rsc_table[i].refcnt = 0;
for (i = 0; i < MLX5_QP_TABLE_SIZE; ++i)
context->uidx_table[i].refcnt = 0;
context->db_list = NULL;
pthread_mutex_init(&context->db_list_mutex, NULL);
context->prefer_bf = get_always_bf(&context->ibv_ctx);
context->shut_up_bf = get_shut_up_bf(&context->ibv_ctx);
context->enable_cqe_comp = get_cqe_comp(&context->ibv_ctx);
mlx5_use_mutex = get_use_mutex(&context->ibv_ctx);
offset = 0;
set_command(MLX5_MMAP_MAP_DC_INFO_PAGE, &offset);
context->cc.buf = mmap(NULL, 4096 * context->num_ports, PROT_READ,
MAP_PRIVATE, cmd_fd, page_size * offset);
if (context->cc.buf == MAP_FAILED)
context->cc.buf = NULL;
mlx5_single_threaded = single_threaded_app(&context->ibv_ctx);
for (i = 0; i < resp.tot_uuars / MLX5_NUM_UUARS_PER_PAGE; ++i) {
uar_mapped = 0;
/* Don't map UAR to WC if BF is not used */
if (!context->shut_up_bf) {
context->uar[i].regs = mlx5_uar_mmap(i, MLX5_MMAP_GET_WC_PAGES_CMD, page_size, cmd_fd);
if (context->uar[i].regs != MAP_FAILED) {
context->uar[i].map_type = MLX5_UAR_MAP_WC;
uar_mapped = 1;
}
}
if (!uar_mapped) {
context->uar[i].regs = mlx5_uar_mmap(i, MLX5_MMAP_GET_NC_PAGES_CMD, page_size, cmd_fd);
if (context->uar[i].regs != MAP_FAILED) {
context->uar[i].map_type = MLX5_UAR_MAP_NC;
uar_mapped = 1;
}
}
if (!uar_mapped) {
/* for backward compatibility with old kernel driver */
context->uar[i].regs = mlx5_uar_mmap(i, MLX5_MMAP_GET_REGULAR_PAGES_CMD, page_size, cmd_fd);
if (context->uar[i].regs != MAP_FAILED) {
context->uar[i].map_type = MLX5_UAR_MAP_WC;
uar_mapped = 1;
}
}
if (!uar_mapped) {
context->uar[i].regs = NULL;
goto err_free_cc;
}
}
for (j = 0; j < gross_uuars; ++j) {
context->bfs[j].reg = context->uar[j / 4].regs +
MLX5_BF_OFFSET + (j % 4) * context->bf_reg_size;
context->bfs[j].need_lock = need_uuar_lock(context, j) &&
context->uar[j / 4].map_type == MLX5_UAR_MAP_WC;
mlx5_lock_init(&context->bfs[j].lock,
!mlx5_single_threaded,
mlx5_get_locktype());
context->bfs[j].offset = 0;
if (context->uar[j / 4].map_type == MLX5_UAR_MAP_WC) {
context->bfs[j].buf_size = context->bf_reg_size / 2;
context->bfs[j].db_method = (context->bfs[j].need_lock && !mlx5_single_threaded) ?
MLX5_DB_METHOD_BF :
(mlx5_single_threaded && wc_auto_evict_size() == 64 ?
MLX5_DB_METHOD_DEDIC_BF_1_THREAD :
MLX5_DB_METHOD_DEDIC_BF);
} else {
context->bfs[j].db_method = MLX5_DB_METHOD_DB;
}
context->bfs[j].uuarn = j;
}
mlx5_lock_init(&context->lock32,
!mlx5_single_threaded,
mlx5_get_locktype());
mlx5_spinlock_init(&context->hugetlb_lock, !mlx5_single_threaded);
INIT_LIST_HEAD(&context->hugetlb_list);
pthread_mutex_init(&context->task_mutex, NULL);
set_extended(verbs_ctx);
set_experimental(ctx);
for (i = 0; i < MLX5_MAX_PORTS_NUM; ++i)
context->port_query_cache[i].valid = 0;
return 0;
err_free_cc:
if (context->cc.buf)
munmap(context->cc.buf, 4096 * context->num_ports);
if (context->hca_core_clock)
munmap(context->hca_core_clock - context->core_clock.offset,
to_mdev(ibdev)->page_size);
err_free_bf:
free(context->bfs);
err_free:
for (i = 0; i < MLX5_MAX_UAR_PAGES; ++i) {
if (context->uar[i].regs)
munmap(context->uar[i].regs, page_size);
}
close_debug_file(context);
return errno;
}
static int mlx5_alloc_context(struct verbs_device *vdev,
struct ibv_context *ctx, int cmd_fd)
{
struct mlx5_context *context;
struct mlx5_alloc_ucontext req;
struct mlx5_alloc_ucontext_resp resp;
struct ibv_device *ibdev = &vdev->device;
struct verbs_context *verbs_ctx = verbs_get_ctx(ctx);
int i;
int page_size = to_mdev(ibdev)->page_size;
int tot_uuars;
int low_lat_uuars;
int gross_uuars;
int j;
off_t offset;
mlx5_single_threaded = single_threaded_app();
context = to_mctx(ctx);
context->ibv_ctx.cmd_fd = cmd_fd;
memset(&resp, 0, sizeof(resp));
open_debug_file(context);
set_debug_mask();
set_freeze_on_error();
if (gethostname(context->hostname, sizeof(context->hostname)))
strcpy(context->hostname, "host_unknown");
tot_uuars = get_total_uuars();
if (tot_uuars <= 0) {
if (tot_uuars == 0)
errno = EINVAL;
else
errno = -tot_uuars;
goto err_free;
}
gross_uuars = tot_uuars / MLX5_NUM_UUARS_PER_PAGE * 4;
context->bfs = calloc(gross_uuars, sizeof *context->bfs);
if (!context->bfs) {
errno = ENOMEM;
goto err_free;
}
low_lat_uuars = get_num_low_lat_uuars();
if (low_lat_uuars < 0) {
errno = ENOMEM;
goto err_free_bf;
}
if (low_lat_uuars > tot_uuars - 1) {
errno = ENOMEM;
goto err_free_bf;
}
memset(&req, 0, sizeof(req));
req.total_num_uuars = tot_uuars;
req.num_low_latency_uuars = low_lat_uuars;
if (ibv_cmd_get_context(&context->ibv_ctx, &req.ibv_req, sizeof req,
&resp.ibv_resp, sizeof resp))
goto err_free_bf;
context->max_num_qps = resp.qp_tab_size;
context->bf_reg_size = resp.bf_reg_size;
context->tot_uuars = resp.tot_uuars;
context->low_lat_uuars = low_lat_uuars;
context->cache_line_size = resp.cache_line_size;
context->max_sq_desc_sz = resp.max_sq_desc_sz;
context->max_rq_desc_sz = resp.max_rq_desc_sz;
context->max_send_wqebb = resp.max_send_wqebb;
context->num_ports = resp.num_ports;
context->max_recv_wr = resp.max_recv_wr;
context->max_srq_recv_wr = resp.max_srq_recv_wr;
context->max_desc_sz_sq_dc = resp.max_desc_sz_sq_dc;
context->atomic_sizes_dc = resp.atomic_sizes_dc;
pthread_mutex_init(&context->rsc_table_mutex, NULL);
pthread_mutex_init(&context->srq_table_mutex, NULL);
for (i = 0; i < MLX5_QP_TABLE_SIZE; ++i)
context->rsc_table[i].refcnt = 0;
context->db_list = NULL;
pthread_mutex_init(&context->db_list_mutex, NULL);
for (i = 0; i < resp.tot_uuars / MLX5_NUM_UUARS_PER_PAGE; ++i) {
offset = 0;
set_command(MLX5_MMAP_GET_REGULAR_PAGES_CMD, &offset);
set_index(i, &offset);
context->uar[i] = mmap(NULL, to_mdev(ibdev)->page_size, PROT_WRITE,
MAP_SHARED, cmd_fd,
page_size * offset);
if (context->uar[i] == MAP_FAILED) {
context->uar[i] = NULL;
goto err_free_bf;
}
}
for (j = 0; j < gross_uuars; ++j) {
context->bfs[j].reg = context->uar[j / 4] +
MLX5_BF_OFFSET + (j % 4) * context->bf_reg_size;
context->bfs[j].need_lock = need_uuar_lock(context, j);
mlx5_spinlock_init(&context->bfs[j].lock);
context->bfs[j].offset = 0;
if (j)
context->bfs[j].buf_size = context->bf_reg_size / 2;
context->bfs[j].uuarn = j;
}
mlx5_spinlock_init(&context->lock32);
context->prefer_bf = get_always_bf();
context->shut_up_bf = get_shut_up_bf();
mlx5_read_env(ibdev, context);
mlx5_spinlock_init(&context->hugetlb_lock);
INIT_LIST_HEAD(&context->hugetlb_list);
pthread_mutex_init(&context->task_mutex, NULL);
ctx->ops = mlx5_ctx_ops;
set_extended(verbs_ctx);
set_experimental(ctx);
return 0;
err_free_bf:
free(context->bfs);
err_free:
for (i = 0; i < MLX5_MAX_UAR_PAGES; ++i) {
if (context->uar[i])
munmap(context->uar[i], page_size);
}
close_debug_file(context);
return errno;
}