int __mlx4_srq_alloc_icm(struct mlx4_dev *dev, int *srqn)
{
struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table;
int err;
*srqn = mlx4_bitmap_alloc(&srq_table->bitmap);
if (*srqn == -1)
return -ENOMEM;
err = mlx4_table_get(dev, &srq_table->table, *srqn);
if (err)
goto err_out;
err = mlx4_table_get(dev, &srq_table->cmpt_table, *srqn);
if (err)
goto err_put;
return 0;
err_put:
mlx4_table_put(dev, &srq_table->table, *srqn);
err_out:
mlx4_bitmap_free(&srq_table->bitmap, *srqn);
return err;
}
int __mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn)
{
struct mlx4_priv *priv = mlx4_priv(dev);
struct mlx4_cq_table *cq_table = &priv->cq_table;
int err;
*cqn = mlx4_bitmap_alloc(&cq_table->bitmap);
if (*cqn == -1)
return -ENOMEM;
err = mlx4_table_get(dev, &cq_table->table, *cqn);
if (err)
goto err_out;
err = mlx4_table_get(dev, &cq_table->cmpt_table, *cqn);
if (err)
goto err_put;
return 0;
err_put:
mlx4_table_put(dev, &cq_table->table, *cqn);
err_out:
mlx4_bitmap_free(&cq_table->bitmap, *cqn, MLX4_NO_RR);
return err;
}
u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align)
{
u32 obj;
if (likely(cnt == 1 && align == 1))
return mlx4_bitmap_alloc(bitmap);
spin_lock(&bitmap->lock);
obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max,
bitmap->last, cnt, align - 1);
if (obj >= bitmap->max) {
bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
& bitmap->mask;
obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max,
0, cnt, align - 1);
}
if (obj < bitmap->max) {
bitmap_set(bitmap->table, obj, cnt);
if (obj == bitmap->last) {
bitmap->last = (obj + cnt);
if (bitmap->last >= bitmap->max)
bitmap->last = 0;
}
obj |= bitmap->top;
} else
obj = -1;
spin_unlock(&bitmap->lock);
return obj;
}
u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align)
{
u32 obj, i;
if (likely(cnt == 1 && align == 1))
return mlx4_bitmap_alloc(bitmap);
spin_lock(&bitmap->lock);
obj = find_aligned_range(bitmap->table, bitmap->last,
bitmap->max, cnt, align);
if (obj >= bitmap->max) {
bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
& bitmap->mask;
obj = find_aligned_range(bitmap->table, 0, bitmap->max,
cnt, align);
}
if (obj < bitmap->max) {
for (i = 0; i < cnt; i++)
set_bit(obj + i, bitmap->table);
if (obj == bitmap->last) {
bitmap->last = (obj + cnt);
if (bitmap->last >= bitmap->max)
bitmap->last = 0;
}
obj |= bitmap->top;
} else
obj = -1;
spin_unlock(&bitmap->lock);
return obj;
}
int mlx4_pd_alloc(struct mlx4_dev *dev, u32 *pdn) {
struct mlx4_priv *priv = mlx4_priv(dev);
*pdn = mlx4_bitmap_alloc(&priv->pd_bitmap);
if (*pdn == -1)
return -ENOMEM;
return 0;
}
int mlx4_uar_alloc(struct mlx4_dev *dev, struct mlx4_uar *uar)
{
uar->index = mlx4_bitmap_alloc(&mlx4_priv(dev)->uar_table.bitmap);
if (uar->index == -1)
return -ENOMEM;
uar->pfn = (pci_resource_start(dev->pdev, 2) >> PAGE_SHIFT) + uar->index;
return 0;
}
int __mlx4_xrcd_alloc(struct mlx4_dev *dev, u32 *xrcdn)
{
struct mlx4_priv *priv = mlx4_priv(dev);
*xrcdn = mlx4_bitmap_alloc(&priv->xrcd_bitmap);
if (*xrcdn == -1)
return -ENOMEM;
return 0;
}
int mlx4_pd_alloc(struct mlx4_dev *dev, u32 *pdn)
{
struct mlx4_priv *priv = mlx4_priv(dev);
*pdn = mlx4_bitmap_alloc(&priv->pd_bitmap);
if (*pdn == -1)
return -ENOMEM;
*pdn |= dev->caps.pd_base << dev->caps.slave_pd_shift;
return 0;
}
int mlx4_uar_alloc(struct mlx4_dev *dev, struct mlx4_uar *uar)
{
int offset;
uar->index = mlx4_bitmap_alloc(&mlx4_priv(dev)->uar_table.bitmap);
if (uar->index == -1)
return -ENOMEM;
if (mlx4_is_slave(dev))
offset = uar->index % ((int) pci_resource_len(dev->pdev, 2) /
dev->caps.uar_page_size);
else
offset = uar->index;
uar->pfn = (pci_resource_start(dev->pdev, 2) >> PAGE_SHIFT) + offset;
return 0;
}
int mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn)
{
struct mlx4_priv *priv = mlx4_priv(dev);
struct mlx4_cq_table *cq_table = &priv->cq_table;
u64 out_param;
int err;
if (mlx4_is_slave(dev)) {
err = mlx4_cmd_imm(dev, 0, &out_param, RES_CQ,
ICM_RESERVE_AND_ALLOC,
MLX4_CMD_ALLOC_RES,
MLX4_CMD_TIME_CLASS_A);
if (err) {
*cqn = -1;
return err;
} else {
*cqn = out_param;
return 0;
}
}
*cqn = mlx4_bitmap_alloc(&cq_table->bitmap);
if (*cqn == -1)
return -ENOMEM;
err = mlx4_table_get(dev, &cq_table->table, *cqn);
if (err)
goto err_out;
err = mlx4_table_get(dev, &cq_table->cmpt_table, *cqn);
if (err)
goto err_put;
return 0;
err_put:
mlx4_table_put(dev, &cq_table->table, *cqn);
err_out:
mlx4_bitmap_free(&cq_table->bitmap, *cqn);
return err;
}
/*
void mlx4_cleanup_xrcd_table(struct mlx4_dev *dev)
{
mlx4_bitmap_cleanup(&mlx4_priv(dev)->xrcd_bitmap);
}
*/
#define PAGE_SHIFT 12 /* LOG2(PAGE_SIZE) */
int mlx4_uar_alloc(struct mlx4_dev *dev, struct mlx4_uar *uar) {
struct mlx4_priv *priv = mlx4_priv(dev);
int offset;
uar->index = mlx4_bitmap_alloc(&priv->uar_table.bitmap);
if (uar->index == -1)
return -ENOMEM;
if (mlx4_is_slave(&priv->dev))
offset = uar->index
% ((int) priv->dev.bar_info[1].bytes
/ priv->dev.caps.uar_page_size);
else
offset = uar->index;
/*MLX4_DEBUG("%p, %d\n", priv->dev.bar_info[1].vaddr, offset);*/
uar->pfn = ((volatile uint64_t) priv->dev.bar_info[1].vaddr >> PAGE_SHIFT)
+ offset;
uar->map = NULL;
return 0;
}
u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt,
int align, u32 skip_mask)
{
u32 obj;
if (likely(cnt == 1 && align == 1 && !skip_mask))
return mlx4_bitmap_alloc(bitmap);
spin_lock(&bitmap->lock);
obj = find_aligned_range(bitmap->table, bitmap->last,
bitmap->max, cnt, align, skip_mask);
if (obj >= bitmap->max) {
bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
& bitmap->mask;
obj = find_aligned_range(bitmap->table, 0, bitmap->max,
cnt, align, skip_mask);
}
if (obj < bitmap->max) {
bitmap_set(bitmap->table, obj, cnt);
if (obj == bitmap->last) {
bitmap->last = (obj + cnt);
if (bitmap->last >= bitmap->max)
bitmap->last = 0;
}
obj |= bitmap->top;
} else
obj = -1;
if (obj != -1)
bitmap->avail -= cnt;
spin_unlock(&bitmap->lock);
return obj;
}
int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, struct mlx4_mtt *mtt,
struct mlx4_uar *uar, u64 db_rec, struct mlx4_cq *cq,
int collapsed)
{
struct mlx4_priv *priv = mlx4_priv(dev);
struct mlx4_cq_table *cq_table = &priv->cq_table;
struct mlx4_cmd_mailbox *mailbox;
struct mlx4_cq_context *cq_context;
u64 mtt_addr;
int err;
cq->cqn = mlx4_bitmap_alloc(&cq_table->bitmap);
if (cq->cqn == -1)
return -ENOMEM;
err = mlx4_table_get(dev, &cq_table->table, cq->cqn);
if (err)
goto err_out;
err = mlx4_table_get(dev, &cq_table->cmpt_table, cq->cqn);
if (err)
goto err_put;
spin_lock_irq(&cq_table->lock);
err = radix_tree_insert(&cq_table->tree, cq->cqn, cq);
spin_unlock_irq(&cq_table->lock);
if (err)
goto err_cmpt_put;
mailbox = mlx4_alloc_cmd_mailbox(dev);
if (IS_ERR(mailbox)) {
err = PTR_ERR(mailbox);
goto err_radix;
}
cq_context = mailbox->buf;
memset(cq_context, 0, sizeof *cq_context);
cq_context->flags = cpu_to_be32(!!collapsed << 18);
cq_context->logsize_usrpage = cpu_to_be32((ilog2(nent) << 24) | uar->index);
cq_context->comp_eqn = priv->eq_table.eq[MLX4_EQ_COMP].eqn;
cq_context->log_page_size = mtt->page_shift - MLX4_ICM_PAGE_SHIFT;
mtt_addr = mlx4_mtt_addr(dev, mtt);
cq_context->mtt_base_addr_h = mtt_addr >> 32;
cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
cq_context->db_rec_addr = cpu_to_be64(db_rec);
err = mlx4_SW2HW_CQ(dev, mailbox, cq->cqn);
mlx4_free_cmd_mailbox(dev, mailbox);
if (err)
goto err_radix;
cq->cons_index = 0;
cq->arm_sn = 1;
cq->uar = uar;
atomic_set(&cq->refcount, 1);
init_completion(&cq->free);
return 0;
err_radix:
spin_lock_irq(&cq_table->lock);
radix_tree_delete(&cq_table->tree, cq->cqn);
spin_unlock_irq(&cq_table->lock);
err_cmpt_put:
mlx4_table_put(dev, &cq_table->cmpt_table, cq->cqn);
err_put:
mlx4_table_put(dev, &cq_table->table, cq->cqn);
err_out:
mlx4_bitmap_free(&cq_table->bitmap, cq->cqn);
return err;
}
int mlx4_srq_alloc(struct mlx4_dev *dev, u32 pdn, struct mlx4_mtt *mtt,
u64 db_rec, struct mlx4_srq *srq)
{
struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table;
struct mlx4_cmd_mailbox *mailbox;
struct mlx4_srq_context *srq_context;
u64 mtt_addr;
int err;
srq->srqn = mlx4_bitmap_alloc(&srq_table->bitmap);
if (srq->srqn == -1)
return -ENOMEM;
err = mlx4_table_get(dev, &srq_table->table, srq->srqn);
if (err)
goto err_out;
err = mlx4_table_get(dev, &srq_table->cmpt_table, srq->srqn);
if (err)
goto err_put;
spin_lock_irq(&srq_table->lock);
err = radix_tree_insert(&srq_table->tree, srq->srqn, srq);
spin_unlock_irq(&srq_table->lock);
if (err)
goto err_cmpt_put;
mailbox = mlx4_alloc_cmd_mailbox(dev);
if (IS_ERR(mailbox)) {
err = PTR_ERR(mailbox);
goto err_radix;
}
srq_context = mailbox->buf;
memset(srq_context, 0, sizeof *srq_context);
srq_context->state_logsize_srqn = cpu_to_be32((ilog2(srq->max) << 24) |
srq->srqn);
srq_context->logstride = srq->wqe_shift - 4;
srq_context->log_page_size = mtt->page_shift - MLX4_ICM_PAGE_SHIFT;
mtt_addr = mlx4_mtt_addr(dev, mtt);
srq_context->mtt_base_addr_h = mtt_addr >> 32;
srq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
srq_context->pd = cpu_to_be32(pdn);
srq_context->db_rec_addr = cpu_to_be64(db_rec);
err = mlx4_SW2HW_SRQ(dev, mailbox, srq->srqn);
mlx4_free_cmd_mailbox(dev, mailbox);
if (err)
goto err_radix;
atomic_set(&srq->refcount, 1);
init_completion(&srq->free);
return 0;
err_radix:
spin_lock_irq(&srq_table->lock);
radix_tree_delete(&srq_table->tree, srq->srqn);
spin_unlock_irq(&srq_table->lock);
err_cmpt_put:
mlx4_table_put(dev, &srq_table->cmpt_table, srq->srqn);
err_put:
mlx4_table_put(dev, &srq_table->table, srq->srqn);
err_out:
mlx4_bitmap_free(&srq_table->bitmap, srq->srqn);
return err;
}
/*
static void mlx4_unmap_uar(struct mlx4_priv *priv)
{
struct mlx4_priv *priv = mlx4_priv(&priv->dev);
int i;
for (i = 0; i < mlx4_num_eq_uar(&priv->dev); ++i)
if (priv->eq_table.uar_map[i]) {
iounmap(priv->eq_table.uar_map[i]);
priv->eq_table.uar_map[i] = NULL;
}
}
*/
static int mlx4_create_eq(struct mlx4_priv *priv, int nent, u8 intr,
struct mlx4_eq *eq) {
struct mlx4_cmd_mailbox *mailbox;
struct mlx4_eq_context *eq_context;
int npages;
u64 *dma_list = NULL;
genpaddr_t t = 0;
u64 mtt_addr;
int err = -ENOMEM;
int i;
eq->priv = priv;
eq->nent = roundup_pow_of_two(max(nent, 2));
/* CX3 is capable of extending the CQE\EQE from 32 to 64 bytes*/
npages = PAGE_ALIGN(
eq->nent * (MLX4_EQ_ENTRY_SIZE << priv->dev.caps.eqe_factor))
/ BASE_PAGE_SIZE;
eq->page_list = malloc(npages * sizeof *eq->page_list);
if (!eq->page_list)
goto err_out;
for (i = 0; i < npages; ++i)
eq->page_list[i].buf = NULL;
dma_list = malloc(npages * sizeof *dma_list);
if (!dma_list)
goto err_out_free;
mailbox = mlx4_alloc_cmd_mailbox();
if (IS_ERR(mailbox))
goto err_out_free;
eq_context = mailbox->buf;
for (i = 0; i < npages; ++i) {
eq->page_list[i].buf = dma_alloc(BASE_PAGE_SIZE, &t);
if (!eq->page_list[i].buf)
goto err_out_free_pages;
dma_list[i] = t;
eq->page_list[i].map = t;
memset(eq->page_list[i].buf, 0, BASE_PAGE_SIZE);
}
eq->eqn = mlx4_bitmap_alloc(&priv->eq_table.bitmap);
if (eq->eqn == -1)
goto err_out_free_pages;
eq->doorbell = mlx4_get_eq_uar(priv, eq);
if (!eq->doorbell) {
err = -ENOMEM;
goto err_out_free_eq;
}
err = mlx4_mtt_init(&priv->dev, npages, PAGE_SHIFT, &eq->mtt);
if (err)
goto err_out_free_eq;
err = mlx4_write_mtt(&priv->dev, &eq->mtt, 0, npages, dma_list);
if (err)
goto err_out_free_mtt;
memset(eq_context, 0, sizeof *eq_context);
eq_context->flags = cpu_to_be32(MLX4_EQ_STATUS_OK |
MLX4_EQ_STATE_ARMED);
eq_context->log_eq_size = ilog2(eq->nent);
eq_context->intr = intr;
eq_context->log_page_size = PAGE_SHIFT - MLX4_ICM_PAGE_SHIFT;
/*printf("mtt_addr: %lx\n", mlx4_mtt_addr(&priv->dev, &eq->mtt));
printf("off: %d\n", eq->mtt.offset);
printf("size: %d\n", priv->dev.caps.mtt_entry_sz);*/
mtt_addr = mlx4_mtt_addr(&priv->dev, &eq->mtt);
eq_context->mtt_base_addr_h = mtt_addr >> 32;
eq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
err = mlx4_SW2HW_EQ(priv, mailbox, eq->eqn);
if (err) {
MLX4_DEBUG("SW2HW_EQ failed (%d)\n", err);
goto err_out_free_mtt;
}
free(dma_list);
mlx4_free_cmd_mailbox(mailbox);
eq->cons_index = 0;
return err;
/*TODO*/
err_out_free_mtt: /*mlx4_mtt_cleanup(&priv->dev, &eq->mtt);*/
err_out_free_eq: /*mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn,
MLX4_USE_RR);*/
err_out_free_pages: /*for (i = 0; i < npages; ++i)
if (eq->page_list[i].buf)
dma_free(&priv->dev.pdev->dev, PAGE_SIZE,
eq->page_list[i].buf, eq->page_list[i].map);*/
mlx4_free_cmd_mailbox(mailbox);
err_out_free: free(eq->page_list);
free(dma_list);
err_out: return err;
}
