int __mlx4_srq_alloc_icm(struct mlx4_dev *dev, int *srqn) { struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; int err; *srqn = mlx4_bitmap_alloc(&srq_table->bitmap); if (*srqn == -1) return -ENOMEM; err = mlx4_table_get(dev, &srq_table->table, *srqn); if (err) goto err_out; err = mlx4_table_get(dev, &srq_table->cmpt_table, *srqn); if (err) goto err_put; return 0; err_put: mlx4_table_put(dev, &srq_table->table, *srqn); err_out: mlx4_bitmap_free(&srq_table->bitmap, *srqn); return err; }
int __mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_cq_table *cq_table = &priv->cq_table; int err; *cqn = mlx4_bitmap_alloc(&cq_table->bitmap); if (*cqn == -1) return -ENOMEM; err = mlx4_table_get(dev, &cq_table->table, *cqn); if (err) goto err_out; err = mlx4_table_get(dev, &cq_table->cmpt_table, *cqn); if (err) goto err_put; return 0; err_put: mlx4_table_put(dev, &cq_table->table, *cqn); err_out: mlx4_bitmap_free(&cq_table->bitmap, *cqn, MLX4_NO_RR); return err; }
u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align) { u32 obj; if (likely(cnt == 1 && align == 1)) return mlx4_bitmap_alloc(bitmap); spin_lock(&bitmap->lock); obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, bitmap->last, cnt, align - 1); if (obj >= bitmap->max) { bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) & bitmap->mask; obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, 0, cnt, align - 1); } if (obj < bitmap->max) { bitmap_set(bitmap->table, obj, cnt); if (obj == bitmap->last) { bitmap->last = (obj + cnt); if (bitmap->last >= bitmap->max) bitmap->last = 0; } obj |= bitmap->top; } else obj = -1; spin_unlock(&bitmap->lock); return obj; }
u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align) { u32 obj, i; if (likely(cnt == 1 && align == 1)) return mlx4_bitmap_alloc(bitmap); spin_lock(&bitmap->lock); obj = find_aligned_range(bitmap->table, bitmap->last, bitmap->max, cnt, align); if (obj >= bitmap->max) { bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) & bitmap->mask; obj = find_aligned_range(bitmap->table, 0, bitmap->max, cnt, align); } if (obj < bitmap->max) { for (i = 0; i < cnt; i++) set_bit(obj + i, bitmap->table); if (obj == bitmap->last) { bitmap->last = (obj + cnt); if (bitmap->last >= bitmap->max) bitmap->last = 0; } obj |= bitmap->top; } else obj = -1; spin_unlock(&bitmap->lock); return obj; }
int mlx4_pd_alloc(struct mlx4_dev *dev, u32 *pdn) { struct mlx4_priv *priv = mlx4_priv(dev); *pdn = mlx4_bitmap_alloc(&priv->pd_bitmap); if (*pdn == -1) return -ENOMEM; return 0; }
int mlx4_uar_alloc(struct mlx4_dev *dev, struct mlx4_uar *uar) { uar->index = mlx4_bitmap_alloc(&mlx4_priv(dev)->uar_table.bitmap); if (uar->index == -1) return -ENOMEM; uar->pfn = (pci_resource_start(dev->pdev, 2) >> PAGE_SHIFT) + uar->index; return 0; }
int __mlx4_xrcd_alloc(struct mlx4_dev *dev, u32 *xrcdn) { struct mlx4_priv *priv = mlx4_priv(dev); *xrcdn = mlx4_bitmap_alloc(&priv->xrcd_bitmap); if (*xrcdn == -1) return -ENOMEM; return 0; }
int mlx4_pd_alloc(struct mlx4_dev *dev, u32 *pdn) { struct mlx4_priv *priv = mlx4_priv(dev); *pdn = mlx4_bitmap_alloc(&priv->pd_bitmap); if (*pdn == -1) return -ENOMEM; *pdn |= dev->caps.pd_base << dev->caps.slave_pd_shift; return 0; }
int mlx4_uar_alloc(struct mlx4_dev *dev, struct mlx4_uar *uar) { int offset; uar->index = mlx4_bitmap_alloc(&mlx4_priv(dev)->uar_table.bitmap); if (uar->index == -1) return -ENOMEM; if (mlx4_is_slave(dev)) offset = uar->index % ((int) pci_resource_len(dev->pdev, 2) / dev->caps.uar_page_size); else offset = uar->index; uar->pfn = (pci_resource_start(dev->pdev, 2) >> PAGE_SHIFT) + offset; return 0; }
int mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_cq_table *cq_table = &priv->cq_table; u64 out_param; int err; if (mlx4_is_slave(dev)) { err = mlx4_cmd_imm(dev, 0, &out_param, RES_CQ, ICM_RESERVE_AND_ALLOC, MLX4_CMD_ALLOC_RES, MLX4_CMD_TIME_CLASS_A); if (err) { *cqn = -1; return err; } else { *cqn = out_param; return 0; } } *cqn = mlx4_bitmap_alloc(&cq_table->bitmap); if (*cqn == -1) return -ENOMEM; err = mlx4_table_get(dev, &cq_table->table, *cqn); if (err) goto err_out; err = mlx4_table_get(dev, &cq_table->cmpt_table, *cqn); if (err) goto err_put; return 0; err_put: mlx4_table_put(dev, &cq_table->table, *cqn); err_out: mlx4_bitmap_free(&cq_table->bitmap, *cqn); return err; }
/* void mlx4_cleanup_xrcd_table(struct mlx4_dev *dev) { mlx4_bitmap_cleanup(&mlx4_priv(dev)->xrcd_bitmap); } */ #define PAGE_SHIFT 12 /* LOG2(PAGE_SIZE) */ int mlx4_uar_alloc(struct mlx4_dev *dev, struct mlx4_uar *uar) { struct mlx4_priv *priv = mlx4_priv(dev); int offset; uar->index = mlx4_bitmap_alloc(&priv->uar_table.bitmap); if (uar->index == -1) return -ENOMEM; if (mlx4_is_slave(&priv->dev)) offset = uar->index % ((int) priv->dev.bar_info[1].bytes / priv->dev.caps.uar_page_size); else offset = uar->index; /*MLX4_DEBUG("%p, %d\n", priv->dev.bar_info[1].vaddr, offset);*/ uar->pfn = ((volatile uint64_t) priv->dev.bar_info[1].vaddr >> PAGE_SHIFT) + offset; uar->map = NULL; return 0; }
u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align, u32 skip_mask) { u32 obj; if (likely(cnt == 1 && align == 1 && !skip_mask)) return mlx4_bitmap_alloc(bitmap); spin_lock(&bitmap->lock); obj = find_aligned_range(bitmap->table, bitmap->last, bitmap->max, cnt, align, skip_mask); if (obj >= bitmap->max) { bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) & bitmap->mask; obj = find_aligned_range(bitmap->table, 0, bitmap->max, cnt, align, skip_mask); } if (obj < bitmap->max) { bitmap_set(bitmap->table, obj, cnt); if (obj == bitmap->last) { bitmap->last = (obj + cnt); if (bitmap->last >= bitmap->max) bitmap->last = 0; } obj |= bitmap->top; } else obj = -1; if (obj != -1) bitmap->avail -= cnt; spin_unlock(&bitmap->lock); return obj; }
int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, struct mlx4_mtt *mtt, struct mlx4_uar *uar, u64 db_rec, struct mlx4_cq *cq, int collapsed) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_cq_table *cq_table = &priv->cq_table; struct mlx4_cmd_mailbox *mailbox; struct mlx4_cq_context *cq_context; u64 mtt_addr; int err; cq->cqn = mlx4_bitmap_alloc(&cq_table->bitmap); if (cq->cqn == -1) return -ENOMEM; err = mlx4_table_get(dev, &cq_table->table, cq->cqn); if (err) goto err_out; err = mlx4_table_get(dev, &cq_table->cmpt_table, cq->cqn); if (err) goto err_put; spin_lock_irq(&cq_table->lock); err = radix_tree_insert(&cq_table->tree, cq->cqn, cq); spin_unlock_irq(&cq_table->lock); if (err) goto err_cmpt_put; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) { err = PTR_ERR(mailbox); goto err_radix; } cq_context = mailbox->buf; memset(cq_context, 0, sizeof *cq_context); cq_context->flags = cpu_to_be32(!!collapsed << 18); cq_context->logsize_usrpage = cpu_to_be32((ilog2(nent) << 24) | uar->index); cq_context->comp_eqn = priv->eq_table.eq[MLX4_EQ_COMP].eqn; cq_context->log_page_size = mtt->page_shift - MLX4_ICM_PAGE_SHIFT; mtt_addr = mlx4_mtt_addr(dev, mtt); cq_context->mtt_base_addr_h = mtt_addr >> 32; cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); cq_context->db_rec_addr = cpu_to_be64(db_rec); err = mlx4_SW2HW_CQ(dev, mailbox, cq->cqn); mlx4_free_cmd_mailbox(dev, mailbox); if (err) goto err_radix; cq->cons_index = 0; cq->arm_sn = 1; cq->uar = uar; atomic_set(&cq->refcount, 1); init_completion(&cq->free); return 0; err_radix: spin_lock_irq(&cq_table->lock); radix_tree_delete(&cq_table->tree, cq->cqn); spin_unlock_irq(&cq_table->lock); err_cmpt_put: mlx4_table_put(dev, &cq_table->cmpt_table, cq->cqn); err_put: mlx4_table_put(dev, &cq_table->table, cq->cqn); err_out: mlx4_bitmap_free(&cq_table->bitmap, cq->cqn); return err; }
int mlx4_srq_alloc(struct mlx4_dev *dev, u32 pdn, struct mlx4_mtt *mtt, u64 db_rec, struct mlx4_srq *srq) { struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; struct mlx4_cmd_mailbox *mailbox; struct mlx4_srq_context *srq_context; u64 mtt_addr; int err; srq->srqn = mlx4_bitmap_alloc(&srq_table->bitmap); if (srq->srqn == -1) return -ENOMEM; err = mlx4_table_get(dev, &srq_table->table, srq->srqn); if (err) goto err_out; err = mlx4_table_get(dev, &srq_table->cmpt_table, srq->srqn); if (err) goto err_put; spin_lock_irq(&srq_table->lock); err = radix_tree_insert(&srq_table->tree, srq->srqn, srq); spin_unlock_irq(&srq_table->lock); if (err) goto err_cmpt_put; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) { err = PTR_ERR(mailbox); goto err_radix; } srq_context = mailbox->buf; memset(srq_context, 0, sizeof *srq_context); srq_context->state_logsize_srqn = cpu_to_be32((ilog2(srq->max) << 24) | srq->srqn); srq_context->logstride = srq->wqe_shift - 4; srq_context->log_page_size = mtt->page_shift - MLX4_ICM_PAGE_SHIFT; mtt_addr = mlx4_mtt_addr(dev, mtt); srq_context->mtt_base_addr_h = mtt_addr >> 32; srq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); srq_context->pd = cpu_to_be32(pdn); srq_context->db_rec_addr = cpu_to_be64(db_rec); err = mlx4_SW2HW_SRQ(dev, mailbox, srq->srqn); mlx4_free_cmd_mailbox(dev, mailbox); if (err) goto err_radix; atomic_set(&srq->refcount, 1); init_completion(&srq->free); return 0; err_radix: spin_lock_irq(&srq_table->lock); radix_tree_delete(&srq_table->tree, srq->srqn); spin_unlock_irq(&srq_table->lock); err_cmpt_put: mlx4_table_put(dev, &srq_table->cmpt_table, srq->srqn); err_put: mlx4_table_put(dev, &srq_table->table, srq->srqn); err_out: mlx4_bitmap_free(&srq_table->bitmap, srq->srqn); return err; }
/* static void mlx4_unmap_uar(struct mlx4_priv *priv) { struct mlx4_priv *priv = mlx4_priv(&priv->dev); int i; for (i = 0; i < mlx4_num_eq_uar(&priv->dev); ++i) if (priv->eq_table.uar_map[i]) { iounmap(priv->eq_table.uar_map[i]); priv->eq_table.uar_map[i] = NULL; } } */ static int mlx4_create_eq(struct mlx4_priv *priv, int nent, u8 intr, struct mlx4_eq *eq) { struct mlx4_cmd_mailbox *mailbox; struct mlx4_eq_context *eq_context; int npages; u64 *dma_list = NULL; genpaddr_t t = 0; u64 mtt_addr; int err = -ENOMEM; int i; eq->priv = priv; eq->nent = roundup_pow_of_two(max(nent, 2)); /* CX3 is capable of extending the CQE\EQE from 32 to 64 bytes*/ npages = PAGE_ALIGN( eq->nent * (MLX4_EQ_ENTRY_SIZE << priv->dev.caps.eqe_factor)) / BASE_PAGE_SIZE; eq->page_list = malloc(npages * sizeof *eq->page_list); if (!eq->page_list) goto err_out; for (i = 0; i < npages; ++i) eq->page_list[i].buf = NULL; dma_list = malloc(npages * sizeof *dma_list); if (!dma_list) goto err_out_free; mailbox = mlx4_alloc_cmd_mailbox(); if (IS_ERR(mailbox)) goto err_out_free; eq_context = mailbox->buf; for (i = 0; i < npages; ++i) { eq->page_list[i].buf = dma_alloc(BASE_PAGE_SIZE, &t); if (!eq->page_list[i].buf) goto err_out_free_pages; dma_list[i] = t; eq->page_list[i].map = t; memset(eq->page_list[i].buf, 0, BASE_PAGE_SIZE); } eq->eqn = mlx4_bitmap_alloc(&priv->eq_table.bitmap); if (eq->eqn == -1) goto err_out_free_pages; eq->doorbell = mlx4_get_eq_uar(priv, eq); if (!eq->doorbell) { err = -ENOMEM; goto err_out_free_eq; } err = mlx4_mtt_init(&priv->dev, npages, PAGE_SHIFT, &eq->mtt); if (err) goto err_out_free_eq; err = mlx4_write_mtt(&priv->dev, &eq->mtt, 0, npages, dma_list); if (err) goto err_out_free_mtt; memset(eq_context, 0, sizeof *eq_context); eq_context->flags = cpu_to_be32(MLX4_EQ_STATUS_OK | MLX4_EQ_STATE_ARMED); eq_context->log_eq_size = ilog2(eq->nent); eq_context->intr = intr; eq_context->log_page_size = PAGE_SHIFT - MLX4_ICM_PAGE_SHIFT; /*printf("mtt_addr: %lx\n", mlx4_mtt_addr(&priv->dev, &eq->mtt)); printf("off: %d\n", eq->mtt.offset); printf("size: %d\n", priv->dev.caps.mtt_entry_sz);*/ mtt_addr = mlx4_mtt_addr(&priv->dev, &eq->mtt); eq_context->mtt_base_addr_h = mtt_addr >> 32; eq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); err = mlx4_SW2HW_EQ(priv, mailbox, eq->eqn); if (err) { MLX4_DEBUG("SW2HW_EQ failed (%d)\n", err); goto err_out_free_mtt; } free(dma_list); mlx4_free_cmd_mailbox(mailbox); eq->cons_index = 0; return err; /*TODO*/ err_out_free_mtt: /*mlx4_mtt_cleanup(&priv->dev, &eq->mtt);*/ err_out_free_eq: /*mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn, MLX4_USE_RR);*/ err_out_free_pages: /*for (i = 0; i < npages; ++i) if (eq->page_list[i].buf) dma_free(&priv->dev.pdev->dev, PAGE_SIZE, eq->page_list[i].buf, eq->page_list[i].map);*/ mlx4_free_cmd_mailbox(mailbox); err_out_free: free(eq->page_list); free(dma_list); err_out: return err; }