int mthca_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr, struct mthca_srq *srq) { struct mthca_data_seg *scatter; void *wqe; int size; int i; srq->wrid = malloc(srq->max * sizeof (uint64_t)); if (!srq->wrid) return -1; size = sizeof (struct mthca_next_seg) + srq->max_gs * sizeof (struct mthca_data_seg); for (srq->wqe_shift = 6; 1 << srq->wqe_shift < size; ++srq->wqe_shift) ; /* nothing */ srq->buf_size = srq->max << srq->wqe_shift; if (mthca_alloc_buf(&srq->buf, align(srq->buf_size, to_mdev(pd->context->device)->page_size), to_mdev(pd->context->device)->page_size)) { free(srq->wrid); return -1; } memset(srq->buf.buf, 0, srq->buf_size); /* * Now initialize the SRQ buffer so that all of the WQEs are * linked into the list of free WQEs. In addition, set the * scatter list L_Keys to the sentry value of 0x100. */ for (i = 0; i < srq->max; ++i) { struct mthca_next_seg *next; next = wqe = get_wqe(srq, i); if (i < srq->max - 1) { *wqe_to_link(wqe) = i + 1; next->nda_op = htonl(((i + 1) << srq->wqe_shift) | 1); } else { *wqe_to_link(wqe) = -1; next->nda_op = 0; } for (scatter = wqe + sizeof (struct mthca_next_seg); (void *) scatter < wqe + (1 << srq->wqe_shift); ++scatter) scatter->lkey = htonl(MTHCA_INVAL_LKEY); } srq->first_free = 0; srq->last_free = srq->max - 1; srq->last = get_wqe(srq, srq->max - 1); return 0; }
void mthca_free_srq_wqe(struct mthca_srq *srq, int ind) { struct mthca_next_seg *last_free; pthread_spin_lock(&srq->lock); last_free = get_wqe(srq, srq->last_free); *wqe_to_link(last_free) = ind; last_free->nda_op = htonl((ind << srq->wqe_shift) | 1); *wqe_to_link(get_wqe(srq, ind)) = -1; srq->last_free = ind; pthread_spin_unlock(&srq->lock); }
int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, struct ib_recv_wr **bad_wr) { struct mlx5_ib_srq *srq = to_msrq(ibsrq); struct mlx5_wqe_srq_next_seg *next; struct mlx5_wqe_data_seg *scat; unsigned long flags; int err = 0; int nreq; int i; spin_lock_irqsave(&srq->lock, flags); for (nreq = 0; wr; nreq++, wr = wr->next) { if (unlikely(wr->num_sge > srq->msrq.max_gs)) { err = -EINVAL; *bad_wr = wr; break; } if (unlikely(srq->head == srq->tail)) { err = -ENOMEM; *bad_wr = wr; break; } srq->wrid[srq->head] = wr->wr_id; next = get_wqe(srq, srq->head); srq->head = be16_to_cpu(next->next_wqe_index); scat = (struct mlx5_wqe_data_seg *)(next + 1); for (i = 0; i < wr->num_sge; i++) { scat[i].byte_count = cpu_to_be32(wr->sg_list[i].length); scat[i].lkey = cpu_to_be32(wr->sg_list[i].lkey); scat[i].addr = cpu_to_be64(wr->sg_list[i].addr); } if (i < srq->msrq.max_avail_gather) { scat[i].byte_count = 0; scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY); scat[i].addr = 0; } } if (likely(nreq)) { srq->wqe_ctr += nreq; /* Make sure that descriptors are written before * doorbell record. */ wmb(); *srq->db.db = cpu_to_be32(srq->wqe_ctr); } spin_unlock_irqrestore(&srq->lock, flags); return err; }
int mlx5_alloc_srq_buf(struct ibv_context *context, struct mlx5_srq *srq) { struct mlx5_wqe_srq_next_seg *next; int size; int buf_size; int i; struct mlx5_context *ctx; ctx = to_mctx(context); if (srq->max_gs < 0) { errno = EINVAL; return -1; } srq->wrid = malloc(srq->max * sizeof *srq->wrid); if (!srq->wrid) return -1; size = sizeof(struct mlx5_wqe_srq_next_seg) + srq->max_gs * sizeof(struct mlx5_wqe_data_seg); size = max(32, size); size = mlx5_round_up_power_of_two(size); if (size > ctx->max_recv_wr) { errno = EINVAL; return -1; } srq->max_gs = (size - sizeof(struct mlx5_wqe_srq_next_seg)) / sizeof(struct mlx5_wqe_data_seg); srq->wqe_shift = mlx5_ilog2(size); buf_size = srq->max * size; if (mlx5_alloc_buf(&srq->buf, buf_size, to_mdev(context->device)->page_size)) { free(srq->wrid); return -1; } memset(srq->buf.buf, 0, buf_size); /* * Now initialize the SRQ buffer so that all of the WQEs are * linked into the list of free WQEs. */ for (i = 0; i < srq->max; ++i) { next = get_wqe(srq, i); next->next_wqe_index = htobe16((i + 1) & (srq->max - 1)); } srq->head = 0; srq->tail = srq->max - 1; return 0; }
void mlx4_free_srq_wqe(struct mlx4_srq *srq, int ind) { struct mlx4_wqe_srq_next_seg *next; pthread_spin_lock(&srq->lock); next = get_wqe(srq, srq->tail); next->next_wqe_index = htobe16(ind); srq->tail = ind; pthread_spin_unlock(&srq->lock); }
void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index) { struct mlx4_wqe_srq_next_seg *next; spin_lock(&srq->lock); next = get_wqe(srq, srq->tail); next->next_wqe_index = cpu_to_be16(wqe_index); srq->tail = wqe_index; spin_unlock(&srq->lock); }
void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index) { struct mlx4_wqe_srq_next_seg *next; /* always called with interrupts disabled. */ spin_lock(&srq->lock); next = get_wqe(srq, srq->tail); next->next_wqe_index = cpu_to_be16(wqe_index); srq->tail = wqe_index; spin_unlock(&srq->lock); }
int mlx4_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr, struct mlx4_srq *srq) { struct mlx4_wqe_srq_next_seg *next; struct mlx4_wqe_data_seg *scatter; int size; int buf_size; int i; srq->wrid = malloc(srq->max * sizeof (uint64_t)); if (!srq->wrid) return -1; size = sizeof (struct mlx4_wqe_srq_next_seg) + srq->max_gs * sizeof (struct mlx4_wqe_data_seg); for (srq->wqe_shift = 5; 1 << srq->wqe_shift < size; ++srq->wqe_shift) ; /* nothing */ buf_size = srq->max << srq->wqe_shift; if (mlx4_alloc_buf(&srq->buf, buf_size, to_mdev(pd->context->device)->page_size)) { free(srq->wrid); return -1; } memset(srq->buf.buf, 0, buf_size); /* * Now initialize the SRQ buffer so that all of the WQEs are * linked into the list of free WQEs. */ for (i = 0; i < srq->max; ++i) { next = get_wqe(srq, i); next->next_wqe_index = htobe16((i + 1) & (srq->max - 1)); for (scatter = (void *) (next + 1); (void *) scatter < (void *) next + (1 << srq->wqe_shift); ++scatter) scatter->lkey = htobe32(MLX4_INVALID_LKEY); } srq->head = 0; srq->tail = srq->max - 1; return 0; }
int mlx5_copy_to_recv_srq(struct mlx5_srq *srq, int idx, void *buf, int size) { struct mlx5_wqe_srq_next_seg *next; struct mlx5_wqe_data_seg *scat; int copy; int i; int max = 1 << (srq->wqe_shift - 4); next = get_wqe(srq, idx); scat = (struct mlx5_wqe_data_seg *) (next + 1); for (i = 0; i < max; ++i) { copy = min_t(long, size, be32toh(scat->byte_count)); memcpy((void *)(unsigned long)be64toh(scat->addr), buf, copy); size -= copy; if (size <= 0) return IBV_WC_SUCCESS; buf += copy; ++scat; } return IBV_WC_LOC_LEN_ERR; }
struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd, struct ib_srq_init_attr *init_attr, struct ib_udata *udata) { struct mlx4_ib_dev *dev = to_mdev(pd->device); struct mlx4_ib_srq *srq; struct mlx4_wqe_srq_next_seg *next; int desc_size; int buf_size; int err; int i; if (init_attr->attr.max_wr >= dev->dev->caps.max_srq_wqes || init_attr->attr.max_sge > dev->dev->caps.max_srq_sge) return ERR_PTR(-EINVAL); srq = kmalloc(sizeof *srq, GFP_KERNEL); if (!srq) return ERR_PTR(-ENOMEM); mutex_init(&srq->mutex); spin_lock_init(&srq->lock); srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1); srq->msrq.max_gs = init_attr->attr.max_sge; desc_size = max(32UL, roundup_pow_of_two(sizeof (struct mlx4_wqe_srq_next_seg) + srq->msrq.max_gs * sizeof (struct mlx4_wqe_data_seg))); srq->msrq.wqe_shift = ilog2(desc_size); buf_size = srq->msrq.max * desc_size; if (pd->uobject) { struct mlx4_ib_create_srq ucmd; if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { err = -EFAULT; goto err_srq; } srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr, buf_size, 0, 0); if (IS_ERR(srq->umem)) { err = PTR_ERR(srq->umem); goto err_srq; } err = mlx4_mtt_init(dev->dev, ib_umem_page_count(srq->umem), ilog2(srq->umem->page_size), &srq->mtt); if (err) goto err_buf; err = mlx4_ib_umem_write_mtt(dev, &srq->mtt, srq->umem); if (err) goto err_mtt; err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context), ucmd.db_addr, &srq->db); if (err) goto err_mtt; } else { err = mlx4_db_alloc(dev->dev, &srq->db, 0); if (err) goto err_srq; *srq->db.db = 0; if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2, &srq->buf)) { err = -ENOMEM; goto err_db; } srq->head = 0; srq->tail = srq->msrq.max - 1; srq->wqe_ctr = 0; for (i = 0; i < srq->msrq.max; ++i) { next = get_wqe(srq, i); next->next_wqe_index = cpu_to_be16((i + 1) & (srq->msrq.max - 1)); } err = mlx4_mtt_init(dev->dev, srq->buf.npages, srq->buf.page_shift, &srq->mtt); if (err) goto err_buf; err = mlx4_buf_write_mtt(dev->dev, &srq->mtt, &srq->buf); if (err) goto err_mtt; srq->wrid = kmalloc(srq->msrq.max * sizeof (u64), GFP_KERNEL); if (!srq->wrid) { err = -ENOMEM; goto err_mtt; } } err = mlx4_srq_alloc(dev->dev, to_mpd(pd)->pdn, &srq->mtt, srq->db.dma, &srq->msrq); if (err) goto err_wrid; srq->msrq.event = mlx4_ib_srq_event; if (pd->uobject) if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) { err = -EFAULT; goto err_wrid; } init_attr->attr.max_wr = srq->msrq.max - 1; return &srq->ibsrq; err_wrid: if (pd->uobject) mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &srq->db); else kfree(srq->wrid); err_mtt: mlx4_mtt_cleanup(dev->dev, &srq->mtt); err_buf: if (pd->uobject) ib_umem_release(srq->umem); else mlx4_buf_free(dev->dev, buf_size, &srq->buf); err_db: if (!pd->uobject) mlx4_db_free(dev->dev, &srq->db); err_srq: kfree(srq); return ERR_PTR(err); }
static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq, struct mlx5_srq_attr *in, int buf_size) { int err; int i; struct mlx5_wqe_srq_next_seg *next; int page_shift; int npages; err = mlx5_db_alloc(dev->mdev, &srq->db); if (err) { mlx5_ib_warn(dev, "alloc dbell rec failed\n"); return err; } if (mlx5_buf_alloc(dev->mdev, buf_size, &srq->buf)) { mlx5_ib_dbg(dev, "buf alloc failed\n"); err = -ENOMEM; goto err_db; } page_shift = srq->buf.page_shift; srq->head = 0; srq->tail = srq->msrq.max - 1; srq->wqe_ctr = 0; for (i = 0; i < srq->msrq.max; i++) { next = get_wqe(srq, i); next->next_wqe_index = cpu_to_be16((i + 1) & (srq->msrq.max - 1)); } npages = DIV_ROUND_UP(srq->buf.npages, 1 << (page_shift - PAGE_SHIFT)); mlx5_ib_dbg(dev, "buf_size %d, page_shift %d, npages %d, calc npages %d\n", buf_size, page_shift, srq->buf.npages, npages); in->pas = mlx5_vzalloc(sizeof(*in->pas) * npages); if (!in->pas) { err = -ENOMEM; goto err_buf; } mlx5_fill_page_array(&srq->buf, in->pas); srq->wrid = kmalloc(srq->msrq.max * sizeof(u64), GFP_KERNEL); if (!srq->wrid) { mlx5_ib_dbg(dev, "kmalloc failed %lu\n", (unsigned long)(srq->msrq.max * sizeof(u64))); err = -ENOMEM; goto err_in; } srq->wq_sig = !!srq_signature; in->log_page_size = page_shift - MLX5_ADAPTER_PAGE_SHIFT; if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 && in->type == IB_SRQT_XRC) in->user_index = MLX5_IB_DEFAULT_UIDX; return 0; err_in: kvfree(in->pas); err_buf: mlx5_buf_free(dev->mdev, &srq->buf); err_db: mlx5_db_free(dev->mdev, &srq->db); return err; }
int mthca_tavor_post_srq_recv(struct ibv_srq *ibsrq, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad_wr) { struct mthca_srq *srq = to_msrq(ibsrq); uint32_t doorbell[2]; int err = 0; int first_ind; int ind; int next_ind; int nreq; int i; void *wqe; void *prev_wqe; pthread_spin_lock(&srq->lock); first_ind = srq->first_free; for (nreq = 0; wr; wr = wr->next) { ind = srq->first_free; wqe = get_wqe(srq, ind); next_ind = *wqe_to_link(wqe); if (next_ind < 0) { err = -1; *bad_wr = wr; break; } prev_wqe = srq->last; srq->last = wqe; ((struct mthca_next_seg *) wqe)->ee_nds = 0; /* flags field will always remain 0 */ wqe += sizeof (struct mthca_next_seg); if (wr->num_sge > srq->max_gs) { err = -1; *bad_wr = wr; srq->last = prev_wqe; break; } for (i = 0; i < wr->num_sge; ++i) { ((struct mthca_data_seg *) wqe)->byte_count = htonl(wr->sg_list[i].length); ((struct mthca_data_seg *) wqe)->lkey = htonl(wr->sg_list[i].lkey); ((struct mthca_data_seg *) wqe)->addr = htonll(wr->sg_list[i].addr); wqe += sizeof (struct mthca_data_seg); } if (i < srq->max_gs) { ((struct mthca_data_seg *) wqe)->byte_count = 0; ((struct mthca_data_seg *) wqe)->lkey = htonl(MTHCA_INVAL_LKEY); ((struct mthca_data_seg *) wqe)->addr = 0; } ((struct mthca_next_seg *) prev_wqe)->ee_nds = htonl(MTHCA_NEXT_DBD); srq->wrid[ind] = wr->wr_id; srq->first_free = next_ind; if (++nreq == MTHCA_TAVOR_MAX_WQES_PER_RECV_DB) { nreq = 0; doorbell[0] = htonl(first_ind << srq->wqe_shift); doorbell[1] = htonl(srq->srqn << 8); /* * Make sure that descriptors are written * before doorbell is rung. */ wmb(); mthca_write64(doorbell, to_mctx(ibsrq->context), MTHCA_RECV_DOORBELL); first_ind = srq->first_free; } } if (nreq) { doorbell[0] = htonl(first_ind << srq->wqe_shift); doorbell[1] = htonl((srq->srqn << 8) | nreq); /* * Make sure that descriptors are written before * doorbell is rung. */ wmb(); mthca_write64(doorbell, to_mctx(ibsrq->context), MTHCA_RECV_DOORBELL); } pthread_spin_unlock(&srq->lock); return err; }
int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, struct ib_recv_wr **bad_wr) { struct mlx4_ib_srq *srq = to_msrq(ibsrq); struct mlx4_wqe_srq_next_seg *next; struct mlx4_wqe_data_seg *scat; unsigned long flags; int err = 0; int nreq; int i; struct mlx4_ib_dev *mdev = to_mdev(ibsrq->device); spin_lock_irqsave(&srq->lock, flags); if (mdev->dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) { err = -EIO; *bad_wr = wr; nreq = 0; goto out; } for (nreq = 0; wr; ++nreq, wr = wr->next) { if (unlikely(wr->num_sge > srq->msrq.max_gs)) { err = -EINVAL; *bad_wr = wr; break; } if (unlikely(srq->head == srq->tail)) { err = -ENOMEM; *bad_wr = wr; break; } srq->wrid[srq->head] = wr->wr_id; next = get_wqe(srq, srq->head); srq->head = be16_to_cpu(next->next_wqe_index); scat = (struct mlx4_wqe_data_seg *) (next + 1); for (i = 0; i < wr->num_sge; ++i) { scat[i].byte_count = cpu_to_be32(wr->sg_list[i].length); scat[i].lkey = cpu_to_be32(wr->sg_list[i].lkey); scat[i].addr = cpu_to_be64(wr->sg_list[i].addr); } if (i < srq->msrq.max_gs) { scat[i].byte_count = 0; scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY); scat[i].addr = 0; } } if (likely(nreq)) { srq->wqe_ctr += nreq; /* * Make sure that descriptors are written before * doorbell record. */ wmb(); *srq->db.db = cpu_to_be32(srq->wqe_ctr); } out: spin_unlock_irqrestore(&srq->lock, flags); return err; }
int mthca_arbel_post_srq_recv(struct ibv_srq *ibsrq, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad_wr) { struct mthca_srq *srq = to_msrq(ibsrq); int err = 0; int ind; int next_ind; int nreq; int i; void *wqe; pthread_spin_lock(&srq->lock); for (nreq = 0; wr; ++nreq, wr = wr->next) { ind = srq->first_free; wqe = get_wqe(srq, ind); next_ind = *wqe_to_link(wqe); if (next_ind < 0) { err = -1; *bad_wr = wr; break; } ((struct mthca_next_seg *) wqe)->ee_nds = 0; /* flags field will always remain 0 */ wqe += sizeof (struct mthca_next_seg); if (wr->num_sge > srq->max_gs) { err = -1; *bad_wr = wr; break; } for (i = 0; i < wr->num_sge; ++i) { ((struct mthca_data_seg *) wqe)->byte_count = htonl(wr->sg_list[i].length); ((struct mthca_data_seg *) wqe)->lkey = htonl(wr->sg_list[i].lkey); ((struct mthca_data_seg *) wqe)->addr = htonll(wr->sg_list[i].addr); wqe += sizeof (struct mthca_data_seg); } if (i < srq->max_gs) { ((struct mthca_data_seg *) wqe)->byte_count = 0; ((struct mthca_data_seg *) wqe)->lkey = htonl(MTHCA_INVAL_LKEY); ((struct mthca_data_seg *) wqe)->addr = 0; } srq->wrid[ind] = wr->wr_id; srq->first_free = next_ind; } if (nreq) { srq->counter += nreq; /* * Make sure that descriptors are written before * we write doorbell record. */ wmb(); *srq->db = htonl(srq->counter); } pthread_spin_unlock(&srq->lock); return err; }
int mlx4_post_srq_recv(struct ibv_srq *ibsrq, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad_wr) { struct mlx4_srq *srq = to_msrq(ibsrq); struct mlx4_wqe_srq_next_seg *next; struct mlx4_wqe_data_seg *scat; int err = 0; int nreq; int i; pthread_spin_lock(&srq->lock); for (nreq = 0; wr; ++nreq, wr = wr->next) { if (wr->num_sge > srq->max_gs) { err = -1; *bad_wr = wr; break; } if (srq->head == srq->tail) { /* SRQ is full*/ err = -1; *bad_wr = wr; break; } srq->wrid[srq->head] = wr->wr_id; next = get_wqe(srq, srq->head); srq->head = be16toh(next->next_wqe_index); scat = (struct mlx4_wqe_data_seg *) (next + 1); for (i = 0; i < wr->num_sge; ++i) { scat[i].byte_count = htobe32(wr->sg_list[i].length); scat[i].lkey = htobe32(wr->sg_list[i].lkey); scat[i].addr = htobe64(wr->sg_list[i].addr); } if (i < srq->max_gs) { scat[i].byte_count = 0; scat[i].lkey = htobe32(MLX4_INVALID_LKEY); scat[i].addr = 0; } } if (nreq) { srq->counter += nreq; /* * Make sure that descriptors are written before * we write doorbell record. */ udma_to_device_barrier(); *srq->db = htobe32(srq->counter); } pthread_spin_unlock(&srq->lock); return err; }
int rxe_completer(void *arg) { struct rxe_qp *qp = (struct rxe_qp *)arg; struct rxe_dev *rxe = to_rdev(qp->ibqp.device); struct rxe_send_wqe *wqe = wqe; struct sk_buff *skb = NULL; struct rxe_pkt_info *pkt = NULL; enum comp_state state; rxe_add_ref(qp); if (!qp->valid || qp->req.state == QP_STATE_ERROR || qp->req.state == QP_STATE_RESET) { rxe_drain_resp_pkts(qp, qp->valid && qp->req.state == QP_STATE_ERROR); goto exit; } if (qp->comp.timeout) { qp->comp.timeout_retry = 1; qp->comp.timeout = 0; } else { qp->comp.timeout_retry = 0; } if (qp->req.need_retry) goto exit; state = COMPST_GET_ACK; while (1) { pr_debug("qp#%d state = %s\n", qp_num(qp), comp_state_name[state]); switch (state) { case COMPST_GET_ACK: skb = skb_dequeue(&qp->resp_pkts); if (skb) { pkt = SKB_TO_PKT(skb); qp->comp.timeout_retry = 0; } state = COMPST_GET_WQE; break; case COMPST_GET_WQE: state = get_wqe(qp, pkt, &wqe); break; case COMPST_CHECK_PSN: state = check_psn(qp, pkt, wqe); break; case COMPST_CHECK_ACK: state = check_ack(qp, pkt, wqe); break; case COMPST_READ: state = do_read(qp, pkt, wqe); break; case COMPST_ATOMIC: state = do_atomic(qp, pkt, wqe); break; case COMPST_WRITE_SEND: if (wqe->state == wqe_state_pending && wqe->last_psn == pkt->psn) state = COMPST_COMP_ACK; else state = COMPST_UPDATE_COMP; break; case COMPST_COMP_ACK: state = complete_ack(qp, pkt, wqe); break; case COMPST_COMP_WQE: state = complete_wqe(qp, pkt, wqe); break; case COMPST_UPDATE_COMP: if (pkt->mask & RXE_END_MASK) qp->comp.opcode = -1; else qp->comp.opcode = pkt->opcode; if (psn_compare(pkt->psn, qp->comp.psn) >= 0) qp->comp.psn = (pkt->psn + 1) & BTH_PSN_MASK; if (qp->req.wait_psn) { qp->req.wait_psn = 0; rxe_run_task(&qp->req.task, 1); } state = COMPST_DONE; break; case COMPST_DONE: if (pkt) { rxe_drop_ref(pkt->qp); kfree_skb(skb); skb = NULL; } goto done; case COMPST_EXIT: if (qp->comp.timeout_retry && wqe) { state = COMPST_ERROR_RETRY; break; } /* re reset the timeout counter if * (1) QP is type RC * (2) the QP is alive * (3) there is a packet sent by the requester that * might be acked (we still might get spurious * timeouts but try to keep them as few as possible) * (4) the timeout parameter is set */ if ((qp_type(qp) == IB_QPT_RC) && (qp->req.state == QP_STATE_READY) && (psn_compare(qp->req.psn, qp->comp.psn) > 0) && qp->qp_timeout_jiffies) mod_timer(&qp->retrans_timer, jiffies + qp->qp_timeout_jiffies); WARN_ON_ONCE(skb); goto exit; case COMPST_ERROR_RETRY: /* we come here if the retry timer fired and we did * not receive a response packet. try to retry the send * queue if that makes sense and the limits have not * been exceeded. remember that some timeouts are * spurious since we do not reset the timer but kick * it down the road or let it expire */ /* there is nothing to retry in this case */ if (!wqe || (wqe->state == wqe_state_posted)) { WARN_ON_ONCE(skb); goto exit; } if (qp->comp.retry_cnt > 0) { if (qp->comp.retry_cnt != 7) qp->comp.retry_cnt--; /* no point in retrying if we have already * seen the last ack that the requester could * have caused */ if (psn_compare(qp->req.psn, qp->comp.psn) > 0) { /* tell the requester to retry the * send queue next time around */ rxe_counter_inc(rxe, RXE_CNT_COMP_RETRY); qp->req.need_retry = 1; rxe_run_task(&qp->req.task, 1); } if (pkt) { rxe_drop_ref(pkt->qp); kfree_skb(skb); skb = NULL; } WARN_ON_ONCE(skb); goto exit; } else { rxe_counter_inc(rxe, RXE_CNT_RETRY_EXCEEDED); wqe->status = IB_WC_RETRY_EXC_ERR; state = COMPST_ERROR; } break; case COMPST_RNR_RETRY: if (qp->comp.rnr_retry > 0) { if (qp->comp.rnr_retry != 7) qp->comp.rnr_retry--; qp->req.need_retry = 1; pr_debug("qp#%d set rnr nak timer\n", qp_num(qp)); mod_timer(&qp->rnr_nak_timer, jiffies + rnrnak_jiffies(aeth_syn(pkt) & ~AETH_TYPE_MASK)); rxe_drop_ref(pkt->qp); kfree_skb(skb); skb = NULL; goto exit; } else { rxe_counter_inc(rxe, RXE_CNT_RNR_RETRY_EXCEEDED); wqe->status = IB_WC_RNR_RETRY_EXC_ERR; state = COMPST_ERROR; } break; case COMPST_ERROR: WARN_ON_ONCE(wqe->status == IB_WC_SUCCESS); do_complete(qp, wqe); rxe_qp_error(qp); if (pkt) { rxe_drop_ref(pkt->qp); kfree_skb(skb); skb = NULL; } WARN_ON_ONCE(skb); goto exit; } } exit: /* we come here if we are done with processing and want the task to * exit from the loop calling us */ WARN_ON_ONCE(skb); rxe_drop_ref(qp); return -EAGAIN; done: /* we come here if we have processed a packet we want the task to call * us again to see if there is anything else to do */ WARN_ON_ONCE(skb); rxe_drop_ref(qp); return 0; }
static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq, struct mlx5_create_srq_mbox_in **in, int buf_size, int *inlen) { int err; int i; struct mlx5_wqe_srq_next_seg *next; int page_shift; int npages; err = mlx5_db_alloc(dev->mdev, &srq->db); if (err) { mlx5_ib_warn(dev, "alloc dbell rec failed\n"); return err; } *srq->db.db = 0; if (mlx5_buf_alloc(dev->mdev, buf_size, PAGE_SIZE * 2, &srq->buf)) { mlx5_ib_dbg(dev, "buf alloc failed\n"); err = -ENOMEM; goto err_db; } page_shift = srq->buf.page_shift; srq->head = 0; srq->tail = srq->msrq.max - 1; srq->wqe_ctr = 0; for (i = 0; i < srq->msrq.max; i++) { next = get_wqe(srq, i); next->next_wqe_index = cpu_to_be16((i + 1) & (srq->msrq.max - 1)); } npages = DIV_ROUND_UP(srq->buf.npages, 1 << (page_shift - PAGE_SHIFT)); mlx5_ib_dbg(dev, "buf_size %d, page_shift %d, npages %d, calc npages %d\n", buf_size, page_shift, srq->buf.npages, npages); *inlen = sizeof(**in) + sizeof(*(*in)->pas) * npages; *in = mlx5_vzalloc(*inlen); if (!*in) { err = -ENOMEM; goto err_buf; } mlx5_fill_page_array(&srq->buf, (*in)->pas); srq->wrid = kmalloc(srq->msrq.max * sizeof(u64), GFP_KERNEL); if (!srq->wrid) { mlx5_ib_dbg(dev, "kmalloc failed %lu\n", (unsigned long)(srq->msrq.max * sizeof(u64))); err = -ENOMEM; goto err_in; } srq->wq_sig = !!srq_signature; (*in)->ctx.log_pg_sz = page_shift - MLX5_ADAPTER_PAGE_SHIFT; return 0; err_in: mlx5_vfree(*in); err_buf: mlx5_buf_free(dev->mdev, &srq->buf); err_db: mlx5_db_free(dev->mdev, &srq->db); return err; }
struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd, struct ib_srq_init_attr *init_attr, struct ib_udata *udata) { struct mlx4_ib_dev *dev = to_mdev(pd->device); struct mlx4_ib_srq *srq; struct mlx4_wqe_srq_next_seg *next; struct mlx4_wqe_data_seg *scatter; u32 cqn; u16 xrcdn; int desc_size; int buf_size; int err; int i; /* Sanity check SRQ size before proceeding */ if (init_attr->attr.max_wr >= dev->dev->caps.max_srq_wqes || init_attr->attr.max_sge > dev->dev->caps.max_srq_sge) return ERR_PTR(-EINVAL); srq = kmalloc(sizeof *srq, GFP_KERNEL); if (!srq) return ERR_PTR(-ENOMEM); mutex_init(&srq->mutex); spin_lock_init(&srq->lock); srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1); srq->msrq.max_gs = init_attr->attr.max_sge; desc_size = max(32UL, roundup_pow_of_two(sizeof (struct mlx4_wqe_srq_next_seg) + srq->msrq.max_gs * sizeof (struct mlx4_wqe_data_seg))); srq->msrq.wqe_shift = ilog2(desc_size); buf_size = srq->msrq.max * desc_size; if (pd->uobject) { struct mlx4_ib_create_srq ucmd; if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { err = -EFAULT; goto err_srq; } srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr, buf_size, 0, 0); if (IS_ERR(srq->umem)) { err = PTR_ERR(srq->umem); goto err_srq; } err = mlx4_mtt_init(dev->dev, ib_umem_page_count(srq->umem), ilog2(srq->umem->page_size), &srq->mtt); if (err) goto err_buf; err = mlx4_ib_umem_write_mtt(dev, &srq->mtt, srq->umem); if (err) goto err_mtt; err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context), ucmd.db_addr, &srq->db); if (err) goto err_mtt; } else { err = mlx4_db_alloc(dev->dev, &srq->db, 0, GFP_KERNEL); if (err) goto err_srq; *srq->db.db = 0; if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2, &srq->buf, GFP_KERNEL)) { err = -ENOMEM; goto err_db; } srq->head = 0; srq->tail = srq->msrq.max - 1; srq->wqe_ctr = 0; for (i = 0; i < srq->msrq.max; ++i) { next = get_wqe(srq, i); next->next_wqe_index = cpu_to_be16((i + 1) & (srq->msrq.max - 1)); for (scatter = (void *) (next + 1); (void *) scatter < (void *) next + desc_size; ++scatter) scatter->lkey = cpu_to_be32(MLX4_INVALID_LKEY); } err = mlx4_mtt_init(dev->dev, srq->buf.npages, srq->buf.page_shift, &srq->mtt); if (err) goto err_buf; err = mlx4_buf_write_mtt(dev->dev, &srq->mtt, &srq->buf, GFP_KERNEL); if (err) goto err_mtt; srq->wrid = kmalloc(srq->msrq.max * sizeof (u64), GFP_KERNEL); if (!srq->wrid) { err = -ENOMEM; goto err_mtt; } } cqn = (init_attr->srq_type == IB_SRQT_XRC) ? to_mcq(init_attr->ext.xrc.cq)->mcq.cqn : 0; xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ? to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn : (u16) dev->dev->caps.reserved_xrcds; err = mlx4_srq_alloc(dev->dev, to_mpd(pd)->pdn, cqn, xrcdn, &srq->mtt, srq->db.dma, &srq->msrq); if (err) goto err_wrid; srq->msrq.event = mlx4_ib_srq_event; srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn; if (pd->uobject) if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) { err = -EFAULT; goto err_wrid; } init_attr->attr.max_wr = srq->msrq.max - 1; return &srq->ibsrq; err_wrid: if (pd->uobject) mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &srq->db); else kfree(srq->wrid); err_mtt: mlx4_mtt_cleanup(dev->dev, &srq->mtt); err_buf: if (pd->uobject) ib_umem_release(srq->umem); else mlx4_buf_free(dev->dev, buf_size, &srq->buf); err_db: if (!pd->uobject) mlx4_db_free(dev->dev, &srq->db); err_srq: kfree(srq); return ERR_PTR(err); }
int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv) { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_wqe_srq_next_seg *next; struct mlx4_en_rx_ring *ring; int i; int ring_ind; int err; int stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + DS_SIZE * priv->num_frags); int max_gs = (stride - sizeof(struct mlx4_wqe_srq_next_seg)) / DS_SIZE; for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { ring = &priv->rx_ring[ring_ind]; ring->prod = 0; ring->cons = 0; ring->actual_size = 0; ring->cqn = priv->rx_cq[ring_ind].mcq.cqn; ring->stride = stride; ring->log_stride = ffs(ring->stride) - 1; ring->buf_size = ring->size * ring->stride; memset(ring->rx_info, 0, sizeof(*(ring->rx_info))); memset(ring->buf, 0, ring->buf_size); mlx4_en_update_rx_prod_db(ring); /* Initailize all descriptors */ for (i = 0; i < ring->size; i++) mlx4_en_init_rx_desc(priv, ring, i); /* Initialize page allocators */ err = mlx4_en_init_allocator(priv, ring); if (err) { mlx4_err(mdev, "Failed initializing ring allocator\n"); goto err_allocator; } /* Fill Rx buffers */ ring->full = 0; } if (mlx4_en_fill_rx_buffers(priv)) goto err_buffers; for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { ring = &priv->rx_ring[ring_ind]; mlx4_en_update_rx_prod_db(ring); /* Configure SRQ representing the ring */ ring->srq.max = ring->size; ring->srq.max_gs = max_gs; ring->srq.wqe_shift = ilog2(ring->stride); for (i = 0; i < ring->srq.max; ++i) { next = get_wqe(ring, i); next->next_wqe_index = cpu_to_be16((i + 1) & (ring->srq.max - 1)); } err = mlx4_srq_alloc(mdev->dev, mdev->priv_pdn, &ring->wqres.mtt, ring->wqres.db.dma, &ring->srq); if (err){ mlx4_err(mdev, "Failed to allocate srq\n"); goto err_srq; } ring->srq.event = mlx4_en_srq_event; } return 0; err_srq: while (ring_ind >= 0) { ring = &priv->rx_ring[ring_ind]; mlx4_srq_free(mdev->dev, &ring->srq); ring_ind--; } err_buffers: for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) mlx4_en_free_rx_buf(priv, &priv->rx_ring[ring_ind]); ring_ind = priv->rx_ring_num - 1; err_allocator: while (ring_ind >= 0) { mlx4_en_destroy_allocator(priv, &priv->rx_ring[ring_ind]); ring_ind--; } return err; }