/* called when the last reference to the qp is dropped */ static void rxe_qp_do_cleanup(struct work_struct *work) { struct rxe_qp *qp = container_of(work, typeof(*qp), cleanup_work.work); rxe_drop_all_mcast_groups(qp); if (qp->sq.queue) rxe_queue_cleanup(qp->sq.queue); if (qp->srq) rxe_drop_ref(qp->srq); if (qp->rq.queue) rxe_queue_cleanup(qp->rq.queue); if (qp->scq) rxe_drop_ref(qp->scq); if (qp->rcq) rxe_drop_ref(qp->rcq); if (qp->pd) rxe_drop_ref(qp->pd); if (qp->resp.mr) { rxe_drop_ref(qp->resp.mr); qp->resp.mr = NULL; } if (qp_type(qp) == IB_QPT_RC) sk_dst_reset(qp->sk->sk); free_rd_atomic_resources(qp); kernel_sock_shutdown(qp->sk, SHUT_RDWR); sock_release(qp->sk); }
/* called when the last reference to the qp is dropped */ void rxe_qp_cleanup(void *arg) { struct rxe_qp *qp = arg; rxe_drop_all_mcast_groups(qp); if (qp->sq.queue) rxe_queue_cleanup(qp->sq.queue); if (qp->srq) rxe_drop_ref(qp->srq); if (qp->rq.queue) rxe_queue_cleanup(qp->rq.queue); if (qp->scq) rxe_drop_ref(qp->scq); if (qp->rcq) rxe_drop_ref(qp->rcq); if (qp->pd) rxe_drop_ref(qp->pd); if (qp->resp.mr) { rxe_drop_ref(qp->resp.mr); qp->resp.mr = NULL; } free_rd_atomic_resources(qp); kernel_sock_shutdown(qp->sk, SHUT_RDWR); }
static struct ib_mr *rxe_get_dma_mr(struct ib_pd *ibpd, int access) { struct rxe_dev *rxe = to_rdev(ibpd->device); struct rxe_pd *pd = to_rpd(ibpd); struct rxe_mem *mr; int err; mr = rxe_alloc(&rxe->mr_pool); if (!mr) { err = -ENOMEM; goto err1; } rxe_add_index(mr); rxe_add_ref(pd); err = rxe_mem_init_dma(pd, access, mr); if (err) goto err2; return &mr->ibmr; err2: rxe_drop_ref(pd); rxe_drop_index(mr); rxe_drop_ref(mr); err1: return ERR_PTR(err); }
static struct ib_ah *rxe_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr) { int err; struct rxe_dev *rxe = to_rdev(ibpd->device); struct rxe_pd *pd = to_rpd(ibpd); struct rxe_ah *ah; err = rxe_av_chk_attr(rxe, attr); if (err) goto err1; ah = rxe_alloc(&rxe->ah_pool); if (!ah) { err = -ENOMEM; goto err1; } rxe_add_ref(pd); ah->pd = pd; err = rxe_init_av(rxe, attr, &ah->av); if (err) goto err2; return &ah->ibah; err2: rxe_drop_ref(pd); rxe_drop_ref(ah); err1: return ERR_PTR(err); }
static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, u64 iova, int access, struct ib_udata *udata) { int err; struct rxe_dev *rxe = to_rdev(ibpd->device); struct rxe_pd *pd = to_rpd(ibpd); struct rxe_mem *mr; mr = rxe_alloc(&rxe->mr_pool); if (!mr) { err = -ENOMEM; goto err2; } rxe_add_index(mr); rxe_add_ref(pd); err = rxe_mem_init_user(pd, start, length, iova, access, udata, mr); if (err) goto err3; return &mr->ibmr; err3: rxe_drop_ref(pd); rxe_drop_index(mr); rxe_drop_ref(mr); err2: return ERR_PTR(err); }
static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, u32 max_num_sg, struct ib_udata *udata) { struct rxe_dev *rxe = to_rdev(ibpd->device); struct rxe_pd *pd = to_rpd(ibpd); struct rxe_mem *mr; int err; if (mr_type != IB_MR_TYPE_MEM_REG) return ERR_PTR(-EINVAL); mr = rxe_alloc(&rxe->mr_pool); if (!mr) { err = -ENOMEM; goto err1; } rxe_add_index(mr); rxe_add_ref(pd); err = rxe_mem_init_fast(pd, max_num_sg, mr); if (err) goto err2; return &mr->ibmr; err2: rxe_drop_ref(pd); rxe_drop_index(mr); rxe_drop_ref(mr); err1: return ERR_PTR(err); }
static int rxe_destroy_ah(struct ib_ah *ibah) { struct rxe_ah *ah = to_rah(ibah); rxe_drop_ref(ah->pd); rxe_drop_ref(ah); return 0; }
static void rxe_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) { struct rxe_srq *srq = to_rsrq(ibsrq); if (srq->rq.queue) rxe_queue_cleanup(srq->rq.queue); rxe_drop_ref(srq->pd); rxe_drop_ref(srq); }
static int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) { struct rxe_mem *mr = to_rmr(ibmr); mr->state = RXE_MEM_STATE_ZOMBIE; rxe_drop_ref(mr->pd); rxe_drop_index(mr); rxe_drop_ref(mr); return 0; }
void free_rd_atomic_resource(struct rxe_qp *qp, struct resp_res *res) { if (res->type == RXE_ATOMIC_MASK) { rxe_drop_ref(qp); kfree_skb(res->atomic.skb); } else if (res->type == RXE_READ_MASK) { if (res->read.mr) rxe_drop_ref(res->read.mr); } res->type = 0; }
static int rxe_destroy_srq(struct ib_srq *ibsrq) { struct rxe_srq *srq = to_rsrq(ibsrq); if (srq->rq.queue) rxe_queue_cleanup(srq->rq.queue); rxe_drop_ref(srq->pd); rxe_drop_index(srq); rxe_drop_ref(srq); return 0; }
void free_rd_atomic_resource(struct rxe_qp *qp, struct resp_res *res) { struct rxe_dev *rxe = to_rdev(qp->ibqp.device); if (res->type == RXE_ATOMIC_MASK) { rxe_drop_ref(qp); kfree_skb(res->atomic.skb); atomic_dec(&rxe->resp_skb_out); } else if (res->type == RXE_READ_MASK) { if (res->read.mr) rxe_drop_ref(res->read.mr); } res->type = 0; }
/* called by the create qp verb */ int rxe_qp_from_init(struct rxe_dev *rxe, struct rxe_qp *qp, struct rxe_pd *pd, struct ib_qp_init_attr *init, struct ib_udata *udata, struct ib_pd *ibpd) { int err; struct rxe_cq *rcq = to_rcq(init->recv_cq); struct rxe_cq *scq = to_rcq(init->send_cq); struct rxe_srq *srq = init->srq ? to_rsrq(init->srq) : NULL; struct ib_ucontext *context = udata ? ibpd->uobject->context : NULL; rxe_add_ref(pd); rxe_add_ref(rcq); rxe_add_ref(scq); if (srq) rxe_add_ref(srq); qp->pd = pd; qp->rcq = rcq; qp->scq = scq; qp->srq = srq; qp->udata = udata; rxe_qp_init_misc(rxe, qp, init); err = rxe_qp_init_req(rxe, qp, init, context, udata); if (err) goto err1; err = rxe_qp_init_resp(rxe, qp, init, context, udata); if (err) goto err2; qp->attr.qp_state = IB_QPS_RESET; qp->valid = 1; return 0; err2: rxe_queue_cleanup(qp->sq.queue); err1: if (srq) rxe_drop_ref(srq); rxe_drop_ref(scq); rxe_drop_ref(rcq); rxe_drop_ref(pd); return err; }
static int rxe_dealloc_pd(struct ib_pd *ibpd) { struct rxe_pd *pd = to_rpd(ibpd); rxe_drop_ref(pd); return 0; }
int rxe_send(struct rxe_pkt_info *pkt, struct sk_buff *skb) { int err; skb->destructor = rxe_skb_tx_dtor; skb->sk = pkt->qp->sk->sk; rxe_add_ref(pkt->qp); atomic_inc(&pkt->qp->skb_out); if (skb->protocol == htons(ETH_P_IP)) { err = ip_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb); } else if (skb->protocol == htons(ETH_P_IPV6)) { err = ip6_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb); } else { pr_err("Unknown layer 3 protocol: %d\n", skb->protocol); atomic_dec(&pkt->qp->skb_out); rxe_drop_ref(pkt->qp); kfree_skb(skb); return -EINVAL; } if (unlikely(net_xmit_eval(err))) { pr_debug("error sending packet: %d\n", err); return -EAGAIN; } return 0; }
static struct ib_cq *rxe_create_cq(struct ib_device *dev, const struct ib_cq_init_attr *attr, struct ib_ucontext *context, struct ib_udata *udata) { int err; struct rxe_dev *rxe = to_rdev(dev); struct rxe_cq *cq; if (attr->flags) return ERR_PTR(-EINVAL); err = rxe_cq_chk_attr(rxe, NULL, attr->cqe, attr->comp_vector, udata); if (err) goto err1; cq = rxe_alloc(&rxe->cq_pool); if (!cq) { err = -ENOMEM; goto err1; } err = rxe_cq_from_init(rxe, cq, attr->cqe, attr->comp_vector, context, udata); if (err) goto err2; return &cq->ibcq; err2: rxe_drop_ref(cq); err1: return ERR_PTR(err); }
static int rxe_dealloc_ucontext(struct ib_ucontext *ibuc) { struct rxe_ucontext *uc = to_ruc(ibuc); rxe_drop_ref(uc); return 0; }
static int rxe_destroy_cq(struct ib_cq *ibcq) { struct rxe_cq *cq = to_rcq(ibcq); rxe_drop_ref(cq); return 0; }
static inline enum resp_states get_req(struct rxe_qp *qp, struct rxe_pkt_info **pkt_p) { struct sk_buff *skb; if (qp->resp.state == QP_STATE_ERROR) { skb = skb_dequeue(&qp->req_pkts); if (skb) { /* drain request packet queue */ rxe_drop_ref(qp); kfree_skb(skb); return RESPST_GET_REQ; } /* go drain recv wr queue */ return RESPST_CHK_RESOURCE; } skb = skb_peek(&qp->req_pkts); if (!skb) return RESPST_EXIT; *pkt_p = SKB_TO_PKT(skb); return (qp->resp.res) ? RESPST_READ_REPLY : RESPST_CHK_PSN; }
static int rxe_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) { struct rxe_cq *cq = to_rcq(ibcq); rxe_cq_disable(cq); rxe_drop_ref(cq); return 0; }
static int rxe_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) { struct rxe_qp *qp = to_rqp(ibqp); rxe_qp_destroy(qp); rxe_drop_index(qp); rxe_drop_ref(qp); return 0; }
/* move the qp to the reset state */ static void rxe_qp_reset(struct rxe_qp *qp) { /* stop tasks from running */ rxe_disable_task(&qp->resp.task); /* stop request/comp */ if (qp->sq.queue) { if (qp_type(qp) == IB_QPT_RC) rxe_disable_task(&qp->comp.task); rxe_disable_task(&qp->req.task); } /* move qp to the reset state */ qp->req.state = QP_STATE_RESET; qp->resp.state = QP_STATE_RESET; /* let state machines reset themselves drain work and packet queues * etc. */ __rxe_do_task(&qp->resp.task); if (qp->sq.queue) { __rxe_do_task(&qp->comp.task); __rxe_do_task(&qp->req.task); rxe_queue_reset(qp->sq.queue); } /* cleanup attributes */ atomic_set(&qp->ssn, 0); qp->req.opcode = -1; qp->req.need_retry = 0; qp->req.noack_pkts = 0; qp->resp.msn = 0; qp->resp.opcode = -1; qp->resp.drop_msg = 0; qp->resp.goto_error = 0; qp->resp.sent_psn_nak = 0; if (qp->resp.mr) { rxe_drop_ref(qp->resp.mr); qp->resp.mr = NULL; } cleanup_rd_atomic_resources(qp); /* reenable tasks */ rxe_enable_task(&qp->resp.task); if (qp->sq.queue) { if (qp_type(qp) == IB_QPT_RC) rxe_enable_task(&qp->comp.task); rxe_enable_task(&qp->req.task); } }
static void rxe_skb_tx_dtor(struct sk_buff *skb) { struct sock *sk = skb->sk; struct rxe_qp *qp = sk->sk_user_data; int skb_out = atomic_dec_return(&qp->skb_out); if (unlikely(qp->need_req_skb && skb_out < RXE_INFLIGHT_SKBS_PER_QP_LOW)) rxe_run_task(&qp->req.task, 1); rxe_drop_ref(qp); }
static int rxe_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init, struct ib_udata *udata) { int err; struct rxe_dev *rxe = to_rdev(ibsrq->device); struct rxe_pd *pd = to_rpd(ibsrq->pd); struct rxe_srq *srq = to_rsrq(ibsrq); struct rxe_create_srq_resp __user *uresp = NULL; if (udata) { if (udata->outlen < sizeof(*uresp)) return -EINVAL; uresp = udata->outbuf; } err = rxe_srq_chk_attr(rxe, NULL, &init->attr, IB_SRQ_INIT_MASK); if (err) goto err1; err = rxe_add_to_pool(&rxe->srq_pool, &srq->pelem); if (err) goto err1; rxe_add_ref(pd); srq->pd = pd; err = rxe_srq_from_init(rxe, srq, init, udata, uresp); if (err) goto err2; return 0; err2: rxe_drop_ref(pd); rxe_drop_ref(srq); err1: return err; }
static struct ib_srq *rxe_create_srq(struct ib_pd *ibpd, struct ib_srq_init_attr *init, struct ib_udata *udata) { int err; struct rxe_dev *rxe = to_rdev(ibpd->device); struct rxe_pd *pd = to_rpd(ibpd); struct rxe_srq *srq; struct ib_ucontext *context = udata ? ibpd->uobject->context : NULL; err = rxe_srq_chk_attr(rxe, NULL, &init->attr, IB_SRQ_INIT_MASK); if (err) goto err1; srq = rxe_alloc(&rxe->srq_pool); if (!srq) { err = -ENOMEM; goto err1; } rxe_add_index(srq); rxe_add_ref(pd); srq->pd = pd; err = rxe_srq_from_init(rxe, srq, init, context, udata); if (err) goto err2; return &srq->ibsrq; err2: rxe_drop_ref(pd); rxe_drop_index(srq); rxe_drop_ref(srq); err1: return ERR_PTR(err); }
static struct ib_qp *rxe_create_qp(struct ib_pd *ibpd, struct ib_qp_init_attr *init, struct ib_udata *udata) { int err; struct rxe_dev *rxe = to_rdev(ibpd->device); struct rxe_pd *pd = to_rpd(ibpd); struct rxe_qp *qp; struct rxe_create_qp_resp __user *uresp = NULL; if (udata) { if (udata->outlen < sizeof(*uresp)) return ERR_PTR(-EINVAL); uresp = udata->outbuf; } err = rxe_qp_chk_init(rxe, init); if (err) goto err1; qp = rxe_alloc(&rxe->qp_pool); if (!qp) { err = -ENOMEM; goto err1; } if (udata) { if (udata->inlen) { err = -EINVAL; goto err2; } qp->is_user = 1; } rxe_add_index(qp); err = rxe_qp_from_init(rxe, qp, pd, init, uresp, ibpd, udata); if (err) goto err3; return &qp->ibqp; err3: rxe_drop_index(qp); err2: rxe_drop_ref(qp); err1: return ERR_PTR(err); }
static int rxe_attach_mcast(struct ib_qp *ibqp, union ib_gid *mgid, u16 mlid) { int err; struct rxe_dev *rxe = to_rdev(ibqp->device); struct rxe_qp *qp = to_rqp(ibqp); struct rxe_mc_grp *grp; /* takes a ref on grp if successful */ err = rxe_mcast_get_grp(rxe, mgid, &grp); if (err) return err; err = rxe_mcast_add_grp_elem(rxe, qp, grp); rxe_drop_ref(grp); return err; }
static void rxe_drain_resp_pkts(struct rxe_qp *qp, bool notify) { struct sk_buff *skb; struct rxe_send_wqe *wqe; while ((skb = skb_dequeue(&qp->resp_pkts))) { rxe_drop_ref(qp); kfree_skb(skb); } while ((wqe = queue_head(qp->sq.queue))) { if (notify) { wqe->status = IB_WC_WR_FLUSH_ERR; do_complete(qp, wqe); } else { advance_consumer(qp->sq.queue); } } }
static struct ib_cq *rxe_create_cq(struct ib_device *dev, const struct ib_cq_init_attr *attr, struct ib_udata *udata) { int err; struct rxe_dev *rxe = to_rdev(dev); struct rxe_cq *cq; struct rxe_create_cq_resp __user *uresp = NULL; if (udata) { if (udata->outlen < sizeof(*uresp)) return ERR_PTR(-EINVAL); uresp = udata->outbuf; } if (attr->flags) return ERR_PTR(-EINVAL); err = rxe_cq_chk_attr(rxe, NULL, attr->cqe, attr->comp_vector); if (err) goto err1; cq = rxe_alloc(&rxe->cq_pool); if (!cq) { err = -ENOMEM; goto err1; } err = rxe_cq_from_init(rxe, cq, attr->cqe, attr->comp_vector, udata, uresp); if (err) goto err2; return &cq->ibcq; err2: rxe_drop_ref(cq); err1: return ERR_PTR(err); }
int rxe_requester(void *arg) { struct rxe_qp *qp = (struct rxe_qp *)arg; struct rxe_pkt_info pkt; struct sk_buff *skb; struct rxe_send_wqe *wqe; enum rxe_hdr_mask mask; int payload; int mtu; int opcode; int ret; struct rxe_send_wqe rollback_wqe; u32 rollback_psn; rxe_add_ref(qp); next_wqe: if (unlikely(!qp->valid || qp->req.state == QP_STATE_ERROR)) goto exit; if (unlikely(qp->req.state == QP_STATE_RESET)) { qp->req.wqe_index = consumer_index(qp->sq.queue); qp->req.opcode = -1; qp->req.need_rd_atomic = 0; qp->req.wait_psn = 0; qp->req.need_retry = 0; goto exit; } if (unlikely(qp->req.need_retry)) { req_retry(qp); qp->req.need_retry = 0; } wqe = req_next_wqe(qp); if (unlikely(!wqe)) goto exit; if (wqe->mask & WR_REG_MASK) { if (wqe->wr.opcode == IB_WR_LOCAL_INV) { struct rxe_dev *rxe = to_rdev(qp->ibqp.device); struct rxe_mem *rmr; rmr = rxe_pool_get_index(&rxe->mr_pool, wqe->wr.ex.invalidate_rkey >> 8); if (!rmr) { pr_err("No mr for key %#x\n", wqe->wr.ex.invalidate_rkey); wqe->state = wqe_state_error; wqe->status = IB_WC_MW_BIND_ERR; goto exit; } rmr->state = RXE_MEM_STATE_FREE; rxe_drop_ref(rmr); wqe->state = wqe_state_done; wqe->status = IB_WC_SUCCESS; } else if (wqe->wr.opcode == IB_WR_REG_MR) { struct rxe_mem *rmr = to_rmr(wqe->wr.wr.reg.mr); rmr->state = RXE_MEM_STATE_VALID; rmr->access = wqe->wr.wr.reg.access; rmr->lkey = wqe->wr.wr.reg.key; rmr->rkey = wqe->wr.wr.reg.key; wqe->state = wqe_state_done; wqe->status = IB_WC_SUCCESS; } else { goto exit; } qp->req.wqe_index = next_index(qp->sq.queue, qp->req.wqe_index); goto next_wqe; }