/* drain the send queue */ static void rxe_qp_drain(struct rxe_qp *qp) { if (qp->sq.queue) { if (qp->req.state != QP_STATE_DRAINED) { qp->req.state = QP_STATE_DRAIN; if (qp_type(qp) == IB_QPT_RC) rxe_run_task(&qp->comp.task, 1); else __rxe_do_task(&qp->comp.task); rxe_run_task(&qp->req.task, 1); } } }
/* move the qp to the error state */ void rxe_qp_error(struct rxe_qp *qp) { qp->req.state = QP_STATE_ERROR; qp->resp.state = QP_STATE_ERROR; /* drain work and packet queues */ rxe_run_task(&qp->resp.task, 1); if (qp_type(qp) == IB_QPT_RC) rxe_run_task(&qp->comp.task, 1); else __rxe_do_task(&qp->comp.task); rxe_run_task(&qp->req.task, 1); }
void rnr_nak_timer(unsigned long data) { struct rxe_qp *qp = (struct rxe_qp *)data; pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp)); rxe_run_task(&qp->req.task, 1); }
void rnr_nak_timer(struct timer_list *t) { struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer); pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp)); rxe_run_task(&qp->req.task, 1); }
static int rxe_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, struct ib_send_wr **bad_wr) { int err = 0; struct rxe_qp *qp = to_rqp(ibqp); unsigned int mask; unsigned int length = 0; int i; int must_sched; if (unlikely(!qp->valid)) { *bad_wr = wr; return -EINVAL; } if (unlikely(qp->req.state < QP_STATE_READY)) { *bad_wr = wr; return -EINVAL; } while (wr) { mask = wr_opcode_mask(wr->opcode, qp); if (unlikely(!mask)) { err = -EINVAL; *bad_wr = wr; break; } if (unlikely((wr->send_flags & IB_SEND_INLINE) && !(mask & WR_INLINE_MASK))) { err = -EINVAL; *bad_wr = wr; break; } length = 0; for (i = 0; i < wr->num_sge; i++) length += wr->sg_list[i].length; err = post_one_send(qp, wr, mask, length); if (err) { *bad_wr = wr; break; } wr = wr->next; } /* * Must sched in case of GSI QP because ib_send_mad() hold irq lock, * and the requester call ip_local_out_sk() that takes spin_lock_bh. */ must_sched = (qp_type(qp) == IB_QPT_GSI) || (queue_count(qp->sq.queue) > 1); rxe_run_task(&qp->req.task, must_sched); return err; }
void retransmit_timer(unsigned long data) { struct rxe_qp *qp = (struct rxe_qp *)data; if (qp->valid) { qp->comp.timeout = 1; rxe_run_task(&qp->comp.task, 1); } }
static int rxe_post_send_kernel(struct rxe_qp *qp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) { int err = 0; unsigned int mask; unsigned int length = 0; int i; while (wr) { mask = wr_opcode_mask(wr->opcode, qp); if (unlikely(!mask)) { err = -EINVAL; *bad_wr = wr; break; } if (unlikely((wr->send_flags & IB_SEND_INLINE) && !(mask & WR_INLINE_MASK))) { err = -EINVAL; *bad_wr = wr; break; } length = 0; for (i = 0; i < wr->num_sge; i++) length += wr->sg_list[i].length; err = post_one_send(qp, wr, mask, length); if (err) { *bad_wr = wr; break; } wr = wr->next; } rxe_run_task(&qp->req.task, 1); if (unlikely(qp->req.state == QP_STATE_ERROR)) rxe_run_task(&qp->comp.task, 1); return err; }
static void rxe_skb_tx_dtor(struct sk_buff *skb) { struct sock *sk = skb->sk; struct rxe_qp *qp = sk->sk_user_data; int skb_out = atomic_dec_return(&qp->skb_out); if (unlikely(qp->need_req_skb && skb_out < RXE_INFLIGHT_SKBS_PER_QP_LOW)) rxe_run_task(&qp->req.task, 1); rxe_drop_ref(qp); }
/* rxe_recv calls here to add a request packet to the input queue */ void rxe_resp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb) { int must_sched; struct rxe_pkt_info *pkt = SKB_TO_PKT(skb); skb_queue_tail(&qp->req_pkts, skb); must_sched = (pkt->opcode == IB_OPCODE_RC_RDMA_READ_REQUEST) || (skb_queue_len(&qp->req_pkts) > 1); rxe_run_task(&qp->resp.task, must_sched); }
void rxe_comp_queue_pkt(struct rxe_dev *rxe, struct rxe_qp *qp, struct sk_buff *skb) { int must_sched; skb_queue_tail(&qp->resp_pkts, skb); must_sched = skb_queue_len(&qp->resp_pkts) > 1; if (must_sched != 0) rxe_counter_inc(rxe, RXE_CNT_COMPLETER_SCHED); rxe_run_task(&qp->comp.task, must_sched); }
static inline enum comp_state complete_ack(struct rxe_qp *qp, struct rxe_pkt_info *pkt, struct rxe_send_wqe *wqe) { unsigned long flags; if (wqe->has_rd_atomic) { wqe->has_rd_atomic = 0; atomic_inc(&qp->req.rd_atomic); if (qp->req.need_rd_atomic) { qp->comp.timeout_retry = 0; qp->req.need_rd_atomic = 0; rxe_run_task(&qp->req.task, 1); } } if (unlikely(qp->req.state == QP_STATE_DRAIN)) { /* state_lock used by requester & completer */ spin_lock_irqsave(&qp->state_lock, flags); if ((qp->req.state == QP_STATE_DRAIN) && (qp->comp.psn == qp->req.psn)) { qp->req.state = QP_STATE_DRAINED; spin_unlock_irqrestore(&qp->state_lock, flags); if (qp->ibqp.event_handler) { struct ib_event ev; ev.device = qp->ibqp.device; ev.element.qp = &qp->ibqp; ev.event = IB_EVENT_SQ_DRAINED; qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); } } else { spin_unlock_irqrestore(&qp->state_lock, flags); } } do_complete(qp, wqe); if (psn_compare(pkt->psn, qp->comp.psn) >= 0) return COMPST_UPDATE_COMP; else return COMPST_DONE; }
static inline enum comp_state complete_wqe(struct rxe_qp *qp, struct rxe_pkt_info *pkt, struct rxe_send_wqe *wqe) { qp->comp.opcode = -1; if (pkt) { if (psn_compare(pkt->psn, qp->comp.psn) >= 0) qp->comp.psn = (pkt->psn + 1) & BTH_PSN_MASK; if (qp->req.wait_psn) { qp->req.wait_psn = 0; rxe_run_task(&qp->req.task, 1); } } do_complete(qp, wqe); return COMPST_GET_WQE; }
static int rxe_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr) { int err = 0; struct rxe_qp *qp = to_rqp(ibqp); struct rxe_rq *rq = &qp->rq; unsigned long flags; if (unlikely((qp_state(qp) < IB_QPS_INIT) || !qp->valid)) { *bad_wr = wr; err = -EINVAL; goto err1; } if (unlikely(qp->srq)) { *bad_wr = wr; err = -EINVAL; goto err1; } spin_lock_irqsave(&rq->producer_lock, flags); while (wr) { err = post_one_recv(rq, wr); if (unlikely(err)) { *bad_wr = wr; break; } wr = wr->next; } spin_unlock_irqrestore(&rq->producer_lock, flags); if (qp->resp.state == QP_STATE_ERROR) rxe_run_task(&qp->resp.task, 1); err1: return err; }
static int rxe_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) { struct rxe_qp *qp = to_rqp(ibqp); if (unlikely(!qp->valid)) { *bad_wr = wr; return -EINVAL; } if (unlikely(qp->req.state < QP_STATE_READY)) { *bad_wr = wr; return -EINVAL; } if (qp->is_user) { /* Utilize process context to do protocol processing */ rxe_run_task(&qp->req.task, 0); return 0; } else return rxe_post_send_kernel(qp, wr, bad_wr); }
/* * IBA Spec. Section 10.7.3.1 SIGNALED COMPLETIONS * ---------8<---------8<------------- * ...Note that if a completion error occurs, a Work Completion * will always be generated, even if the signaling * indicator requests an Unsignaled Completion. * ---------8<---------8<------------- */ static void do_complete(struct rxe_qp *qp, struct rxe_send_wqe *wqe) { struct rxe_cqe cqe; if ((qp->sq_sig_type == IB_SIGNAL_ALL_WR) || (wqe->wr.send_flags & IB_SEND_SIGNALED) || wqe->status != IB_WC_SUCCESS) { make_send_cqe(qp, wqe, &cqe); advance_consumer(qp->sq.queue); rxe_cq_post(qp->scq, &cqe, 0); } else { advance_consumer(qp->sq.queue); } /* * we completed something so let req run again * if it is trying to fence */ if (qp->req.wait_fence) { qp->req.wait_fence = 0; rxe_run_task(&qp->req.task, 1); } }
int rxe_completer(void *arg) { struct rxe_qp *qp = (struct rxe_qp *)arg; struct rxe_dev *rxe = to_rdev(qp->ibqp.device); struct rxe_send_wqe *wqe = wqe; struct sk_buff *skb = NULL; struct rxe_pkt_info *pkt = NULL; enum comp_state state; rxe_add_ref(qp); if (!qp->valid || qp->req.state == QP_STATE_ERROR || qp->req.state == QP_STATE_RESET) { rxe_drain_resp_pkts(qp, qp->valid && qp->req.state == QP_STATE_ERROR); goto exit; } if (qp->comp.timeout) { qp->comp.timeout_retry = 1; qp->comp.timeout = 0; } else { qp->comp.timeout_retry = 0; } if (qp->req.need_retry) goto exit; state = COMPST_GET_ACK; while (1) { pr_debug("qp#%d state = %s\n", qp_num(qp), comp_state_name[state]); switch (state) { case COMPST_GET_ACK: skb = skb_dequeue(&qp->resp_pkts); if (skb) { pkt = SKB_TO_PKT(skb); qp->comp.timeout_retry = 0; } state = COMPST_GET_WQE; break; case COMPST_GET_WQE: state = get_wqe(qp, pkt, &wqe); break; case COMPST_CHECK_PSN: state = check_psn(qp, pkt, wqe); break; case COMPST_CHECK_ACK: state = check_ack(qp, pkt, wqe); break; case COMPST_READ: state = do_read(qp, pkt, wqe); break; case COMPST_ATOMIC: state = do_atomic(qp, pkt, wqe); break; case COMPST_WRITE_SEND: if (wqe->state == wqe_state_pending && wqe->last_psn == pkt->psn) state = COMPST_COMP_ACK; else state = COMPST_UPDATE_COMP; break; case COMPST_COMP_ACK: state = complete_ack(qp, pkt, wqe); break; case COMPST_COMP_WQE: state = complete_wqe(qp, pkt, wqe); break; case COMPST_UPDATE_COMP: if (pkt->mask & RXE_END_MASK) qp->comp.opcode = -1; else qp->comp.opcode = pkt->opcode; if (psn_compare(pkt->psn, qp->comp.psn) >= 0) qp->comp.psn = (pkt->psn + 1) & BTH_PSN_MASK; if (qp->req.wait_psn) { qp->req.wait_psn = 0; rxe_run_task(&qp->req.task, 1); } state = COMPST_DONE; break; case COMPST_DONE: if (pkt) { rxe_drop_ref(pkt->qp); kfree_skb(skb); skb = NULL; } goto done; case COMPST_EXIT: if (qp->comp.timeout_retry && wqe) { state = COMPST_ERROR_RETRY; break; } /* re reset the timeout counter if * (1) QP is type RC * (2) the QP is alive * (3) there is a packet sent by the requester that * might be acked (we still might get spurious * timeouts but try to keep them as few as possible) * (4) the timeout parameter is set */ if ((qp_type(qp) == IB_QPT_RC) && (qp->req.state == QP_STATE_READY) && (psn_compare(qp->req.psn, qp->comp.psn) > 0) && qp->qp_timeout_jiffies) mod_timer(&qp->retrans_timer, jiffies + qp->qp_timeout_jiffies); WARN_ON_ONCE(skb); goto exit; case COMPST_ERROR_RETRY: /* we come here if the retry timer fired and we did * not receive a response packet. try to retry the send * queue if that makes sense and the limits have not * been exceeded. remember that some timeouts are * spurious since we do not reset the timer but kick * it down the road or let it expire */ /* there is nothing to retry in this case */ if (!wqe || (wqe->state == wqe_state_posted)) { WARN_ON_ONCE(skb); goto exit; } if (qp->comp.retry_cnt > 0) { if (qp->comp.retry_cnt != 7) qp->comp.retry_cnt--; /* no point in retrying if we have already * seen the last ack that the requester could * have caused */ if (psn_compare(qp->req.psn, qp->comp.psn) > 0) { /* tell the requester to retry the * send queue next time around */ rxe_counter_inc(rxe, RXE_CNT_COMP_RETRY); qp->req.need_retry = 1; rxe_run_task(&qp->req.task, 1); } if (pkt) { rxe_drop_ref(pkt->qp); kfree_skb(skb); skb = NULL; } WARN_ON_ONCE(skb); goto exit; } else { rxe_counter_inc(rxe, RXE_CNT_RETRY_EXCEEDED); wqe->status = IB_WC_RETRY_EXC_ERR; state = COMPST_ERROR; } break; case COMPST_RNR_RETRY: if (qp->comp.rnr_retry > 0) { if (qp->comp.rnr_retry != 7) qp->comp.rnr_retry--; qp->req.need_retry = 1; pr_debug("qp#%d set rnr nak timer\n", qp_num(qp)); mod_timer(&qp->rnr_nak_timer, jiffies + rnrnak_jiffies(aeth_syn(pkt) & ~AETH_TYPE_MASK)); rxe_drop_ref(pkt->qp); kfree_skb(skb); skb = NULL; goto exit; } else { rxe_counter_inc(rxe, RXE_CNT_RNR_RETRY_EXCEEDED); wqe->status = IB_WC_RNR_RETRY_EXC_ERR; state = COMPST_ERROR; } break; case COMPST_ERROR: WARN_ON_ONCE(wqe->status == IB_WC_SUCCESS); do_complete(qp, wqe); rxe_qp_error(qp); if (pkt) { rxe_drop_ref(pkt->qp); kfree_skb(skb); skb = NULL; } WARN_ON_ONCE(skb); goto exit; } } exit: /* we come here if we are done with processing and want the task to * exit from the loop calling us */ WARN_ON_ONCE(skb); rxe_drop_ref(qp); return -EAGAIN; done: /* we come here if we have processed a packet we want the task to call * us again to see if there is anything else to do */ WARN_ON_ONCE(skb); rxe_drop_ref(qp); return 0; }
static inline enum comp_state check_ack(struct rxe_qp *qp, struct rxe_pkt_info *pkt, struct rxe_send_wqe *wqe) { unsigned int mask = pkt->mask; u8 syn; struct rxe_dev *rxe = to_rdev(qp->ibqp.device); /* Check the sequence only */ switch (qp->comp.opcode) { case -1: /* Will catch all *_ONLY cases. */ if (!(mask & RXE_START_MASK)) return COMPST_ERROR; break; case IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST: case IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE: if (pkt->opcode != IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE && pkt->opcode != IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST) { return COMPST_ERROR; } break; default: WARN_ON_ONCE(1); } /* Check operation validity. */ switch (pkt->opcode) { case IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST: case IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST: case IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY: syn = aeth_syn(pkt); if ((syn & AETH_TYPE_MASK) != AETH_ACK) return COMPST_ERROR; /* Fall through (IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE * doesn't have an AETH) */ case IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE: if (wqe->wr.opcode != IB_WR_RDMA_READ && wqe->wr.opcode != IB_WR_RDMA_READ_WITH_INV) { return COMPST_ERROR; } reset_retry_counters(qp); return COMPST_READ; case IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE: syn = aeth_syn(pkt); if ((syn & AETH_TYPE_MASK) != AETH_ACK) return COMPST_ERROR; if (wqe->wr.opcode != IB_WR_ATOMIC_CMP_AND_SWP && wqe->wr.opcode != IB_WR_ATOMIC_FETCH_AND_ADD) return COMPST_ERROR; reset_retry_counters(qp); return COMPST_ATOMIC; case IB_OPCODE_RC_ACKNOWLEDGE: syn = aeth_syn(pkt); switch (syn & AETH_TYPE_MASK) { case AETH_ACK: reset_retry_counters(qp); return COMPST_WRITE_SEND; case AETH_RNR_NAK: rxe_counter_inc(rxe, RXE_CNT_RCV_RNR); return COMPST_RNR_RETRY; case AETH_NAK: switch (syn) { case AETH_NAK_PSN_SEQ_ERROR: /* a nak implicitly acks all packets with psns * before */ if (psn_compare(pkt->psn, qp->comp.psn) > 0) { rxe_counter_inc(rxe, RXE_CNT_RCV_SEQ_ERR); qp->comp.psn = pkt->psn; if (qp->req.wait_psn) { qp->req.wait_psn = 0; rxe_run_task(&qp->req.task, 1); } } return COMPST_ERROR_RETRY; case AETH_NAK_INVALID_REQ: wqe->status = IB_WC_REM_INV_REQ_ERR; return COMPST_ERROR; case AETH_NAK_REM_ACC_ERR: wqe->status = IB_WC_REM_ACCESS_ERR; return COMPST_ERROR; case AETH_NAK_REM_OP_ERR: wqe->status = IB_WC_REM_OP_ERR; return COMPST_ERROR; default: pr_warn("unexpected nak %x\n", syn); wqe->status = IB_WC_REM_OP_ERR; return COMPST_ERROR; } default: return COMPST_ERROR; } break; default: pr_warn("unexpected opcode\n"); } return COMPST_ERROR; }