static void insert_recv_cqe(struct t4_wq *wq, struct t4_cq *cq) { struct t4_cqe cqe; PDBG("%s wq %p cq %p sw_cidx %u sw_pidx %u\n", __func__, wq, cq, cq->sw_cidx, cq->sw_pidx); memset(&cqe, 0, sizeof(cqe)); cqe.header = cpu_to_be32(V_CQE_STATUS(T4_ERR_SWFLUSH) | V_CQE_OPCODE(FW_RI_SEND) | V_CQE_TYPE(0) | V_CQE_SWCQE(1) | V_CQE_QPID(wq->rq.qid)); cqe.bits_type_ts = cpu_to_be64(V_CQE_GENBIT((u64)cq->gen)); cq->sw_queue[cq->sw_pidx] = cqe; t4_swcq_produce(cq); }
static void insert_sq_cqe(struct t4_wq *wq, struct t4_cq *cq, struct t4_swsqe *swcqe) { struct t4_cqe cqe; PDBG("%s wq %p cq %p sw_cidx %u sw_pidx %u\n", __func__, wq, cq, cq->sw_cidx, cq->sw_pidx); memset(&cqe, 0, sizeof(cqe)); cqe.header = cpu_to_be32(V_CQE_STATUS(T4_ERR_SWFLUSH) | V_CQE_OPCODE(swcqe->opcode) | V_CQE_TYPE(1) | V_CQE_SWCQE(1) | V_CQE_QPID(wq->sq.qid)); CQE_WRID_SQ_IDX(&cqe) = swcqe->idx; cqe.bits_type_ts = cpu_to_be64(V_CQE_GENBIT((u64)cq->gen)); cq->sw_queue[cq->sw_pidx] = cqe; t4_swcq_produce(cq); }
/* * poll_cq * * Caller must: * check the validity of the first CQE, * supply the wq assicated with the qpid. * * credit: cq credit to return to sge. * cqe_flushed: 1 iff the CQE is flushed. * cqe: copy of the polled CQE. * * return value: * 0 CQE returned ok. * -EAGAIN CQE skipped, try again. * -EOVERFLOW CQ overflow detected. */ static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe, u8 *cqe_flushed, u64 *cookie, u32 *credit) { int ret = 0; struct t4_cqe *hw_cqe, read_cqe; *cqe_flushed = 0; *credit = 0; ret = t4_next_cqe(cq, &hw_cqe); if (ret) return ret; PDBG("%s CQE OVF %u qpid 0x%0x genbit %u type %u status 0x%0x" " opcode 0x%0x len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x\n", __func__, CQE_OVFBIT(hw_cqe), CQE_QPID(hw_cqe), CQE_GENBIT(hw_cqe), CQE_TYPE(hw_cqe), CQE_STATUS(hw_cqe), CQE_OPCODE(hw_cqe), CQE_LEN(hw_cqe), CQE_WRID_HI(hw_cqe), CQE_WRID_LOW(hw_cqe)); /* * skip cqe's not affiliated with a QP. */ if (wq == NULL) { ret = -EAGAIN; goto skip_cqe; } /* * Gotta tweak READ completions: * 1) the cqe doesn't contain the sq_wptr from the wr. * 2) opcode not reflected from the wr. * 3) read_len not reflected from the wr. * 4) cq_type is RQ_TYPE not SQ_TYPE. */ if (RQ_TYPE(hw_cqe) && (CQE_OPCODE(hw_cqe) == FW_RI_READ_RESP)) { /* * If this is an unsolicited read response, then the read * was generated by the kernel driver as part of peer-2-peer * connection setup. So ignore the completion. */ if (!wq->sq.oldest_read) { if (CQE_STATUS(hw_cqe)) t4_set_wq_in_error(wq); ret = -EAGAIN; goto skip_cqe; } /* * Don't write to the HWCQ, so create a new read req CQE * in local memory. */ create_read_req_cqe(wq, hw_cqe, &read_cqe); hw_cqe = &read_cqe; advance_oldest_read(wq); } if (CQE_STATUS(hw_cqe) || t4_wq_in_error(wq)) { *cqe_flushed = t4_wq_in_error(wq); t4_set_wq_in_error(wq); goto proc_cqe; } if (CQE_OPCODE(hw_cqe) == FW_RI_TERMINATE) { ret = -EAGAIN; goto skip_cqe; } /* * RECV completion. */ if (RQ_TYPE(hw_cqe)) { /* * HW only validates 4 bits of MSN. So we must validate that * the MSN in the SEND is the next expected MSN. If its not, * then we complete this with T4_ERR_MSN and mark the wq in * error. */ if (t4_rq_empty(wq)) { t4_set_wq_in_error(wq); ret = -EAGAIN; goto skip_cqe; } if (unlikely((CQE_WRID_MSN(hw_cqe) != (wq->rq.msn)))) { t4_set_wq_in_error(wq); hw_cqe->header |= htonl(V_CQE_STATUS(T4_ERR_MSN)); goto proc_cqe; } goto proc_cqe; } /* * If we get here its a send completion. * * Handle out of order completion. These get stuffed * in the SW SQ. Then the SW SQ is walked to move any * now in-order completions into the SW CQ. This handles * 2 cases: * 1) reaping unsignaled WRs when the first subsequent * signaled WR is completed. * 2) out of order read completions. */ if (!SW_CQE(hw_cqe) && (CQE_WRID_SQ_IDX(hw_cqe) != wq->sq.cidx)) { struct t4_swsqe *swsqe; PDBG("%s out of order completion going in sw_sq at idx %u\n", __func__, CQE_WRID_SQ_IDX(hw_cqe)); swsqe = &wq->sq.sw_sq[CQE_WRID_SQ_IDX(hw_cqe)]; swsqe->cqe = *hw_cqe; swsqe->complete = 1; ret = -EAGAIN; goto flush_wq; } proc_cqe: *cqe = *hw_cqe; /* * Reap the associated WR(s) that are freed up with this * completion. */ if (SQ_TYPE(hw_cqe)) { wq->sq.cidx = CQE_WRID_SQ_IDX(hw_cqe); PDBG("%s completing sq idx %u\n", __func__, wq->sq.cidx); *cookie = wq->sq.sw_sq[wq->sq.cidx].wr_id; t4_sq_consume(wq); } else { PDBG("%s completing rq idx %u\n", __func__, wq->rq.cidx); *cookie = wq->rq.sw_rq[wq->rq.cidx].wr_id; BUG_ON(t4_rq_empty(wq)); t4_rq_consume(wq); } flush_wq: /* * Flush any completed cqes that are now in-order. */ flush_completed_wrs(wq, cq); skip_cqe: if (SW_CQE(hw_cqe)) { PDBG("%s cq %p cqid 0x%x skip sw cqe cidx %u\n", __func__, cq, cq->cqid, cq->sw_cidx); t4_swcq_consume(cq); } else { PDBG("%s cq %p cqid 0x%x skip hw cqe cidx %u\n", __func__, cq, cq->cqid, cq->cidx); t4_hwcq_consume(cq); } return ret; }
static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe, u8 *cqe_flushed, u64 *cookie, u32 *credit) { int ret = 0; struct t4_cqe *hw_cqe, read_cqe; *cqe_flushed = 0; *credit = 0; ret = t4_next_cqe(cq, &hw_cqe); if (ret) return ret; PDBG("%s CQE OVF %u qpid 0x%0x genbit %u type %u status 0x%0x" " opcode 0x%0x len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x\n", __func__, CQE_OVFBIT(hw_cqe), CQE_QPID(hw_cqe), CQE_GENBIT(hw_cqe), CQE_TYPE(hw_cqe), CQE_STATUS(hw_cqe), CQE_OPCODE(hw_cqe), CQE_LEN(hw_cqe), CQE_WRID_HI(hw_cqe), CQE_WRID_LOW(hw_cqe)); /* */ if (wq == NULL) { ret = -EAGAIN; goto skip_cqe; } /* */ if (RQ_TYPE(hw_cqe) && (CQE_OPCODE(hw_cqe) == FW_RI_READ_RESP)) { /* */ if (!wq->sq.oldest_read) { if (CQE_STATUS(hw_cqe)) t4_set_wq_in_error(wq); ret = -EAGAIN; goto skip_cqe; } /* */ create_read_req_cqe(wq, hw_cqe, &read_cqe); hw_cqe = &read_cqe; advance_oldest_read(wq); } if (CQE_STATUS(hw_cqe) || t4_wq_in_error(wq)) { *cqe_flushed = t4_wq_in_error(wq); t4_set_wq_in_error(wq); goto proc_cqe; } if (CQE_OPCODE(hw_cqe) == FW_RI_TERMINATE) { ret = -EAGAIN; goto skip_cqe; } /* */ if (RQ_TYPE(hw_cqe)) { /* */ if (t4_rq_empty(wq)) { t4_set_wq_in_error(wq); ret = -EAGAIN; goto skip_cqe; } if (unlikely((CQE_WRID_MSN(hw_cqe) != (wq->rq.msn)))) { t4_set_wq_in_error(wq); hw_cqe->header |= htonl(V_CQE_STATUS(T4_ERR_MSN)); goto proc_cqe; } goto proc_cqe; } /* */ if (!SW_CQE(hw_cqe) && (CQE_WRID_SQ_IDX(hw_cqe) != wq->sq.cidx)) { struct t4_swsqe *swsqe; PDBG("%s out of order completion going in sw_sq at idx %u\n", __func__, CQE_WRID_SQ_IDX(hw_cqe)); swsqe = &wq->sq.sw_sq[CQE_WRID_SQ_IDX(hw_cqe)]; swsqe->cqe = *hw_cqe; swsqe->complete = 1; ret = -EAGAIN; goto flush_wq; } proc_cqe: *cqe = *hw_cqe; /* */ if (SQ_TYPE(hw_cqe)) { wq->sq.cidx = CQE_WRID_SQ_IDX(hw_cqe); PDBG("%s completing sq idx %u\n", __func__, wq->sq.cidx); *cookie = wq->sq.sw_sq[wq->sq.cidx].wr_id; t4_sq_consume(wq); } else { PDBG("%s completing rq idx %u\n", __func__, wq->rq.cidx); *cookie = wq->rq.sw_rq[wq->rq.cidx].wr_id; BUG_ON(t4_rq_empty(wq)); t4_rq_consume(wq); } flush_wq: /* */ flush_completed_wrs(wq, cq); skip_cqe: if (SW_CQE(hw_cqe)) { PDBG("%s cq %p cqid 0x%x skip sw cqe cidx %u\n", __func__, cq, cq->cqid, cq->sw_cidx); t4_swcq_consume(cq); } else { PDBG("%s cq %p cqid 0x%x skip hw cqe cidx %u\n", __func__, cq, cq->cqid, cq->cidx); t4_hwcq_consume(cq); } return ret; }