/** * this function polls the CQ, and extracts the needed fields * upon CQE error state it will return -1 * if a bad checksum packet or a filler bit it will return VMA_MP_RQ_BAD_PACKET */ int cq_mgr_mp::poll_mp_cq(uint16_t &size, uint32_t &strides_used, uint32_t &flags, struct mlx5_cqe64 *&out_cqe64) { struct mlx5_cqe64 *cqe= check_cqe(); if (likely(cqe)) { if (unlikely(MLX5_CQE_OPCODE(cqe->op_own) != MLX5_CQE_RESP_SEND)) { cq_logdbg("Warning op_own is %x", MLX5_CQE_OPCODE(cqe->op_own)); // optimize checks in ring by setting size non zero if (MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_RESP_ERR) { cq_logdbg("poll_length, CQE response error, " "syndrome=0x%x, vendor syndrome error=0x%x, " "HW syndrome 0x%x, HW syndrome type 0x%x\n", ((struct mlx5_err_cqe *)cqe)->syndrome, ((struct mlx5_err_cqe *)cqe)->vendor_err_synd, ((struct mlx5_err_cqe *)cqe)->hw_err_synd, ((struct mlx5_err_cqe *)cqe)->hw_synd_type); } size = 1; m_p_cq_stat->n_rx_pkt_drop++; return -1; } m_p_cq_stat->n_rx_pkt_drop += cqe->sop_qpn.sop; out_cqe64 = cqe; uint32_t stride_byte_cnt = ntohl(cqe->byte_cnt); strides_used = (stride_byte_cnt & MP_RQ_NUM_STRIDES_FIELD_MASK) >> MP_RQ_NUM_STRIDES_FIELD_SHIFT; flags = (!!(cqe->hds_ip_ext & MLX5_CQE_L4_OK) * IBV_EXP_CQ_RX_TCP_UDP_CSUM_OK) | (!!(cqe->hds_ip_ext & MLX5_CQE_L3_OK) * IBV_EXP_CQ_RX_IP_CSUM_OK); if (likely(flags == UDP_OK_FLAGS)) { size = stride_byte_cnt & MP_RQ_BYTE_CNT_FIELD_MASK; } else { // if CSUM is bad it can be either filler or bad packet flags = VMA_MP_RQ_BAD_PACKET; size = 1; if (stride_byte_cnt & MP_RQ_FILLER_FIELD_MASK) { m_p_cq_stat->n_rx_pkt_drop++; } } ++m_mlx5_cq.cq_ci; prefetch((uint8_t*)m_mlx5_cq.cq_buf + ((m_mlx5_cq.cq_ci & (m_mlx5_cq.cqe_count - 1)) << m_mlx5_cq.cqe_size_log)); } else {
mem_buf_desc_t* cq_mgr_mlx5::poll(enum buff_status_e& status) { mem_buf_desc_t *buff = NULL; #ifdef RDTSC_MEASURE_RX_VMA_TCP_IDLE_POLL RDTSC_TAKE_END(RDTSC_FLOW_RX_VMA_TCP_IDLE_POLL); #endif //RDTSC_MEASURE_RX_VMA_TCP_IDLE_POLL #if defined(RDTSC_MEASURE_RX_VERBS_READY_POLL) || defined(RDTSC_MEASURE_RX_VERBS_IDLE_POLL) RDTSC_TAKE_START_RX_VERBS_POLL(RDTSC_FLOW_RX_VERBS_READY_POLL, RDTSC_FLOW_RX_VERBS_IDLE_POLL); #endif //RDTSC_MEASURE_RX_VERBS_READY_POLL || RDTSC_MEASURE_RX_VERBS_IDLE_POLL if (unlikely(NULL == m_rx_hot_buffer)) { if (likely(m_rq->tail != m_rq->head)) { uint32_t index = m_rq->tail & (m_qp_rec.qp->m_rx_num_wr - 1); m_rx_hot_buffer = (mem_buf_desc_t *)m_p_rq_wqe_idx_to_wrid[index]; m_p_rq_wqe_idx_to_wrid[index] = 0; prefetch((void*)m_rx_hot_buffer); prefetch((uint8_t*)m_cqes + ((m_cq_cons_index & (m_cq_size - 1)) << m_cqe_log_sz)); } else { #ifdef RDTSC_MEASURE_RX_VERBS_IDLE_POLL RDTSC_TAKE_END(RDTSC_FLOW_RX_VERBS_IDLE_POLL); #endif #if defined(RDTSC_MEASURE_RX_VMA_TCP_IDLE_POLL) || defined(RDTSC_MEASURE_RX_CQE_RECEIVEFROM) RDTSC_TAKE_START_VMA_IDLE_POLL_CQE_TO_RECVFROM(RDTSC_FLOW_RX_VMA_TCP_IDLE_POLL, RDTSC_FLOW_RX_CQE_TO_RECEIVEFROM); #endif //RDTSC_MEASURE_RX_VMA_TCP_IDLE_POLL || RDTSC_MEASURE_RX_CQE_RECEIVEFROM /* If rq_tail and rq_head are pointing to the same wqe, * the wq is empty and there is no cqe to be received */ return NULL; } } mlx5_cqe64 *cqe = check_cqe(); if (likely(cqe)) { /* Update the consumer index */ ++m_cq_cons_index; rmb(); cqe64_to_mem_buff_desc(cqe, m_rx_hot_buffer, status); ++m_rq->tail; *m_cq_dbell = htonl(m_cq_cons_index & 0xffffff); buff = m_rx_hot_buffer; m_rx_hot_buffer = NULL; #ifdef RDTSC_MEASURE_RX_VERBS_READY_POLL RDTSC_TAKE_END(RDTSC_FLOW_RX_VERBS_READY_POLL); #endif //RDTSC_MEASURE_RX_VERBS_READY_POLL #ifdef RDTSC_MEASURE_RX_READY_POLL_TO_LWIP RDTSC_TAKE_START(RDTSC_FLOW_RX_READY_POLL_TO_LWIP); #endif } else { #ifdef RDTSC_MEASURE_RX_VERBS_IDLE_POLL RDTSC_TAKE_END(RDTSC_FLOW_RX_VERBS_IDLE_POLL); #endif #if defined(RDTSC_MEASURE_RX_VMA_TCP_IDLE_POLL) || defined(RDTSC_MEASURE_RX_CQE_RECEIVEFROM) RDTSC_TAKE_START_VMA_IDLE_POLL_CQE_TO_RECVFROM(RDTSC_FLOW_RX_VMA_TCP_IDLE_POLL, RDTSC_FLOW_RX_CQE_TO_RECEIVEFROM); #endif //RDTSC_MEASURE_RX_VMA_TCP_IDLE_POLL || RDTSC_MEASURE_RX_CQE_RECEIVEFROM prefetch((void*)m_rx_hot_buffer); } prefetch((uint8_t*)m_cqes + ((m_cq_cons_index & (m_cq_size - 1)) << m_cqe_log_sz)); return buff; }