static void
vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *txq)
{
int completed = 0;
struct rte_mbuf *mbuf;
vmxnet3_comp_ring_t *comp_ring = &txq->comp_ring;
struct Vmxnet3_TxCompDesc *tcd = (struct Vmxnet3_TxCompDesc *)
(comp_ring->base + comp_ring->next2proc);
while (tcd->gen == comp_ring->gen) {
/* Release cmd_ring descriptor and free mbuf */
VMXNET3_ASSERT(txq->cmd_ring.base[tcd->txdIdx].txd.eop == 1);
while (txq->cmd_ring.next2comp != tcd->txdIdx) {
mbuf = txq->cmd_ring.buf_info[txq->cmd_ring.next2comp].m;
txq->cmd_ring.buf_info[txq->cmd_ring.next2comp].m = NULL;
rte_pktmbuf_free_seg(mbuf);
/* Mark the txd for which tcd was generated as completed */
vmxnet3_cmd_ring_adv_next2comp(&txq->cmd_ring);
completed++;
}
vmxnet3_comp_ring_adv_next2proc(comp_ring);
tcd = (struct Vmxnet3_TxCompDesc *)(comp_ring->base +
comp_ring->next2proc);
}
PMD_TX_LOG(DEBUG, "Processed %d tx comps & command descs.", completed);
}
static inline void
vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *txq)
{
int completed = 0;
struct rte_mbuf *mbuf;
vmxnet3_comp_ring_t *comp_ring = &txq->comp_ring;
struct Vmxnet3_TxCompDesc *tcd = (struct Vmxnet3_TxCompDesc *)
(comp_ring->base + comp_ring->next2proc);
while (tcd->gen == comp_ring->gen) {
/* Release cmd_ring descriptor and free mbuf */
#ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER
VMXNET3_ASSERT(txq->cmd_ring.base[tcd->txdIdx].txd.eop == 1);
#endif
mbuf = txq->cmd_ring.buf_info[tcd->txdIdx].m;
if (unlikely(mbuf == NULL))
rte_panic("EOP desc does not point to a valid mbuf");
else
rte_pktmbuf_free(mbuf);
txq->cmd_ring.buf_info[tcd->txdIdx].m = NULL;
/* Mark the txd for which tcd was generated as completed */
vmxnet3_cmd_ring_adv_next2comp(&txq->cmd_ring);
vmxnet3_comp_ring_adv_next2proc(comp_ring);
tcd = (struct Vmxnet3_TxCompDesc *)(comp_ring->base +
comp_ring->next2proc);
completed++;
}
PMD_TX_LOG(DEBUG, "Processed %d tx comps & command descs.", completed);
}
/*
* Process the Rx Completion Ring of given vmxnet3_rx_queue
* for nb_pkts burst and return the number of packets received
*/
uint16_t
vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
{
uint16_t nb_rx;
uint32_t nb_rxd, idx;
uint8_t ring_idx;
vmxnet3_rx_queue_t *rxq;
Vmxnet3_RxCompDesc *rcd;
vmxnet3_buf_info_t *rbi;
Vmxnet3_RxDesc *rxd;
struct rte_mbuf *rxm = NULL;
struct vmxnet3_hw *hw;
nb_rx = 0;
ring_idx = 0;
nb_rxd = 0;
idx = 0;
rxq = rx_queue;
hw = rxq->hw;
rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
if (unlikely(rxq->stopped)) {
PMD_RX_LOG(DEBUG, "Rx queue is stopped.");
return 0;
}
while (rcd->gen == rxq->comp_ring.gen) {
if (nb_rx >= nb_pkts)
break;
idx = rcd->rxdIdx;
ring_idx = (uint8_t)((rcd->rqID == rxq->qid1) ? 0 : 1);
rxd = (Vmxnet3_RxDesc *)rxq->cmd_ring[ring_idx].base + idx;
rbi = rxq->cmd_ring[ring_idx].buf_info + idx;
if (unlikely(rcd->sop != 1 || rcd->eop != 1)) {
rte_pktmbuf_free_seg(rbi->m);
PMD_RX_LOG(DEBUG, "Packet spread across multiple buffers\n)");
goto rcd_done;
}
PMD_RX_LOG(DEBUG, "rxd idx: %d ring idx: %d.", idx, ring_idx);
VMXNET3_ASSERT(rcd->len <= rxd->len);
VMXNET3_ASSERT(rbi->m);
if (unlikely(rcd->len == 0)) {
PMD_RX_LOG(DEBUG, "Rx buf was skipped. rxring[%d][%d]\n)",
ring_idx, idx);
VMXNET3_ASSERT(rcd->sop && rcd->eop);
rte_pktmbuf_free_seg(rbi->m);
goto rcd_done;
}
/* Assuming a packet is coming in a single packet buffer */
if (unlikely(rxd->btype != VMXNET3_RXD_BTYPE_HEAD)) {
PMD_RX_LOG(DEBUG,
"Alert : Misbehaving device, incorrect "
" buffer type used. iPacket dropped.");
rte_pktmbuf_free_seg(rbi->m);
goto rcd_done;
}
VMXNET3_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_HEAD);
/* Get the packet buffer pointer from buf_info */
rxm = rbi->m;
/* Clear descriptor associated buf_info to be reused */
rbi->m = NULL;
rbi->bufPA = 0;
/* Update the index that we received a packet */
rxq->cmd_ring[ring_idx].next2comp = idx;
/* For RCD with EOP set, check if there is frame error */
if (unlikely(rcd->err)) {
rxq->stats.drop_total++;
rxq->stats.drop_err++;
if (!rcd->fcs) {
rxq->stats.drop_fcs++;
PMD_RX_LOG(ERR, "Recv packet dropped due to frame err.");
}
PMD_RX_LOG(ERR, "Error in received packet rcd#:%d rxd:%d",
(int)(rcd - (struct Vmxnet3_RxCompDesc *)
rxq->comp_ring.base), rcd->rxdIdx);
rte_pktmbuf_free_seg(rxm);
goto rcd_done;
}
/* Initialize newly received packet buffer */
rxm->port = rxq->port_id;
rxm->nb_segs = 1;
rxm->next = NULL;
rxm->pkt_len = (uint16_t)rcd->len;
rxm->data_len = (uint16_t)rcd->len;
rxm->data_off = RTE_PKTMBUF_HEADROOM;
rxm->ol_flags = 0;
rxm->vlan_tci = 0;
vmxnet3_rx_offload(rcd, rxm);
rx_pkts[nb_rx++] = rxm;
rcd_done:
rxq->cmd_ring[ring_idx].next2comp = idx;
VMXNET3_INC_RING_IDX_ONLY(rxq->cmd_ring[ring_idx].next2comp, rxq->cmd_ring[ring_idx].size);
/* It's time to allocate some new buf and renew descriptors */
vmxnet3_post_rx_bufs(rxq, ring_idx);
if (unlikely(rxq->shared->ctrl.updateRxProd)) {
VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN),
rxq->cmd_ring[ring_idx].next2fill);
}
/* Advance to the next descriptor in comp_ring */
vmxnet3_comp_ring_adv_next2proc(&rxq->comp_ring);
rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
nb_rxd++;
if (nb_rxd > rxq->cmd_ring[0].size) {
PMD_RX_LOG(ERR,
"Used up quota of receiving packets,"
" relinquish control.");
break;
}
}
return nb_rx;
}
uint16_t
vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts)
{
uint16_t nb_tx;
Vmxnet3_TxDesc *txd = NULL;
vmxnet3_buf_info_t *tbi = NULL;
struct vmxnet3_hw *hw;
struct rte_mbuf *txm;
vmxnet3_tx_queue_t *txq = tx_queue;
hw = txq->hw;
if (txq->stopped) {
PMD_TX_LOG(DEBUG, "Tx queue is stopped.");
return 0;
}
/* Free up the comp_descriptors aggressively */
vmxnet3_tq_tx_complete(txq);
nb_tx = 0;
while (nb_tx < nb_pkts) {
if (vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring)) {
txm = tx_pkts[nb_tx];
/* Don't support scatter packets yet, free them if met */
if (txm->pkt.nb_segs != 1) {
PMD_TX_LOG(DEBUG, "Don't support scatter packets yet, drop!");
rte_pktmbuf_free(tx_pkts[nb_tx]);
txq->stats.drop_total++;
nb_tx++;
continue;
}
/* Needs to minus ether header len */
if (txm->pkt.data_len > (hw->cur_mtu + ETHER_HDR_LEN)) {
PMD_TX_LOG(DEBUG, "Packet data_len higher than MTU");
rte_pktmbuf_free(tx_pkts[nb_tx]);
txq->stats.drop_total++;
nb_tx++;
continue;
}
txd = (Vmxnet3_TxDesc *)(txq->cmd_ring.base + txq->cmd_ring.next2fill);
/* Fill the tx descriptor */
tbi = txq->cmd_ring.buf_info + txq->cmd_ring.next2fill;
tbi->bufPA = RTE_MBUF_DATA_DMA_ADDR(txm);
txd->addr = tbi->bufPA;
txd->len = txm->pkt.data_len;
/* Mark the last descriptor as End of Packet. */
txd->cq = 1;
txd->eop = 1;
/* Record current mbuf for freeing it later in tx complete */
#ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER
VMXNET3_ASSERT(txm);
#endif
tbi->m = txm;
/* Set the offloading mode to default */
txd->hlen = 0;
txd->om = VMXNET3_OM_NONE;
txd->msscof = 0;
/* finally flip the GEN bit of the SOP desc */
txd->gen = txq->cmd_ring.gen;
txq->shared->ctrl.txNumDeferred++;
/* move to the next2fill descriptor */
vmxnet3_cmd_ring_adv_next2fill(&txq->cmd_ring);
nb_tx++;
} else {
PMD_TX_LOG(DEBUG, "No free tx cmd desc(s)");
txq->stats.drop_total += (nb_pkts - nb_tx);
break;
}
}
PMD_TX_LOG(DEBUG, "vmxnet3 txThreshold: %u", txq->shared->ctrl.txThreshold);
if (txq->shared->ctrl.txNumDeferred >= txq->shared->ctrl.txThreshold) {
txq->shared->ctrl.txNumDeferred = 0;
/* Notify vSwitch that packets are available. */
VMXNET3_WRITE_BAR0_REG(hw, (VMXNET3_REG_TXPROD + txq->queue_id * VMXNET3_REG_ALIGN),
txq->cmd_ring.next2fill);
}
return nb_tx;
}
uint16_t
vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts)
{
uint16_t nb_tx;
Vmxnet3_TxDesc *txd = NULL;
vmxnet3_buf_info_t *tbi = NULL;
struct vmxnet3_hw *hw;
struct rte_mbuf *txm;
vmxnet3_tx_queue_t *txq = tx_queue;
hw = txq->hw;
if (unlikely(txq->stopped)) {
PMD_TX_LOG(DEBUG, "Tx queue is stopped.");
return 0;
}
/* Free up the comp_descriptors aggressively */
vmxnet3_tq_tx_complete(txq);
nb_tx = 0;
while (nb_tx < nb_pkts) {
if (vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring)) {
int copy_size = 0;
txm = tx_pkts[nb_tx];
/* Don't support scatter packets yet, free them if met */
if (txm->nb_segs != 1) {
if (vmxnet3_xmit_convert_callback ){
txm=vmxnet3_xmit_convert_callback(txm);
}else{
txq->stats.drop_total++;
nb_tx++;
rte_pktmbuf_free(txm);
continue;
}
}
if (!txm) {
txq->stats.drop_total++;
nb_tx++;
continue;
}
/* Needs to minus ether header len */
if (txm->data_len > (hw->cur_mtu + ETHER_HDR_LEN)) {
PMD_TX_LOG(DEBUG, "Packet data_len higher than MTU");
rte_pktmbuf_free(txm);
txq->stats.drop_total++;
nb_tx++;
continue;
}
txd = (Vmxnet3_TxDesc *)(txq->cmd_ring.base + txq->cmd_ring.next2fill);
if (rte_pktmbuf_pkt_len(txm) <= VMXNET3_HDR_COPY_SIZE) {
struct Vmxnet3_TxDataDesc *tdd;
tdd = txq->data_ring.base + txq->cmd_ring.next2fill;
copy_size = rte_pktmbuf_pkt_len(txm);
rte_memcpy(tdd->data, rte_pktmbuf_mtod(txm, char *), copy_size);
}
/* Fill the tx descriptor */
tbi = txq->cmd_ring.buf_info + txq->cmd_ring.next2fill;
tbi->bufPA = RTE_MBUF_DATA_DMA_ADDR(txm);
if (copy_size)
txd->addr = rte_cpu_to_le_64(txq->data_ring.basePA +
txq->cmd_ring.next2fill *
sizeof(struct Vmxnet3_TxDataDesc));
else
txd->addr = tbi->bufPA;
txd->len = txm->data_len;
/* Mark the last descriptor as End of Packet. */
txd->cq = 1;
txd->eop = 1;
/* Add VLAN tag if requested */
if (txm->ol_flags & PKT_TX_VLAN_PKT) {
txd->ti = 1;
txd->tci = rte_cpu_to_le_16(txm->vlan_tci);
}
/* Record current mbuf for freeing it later in tx complete */
#ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER
VMXNET3_ASSERT(txm);
#endif
tbi->m = txm;
/* Set the offloading mode to default */
txd->hlen = 0;
txd->om = VMXNET3_OM_NONE;
txd->msscof = 0;
/* finally flip the GEN bit of the SOP desc */
txd->gen = txq->cmd_ring.gen;
txq->shared->ctrl.txNumDeferred++;
/* move to the next2fill descriptor */
vmxnet3_cmd_ring_adv_next2fill(&txq->cmd_ring);
nb_tx++;
} else {
