Exemplo n.º 1
0
/*
 * Initialize a rxq by populating the whole Rx ring with rxbufs.
 */
int
vmxnet3s_rxq_init(vmxnet3s_softc_t *dp, vmxnet3s_rxq_t *rxq)
{
	vmxnet3s_cmdring_t *cmdring = &rxq->cmdring;

	do {
		if (vmxnet3s_rx_populate(dp, rxq, cmdring->next2fill,
		    B_TRUE) != DDI_SUCCESS)
			goto error;
		VMXNET3_INC_RING_IDX(cmdring, cmdring->next2fill);
	} while (cmdring->next2fill);

	dp->rxpool.nbufslimit = vmxnet3s_getprop(dp, "RxBufPoolLimit",
	    0, cmdring->size * 10, cmdring->size * 2);

	return (DDI_SUCCESS);

error:
	while (cmdring->next2fill) {
		VMXNET3_DEC_RING_IDX(cmdring, cmdring->next2fill);
		vmxnet3s_free_rxbuf(dp, rxq->bufring[cmdring->next2fill].rxbuf);
	}

	return (DDI_FAILURE);
}
Exemplo n.º 2
0
/*
 * vmxnet3_tx_complete --
 *
 *    Parse a transmit queue and complete packets.
 *
 * Results:
 *    B_TRUE if Tx must be updated or B_FALSE if no action is required.
 *
 * Side effects:
 *    None.
 */
boolean_t
vmxnet3_tx_complete(vmxnet3_softc_t *dp, vmxnet3_txqueue_t *txq)
{
	vmxnet3_cmdring_t *cmdRing = &txq->cmdRing;
	vmxnet3_compring_t *compRing = &txq->compRing;
	Vmxnet3_GenericDesc *compDesc;
	boolean_t completedTx = B_FALSE;
	boolean_t ret = B_FALSE;

	mutex_enter(&dp->txLock);

	compDesc = VMXNET3_GET_DESC(compRing, compRing->next2comp);
	while (compDesc->tcd.gen == compRing->gen) {
		vmxnet3_metatx_t *sopMetaDesc, *eopMetaDesc;
		uint16_t sopIdx, eopIdx;
		mblk_t *mp;

		eopIdx = compDesc->tcd.txdIdx;
		eopMetaDesc = &txq->metaRing[eopIdx];
		sopIdx = eopMetaDesc->sopIdx;
		sopMetaDesc = &txq->metaRing[sopIdx];

		ASSERT(eopMetaDesc->frags);
		cmdRing->avail += eopMetaDesc->frags;

		ASSERT(sopMetaDesc->mp);
		mp = sopMetaDesc->mp;
		freemsg(mp);

		eopMetaDesc->sopIdx = 0;
		eopMetaDesc->frags = 0;
		sopMetaDesc->mp = NULL;

		completedTx = B_TRUE;

		vmxnet3_debug(dp, 3, "cp 0x%p on [%u;%u]\n", mp, sopIdx,
		    eopIdx);

		VMXNET3_INC_RING_IDX(compRing, compRing->next2comp);
		compDesc = VMXNET3_GET_DESC(compRing, compRing->next2comp);
	}

	if (dp->txMustResched && completedTx) {
		dp->txMustResched = B_FALSE;
		ret = B_TRUE;
	}

	mutex_exit(&dp->txLock);

	return (ret);
}
Exemplo n.º 3
0
/*
 * Parse a transmit queue and complete packets.
 */
boolean_t
vmxnet3s_tx_complete(vmxnet3s_softc_t *dp, vmxnet3s_txq_t *txq)
{
	vmxnet3s_cmdring_t *cmdring = &txq->cmdring;
	vmxnet3s_compring_t *compring = &txq->compring;
	vmxnet3s_gendesc_t *compdesc;
	boolean_t	completedtx = B_FALSE;
	boolean_t	ret = B_FALSE;

	mutex_enter(&dp->txlock);

	compdesc = VMXNET3_GET_DESC(compring, compring->next2comp);
	while (compdesc->tcd.gen == compring->gen) {
		vmxnet3s_metatx_t *sopmetadesc, *eopmetadesc;
		uint16_t	sopidx;
		uint16_t	eopidx;
		mblk_t		*mp;

		eopidx = compdesc->tcd.txdidx;
		eopmetadesc = &txq->metaring[eopidx];
		sopidx = eopmetadesc->sopidx;
		sopmetadesc = &txq->metaring[sopidx];

		ASSERT(eopmetadesc->frags);
		cmdring->avail += eopmetadesc->frags;

		ASSERT(sopmetadesc->mp);
		mp = sopmetadesc->mp;
		freemsg(mp);

		eopmetadesc->sopidx = 0;
		eopmetadesc->frags = 0;
		sopmetadesc->mp = NULL;

		completedtx = B_TRUE;

		VMXNET3_INC_RING_IDX(compring, compring->next2comp);
		compdesc = VMXNET3_GET_DESC(compring, compring->next2comp);
	}

	if (dp->txmustresched && completedtx) {
		dp->txmustresched = B_FALSE;
		ret = B_TRUE;
	}

	mutex_exit(&dp->txlock);

	return (ret);
}
Exemplo n.º 4
0
/*
 *---------------------------------------------------------------------------
 *
 * vmxnet3_tx_one --
 *
 *    Map a msg into the Tx command ring of a vmxnet3 device.
 *
 * Results:
 *    VMXNET3_TX_OK if everything went well.
 *    VMXNET3_TX_RINGFULL if the ring is nearly full.
 *    VMXNET3_TX_PULLUP if the msg is overfragmented.
 *    VMXNET3_TX_FAILURE if there was a DMA or offload error.
 *
 * Side effects:
 *    The ring is filled if VMXNET3_TX_OK is returned.
 *
 *---------------------------------------------------------------------------
 */
static vmxnet3_txstatus
vmxnet3_tx_one(vmxnet3_softc_t *dp,
               vmxnet3_txqueue_t *txq,
               vmxnet3_offload_t *ol,
               mblk_t *mp,
               boolean_t retry)
{
   int ret = VMXNET3_TX_OK;
   unsigned int frags = 0, totLen = 0;
   vmxnet3_cmdring_t *cmdRing = &txq->cmdRing;
   Vmxnet3_TxQueueCtrl *txqCtrl = txq->sharedCtrl;
   Vmxnet3_GenericDesc *txDesc;
   uint16_t sopIdx, eopIdx;
   uint8_t sopGen, curGen;
   mblk_t *mblk;

   mutex_enter(&dp->txLock);

   sopIdx = eopIdx = cmdRing->next2fill;
   sopGen = cmdRing->gen;
   curGen = !cmdRing->gen;

   for (mblk = mp; mblk != NULL; mblk = mblk->b_cont) {
      unsigned int len = MBLKL(mblk);
      ddi_dma_cookie_t cookie;
      uint_t cookieCount;

      if (len) {
         totLen += len;
      } else {
         continue;
      }

      if (ddi_dma_addr_bind_handle(dp->txDmaHandle, NULL,
                                   (caddr_t) mblk->b_rptr, len,
                                   DDI_DMA_RDWR | DDI_DMA_STREAMING,
                                   DDI_DMA_DONTWAIT, NULL,
                                   &cookie, &cookieCount) != DDI_DMA_MAPPED) {
         VMXNET3_WARN(dp, "ddi_dma_addr_bind_handle() failed\n");
         ret = VMXNET3_TX_FAILURE;
         goto error;
      }

      ASSERT(cookieCount);

      do {
         uint64_t addr = cookie.dmac_laddress;
         size_t len = cookie.dmac_size;

         do {
            uint32_t dw2, dw3;
            size_t chunkLen;

            ASSERT(!txq->metaRing[eopIdx].mp);
            ASSERT(cmdRing->avail - frags);

            if (frags >= cmdRing->size - 1 ||
                (ol->om != VMXNET3_OM_TSO && frags >= VMXNET3_MAX_TXD_PER_PKT)) {

               if (retry) {
                  VMXNET3_DEBUG(dp, 2, "overfragmented, frags=%u ring=%hu om=%hu\n",
                                frags, cmdRing->size, ol->om);
               }
               ddi_dma_unbind_handle(dp->txDmaHandle);
               ret = VMXNET3_TX_PULLUP;
               goto error;
            }
            if (cmdRing->avail - frags <= 1) {
               dp->txMustResched = B_TRUE;
               ddi_dma_unbind_handle(dp->txDmaHandle);
               ret = VMXNET3_TX_RINGFULL;
               goto error;
            }

            if (len > VMXNET3_MAX_TX_BUF_SIZE) {
               chunkLen = VMXNET3_MAX_TX_BUF_SIZE;
            } else {
               chunkLen = len;
            }

            frags++;
            eopIdx = cmdRing->next2fill;

            txDesc = VMXNET3_GET_DESC(cmdRing, eopIdx);
            ASSERT(txDesc->txd.gen != cmdRing->gen);

            // txd.addr
            txDesc->txd.addr = addr;
            // txd.dw2
            dw2 = chunkLen == VMXNET3_MAX_TX_BUF_SIZE ? 0 : chunkLen;
            dw2 |= curGen << VMXNET3_TXD_GEN_SHIFT;
            txDesc->dword[2] = dw2;
            ASSERT(txDesc->txd.len == len || txDesc->txd.len == 0);
            // txd.dw3
            dw3 = 0;
            txDesc->dword[3] = dw3;

            VMXNET3_INC_RING_IDX(cmdRing, cmdRing->next2fill);
            curGen = cmdRing->gen;

            addr += chunkLen;
            len -= chunkLen;
         } while (len);

         if (--cookieCount) {
            ddi_dma_nextcookie(dp->txDmaHandle, &cookie);
         }
      } while (cookieCount);

      ddi_dma_unbind_handle(dp->txDmaHandle);
   }

   /* Update the EOP descriptor */
   txDesc = VMXNET3_GET_DESC(cmdRing, eopIdx);
   txDesc->dword[3] |= VMXNET3_TXD_CQ | VMXNET3_TXD_EOP;

   /* Update the SOP descriptor. Must be done last */
   txDesc = VMXNET3_GET_DESC(cmdRing, sopIdx);
   if (ol->om == VMXNET3_OM_TSO &&
       txDesc->txd.len != 0 &&
       txDesc->txd.len < ol->hlen) {
      ret = VMXNET3_TX_FAILURE;
      goto error;
   }
   txDesc->txd.om = ol->om;
   txDesc->txd.hlen = ol->hlen;
   txDesc->txd.msscof = ol->msscof;
   membar_producer();
   txDesc->txd.gen = sopGen;

   /* Update the meta ring & metadata */
   txq->metaRing[sopIdx].mp = mp;
   txq->metaRing[eopIdx].sopIdx = sopIdx;
   txq->metaRing[eopIdx].frags = frags;
   cmdRing->avail -= frags;
   if (ol->om == VMXNET3_OM_TSO) {
      txqCtrl->txNumDeferred +=
         (totLen - ol->hlen + ol->msscof - 1) / ol->msscof;
   } else {
      txqCtrl->txNumDeferred++;
   }

   VMXNET3_DEBUG(dp, 3, "tx 0x%p on [%u;%u]\n", mp, sopIdx, eopIdx);

   goto done;

error:
   /* Reverse the generation bits */
   while (sopIdx != cmdRing->next2fill) {
      VMXNET3_DEC_RING_IDX(cmdRing, cmdRing->next2fill);
      txDesc = VMXNET3_GET_DESC(cmdRing, cmdRing->next2fill);
      txDesc->txd.gen = !cmdRing->gen;
   }

done:
   mutex_exit(&dp->txLock);

   return ret;
}
Exemplo n.º 5
0
/*
 *---------------------------------------------------------------------------
 *
 * vmxnet3_tx_prepare_offload --
 *
 *    Build the offload context of a msg.
 *
 * Results:
 *    0 if everything went well.
 *    +n if n bytes need to be pulled up.
 *    -1 in case of error (not used).
 *
 * Side effects:
 *    None.
 *
 *---------------------------------------------------------------------------
 */
static int
vmxnet3_tx_prepare_offload(vmxnet3_softc_t *dp,
                           vmxnet3_offload_t *ol,
                           mblk_t *mp)
{
   int ret = 0;
   uint32_t start, stuff, value, flags;
#if defined(OPEN_SOLARIS) || defined(SOL11)
   uint32_t lso_flag, mss;
#endif

   ol->om = VMXNET3_OM_NONE;
   ol->hlen = 0;
   ol->msscof = 0;

   hcksum_retrieve(mp, NULL, NULL, &start, &stuff, NULL, &value, &flags);
#if defined(OPEN_SOLARIS) || defined(SOL11)
   mac_lso_get(mp, &mss, &lso_flag);

   if (flags || lso_flag) {
#else
   if (flags) {
#endif
      struct ether_vlan_header *eth = (void *) mp->b_rptr;
      uint8_t ethLen;

      if (eth->ether_tpid == htons(ETHERTYPE_VLAN)) {
         ethLen = sizeof(struct ether_vlan_header);
      } else {
         ethLen = sizeof(struct ether_header);
      }

      VMXNET3_DEBUG(dp, 4, "flags=0x%x, ethLen=%u, start=%u, stuff=%u, value=%u\n",
                            flags,      ethLen,    start,    stuff,    value);

#if defined(OPEN_SOLARIS) || defined(SOL11)
      if (lso_flag & HW_LSO) {
#else
      if (flags & HCK_PARTIALCKSUM) {
         ol->om = VMXNET3_OM_CSUM;
         ol->hlen = start + ethLen;
         ol->msscof = stuff + ethLen;
      }
      if (flags & HW_LSO) {
#endif
         mblk_t *mblk = mp;
         uint8_t *ip, *tcp;
         uint8_t ipLen, tcpLen;

         /*
          * Copy e1000g's behavior:
          * - Do not assume all the headers are in the same mblk.
          * - Assume each header is always within one mblk.
          * - Assume the ethernet header is in the first mblk.
          */
         ip = mblk->b_rptr + ethLen;
         if (ip >= mblk->b_wptr) {
            mblk = mblk->b_cont;
            ip = mblk->b_rptr;
         }
         ipLen = IPH_HDR_LENGTH((ipha_t *) ip);
         tcp = ip + ipLen;
         if (tcp >= mblk->b_wptr) {
            mblk = mblk->b_cont;
            tcp = mblk->b_rptr;
         }
         tcpLen = TCP_HDR_LENGTH((tcph_t *) tcp);
         if (tcp + tcpLen > mblk->b_wptr) { // careful, '>' instead of '>=' here
            mblk = mblk->b_cont;
         }

         ol->om = VMXNET3_OM_TSO;
         ol->hlen = ethLen + ipLen + tcpLen;
#if defined(OPEN_SOLARIS) || defined(SOL11)
         ol->msscof = mss;
#else
         /* OpenSolaris fills 'value' with the MSS but Solaris doesn't. */
         ol->msscof = DB_LSOMSS(mp);
#endif
         if (mblk != mp) {
            ret = ol->hlen;
         }
      }
#if defined(OPEN_SOLARIS) || defined(SOL11)
      else if (flags & HCK_PARTIALCKSUM) {
         ol->om = VMXNET3_OM_CSUM;
         ol->hlen = start + ethLen;
         ol->msscof = stuff + ethLen;
      }
#endif
   }

   return ret;
}

/*
 *---------------------------------------------------------------------------
 *
 * vmxnet3_tx_one --
 *
 *    Map a msg into the Tx command ring of a vmxnet3 device.
 *
 * Results:
 *    VMXNET3_TX_OK if everything went well.
 *    VMXNET3_TX_RINGFULL if the ring is nearly full.
 *    VMXNET3_TX_PULLUP if the msg is overfragmented.
 *    VMXNET3_TX_FAILURE if there was a DMA or offload error.
 *
 * Side effects:
 *    The ring is filled if VMXNET3_TX_OK is returned.
 *
 *---------------------------------------------------------------------------
 */
static vmxnet3_txstatus
vmxnet3_tx_one(vmxnet3_softc_t *dp,
               vmxnet3_txqueue_t *txq,
               vmxnet3_offload_t *ol,
               mblk_t *mp,
               boolean_t retry)
{
   int ret = VMXNET3_TX_OK;
   unsigned int frags = 0, totLen = 0;
   vmxnet3_cmdring_t *cmdRing = &txq->cmdRing;
   Vmxnet3_TxQueueCtrl *txqCtrl = txq->sharedCtrl;
   Vmxnet3_GenericDesc *txDesc;
   uint16_t sopIdx, eopIdx;
   uint8_t sopGen, curGen;
   mblk_t *mblk;

   mutex_enter(&dp->txLock);

   sopIdx = eopIdx = cmdRing->next2fill;
   sopGen = cmdRing->gen;
   curGen = !cmdRing->gen;

   for (mblk = mp; mblk != NULL; mblk = mblk->b_cont) {
      unsigned int len = MBLKL(mblk);
      ddi_dma_cookie_t cookie;
      uint_t cookieCount;

      if (len) {
         totLen += len;
      } else {
         continue;
      }

      if (ddi_dma_addr_bind_handle(dp->txDmaHandle, NULL,
                                   (caddr_t) mblk->b_rptr, len,
                                   DDI_DMA_RDWR | DDI_DMA_STREAMING,
                                   DDI_DMA_DONTWAIT, NULL,
                                   &cookie, &cookieCount) != DDI_DMA_MAPPED) {
         VMXNET3_WARN(dp, "ddi_dma_addr_bind_handle() failed\n");
         ret = VMXNET3_TX_FAILURE;
         goto error;
      }

      ASSERT(cookieCount);

      do {
         uint64_t addr = cookie.dmac_laddress;
         size_t len = cookie.dmac_size;

         do {
            uint32_t dw2, dw3;
            size_t chunkLen;

            ASSERT(!txq->metaRing[eopIdx].mp);
            ASSERT(cmdRing->avail - frags);

            if (frags >= cmdRing->size - 1 ||
                (ol->om != VMXNET3_OM_TSO && frags >= VMXNET3_MAX_TXD_PER_PKT)) {

               if (retry) {
                  VMXNET3_DEBUG(dp, 2, "overfragmented, frags=%u ring=%hu om=%hu\n",
                                frags, cmdRing->size, ol->om);
               }
               ddi_dma_unbind_handle(dp->txDmaHandle);
               ret = VMXNET3_TX_PULLUP;
               goto error;
            }
            if (cmdRing->avail - frags <= 1) {
               dp->txMustResched = B_TRUE;
               ddi_dma_unbind_handle(dp->txDmaHandle);
               ret = VMXNET3_TX_RINGFULL;
               goto error;
            }

            if (len > VMXNET3_MAX_TX_BUF_SIZE) {
               chunkLen = VMXNET3_MAX_TX_BUF_SIZE;
            } else {
               chunkLen = len;
            }

            frags++;
            eopIdx = cmdRing->next2fill;

            txDesc = VMXNET3_GET_DESC(cmdRing, eopIdx);
            ASSERT(txDesc->txd.gen != cmdRing->gen);

            // txd.addr
            txDesc->txd.addr = addr;
            // txd.dw2
            dw2 = chunkLen == VMXNET3_MAX_TX_BUF_SIZE ? 0 : chunkLen;
            dw2 |= curGen << VMXNET3_TXD_GEN_SHIFT;
            txDesc->dword[2] = dw2;
            ASSERT(txDesc->txd.len == len || txDesc->txd.len == 0);
            // txd.dw3
            dw3 = 0;
            txDesc->dword[3] = dw3;

            VMXNET3_INC_RING_IDX(cmdRing, cmdRing->next2fill);
            curGen = cmdRing->gen;

            addr += chunkLen;
            len -= chunkLen;
         } while (len);

         if (--cookieCount) {
            ddi_dma_nextcookie(dp->txDmaHandle, &cookie);
         }
      } while (cookieCount);

      ddi_dma_unbind_handle(dp->txDmaHandle);
   }

   /* Update the EOP descriptor */
   txDesc = VMXNET3_GET_DESC(cmdRing, eopIdx);
   txDesc->dword[3] |= VMXNET3_TXD_CQ | VMXNET3_TXD_EOP;

   /* Update the SOP descriptor. Must be done last */
   txDesc = VMXNET3_GET_DESC(cmdRing, sopIdx);
   if (ol->om == VMXNET3_OM_TSO &&
       txDesc->txd.len != 0 &&
       txDesc->txd.len < ol->hlen) {
      ret = VMXNET3_TX_FAILURE;
      goto error;
   }
   txDesc->txd.om = ol->om;
   txDesc->txd.hlen = ol->hlen;
   txDesc->txd.msscof = ol->msscof;
   membar_producer();
   txDesc->txd.gen = sopGen;

   /* Update the meta ring & metadata */
   txq->metaRing[sopIdx].mp = mp;
   txq->metaRing[eopIdx].sopIdx = sopIdx;
   txq->metaRing[eopIdx].frags = frags;
   cmdRing->avail -= frags;
   if (ol->om == VMXNET3_OM_TSO) {
      txqCtrl->txNumDeferred +=
         (totLen - ol->hlen + ol->msscof - 1) / ol->msscof;
   } else {
      txqCtrl->txNumDeferred++;
   }

   VMXNET3_DEBUG(dp, 3, "tx 0x%p on [%u;%u]\n", mp, sopIdx, eopIdx);

   goto done;

error:
   /* Reverse the generation bits */
   while (sopIdx != cmdRing->next2fill) {
      VMXNET3_DEC_RING_IDX(cmdRing, cmdRing->next2fill);
      txDesc = VMXNET3_GET_DESC(cmdRing, cmdRing->next2fill);
      txDesc->txd.gen = !cmdRing->gen;
   }

done:
   mutex_exit(&dp->txLock);

   return ret;
}

/*
 *---------------------------------------------------------------------------
 *
 * vmxnet3_tx --
 *
 *    Send packets on a vmxnet3 device.
 *
 * Results:
 *    NULL in case of success or failure.
 *    The mps to be retransmitted later if the ring is full.
 *
 * Side effects:
 *    None.
 *
 *---------------------------------------------------------------------------
 */
mblk_t *
vmxnet3_tx(void *data, mblk_t *mps)
{
   vmxnet3_softc_t *dp = data;
   vmxnet3_txqueue_t *txq = &dp->txQueue;
   vmxnet3_cmdring_t *cmdRing = &txq->cmdRing;
   Vmxnet3_TxQueueCtrl *txqCtrl = txq->sharedCtrl;
   vmxnet3_txstatus status = VMXNET3_TX_OK;
   mblk_t *mp;

   ASSERT(mps != NULL);

   do {
      vmxnet3_offload_t ol;
      int pullup;

      mp = mps;
      mps = mp->b_next;
      mp->b_next = NULL;

      if (DB_TYPE(mp) != M_DATA) {
         /*
          * PR #315560: Solaris might pass M_PROTO mblks for some reason.
          * Drop them because we don't understand them and because their
          * contents are not Ethernet frames anyway.
          */
         ASSERT(B_FALSE);
         freemsg(mp);
         continue;
      }

      /*
       * Prepare the offload while we're still handling the original
       * message -- msgpullup() discards the metadata afterwards.
       */
      pullup = vmxnet3_tx_prepare_offload(dp, &ol, mp);
      if (pullup) {
         mblk_t *new_mp = msgpullup(mp, pullup);
         freemsg(mp);
         if (new_mp) {
            mp = new_mp;
         } else {
            continue;
         }
      }

      /*
       * Try to map the message in the Tx ring.
       * This call might fail for non-fatal reasons.
       */
      status = vmxnet3_tx_one(dp, txq, &ol, mp, B_FALSE);
      if (status == VMXNET3_TX_PULLUP) {
         /*
          * Try one more time after flattening
          * the message with msgpullup().
          */
         if (mp->b_cont != NULL) {
            mblk_t *new_mp = msgpullup(mp, -1);
            freemsg(mp);
            if (new_mp) {
               mp = new_mp;
               status = vmxnet3_tx_one(dp, txq, &ol, mp, B_TRUE);
            } else {
               continue;
            }
         }
      }
      if (status != VMXNET3_TX_OK && status != VMXNET3_TX_RINGFULL) {
         /* Fatal failure, drop it */
         freemsg(mp);
      }
   } while (mps && status != VMXNET3_TX_RINGFULL);

   if (status == VMXNET3_TX_RINGFULL) {
      mp->b_next = mps;
      mps = mp;
   } else {
      ASSERT(!mps);
   }

   /* Notify the device */
   mutex_enter(&dp->txLock);
   if (txqCtrl->txNumDeferred >= txqCtrl->txThreshold) {
      txqCtrl->txNumDeferred = 0;
      VMXNET3_BAR0_PUT32(dp, VMXNET3_REG_TXPROD, cmdRing->next2fill);
   }
   mutex_exit(&dp->txLock);

   return mps;
}
Exemplo n.º 6
0
/*
 * Interrupt handler for Rx. Look if there are any pending Rx and
 * put them in mplist.
 */
mblk_t *
vmxnet3s_rx_intr(vmxnet3s_softc_t *dp, vmxnet3s_rxq_t *rxq)
{
	vmxnet3s_compring_t *compring = &rxq->compring;
	vmxnet3s_cmdring_t *cmdring = &rxq->cmdring;
	vmxnet3s_rxqctrl_t *rxqctrl = rxq->sharedctrl;
	vmxnet3s_gendesc_t *compdesc;
	mblk_t		*mplist = NULL;
	mblk_t		**mplisttail = &mplist;

	ASSERT(mutex_owned(&dp->intrlock));

	compdesc = VMXNET3_GET_DESC(compring, compring->next2comp);
	while (compdesc->rcd.gen == compring->gen) {
		mblk_t		*mp = NULL;
		mblk_t		**mptail = &mp;
		boolean_t	mpvalid = B_TRUE;
		boolean_t	eop;

		ASSERT(compdesc->rcd.sop);

		do {
			uint16_t	rxdidx = compdesc->rcd.rxdidx;
			vmxnet3s_rxbuf_t *rxbuf = rxq->bufring[rxdidx].rxbuf;
			mblk_t		*mblk = rxbuf->mblk;
			vmxnet3s_gendesc_t *rxdesc;

			while (compdesc->rcd.gen != compring->gen) {
				/*
				 * H/W may be still be in the middle of
				 * generating this entry, so hold on until
				 * the gen bit is flipped.
				 */
				membar_consumer();
			}
			ASSERT(compdesc->rcd.gen == compring->gen);
			ASSERT(rxbuf);
			ASSERT(mblk);

			/* Some Rx descriptors may have been skipped */
			while (cmdring->next2fill != rxdidx) {
				rxdesc = VMXNET3_GET_DESC(cmdring,
				    cmdring->next2fill);
				rxdesc->rxd.gen = cmdring->gen;
				VMXNET3_INC_RING_IDX(cmdring,
				    cmdring->next2fill);
			}

			eop = compdesc->rcd.eop;

			/*
			 * Now we have a piece of the packet in the rxdidx
			 * descriptor. Grab it only if we achieve to replace
			 * it with a fresh buffer.
			 */
			if (vmxnet3s_rx_populate(dp, rxq, rxdidx, B_FALSE)
			    == DDI_SUCCESS) {
				/* Success, we can chain the mblk with the mp */
				mblk->b_wptr = mblk->b_rptr + compdesc->rcd.len;
				*mptail = mblk;
				mptail = &mblk->b_cont;
				ASSERT(*mptail == NULL);

				if (eop) {
					if (!compdesc->rcd.err) {
						/*
						 * Tag the mp if it was
						 * checksummed by the H/W
						 */
						vmxnet3s_rx_hwcksum(dp, mp,
						    compdesc);
					} else {
						mpvalid = B_FALSE;
					}
				}
			} else {
				/*
				 * Keep the same buffer, we still need to flip
				 * the gen bit
				 */
				rxdesc = VMXNET3_GET_DESC(cmdring, rxdidx);
				rxdesc->rxd.gen = cmdring->gen;
				mpvalid = B_FALSE;
			}

			VMXNET3_INC_RING_IDX(compring, compring->next2comp);
			VMXNET3_INC_RING_IDX(cmdring, cmdring->next2fill);
			compdesc = VMXNET3_GET_DESC(compring,
			    compring->next2comp);
		} while (!eop);

		if (mp) {
			if (mpvalid) {
				*mplisttail = mp;
				mplisttail = &mp->b_next;
				ASSERT(*mplisttail == NULL);
			} else {
				/* This message got holes, drop it */
				freemsg(mp);
			}
		}
	}

	if (rxqctrl->updaterxprod) {
		uint32_t	rxprod;

		/*
		 * All buffers are actually available, but we can't tell that to
		 * the device because it may interpret that as an empty ring.
		 * So skip one buffer.
		 */
		if (cmdring->next2fill)
			rxprod = cmdring->next2fill - 1;
		else
			rxprod = cmdring->size - 1;
		VMXNET3_BAR0_PUT32(dp, VMXNET3_REG_RXPROD, rxprod);
	}

	return (mplist);
}
Exemplo n.º 7
0
/*
 * Map a msg into the Tx command ring of a vmxnet3s device.
 */
static vmxnet3s_txstatus_t
vmxnet3s_tx_one(vmxnet3s_softc_t *dp, vmxnet3s_txq_t *txq,
    vmxnet3s_offload_t *ol, mblk_t *mp, int to_copy)
{
	int		ret = VMXNET3_TX_OK;
	uint_t	frags = 0, totlen = 0;
	vmxnet3s_cmdring_t *cmdring = &txq->cmdring;
	vmxnet3s_txqctrl_t *txqctrl = txq->sharedctrl;
	vmxnet3s_gendesc_t *txdesc;
	uint16_t	sopidx;
	uint16_t	eopidx;
	uint8_t		sopgen;
	uint8_t		curgen;
	mblk_t		*mblk;
	uint_t	len;
	size_t		offset = 0;

	mutex_enter(&dp->txlock);

	sopidx = eopidx = cmdring->next2fill;
	sopgen = cmdring->gen;
	curgen = !cmdring->gen;

	mblk = mp;
	len = MBLKL(mblk);

	if (to_copy) {
		uint32_t	dw2;
		uint32_t	dw3;

		ASSERT(len >= to_copy);

		if (cmdring->avail <= 1) {
			dp->txmustresched = B_TRUE;
			ret = VMXNET3_TX_RINGFULL;
			goto error;
		}
		totlen += to_copy;

		len -= to_copy;
		offset = to_copy;

		bcopy(mblk->b_rptr, dp->txcache.nodes[sopidx].va, to_copy);

		eopidx = cmdring->next2fill;
		txdesc = VMXNET3_GET_DESC(cmdring, eopidx);

		ASSERT(txdesc->txd.gen != cmdring->gen);

		txdesc->txd.addr = dp->txcache.nodes[sopidx].pa;
		dw2 = to_copy;
		dw2 |= curgen << VMXNET3_TXD_GEN_SHIFT;
		txdesc->dword[2] = dw2;
		ASSERT(txdesc->txd.len == to_copy || txdesc->txd.len == 0);
		dw3 = 0;
		txdesc->dword[3] = dw3;

		VMXNET3_INC_RING_IDX(cmdring, cmdring->next2fill);
		curgen = cmdring->gen;

		frags++;
	}

	for (; mblk != NULL; mblk = mblk->b_cont, len = mblk ? MBLKL(mblk) : 0,
	    offset = 0) {
		ddi_dma_cookie_t cookie;
		uint_t cookiecount;

		if (len)
			totlen += len;
		else
			continue;

		if (ddi_dma_addr_bind_handle(dp->txdmahdl, NULL,
		    (caddr_t)mblk->b_rptr + offset, len,
		    DDI_DMA_RDWR | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL,
		    &cookie, &cookiecount) != DDI_DMA_MAPPED) {
			ret = VMXNET3_TX_FAILURE;
			goto error;
		}

		ASSERT(cookiecount);

		do {
			uint64_t	addr = cookie.dmac_laddress;
			size_t		len = cookie.dmac_size;

			do {
				uint32_t	dw2;
				uint32_t	dw3;
				size_t		chunklen;

				ASSERT(!txq->metaring[eopidx].mp);
				ASSERT(cmdring->avail - frags);

				if (frags >= cmdring->size - 1 || (ol->om !=
				    VMXNET3_OM_TSO &&
				    frags >= VMXNET3_MAX_TXD_PER_PKT)) {
					(void) ddi_dma_unbind_handle(
					    dp->txdmahdl);
					ret = VMXNET3_TX_PULLUP;
					goto error;
				}
				if (cmdring->avail - frags <= 1) {
					dp->txmustresched = B_TRUE;
					(void) ddi_dma_unbind_handle(
					    dp->txdmahdl);
					ret = VMXNET3_TX_RINGFULL;
					goto error;
				}

				if (len > VMXNET3_MAX_TX_BUF_SIZE)
					chunklen = VMXNET3_MAX_TX_BUF_SIZE;
				else
					chunklen = len;

				frags++;
				eopidx = cmdring->next2fill;

				txdesc = VMXNET3_GET_DESC(cmdring, eopidx);
				ASSERT(txdesc->txd.gen != cmdring->gen);

				/* txd.addr */
				txdesc->txd.addr = addr;
				/* txd.dw2 */
				dw2 = chunklen == VMXNET3_MAX_TX_BUF_SIZE ? 0 :
				    chunklen;
				dw2 |= curgen << VMXNET3_TXD_GEN_SHIFT;
				txdesc->dword[2] = dw2;
				ASSERT(txdesc->txd.len == len ||
				    txdesc->txd.len == 0);
				/* txd.dw3 */
				dw3 = 0;
				txdesc->dword[3] = dw3;

				VMXNET3_INC_RING_IDX(cmdring,
				    cmdring->next2fill);
				curgen = cmdring->gen;

				addr += chunklen;
				len -= chunklen;
			} while (len);

			if (--cookiecount)
				ddi_dma_nextcookie(dp->txdmahdl, &cookie);
		} while (cookiecount);

		(void) ddi_dma_unbind_handle(dp->txdmahdl);
	}

	/* Update the EOP descriptor */
	txdesc = VMXNET3_GET_DESC(cmdring, eopidx);
	txdesc->dword[3] |= VMXNET3_TXD_CQ | VMXNET3_TXD_EOP;

	/* Update the SOP descriptor. Must be done last */
	txdesc = VMXNET3_GET_DESC(cmdring, sopidx);
	if (ol->om == VMXNET3_OM_TSO &&
	    txdesc->txd.len != 0 &&
	    txdesc->txd.len < ol->hlen) {
		ret = VMXNET3_TX_FAILURE;
		goto error;
	}
	txdesc->txd.om = ol->om;
	txdesc->txd.hlen = ol->hlen;
	txdesc->txd.msscof = ol->msscof;
	membar_producer();
	txdesc->txd.gen = sopgen;

	/* Update the meta ring & metadata */
	txq->metaring[sopidx].mp = mp;
	txq->metaring[eopidx].sopidx = sopidx;
	txq->metaring[eopidx].frags = frags;
	cmdring->avail -= frags;
	if (ol->om == VMXNET3_OM_TSO) {
		txqctrl->txnumdeferred += (totlen - ol->hlen + ol->msscof - 1) /
		    ol->msscof;
	} else {
		txqctrl->txnumdeferred++;
	}

	goto done;

error:
	/* Reverse the generation bits */
	while (sopidx != cmdring->next2fill) {
		VMXNET3_DEC_RING_IDX(cmdring, cmdring->next2fill);
		txdesc = VMXNET3_GET_DESC(cmdring, cmdring->next2fill);
		txdesc->txd.gen = !cmdring->gen;
	}

done:
	mutex_exit(&dp->txlock);

	return (ret);
}