void virtio_ve_add_cookie(struct vq_entry *qe, ddi_dma_handle_t dma_handle, ddi_dma_cookie_t dma_cookie, unsigned int ncookies, boolean_t write) { int i; for (i = 0; i < ncookies; i++) { virtio_ve_add_indirect_buf(qe, dma_cookie.dmac_laddress, dma_cookie.dmac_size, write); ddi_dma_nextcookie(dma_handle, &dma_cookie); } }
/* * function to copy the packet or dma map on the fly depending on size * * wq - pointer to WQ * wqed - Pointer to WQE descriptor * mp - Pointer to packet chain * * return DDI_SUCCESS=>success, DDI_FAILURE=>error */ static int oce_map_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp, uint32_t pkt_len) { ddi_dma_cookie_t cookie; oce_wq_mdesc_t *wqmd; uint32_t ncookies; int ret; struct oce_dev *dev = wq->parent; wqmd = oce_wqm_alloc(wq); if (wqmd == NULL) { oce_log(dev, CE_WARN, MOD_TX, "%s", "wqm pool empty"); return (ENOMEM); } ret = ddi_dma_addr_bind_handle(wqmd->dma_handle, (struct as *)0, (caddr_t)mp->b_rptr, pkt_len, DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL, &cookie, &ncookies); if (ret != DDI_DMA_MAPPED) { oce_log(dev, CE_WARN, MOD_TX, "MAP FAILED %d", ret); /* free the last one */ oce_wqm_free(wq, wqmd); return (ENOMEM); } do { wqed->frag[wqed->frag_idx].u0.s.frag_pa_hi = ADDR_HI(cookie.dmac_laddress); wqed->frag[wqed->frag_idx].u0.s.frag_pa_lo = ADDR_LO(cookie.dmac_laddress); wqed->frag[wqed->frag_idx].u0.s.frag_len = (uint32_t)cookie.dmac_size; wqed->frag_cnt++; wqed->frag_idx++; if (--ncookies > 0) ddi_dma_nextcookie(wqmd->dma_handle, &cookie); else break; } while (ncookies > 0); wqed->hdesc[wqed->nhdl].hdl = (void *)wqmd; wqed->hdesc[wqed->nhdl].type = MAPPED_WQE; wqed->nhdl++; return (0); } /* oce_map_wqe */
/* * igb_tx_bind * * Bind the mblk fragment with DMA */ static int igb_tx_bind(igb_tx_ring_t *tx_ring, tx_control_block_t *tcb, mblk_t *mp, uint32_t len) { int status, i; ddi_dma_cookie_t dma_cookie; uint_t ncookies; int desc_num; /* * Use DMA binding to process the mblk fragment */ status = ddi_dma_addr_bind_handle(tcb->tx_dma_handle, NULL, (caddr_t)mp->b_rptr, len, DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, 0, &dma_cookie, &ncookies); if (status != DDI_DMA_MAPPED) { IGB_DEBUG_STAT(tx_ring->stat_fail_dma_bind); return (-1); } tcb->frag_num++; tcb->tx_type = USE_DMA; /* * Each fragment can span several cookies. One cookie will have * one tx descriptor to transmit. */ desc_num = 0; for (i = ncookies; i > 0; i--) { /* * Save the address and length to the private data structure * of the tx control block, which will be used to fill the * tx descriptor ring after all the fragments are processed. */ igb_save_desc(tcb, dma_cookie.dmac_laddress, dma_cookie.dmac_size); desc_num++; if (i > 1) ddi_dma_nextcookie(tcb->tx_dma_handle, &dma_cookie); } return (desc_num); }
/* *--------------------------------------------------------------------------- * * vmxnet3_tx_one -- * * Map a msg into the Tx command ring of a vmxnet3 device. * * Results: * VMXNET3_TX_OK if everything went well. * VMXNET3_TX_RINGFULL if the ring is nearly full. * VMXNET3_TX_PULLUP if the msg is overfragmented. * VMXNET3_TX_FAILURE if there was a DMA or offload error. * * Side effects: * The ring is filled if VMXNET3_TX_OK is returned. * *--------------------------------------------------------------------------- */ static vmxnet3_txstatus vmxnet3_tx_one(vmxnet3_softc_t *dp, vmxnet3_txqueue_t *txq, vmxnet3_offload_t *ol, mblk_t *mp, boolean_t retry) { int ret = VMXNET3_TX_OK; unsigned int frags = 0, totLen = 0; vmxnet3_cmdring_t *cmdRing = &txq->cmdRing; Vmxnet3_TxQueueCtrl *txqCtrl = txq->sharedCtrl; Vmxnet3_GenericDesc *txDesc; uint16_t sopIdx, eopIdx; uint8_t sopGen, curGen; mblk_t *mblk; mutex_enter(&dp->txLock); sopIdx = eopIdx = cmdRing->next2fill; sopGen = cmdRing->gen; curGen = !cmdRing->gen; for (mblk = mp; mblk != NULL; mblk = mblk->b_cont) { unsigned int len = MBLKL(mblk); ddi_dma_cookie_t cookie; uint_t cookieCount; if (len) { totLen += len; } else { continue; } if (ddi_dma_addr_bind_handle(dp->txDmaHandle, NULL, (caddr_t) mblk->b_rptr, len, DDI_DMA_RDWR | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL, &cookie, &cookieCount) != DDI_DMA_MAPPED) { VMXNET3_WARN(dp, "ddi_dma_addr_bind_handle() failed\n"); ret = VMXNET3_TX_FAILURE; goto error; } ASSERT(cookieCount); do { uint64_t addr = cookie.dmac_laddress; size_t len = cookie.dmac_size; do { uint32_t dw2, dw3; size_t chunkLen; ASSERT(!txq->metaRing[eopIdx].mp); ASSERT(cmdRing->avail - frags); if (frags >= cmdRing->size - 1 || (ol->om != VMXNET3_OM_TSO && frags >= VMXNET3_MAX_TXD_PER_PKT)) { if (retry) { VMXNET3_DEBUG(dp, 2, "overfragmented, frags=%u ring=%hu om=%hu\n", frags, cmdRing->size, ol->om); } ddi_dma_unbind_handle(dp->txDmaHandle); ret = VMXNET3_TX_PULLUP; goto error; } if (cmdRing->avail - frags <= 1) { dp->txMustResched = B_TRUE; ddi_dma_unbind_handle(dp->txDmaHandle); ret = VMXNET3_TX_RINGFULL; goto error; } if (len > VMXNET3_MAX_TX_BUF_SIZE) { chunkLen = VMXNET3_MAX_TX_BUF_SIZE; } else { chunkLen = len; } frags++; eopIdx = cmdRing->next2fill; txDesc = VMXNET3_GET_DESC(cmdRing, eopIdx); ASSERT(txDesc->txd.gen != cmdRing->gen); // txd.addr txDesc->txd.addr = addr; // txd.dw2 dw2 = chunkLen == VMXNET3_MAX_TX_BUF_SIZE ? 0 : chunkLen; dw2 |= curGen << VMXNET3_TXD_GEN_SHIFT; txDesc->dword[2] = dw2; ASSERT(txDesc->txd.len == len || txDesc->txd.len == 0); // txd.dw3 dw3 = 0; txDesc->dword[3] = dw3; VMXNET3_INC_RING_IDX(cmdRing, cmdRing->next2fill); curGen = cmdRing->gen; addr += chunkLen; len -= chunkLen; } while (len); if (--cookieCount) { ddi_dma_nextcookie(dp->txDmaHandle, &cookie); } } while (cookieCount); ddi_dma_unbind_handle(dp->txDmaHandle); } /* Update the EOP descriptor */ txDesc = VMXNET3_GET_DESC(cmdRing, eopIdx); txDesc->dword[3] |= VMXNET3_TXD_CQ | VMXNET3_TXD_EOP; /* Update the SOP descriptor. Must be done last */ txDesc = VMXNET3_GET_DESC(cmdRing, sopIdx); if (ol->om == VMXNET3_OM_TSO && txDesc->txd.len != 0 && txDesc->txd.len < ol->hlen) { ret = VMXNET3_TX_FAILURE; goto error; } txDesc->txd.om = ol->om; txDesc->txd.hlen = ol->hlen; txDesc->txd.msscof = ol->msscof; membar_producer(); txDesc->txd.gen = sopGen; /* Update the meta ring & metadata */ txq->metaRing[sopIdx].mp = mp; txq->metaRing[eopIdx].sopIdx = sopIdx; txq->metaRing[eopIdx].frags = frags; cmdRing->avail -= frags; if (ol->om == VMXNET3_OM_TSO) { txqCtrl->txNumDeferred += (totLen - ol->hlen + ol->msscof - 1) / ol->msscof; } else { txqCtrl->txNumDeferred++; } VMXNET3_DEBUG(dp, 3, "tx 0x%p on [%u;%u]\n", mp, sopIdx, eopIdx); goto done; error: /* Reverse the generation bits */ while (sopIdx != cmdRing->next2fill) { VMXNET3_DEC_RING_IDX(cmdRing, cmdRing->next2fill); txDesc = VMXNET3_GET_DESC(cmdRing, cmdRing->next2fill); txDesc->txd.gen = !cmdRing->gen; } done: mutex_exit(&dp->txLock); return ret; }
/* *--------------------------------------------------------------------------- * * vmxnet3_tx_prepare_offload -- * * Build the offload context of a msg. * * Results: * 0 if everything went well. * +n if n bytes need to be pulled up. * -1 in case of error (not used). * * Side effects: * None. * *--------------------------------------------------------------------------- */ static int vmxnet3_tx_prepare_offload(vmxnet3_softc_t *dp, vmxnet3_offload_t *ol, mblk_t *mp) { int ret = 0; uint32_t start, stuff, value, flags; #if defined(OPEN_SOLARIS) || defined(SOL11) uint32_t lso_flag, mss; #endif ol->om = VMXNET3_OM_NONE; ol->hlen = 0; ol->msscof = 0; hcksum_retrieve(mp, NULL, NULL, &start, &stuff, NULL, &value, &flags); #if defined(OPEN_SOLARIS) || defined(SOL11) mac_lso_get(mp, &mss, &lso_flag); if (flags || lso_flag) { #else if (flags) { #endif struct ether_vlan_header *eth = (void *) mp->b_rptr; uint8_t ethLen; if (eth->ether_tpid == htons(ETHERTYPE_VLAN)) { ethLen = sizeof(struct ether_vlan_header); } else { ethLen = sizeof(struct ether_header); } VMXNET3_DEBUG(dp, 4, "flags=0x%x, ethLen=%u, start=%u, stuff=%u, value=%u\n", flags, ethLen, start, stuff, value); #if defined(OPEN_SOLARIS) || defined(SOL11) if (lso_flag & HW_LSO) { #else if (flags & HCK_PARTIALCKSUM) { ol->om = VMXNET3_OM_CSUM; ol->hlen = start + ethLen; ol->msscof = stuff + ethLen; } if (flags & HW_LSO) { #endif mblk_t *mblk = mp; uint8_t *ip, *tcp; uint8_t ipLen, tcpLen; /* * Copy e1000g's behavior: * - Do not assume all the headers are in the same mblk. * - Assume each header is always within one mblk. * - Assume the ethernet header is in the first mblk. */ ip = mblk->b_rptr + ethLen; if (ip >= mblk->b_wptr) { mblk = mblk->b_cont; ip = mblk->b_rptr; } ipLen = IPH_HDR_LENGTH((ipha_t *) ip); tcp = ip + ipLen; if (tcp >= mblk->b_wptr) { mblk = mblk->b_cont; tcp = mblk->b_rptr; } tcpLen = TCP_HDR_LENGTH((tcph_t *) tcp); if (tcp + tcpLen > mblk->b_wptr) { // careful, '>' instead of '>=' here mblk = mblk->b_cont; } ol->om = VMXNET3_OM_TSO; ol->hlen = ethLen + ipLen + tcpLen; #if defined(OPEN_SOLARIS) || defined(SOL11) ol->msscof = mss; #else /* OpenSolaris fills 'value' with the MSS but Solaris doesn't. */ ol->msscof = DB_LSOMSS(mp); #endif if (mblk != mp) { ret = ol->hlen; } } #if defined(OPEN_SOLARIS) || defined(SOL11) else if (flags & HCK_PARTIALCKSUM) { ol->om = VMXNET3_OM_CSUM; ol->hlen = start + ethLen; ol->msscof = stuff + ethLen; } #endif } return ret; } /* *--------------------------------------------------------------------------- * * vmxnet3_tx_one -- * * Map a msg into the Tx command ring of a vmxnet3 device. * * Results: * VMXNET3_TX_OK if everything went well. * VMXNET3_TX_RINGFULL if the ring is nearly full. * VMXNET3_TX_PULLUP if the msg is overfragmented. * VMXNET3_TX_FAILURE if there was a DMA or offload error. * * Side effects: * The ring is filled if VMXNET3_TX_OK is returned. * *--------------------------------------------------------------------------- */ static vmxnet3_txstatus vmxnet3_tx_one(vmxnet3_softc_t *dp, vmxnet3_txqueue_t *txq, vmxnet3_offload_t *ol, mblk_t *mp, boolean_t retry) { int ret = VMXNET3_TX_OK; unsigned int frags = 0, totLen = 0; vmxnet3_cmdring_t *cmdRing = &txq->cmdRing; Vmxnet3_TxQueueCtrl *txqCtrl = txq->sharedCtrl; Vmxnet3_GenericDesc *txDesc; uint16_t sopIdx, eopIdx; uint8_t sopGen, curGen; mblk_t *mblk; mutex_enter(&dp->txLock); sopIdx = eopIdx = cmdRing->next2fill; sopGen = cmdRing->gen; curGen = !cmdRing->gen; for (mblk = mp; mblk != NULL; mblk = mblk->b_cont) { unsigned int len = MBLKL(mblk); ddi_dma_cookie_t cookie; uint_t cookieCount; if (len) { totLen += len; } else { continue; } if (ddi_dma_addr_bind_handle(dp->txDmaHandle, NULL, (caddr_t) mblk->b_rptr, len, DDI_DMA_RDWR | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL, &cookie, &cookieCount) != DDI_DMA_MAPPED) { VMXNET3_WARN(dp, "ddi_dma_addr_bind_handle() failed\n"); ret = VMXNET3_TX_FAILURE; goto error; } ASSERT(cookieCount); do { uint64_t addr = cookie.dmac_laddress; size_t len = cookie.dmac_size; do { uint32_t dw2, dw3; size_t chunkLen; ASSERT(!txq->metaRing[eopIdx].mp); ASSERT(cmdRing->avail - frags); if (frags >= cmdRing->size - 1 || (ol->om != VMXNET3_OM_TSO && frags >= VMXNET3_MAX_TXD_PER_PKT)) { if (retry) { VMXNET3_DEBUG(dp, 2, "overfragmented, frags=%u ring=%hu om=%hu\n", frags, cmdRing->size, ol->om); } ddi_dma_unbind_handle(dp->txDmaHandle); ret = VMXNET3_TX_PULLUP; goto error; } if (cmdRing->avail - frags <= 1) { dp->txMustResched = B_TRUE; ddi_dma_unbind_handle(dp->txDmaHandle); ret = VMXNET3_TX_RINGFULL; goto error; } if (len > VMXNET3_MAX_TX_BUF_SIZE) { chunkLen = VMXNET3_MAX_TX_BUF_SIZE; } else { chunkLen = len; } frags++; eopIdx = cmdRing->next2fill; txDesc = VMXNET3_GET_DESC(cmdRing, eopIdx); ASSERT(txDesc->txd.gen != cmdRing->gen); // txd.addr txDesc->txd.addr = addr; // txd.dw2 dw2 = chunkLen == VMXNET3_MAX_TX_BUF_SIZE ? 0 : chunkLen; dw2 |= curGen << VMXNET3_TXD_GEN_SHIFT; txDesc->dword[2] = dw2; ASSERT(txDesc->txd.len == len || txDesc->txd.len == 0); // txd.dw3 dw3 = 0; txDesc->dword[3] = dw3; VMXNET3_INC_RING_IDX(cmdRing, cmdRing->next2fill); curGen = cmdRing->gen; addr += chunkLen; len -= chunkLen; } while (len); if (--cookieCount) { ddi_dma_nextcookie(dp->txDmaHandle, &cookie); } } while (cookieCount); ddi_dma_unbind_handle(dp->txDmaHandle); } /* Update the EOP descriptor */ txDesc = VMXNET3_GET_DESC(cmdRing, eopIdx); txDesc->dword[3] |= VMXNET3_TXD_CQ | VMXNET3_TXD_EOP; /* Update the SOP descriptor. Must be done last */ txDesc = VMXNET3_GET_DESC(cmdRing, sopIdx); if (ol->om == VMXNET3_OM_TSO && txDesc->txd.len != 0 && txDesc->txd.len < ol->hlen) { ret = VMXNET3_TX_FAILURE; goto error; } txDesc->txd.om = ol->om; txDesc->txd.hlen = ol->hlen; txDesc->txd.msscof = ol->msscof; membar_producer(); txDesc->txd.gen = sopGen; /* Update the meta ring & metadata */ txq->metaRing[sopIdx].mp = mp; txq->metaRing[eopIdx].sopIdx = sopIdx; txq->metaRing[eopIdx].frags = frags; cmdRing->avail -= frags; if (ol->om == VMXNET3_OM_TSO) { txqCtrl->txNumDeferred += (totLen - ol->hlen + ol->msscof - 1) / ol->msscof; } else { txqCtrl->txNumDeferred++; } VMXNET3_DEBUG(dp, 3, "tx 0x%p on [%u;%u]\n", mp, sopIdx, eopIdx); goto done; error: /* Reverse the generation bits */ while (sopIdx != cmdRing->next2fill) { VMXNET3_DEC_RING_IDX(cmdRing, cmdRing->next2fill); txDesc = VMXNET3_GET_DESC(cmdRing, cmdRing->next2fill); txDesc->txd.gen = !cmdRing->gen; } done: mutex_exit(&dp->txLock); return ret; } /* *--------------------------------------------------------------------------- * * vmxnet3_tx -- * * Send packets on a vmxnet3 device. * * Results: * NULL in case of success or failure. * The mps to be retransmitted later if the ring is full. * * Side effects: * None. * *--------------------------------------------------------------------------- */ mblk_t * vmxnet3_tx(void *data, mblk_t *mps) { vmxnet3_softc_t *dp = data; vmxnet3_txqueue_t *txq = &dp->txQueue; vmxnet3_cmdring_t *cmdRing = &txq->cmdRing; Vmxnet3_TxQueueCtrl *txqCtrl = txq->sharedCtrl; vmxnet3_txstatus status = VMXNET3_TX_OK; mblk_t *mp; ASSERT(mps != NULL); do { vmxnet3_offload_t ol; int pullup; mp = mps; mps = mp->b_next; mp->b_next = NULL; if (DB_TYPE(mp) != M_DATA) { /* * PR #315560: Solaris might pass M_PROTO mblks for some reason. * Drop them because we don't understand them and because their * contents are not Ethernet frames anyway. */ ASSERT(B_FALSE); freemsg(mp); continue; } /* * Prepare the offload while we're still handling the original * message -- msgpullup() discards the metadata afterwards. */ pullup = vmxnet3_tx_prepare_offload(dp, &ol, mp); if (pullup) { mblk_t *new_mp = msgpullup(mp, pullup); freemsg(mp); if (new_mp) { mp = new_mp; } else { continue; } } /* * Try to map the message in the Tx ring. * This call might fail for non-fatal reasons. */ status = vmxnet3_tx_one(dp, txq, &ol, mp, B_FALSE); if (status == VMXNET3_TX_PULLUP) { /* * Try one more time after flattening * the message with msgpullup(). */ if (mp->b_cont != NULL) { mblk_t *new_mp = msgpullup(mp, -1); freemsg(mp); if (new_mp) { mp = new_mp; status = vmxnet3_tx_one(dp, txq, &ol, mp, B_TRUE); } else { continue; } } } if (status != VMXNET3_TX_OK && status != VMXNET3_TX_RINGFULL) { /* Fatal failure, drop it */ freemsg(mp); } } while (mps && status != VMXNET3_TX_RINGFULL); if (status == VMXNET3_TX_RINGFULL) { mp->b_next = mps; mps = mp; } else { ASSERT(!mps); } /* Notify the device */ mutex_enter(&dp->txLock); if (txqCtrl->txNumDeferred >= txqCtrl->txThreshold) { txqCtrl->txNumDeferred = 0; VMXNET3_BAR0_PUT32(dp, VMXNET3_REG_TXPROD, cmdRing->next2fill); } mutex_exit(&dp->txLock); return mps; }
/** * Virtio Net Xmit hook. * * @param pvArg Pointer to private data. * @param pMsg Pointer to the message. * * @return Pointer to message not Xmited. */ static mblk_t *VirtioNetXmit(void *pvArg, mblk_t *pMsg) { LogFlowFunc((VIRTIOLOGNAME ":VirtioNetXmit pMsg=%p\n", pMsg)); cmn_err(CE_NOTE, "Xmit pMsg=%p\n", pMsg); PVIRTIODEVICE pDevice = pvArg; virtio_net_t *pNet = pDevice->pvDevice; bool fNotify = false; while (pMsg) { mblk_t *pNextMsg = pMsg->b_next; #if 0 mblk_t *pHdr = allocb(sizeof(virtio_net_header_t), BPRI_HI); if (RT_UNLIKELY(!pHdr)) break; virtio_net_header_t *pNetHdr = pHdr->b_rptr; memset(pNetHdr, 0, sizeof(virtio_net_header_t)); pNetHdr->u8Flags = VIRTIO_NET_GUEST_CSUM; pNetHdr->u16HdrLen = sizeof(virtio_net_header_t); pHdr->b_wptr += sizeof(virtio_net_header_t); pHdr->b_cont = pMsg; #endif virtio_net_txbuf_t *pTxBuf = kmem_cache_alloc(pNet->pTxCache, KM_SLEEP); if (!pTxBuf) break; ddi_dma_cookie_t DmaCookie; uint_t cCookies; int rc = ddi_dma_addr_bind_handle(pTxBuf->hDMA, NULL /* addrspace */, (char *)pMsg->b_rptr, MBLKL(pMsg), DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_SLEEP, 0 /* addr */, &DmaCookie, &cCookies); cmn_err(CE_NOTE, "VirtioNetXmit: MBLKL pMsg=%u\n", MBLKL(pMsg)); if (rc != DDI_DMA_MAPPED) { LogRel((VIRTIOLOGNAME ":VirtioNetXmit failed to map address to DMA handle. rc=%d\n", rc)); kmem_cache_free(pNet->pTxCache, pTxBuf); break; } /** @todo get 'cCookies' slots from the ring. */ for (uint_t i = 0; i < cCookies; i++) { uint16_t fFlags = 0; if (i < cCookies - 1) fFlags |= VIRTIO_FLAGS_RING_DESC_NEXT; rc = VirtioRingPush(pNet->pTxQueue, DmaCookie.dmac_laddress, DmaCookie.dmac_size, fFlags); if (RT_FAILURE(rc)) { LogRel((VIRTIOLOGNAME ":VirtioNetXmit failed. rc=%Rrc\n", rc)); break; } ddi_dma_nextcookie(pTxBuf->hDMA, &DmaCookie); } pMsg = pNextMsg; fNotify = true; if (RT_FAILURE(rc)) { ddi_dma_unbind_handle(pTxBuf->hDMA); break; } } if (fNotify) pDevice->pHyperOps->pfnNotifyQueue(pDevice, pNet->pTxQueue); return pMsg; }
/* * dcam_frame_rcv_init */ int dcam_frame_rcv_init(dcam_state_t *softc_p, int vid_mode, int frame_rate, int ring_buff_capacity) { int16_t bytes_per_pkt; /* # pkt bytes + overhead */ int bytes_per_frame; size_t frame; int cookie; int failure; id1394_isoch_dmainfo_t isoch_args; /* for alloc isoch call */ ixl1394_command_t *last_ixlp; /* last ixl in chain, */ /* used for appending ixls */ ixl1394_command_t *new_ixl_cmdp; /* new ixl command */ ixl1394_set_syncwait_t *new_ixl_sswp; /* new ixl set syncwait */ ixl1394_xfer_pkt_t *new_ixl_xfpp; /* new ixl xfer packet */ ixl1394_xfer_buf_t *new_ixl_xfbp; /* new ixl xfer buffer */ ixl1394_callback_t *new_ixl_cbp; /* new ixl callback */ ixl1394_jump_t *new_ixl_jmpp; /* new ixl jump */ int32_t result; /* errno from alloc_isoch_dma */ buff_info_t *buff_info_p; dcam1394_reg_io_t reg_io; uint_t data; size_t num_bytes, num_bytes_left; size_t num_xfer_cmds, xfer_cmd; size_t max_ixl_buff_size; uint64_t ixl_buff_kaddr; caddr_t ixl_buff_vaddr; bytes_per_pkt = g_bytes_per_packet[vid_mode][frame_rate]; if (bytes_per_pkt == -1) { return (1); } bytes_per_frame = g_bytes_per_frame[vid_mode]; if ((softc_p->ring_buff_p = ring_buff_create(softc_p, (size_t)ring_buff_capacity, (size_t)bytes_per_frame)) == NULL) { return (1); } softc_p->ring_buff_p->read_ptr_pos[0] = 0; /* allocate isoch channel */ softc_p->sii.si_channel_mask = 0xFFFF000000000000; softc_p->sii.si_bandwidth = bytes_per_pkt; softc_p->sii.rsrc_fail_target = dcam_rsrc_fail; softc_p->sii.single_evt_arg = softc_p; softc_p->sii.si_speed = softc_p->targetinfo.current_max_speed; if (t1394_alloc_isoch_single(softc_p->sl_handle, &softc_p->sii, 0, &softc_p->sii_output_args, &softc_p->sii_hdl, &failure) != DDI_SUCCESS) { return (1); } /* * At this point, all buffer memory has been allocated and * mapped, and is tracked on a linear linked list. Now need to * build the IXL. Done on a frame-by-frame basis. Could * theoretically have been done at the same time as the mem alloc * above, but hey, no need to be so fancy here. * * ixl buff size is bound by SHRT_MAX and needs to * be a multiple of packet size */ max_ixl_buff_size = (SHRT_MAX / bytes_per_pkt) * bytes_per_pkt; /* for each frame build frame's ixl list */ for (frame = 0; frame < softc_p->ring_buff_p->num_buffs; frame++) { buff_info_p = &(softc_p->ring_buff_p->buff_info_array_p[frame]); /* * if this is the 1st frame, put a IXL label at the top so a * loop can be created later */ if (frame == 0) { new_ixl_cmdp = kmem_zalloc( sizeof (ixl1394_label_t), KM_SLEEP); softc_p->ixlp = new_ixl_cmdp; new_ixl_cmdp->ixl_opcode = IXL1394_OP_LABEL; last_ixlp = softc_p->ixlp; } /* add wait-for-sync IXL command */ new_ixl_sswp = kmem_zalloc( sizeof (ixl1394_set_syncwait_t), KM_SLEEP); new_ixl_sswp->ixl_opcode = IXL1394_OP_SET_SYNCWAIT; last_ixlp->next_ixlp = (ixl1394_command_t *)new_ixl_sswp; last_ixlp = (ixl1394_command_t *)new_ixl_sswp; /* add in each dma cookie */ for (cookie = 0; cookie < buff_info_p->dma_cookie_count; cookie++) { num_xfer_cmds = min(bytes_per_frame, buff_info_p->dma_cookie.dmac_size) / max_ixl_buff_size; if (min(bytes_per_frame, buff_info_p->dma_cookie.dmac_size) % max_ixl_buff_size) { num_xfer_cmds++; } num_bytes_left = min(bytes_per_frame, buff_info_p->dma_cookie.dmac_size); ixl_buff_kaddr = buff_info_p->dma_cookie.dmac_laddress; ixl_buff_vaddr = buff_info_p->kaddr_p; for (xfer_cmd = 0; xfer_cmd < (num_xfer_cmds + 1); xfer_cmd++) { num_bytes = min(num_bytes_left, max_ixl_buff_size); if (xfer_cmd == 0) { new_ixl_xfpp = kmem_zalloc( sizeof (ixl1394_xfer_pkt_t), KM_SLEEP); new_ixl_xfpp->ixl_opcode = IXL1394_OP_RECV_PKT_ST; new_ixl_xfpp->ixl_buf._dmac_ll = ixl_buff_kaddr; new_ixl_xfpp->size = (uint16_t)bytes_per_pkt; new_ixl_xfpp->mem_bufp = ixl_buff_vaddr; last_ixlp->next_ixlp = (ixl1394_command_t *)new_ixl_xfpp; last_ixlp = (ixl1394_command_t *)new_ixl_xfpp; num_bytes_left -= bytes_per_pkt; ixl_buff_kaddr += bytes_per_pkt; ixl_buff_vaddr += bytes_per_pkt; continue; } /* allocate & init an IXL transfer command. */ new_ixl_xfbp = kmem_zalloc(sizeof (ixl1394_xfer_buf_t), KM_SLEEP); new_ixl_xfbp->ixl_opcode = IXL1394_OP_RECV_BUF; new_ixl_xfbp->ixl_buf._dmac_ll = ixl_buff_kaddr; new_ixl_xfbp->size = (uint16_t)num_bytes; new_ixl_xfbp->pkt_size = bytes_per_pkt; new_ixl_xfbp->mem_bufp = ixl_buff_vaddr; last_ixlp->next_ixlp = (ixl1394_command_t *)new_ixl_xfbp; last_ixlp = (ixl1394_command_t *)new_ixl_xfbp; num_bytes_left -= num_bytes; ixl_buff_kaddr += num_bytes; ixl_buff_vaddr += num_bytes; } if (cookie > 0) { ddi_dma_nextcookie(buff_info_p->dma_handle, &(buff_info_p->dma_cookie)); } } /* * at this point, have finished a frame. put in a callback */ new_ixl_cbp = kmem_zalloc( sizeof (ixl1394_callback_t), KM_SLEEP); new_ixl_cbp->ixl_opcode = IXL1394_OP_CALLBACK; new_ixl_cbp->callback = &dcam_frame_is_done; new_ixl_cbp->callback_arg = NULL; last_ixlp->next_ixlp = (ixl1394_command_t *)new_ixl_cbp; last_ixlp = (ixl1394_command_t *)new_ixl_cbp; } /* * for the final touch, put an IXL jump at the end to jump to the * label at the top */ new_ixl_jmpp = kmem_zalloc(sizeof (ixl1394_jump_t), KM_SLEEP); new_ixl_jmpp->ixl_opcode = IXL1394_OP_JUMP; new_ixl_jmpp->label = softc_p->ixlp; last_ixlp->next_ixlp = (ixl1394_command_t *)new_ixl_jmpp; /* don't need this, but it's neater */ last_ixlp = (ixl1394_command_t *)new_ixl_jmpp; /* call fwim routine to alloc an isoch resource */ isoch_args.ixlp = softc_p->ixlp; isoch_args.channel_num = softc_p->sii_output_args.channel_num; /* other misc args. note speed doesn't matter for isoch receive */ isoch_args.idma_options = ID1394_LISTEN_PKT_MODE; isoch_args.default_tag = 0; isoch_args.default_sync = 1; isoch_args.global_callback_arg = softc_p; /* set the ISO channel number */ data = (softc_p->sii_output_args.channel_num & 0xF) << 28; /* set the ISO speed */ data |= (softc_p->targetinfo.current_max_speed << 24); reg_io.offs = DCAM1394_REG_OFFS_CUR_ISO_CHANNEL; reg_io.val = data; if (dcam_reg_write(softc_p, ®_io)) { return (1); } result = 1234; if (t1394_alloc_isoch_dma(softc_p->sl_handle, &isoch_args, 0, &softc_p->isoch_handle, &result) != DDI_SUCCESS) { return (1); } return (0); }
/* * Map a msg into the Tx command ring of a vmxnet3s device. */ static vmxnet3s_txstatus_t vmxnet3s_tx_one(vmxnet3s_softc_t *dp, vmxnet3s_txq_t *txq, vmxnet3s_offload_t *ol, mblk_t *mp, int to_copy) { int ret = VMXNET3_TX_OK; uint_t frags = 0, totlen = 0; vmxnet3s_cmdring_t *cmdring = &txq->cmdring; vmxnet3s_txqctrl_t *txqctrl = txq->sharedctrl; vmxnet3s_gendesc_t *txdesc; uint16_t sopidx; uint16_t eopidx; uint8_t sopgen; uint8_t curgen; mblk_t *mblk; uint_t len; size_t offset = 0; mutex_enter(&dp->txlock); sopidx = eopidx = cmdring->next2fill; sopgen = cmdring->gen; curgen = !cmdring->gen; mblk = mp; len = MBLKL(mblk); if (to_copy) { uint32_t dw2; uint32_t dw3; ASSERT(len >= to_copy); if (cmdring->avail <= 1) { dp->txmustresched = B_TRUE; ret = VMXNET3_TX_RINGFULL; goto error; } totlen += to_copy; len -= to_copy; offset = to_copy; bcopy(mblk->b_rptr, dp->txcache.nodes[sopidx].va, to_copy); eopidx = cmdring->next2fill; txdesc = VMXNET3_GET_DESC(cmdring, eopidx); ASSERT(txdesc->txd.gen != cmdring->gen); txdesc->txd.addr = dp->txcache.nodes[sopidx].pa; dw2 = to_copy; dw2 |= curgen << VMXNET3_TXD_GEN_SHIFT; txdesc->dword[2] = dw2; ASSERT(txdesc->txd.len == to_copy || txdesc->txd.len == 0); dw3 = 0; txdesc->dword[3] = dw3; VMXNET3_INC_RING_IDX(cmdring, cmdring->next2fill); curgen = cmdring->gen; frags++; } for (; mblk != NULL; mblk = mblk->b_cont, len = mblk ? MBLKL(mblk) : 0, offset = 0) { ddi_dma_cookie_t cookie; uint_t cookiecount; if (len) totlen += len; else continue; if (ddi_dma_addr_bind_handle(dp->txdmahdl, NULL, (caddr_t)mblk->b_rptr + offset, len, DDI_DMA_RDWR | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL, &cookie, &cookiecount) != DDI_DMA_MAPPED) { ret = VMXNET3_TX_FAILURE; goto error; } ASSERT(cookiecount); do { uint64_t addr = cookie.dmac_laddress; size_t len = cookie.dmac_size; do { uint32_t dw2; uint32_t dw3; size_t chunklen; ASSERT(!txq->metaring[eopidx].mp); ASSERT(cmdring->avail - frags); if (frags >= cmdring->size - 1 || (ol->om != VMXNET3_OM_TSO && frags >= VMXNET3_MAX_TXD_PER_PKT)) { (void) ddi_dma_unbind_handle( dp->txdmahdl); ret = VMXNET3_TX_PULLUP; goto error; } if (cmdring->avail - frags <= 1) { dp->txmustresched = B_TRUE; (void) ddi_dma_unbind_handle( dp->txdmahdl); ret = VMXNET3_TX_RINGFULL; goto error; } if (len > VMXNET3_MAX_TX_BUF_SIZE) chunklen = VMXNET3_MAX_TX_BUF_SIZE; else chunklen = len; frags++; eopidx = cmdring->next2fill; txdesc = VMXNET3_GET_DESC(cmdring, eopidx); ASSERT(txdesc->txd.gen != cmdring->gen); /* txd.addr */ txdesc->txd.addr = addr; /* txd.dw2 */ dw2 = chunklen == VMXNET3_MAX_TX_BUF_SIZE ? 0 : chunklen; dw2 |= curgen << VMXNET3_TXD_GEN_SHIFT; txdesc->dword[2] = dw2; ASSERT(txdesc->txd.len == len || txdesc->txd.len == 0); /* txd.dw3 */ dw3 = 0; txdesc->dword[3] = dw3; VMXNET3_INC_RING_IDX(cmdring, cmdring->next2fill); curgen = cmdring->gen; addr += chunklen; len -= chunklen; } while (len); if (--cookiecount) ddi_dma_nextcookie(dp->txdmahdl, &cookie); } while (cookiecount); (void) ddi_dma_unbind_handle(dp->txdmahdl); } /* Update the EOP descriptor */ txdesc = VMXNET3_GET_DESC(cmdring, eopidx); txdesc->dword[3] |= VMXNET3_TXD_CQ | VMXNET3_TXD_EOP; /* Update the SOP descriptor. Must be done last */ txdesc = VMXNET3_GET_DESC(cmdring, sopidx); if (ol->om == VMXNET3_OM_TSO && txdesc->txd.len != 0 && txdesc->txd.len < ol->hlen) { ret = VMXNET3_TX_FAILURE; goto error; } txdesc->txd.om = ol->om; txdesc->txd.hlen = ol->hlen; txdesc->txd.msscof = ol->msscof; membar_producer(); txdesc->txd.gen = sopgen; /* Update the meta ring & metadata */ txq->metaring[sopidx].mp = mp; txq->metaring[eopidx].sopidx = sopidx; txq->metaring[eopidx].frags = frags; cmdring->avail -= frags; if (ol->om == VMXNET3_OM_TSO) { txqctrl->txnumdeferred += (totlen - ol->hlen + ol->msscof - 1) / ol->msscof; } else { txqctrl->txnumdeferred++; } goto done; error: /* Reverse the generation bits */ while (sopidx != cmdring->next2fill) { VMXNET3_DEC_RING_IDX(cmdring, cmdring->next2fill); txdesc = VMXNET3_GET_DESC(cmdring, cmdring->next2fill); txdesc->txd.gen = !cmdring->gen; } done: mutex_exit(&dp->txlock); return (ret); }
static int hxge_start(p_hxge_t hxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) { int dma_status, status = 0; p_tx_desc_t tx_desc_ring_vp; hpi_handle_t hpi_desc_handle; hxge_os_dma_handle_t tx_desc_dma_handle; p_tx_desc_t tx_desc_p; p_tx_msg_t tx_msg_ring; p_tx_msg_t tx_msg_p; tx_desc_t tx_desc, *tmp_desc_p; tx_desc_t sop_tx_desc, *sop_tx_desc_p; p_tx_pkt_header_t hdrp; p_tx_pkt_hdr_all_t pkthdrp; uint8_t npads = 0; uint64_t dma_ioaddr; uint32_t dma_flags; int last_bidx; uint8_t *b_rptr; caddr_t kaddr; uint32_t nmblks; uint32_t ngathers; uint32_t clen; int len; uint32_t pkt_len, pack_len, min_len; uint32_t bcopy_thresh; int i, cur_index, sop_index; uint16_t tail_index; boolean_t tail_wrap = B_FALSE; hxge_dma_common_t desc_area; hxge_os_dma_handle_t dma_handle; ddi_dma_cookie_t dma_cookie; hpi_handle_t hpi_handle; p_mblk_t nmp; p_mblk_t t_mp; uint32_t ncookies; boolean_t good_packet; boolean_t mark_mode = B_FALSE; p_hxge_stats_t statsp; p_hxge_tx_ring_stats_t tdc_stats; t_uscalar_t start_offset = 0; t_uscalar_t stuff_offset = 0; t_uscalar_t end_offset = 0; t_uscalar_t value = 0; t_uscalar_t cksum_flags = 0; boolean_t cksum_on = B_FALSE; uint32_t boff = 0; uint64_t tot_xfer_len = 0, tmp_len = 0; boolean_t header_set = B_FALSE; tdc_tdr_kick_t kick; uint32_t offset; #ifdef HXGE_DEBUG p_tx_desc_t tx_desc_ring_pp; p_tx_desc_t tx_desc_pp; tx_desc_t *save_desc_p; int dump_len; int sad_len; uint64_t sad; int xfer_len; uint32_t msgsize; #endif HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: tx dma channel %d", tx_ring_p->tdc)); HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: Starting tdc %d desc pending %d", tx_ring_p->tdc, tx_ring_p->descs_pending)); statsp = hxgep->statsp; if (hxgep->statsp->port_stats.lb_mode == hxge_lb_normal) { if (!statsp->mac_stats.link_up) { freemsg(mp); HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: " "link not up or LB mode")); goto hxge_start_fail1; } } mac_hcksum_get(mp, &start_offset, &stuff_offset, &end_offset, &value, &cksum_flags); if (!HXGE_IS_VLAN_PACKET(mp->b_rptr)) { start_offset += sizeof (ether_header_t); stuff_offset += sizeof (ether_header_t); } else { start_offset += sizeof (struct ether_vlan_header); stuff_offset += sizeof (struct ether_vlan_header); } if (cksum_flags & HCK_PARTIALCKSUM) { HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: mp $%p len %d " "cksum_flags 0x%x (partial checksum) ", mp, MBLKL(mp), cksum_flags)); cksum_on = B_TRUE; } MUTEX_ENTER(&tx_ring_p->lock); start_again: ngathers = 0; sop_index = tx_ring_p->wr_index; #ifdef HXGE_DEBUG if (tx_ring_p->descs_pending) { HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: desc pending %d ", tx_ring_p->descs_pending)); } dump_len = (int)(MBLKL(mp)); dump_len = (dump_len > 128) ? 128: dump_len; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: tdc %d: dumping ...: b_rptr $%p " "(Before header reserve: ORIGINAL LEN %d)", tx_ring_p->tdc, mp->b_rptr, dump_len)); HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: dump packets (IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, hxge_dump_packet((char *)mp->b_rptr, dump_len))); #endif tdc_stats = tx_ring_p->tdc_stats; mark_mode = (tx_ring_p->descs_pending && ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) < hxge_tx_minfree)); HXGE_DEBUG_MSG((hxgep, TX_CTL, "TX Descriptor ring is channel %d mark mode %d", tx_ring_p->tdc, mark_mode)); if (!hxge_txdma_reclaim(hxgep, tx_ring_p, hxge_tx_minfree)) { HXGE_DEBUG_MSG((hxgep, TX_CTL, "TX Descriptor ring is full: channel %d", tx_ring_p->tdc)); HXGE_DEBUG_MSG((hxgep, TX_CTL, "TX Descriptor ring is full: channel %d", tx_ring_p->tdc)); (void) atomic_cas_32((uint32_t *)&tx_ring_p->queueing, 0, 1); tdc_stats->tx_no_desc++; MUTEX_EXIT(&tx_ring_p->lock); status = 1; goto hxge_start_fail1; } nmp = mp; i = sop_index = tx_ring_p->wr_index; nmblks = 0; ngathers = 0; pkt_len = 0; pack_len = 0; clen = 0; last_bidx = -1; good_packet = B_TRUE; desc_area = tx_ring_p->tdc_desc; hpi_handle = desc_area.hpi_handle; hpi_desc_handle.regh = (hxge_os_acc_handle_t) DMA_COMMON_ACC_HANDLE(desc_area); hpi_desc_handle.hxgep = hxgep; tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); #ifdef HXGE_DEBUG #if defined(__i386) tx_desc_ring_pp = (p_tx_desc_t)(uint32_t)DMA_COMMON_IOADDR(desc_area); #else tx_desc_ring_pp = (p_tx_desc_t)DMA_COMMON_IOADDR(desc_area); #endif #endif tx_desc_dma_handle = (hxge_os_dma_handle_t)DMA_COMMON_HANDLE(desc_area); tx_msg_ring = tx_ring_p->tx_msg_ring; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: wr_index %d i %d", sop_index, i)); #ifdef HXGE_DEBUG msgsize = msgdsize(nmp); HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(1): wr_index %d i %d msgdsize %d", sop_index, i, msgsize)); #endif /* * The first 16 bytes of the premapped buffer are reserved * for header. No padding will be used. */ pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; if (hxge_tx_use_bcopy) { bcopy_thresh = (hxge_bcopy_thresh - TX_PKT_HEADER_SIZE); } else { bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); } while (nmp) { good_packet = B_TRUE; b_rptr = nmp->b_rptr; len = MBLKL(nmp); if (len <= 0) { nmp = nmp->b_cont; continue; } nmblks++; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(1): nmblks %d " "len %d pkt_len %d pack_len %d", nmblks, len, pkt_len, pack_len)); /* * Hardware limits the transfer length to 4K. * If len is more than 4K, we need to break * nmp into two chunks: Make first chunk smaller * than 4K. The second chunk will be broken into * less than 4K (if needed) during the next pass. */ if (len > (TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE)) { if ((t_mp = dupb(nmp)) != NULL) { nmp->b_wptr = nmp->b_rptr + (TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE); t_mp->b_rptr = nmp->b_wptr; t_mp->b_cont = nmp->b_cont; nmp->b_cont = t_mp; len = MBLKL(nmp); } else { good_packet = B_FALSE; goto hxge_start_fail2; } } tx_desc.value = 0; tx_desc_p = &tx_desc_ring_vp[i]; #ifdef HXGE_DEBUG tx_desc_pp = &tx_desc_ring_pp[i]; #endif tx_msg_p = &tx_msg_ring[i]; #if defined(__i386) hpi_desc_handle.regp = (uint32_t)tx_desc_p; #else hpi_desc_handle.regp = (uint64_t)tx_desc_p; #endif if (!header_set && ((!hxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || (len >= bcopy_thresh))) { header_set = B_TRUE; bcopy_thresh += TX_PKT_HEADER_SIZE; boff = 0; pack_len = 0; kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); hdrp = (p_tx_pkt_header_t)kaddr; clen = pkt_len; dma_handle = tx_msg_p->buf_dma_handle; dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); offset = tx_msg_p->offset_index * hxge_bcopy_thresh; (void) ddi_dma_sync(dma_handle, offset, hxge_bcopy_thresh, DDI_DMA_SYNC_FORDEV); tx_msg_p->flags.dma_type = USE_BCOPY; goto hxge_start_control_header_only; } pkt_len += len; pack_len += len; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(3): desc entry %d DESC IOADDR $%p " "desc_vp $%p tx_desc_p $%p desc_pp $%p tx_desc_pp $%p " "len %d pkt_len %d pack_len %d", i, DMA_COMMON_IOADDR(desc_area), tx_desc_ring_vp, tx_desc_p, tx_desc_ring_pp, tx_desc_pp, len, pkt_len, pack_len)); if (len < bcopy_thresh) { HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(4): USE BCOPY: ")); if (hxge_tx_tiny_pack) { uint32_t blst = TXDMA_DESC_NEXT_INDEX(i, -1, tx_ring_p->tx_wrap_mask); HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(5): pack")); if ((pack_len <= bcopy_thresh) && (last_bidx == blst)) { HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: pack(6) " "(pkt_len %d pack_len %d)", pkt_len, pack_len)); i = blst; tx_desc_p = &tx_desc_ring_vp[i]; #ifdef HXGE_DEBUG tx_desc_pp = &tx_desc_ring_pp[i]; #endif tx_msg_p = &tx_msg_ring[i]; boff = pack_len - len; ngathers--; } else if (pack_len > bcopy_thresh && header_set) { pack_len = len; boff = 0; bcopy_thresh = hxge_bcopy_thresh; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(7): > max NEW " "bcopy thresh %d " "pkt_len %d pack_len %d(next)", bcopy_thresh, pkt_len, pack_len)); } last_bidx = i; } kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { hdrp = (p_tx_pkt_header_t)kaddr; header_set = B_TRUE; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(7_x2): " "pkt_len %d pack_len %d (new hdrp $%p)", pkt_len, pack_len, hdrp)); } tx_msg_p->flags.dma_type = USE_BCOPY; kaddr += boff; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(8): USE BCOPY: before bcopy " "DESC IOADDR $%p entry %d bcopy packets %d " "bcopy kaddr $%p bcopy ioaddr (SAD) $%p " "bcopy clen %d bcopy boff %d", DMA_COMMON_IOADDR(desc_area), i, tdc_stats->tx_hdr_pkts, kaddr, dma_ioaddr, clen, boff)); HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: 1USE BCOPY: ")); HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: 2USE BCOPY: ")); HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: " "last USE BCOPY: copy from b_rptr $%p " "to KADDR $%p (len %d offset %d", b_rptr, kaddr, len, boff)); bcopy(b_rptr, kaddr, len); #ifdef HXGE_DEBUG dump_len = (len > 128) ? 128: len; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: dump packets " "(After BCOPY len %d)" "(b_rptr $%p): %s", len, nmp->b_rptr, hxge_dump_packet((char *)nmp->b_rptr, dump_len))); #endif dma_handle = tx_msg_p->buf_dma_handle; dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); offset = tx_msg_p->offset_index * hxge_bcopy_thresh; (void) ddi_dma_sync(dma_handle, offset, hxge_bcopy_thresh, DDI_DMA_SYNC_FORDEV); clen = len + boff; tdc_stats->tx_hdr_pkts++; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(9): " "USE BCOPY: DESC IOADDR $%p entry %d " "bcopy packets %d bcopy kaddr $%p " "bcopy ioaddr (SAD) $%p bcopy clen %d " "bcopy boff %d", DMA_COMMON_IOADDR(desc_area), i, tdc_stats->tx_hdr_pkts, kaddr, dma_ioaddr, clen, boff)); } else { HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(12): USE DVMA: len %d", len)); tx_msg_p->flags.dma_type = USE_DMA; dma_flags = DDI_DMA_WRITE; if (len < hxge_dma_stream_thresh) { dma_flags |= DDI_DMA_CONSISTENT; } else { dma_flags |= DDI_DMA_STREAMING; } dma_handle = tx_msg_p->dma_handle; dma_status = ddi_dma_addr_bind_handle(dma_handle, NULL, (caddr_t)b_rptr, len, dma_flags, DDI_DMA_DONTWAIT, NULL, &dma_cookie, &ncookies); if (dma_status == DDI_DMA_MAPPED) { dma_ioaddr = dma_cookie.dmac_laddress; len = (int)dma_cookie.dmac_size; clen = (uint32_t)dma_cookie.dmac_size; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(12_1): " "USE DVMA: len %d clen %d ngathers %d", len, clen, ngathers)); #if defined(__i386) hpi_desc_handle.regp = (uint32_t)tx_desc_p; #else hpi_desc_handle.regp = (uint64_t)tx_desc_p; #endif while (ncookies > 1) { ngathers++; /* * this is the fix for multiple * cookies, which are basically * a descriptor entry, we don't set * SOP bit as well as related fields */ (void) hpi_txdma_desc_gather_set( hpi_desc_handle, &tx_desc, (ngathers -1), mark_mode, ngathers, dma_ioaddr, clen); tx_msg_p->tx_msg_size = clen; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: DMA " "ncookie %d ngathers %d " "dma_ioaddr $%p len %d" "desc $%p descp $%p (%d)", ncookies, ngathers, dma_ioaddr, clen, *tx_desc_p, tx_desc_p, i)); ddi_dma_nextcookie(dma_handle, &dma_cookie); dma_ioaddr = dma_cookie.dmac_laddress; len = (int)dma_cookie.dmac_size; clen = (uint32_t)dma_cookie.dmac_size; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(12_2): " "USE DVMA: len %d clen %d ", len, clen)); i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); tx_desc_p = &tx_desc_ring_vp[i]; hpi_desc_handle.regp = #if defined(__i386) (uint32_t)tx_desc_p; #else (uint64_t)tx_desc_p; #endif tx_msg_p = &tx_msg_ring[i]; tx_msg_p->flags.dma_type = USE_NONE; tx_desc.value = 0; ncookies--; } tdc_stats->tx_ddi_pkts++; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: DMA: ddi packets %d", tdc_stats->tx_ddi_pkts)); } else { HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "dma mapping failed for %d " "bytes addr $%p flags %x (%d)", len, b_rptr, status, status)); good_packet = B_FALSE; tdc_stats->tx_dma_bind_fail++; tx_msg_p->flags.dma_type = USE_NONE; status = 1; goto hxge_start_fail2; } } /* ddi dvma */ nmp = nmp->b_cont; hxge_start_control_header_only: #if defined(__i386) hpi_desc_handle.regp = (uint32_t)tx_desc_p; #else hpi_desc_handle.regp = (uint64_t)tx_desc_p; #endif ngathers++; if (ngathers == 1) { #ifdef HXGE_DEBUG save_desc_p = &sop_tx_desc; #endif sop_tx_desc_p = &sop_tx_desc; sop_tx_desc_p->value = 0; sop_tx_desc_p->bits.tr_len = clen; sop_tx_desc_p->bits.sad = dma_ioaddr >> 32; sop_tx_desc_p->bits.sad_l = dma_ioaddr & 0xffffffff; } else {
/* * SMCG_send() -- send a packet */ static int SMCG_send(gld_mac_info_t *macinfo, mblk_t *mp) { smcg_t *smcg = (smcg_t *)macinfo->gldm_private; Adapter_Struc *pAd = smcg->smcg_pAd; int i = 0, j = 0, totlen = 0, msglen = 0, rc; mblk_t *mptr = mp; Data_Buff_Structure dbuf; ddi_dma_cookie_t cookie; unsigned int ncookies; for (; mptr != NULL; i++, mptr = mptr->b_cont) { if (i >= SMCG_MAX_TX_MBLKS) { if (pullupmsg(mp, -1) == 0) { smcg->smcg_need_gld_sched = 1; return (GLD_NORESOURCES); /* retry send */ } msglen = (mp->b_wptr - mp->b_rptr); break; } msglen += (mptr->b_wptr - mptr->b_rptr); } if (msglen > ETHERMAX) { cmn_err(CE_WARN, SMCG_NAME "%d: dropping oversize packet (%d)", macinfo->gldm_ppa, msglen); return (GLD_BADARG); } mutex_enter(&smcg->txbuf_lock); mutex_enter(&smcg->lm_lock); LM_Reap_Xmits(pAd); mutex_exit(&smcg->lm_lock); if ((smcg->tx_ring_head + 1) % pAd->num_of_tx_buffs == smcg->tx_ring_tail) { smcg->smcg_need_gld_sched = 1; mutex_exit(&smcg->txbuf_lock); return (GLD_NORESOURCES); /* retry send */ } for (mptr = mp, i = 0; mptr != NULL; mptr = mptr->b_cont) { int blocklen = mptr->b_wptr - mptr->b_rptr; if (blocklen == 0) continue; ASSERT(i < SMCG_MAX_TX_MBLKS); rc = ddi_dma_addr_bind_handle( smcg->tx_info[smcg->tx_ring_head].dmahandle[i], NULL, (caddr_t)mptr->b_rptr, (size_t)blocklen, DDI_DMA_WRITE, DDI_DMA_DONTWAIT, 0, &cookie, &ncookies); if (rc != DDI_DMA_MAPPED) { while (--i >= 0) (void) ddi_dma_unbind_handle( smcg->tx_info[smcg->tx_ring_head]. dmahandle[i]); if (rc == DDI_DMA_NORESOURCES) { smcg->smcg_need_gld_sched = 1; mutex_exit(&smcg->txbuf_lock); return (GLD_NORESOURCES); } #ifdef DEBUG if (SMCG_debug & SMCGTRACE) cmn_err(CE_WARN, SMCG_NAME "Send bind handle failure = 0x%x", rc); #endif mutex_exit(&smcg->txbuf_lock); return (GLD_FAILURE); } /* CONSTANTCONDITION */ while (1) { dbuf.fragment_list[j].fragment_length = cookie.dmac_size | PHYSICAL_ADDR; dbuf.fragment_list[j].fragment_ptr = (unsigned char *)(uintptr_t)cookie.dmac_address; j++; if (--ncookies == 0) break; ddi_dma_nextcookie( smcg->tx_info[smcg->tx_ring_head].dmahandle[i], &cookie); } i++; totlen += blocklen; } dbuf.fragment_count = j; smcg->tx_info[smcg->tx_ring_head].handles_bound = i; smcg->tx_info[smcg->tx_ring_head].mptr = mp; if (totlen < ETHERMIN) totlen = ETHERMIN; /* pad if necessary */ mutex_enter(&smcg->lm_lock); pAd->xmit_interrupts = (smcg->smcg_need_gld_sched) ? 1 : 0; rc = LM_Send(&dbuf, pAd, totlen); mutex_exit(&smcg->lm_lock); if (rc != SUCCESS) { for (i = 0; i < smcg->tx_info[smcg->tx_ring_head].handles_bound; i++) (void) ddi_dma_unbind_handle( smcg->tx_info[smcg->tx_ring_head].dmahandle[i]); } else smcg->tx_ring_head = (smcg->tx_ring_head+1) % pAd->num_of_tx_buffs; mutex_exit(&smcg->txbuf_lock); #ifdef DEBUG if (rc != SUCCESS && rc != OUT_OF_RESOURCES) cmn_err(CE_WARN, SMCG_NAME "_send: LM_Send failed %d", rc); #endif if (rc == SUCCESS) { return (GLD_SUCCESS); } else if (rc == OUT_OF_RESOURCES) { smcg->smcg_need_gld_sched = 1; return (GLD_NORESOURCES); } else { return (GLD_FAILURE); } }