/** * xge_hal_fifo_dtr_next_completed - Retrieve next completed descriptor. * @channelh: Channel handle. * @dtrh: Descriptor handle. Returned by HAL. * @t_code: Transfer code, as per Xframe User Guide, * Transmit Descriptor Format. * Returned by HAL. * * Retrieve the _next_ completed descriptor. * HAL uses channel callback (*xge_hal_channel_callback_f) to notifiy * upper-layer driver (ULD) of new completed descriptors. After that * the ULD can use xge_hal_fifo_dtr_next_completed to retrieve the rest * completions (the very first completion is passed by HAL via * xge_hal_channel_callback_f). * * Implementation-wise, the upper-layer driver is free to call * xge_hal_fifo_dtr_next_completed either immediately from inside the * channel callback, or in a deferred fashion and separate (from HAL) * context. * * Non-zero @t_code means failure to process the descriptor. * The failure could happen, for instance, when the link is * down, in which case Xframe completes the descriptor because it * is not able to send the data out. * * For details please refer to Xframe User Guide. * * Returns: XGE_HAL_OK - success. * XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS - No completed descriptors * are currently available for processing. * * See also: xge_hal_channel_callback_f{}, * xge_hal_ring_dtr_next_completed(). * Usage: See ex_tx_compl{}. */ __HAL_STATIC_FIFO __HAL_INLINE_FIFO xge_hal_status_e xge_hal_fifo_dtr_next_completed(xge_hal_channel_h channelh, xge_hal_dtr_h *dtrh, u8 *t_code) { xge_hal_fifo_txd_t *txdp; xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING) xge_hal_fifo_txdl_priv_t *txdl_priv; #endif __hal_channel_dtr_try_complete(channelh, dtrh); txdp = (xge_hal_fifo_txd_t *)*dtrh; if (txdp == NULL) { return XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS; } #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING) txdl_priv = __hal_fifo_txdl_priv(txdp); /* sync TxDL to read the ownership * * Note: 16bytes means Control_1 & Control_2 */ xge_os_dma_sync(fifo->channel.pdev, txdl_priv->dma_handle, txdl_priv->dma_addr, txdl_priv->dma_offset, 16, XGE_OS_DMA_DIR_FROMDEVICE); #endif /* check whether host owns it */ if ( !(txdp->control_1 & XGE_HAL_TXD_LIST_OWN_XENA) ) { xge_assert(txdp->host_control!=0); __hal_channel_dtr_complete(channelh); *t_code = (u8)XGE_HAL_GET_TXD_T_CODE(txdp->control_1); /* see XGE_HAL_SET_TXD_T_CODE() above.. */ xge_assert(*t_code != XGE_HAL_TXD_T_CODE_UNUSED_5); if (fifo->channel.usage_cnt > 0) fifo->channel.usage_cnt--; return XGE_HAL_OK; } /* no more completions */ *dtrh = 0; return XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS; }
/** * xge_hal_stats_hw - Get HW device statistics. * @devh: HAL device handle. * @hw_info: Xframe statistic counters. See xge_hal_stats_hw_info_t. * Returned by HAL. * * Get device and HAL statistics. The latter is part of the in-host statistics * that HAL maintains for _that_ device. * * Returns: XGE_HAL_OK - success. * XGE_HAL_INF_STATS_IS_NOT_READY - Statistics information is not * currently available. * * See also: xge_hal_status_e{}. */ xge_hal_status_e xge_hal_stats_hw(xge_hal_device_h devh, xge_hal_stats_hw_info_t **hw_info) { xge_hal_device_t *hldev = (xge_hal_device_t *)devh; xge_assert(xge_hal_device_check_id(hldev) != XGE_HAL_CARD_TITAN) if (!hldev->stats.is_initialized || !hldev->stats.is_enabled) { *hw_info = NULL; return XGE_HAL_INF_STATS_IS_NOT_READY; } #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_STATS_STREAMING) xge_os_dma_sync(hldev->pdev, hldev->stats.hw_info_dmah, hldev->stats.dma_addr, 0, sizeof(xge_hal_stats_hw_info_t), XGE_OS_DMA_DIR_FROMDEVICE); #endif /* * update hw counters, taking into account * the "reset" or "saved" * values */ __hal_stats_update_latest(devh); /* * statistics HW bug fixups for Xena and Herc */ if (xge_hal_device_check_id(hldev) == XGE_HAL_CARD_XENA || xge_hal_device_check_id(hldev) == XGE_HAL_CARD_HERC) { u64 mcst, bcst; xge_hal_stats_hw_info_t *hwsta = &hldev->stats.hw_info_latest; mcst = ((u64)hwsta->rmac_vld_mcst_frms_oflow << 32) | hwsta->rmac_vld_mcst_frms; bcst = ((u64)hwsta->rmac_vld_bcst_frms_oflow << 32) | hwsta->rmac_vld_bcst_frms; mcst -= bcst; hwsta->rmac_vld_mcst_frms_oflow = (u32)(mcst >> 32); hwsta->rmac_vld_mcst_frms = (u32)mcst; }
static void __hal_ring_rxdblock_link(xge_hal_mempool_h mempoolh, xge_hal_ring_t *ring, int from, int to) { xge_hal_ring_block_t *to_item, *from_item; dma_addr_t to_dma, from_dma; pci_dma_h to_dma_handle, from_dma_handle; /* get "from" RxD block */ from_item = (xge_hal_ring_block_t *) __hal_mempool_item((xge_hal_mempool_t *) mempoolh, from); xge_assert(from_item); /* get "to" RxD block */ to_item = (xge_hal_ring_block_t *) __hal_mempool_item((xge_hal_mempool_t *) mempoolh, to); xge_assert(to_item); /* return address of the beginning of previous RxD block */ to_dma = __hal_ring_item_dma_addr(mempoolh, to_item, &to_dma_handle); /* set next pointer for this RxD block to point on * previous item's DMA start address */ __hal_ring_block_next_pointer_set(from_item, to_dma); /* return "from" RxD block's DMA start address */ from_dma = __hal_ring_item_dma_addr(mempoolh, from_item, &from_dma_handle); #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING) /* we must sync "from" RxD block, so hardware will see it */ xge_os_dma_sync(ring->channel.pdev, from_dma_handle, from_dma + XGE_HAL_RING_NEXT_BLOCK_POINTER_OFFSET, __hal_ring_item_dma_offset(mempoolh, from_item) + XGE_HAL_RING_NEXT_BLOCK_POINTER_OFFSET, sizeof(u64), XGE_OS_DMA_DIR_TODEVICE); #endif xge_debug_ring(XGE_TRACE, "block%d:0x"XGE_OS_LLXFMT" => block%d:0x"XGE_OS_LLXFMT, from, (unsigned long long)from_dma, to, (unsigned long long)to_dma); }
/** * xge_hal_fifo_dtr_buffer_finalize - Prepares a descriptor that contains the * single physically contiguous buffer. * * @channelh: Channel handle. * @dtrh: Descriptor handle. * @frag_idx: Index of the data buffer in the Txdl list. * * This API in conjuction with xge_hal_fifo_dtr_buffer_append() prepares * a descriptor that consists of a single physically contiguous buffer * which inturn contains the contents of one or more virtually contiguous * buffers received from the upper layer. * * See Also: xge_hal_fifo_dtr_buffer_append(). */ __HAL_STATIC_FIFO __HAL_INLINE_FIFO void xge_hal_fifo_dtr_buffer_finalize(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh, int frag_idx) { xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; xge_hal_fifo_txdl_priv_t *txdl_priv; xge_hal_fifo_txd_t *txdp; ptrdiff_t prev_boff; xge_assert(frag_idx < fifo->config->max_frags); txdl_priv = __hal_fifo_txdl_priv(dtrh); txdp = (xge_hal_fifo_txd_t *)dtrh + txdl_priv->frags; if (frag_idx != 0) { txdp->control_1 = txdp->control_2 = 0; } prev_boff = txdl_priv->align_vaddr_start - txdl_priv->align_vaddr; txdp->buffer_pointer = (u64)txdl_priv->align_dma_addr + prev_boff; txdp->control_1 |= XGE_HAL_TXD_BUFFER0_SIZE(txdl_priv->align_dma_offset); txdl_priv->bytes_sent += (unsigned int)txdl_priv->align_dma_offset; fifo->channel.stats.total_buffers++; fifo->channel.stats.copied_buffers++; txdl_priv->frags++; txdl_priv->align_used_frags++; #if defined(XGE_OS_DMA_REQUIRES_SYNC) /* sync pre-mapped buffer */ xge_os_dma_sync(fifo->channel.pdev, txdl_priv->align_dma_handle, txdp->buffer_pointer, 0, txdl_priv->align_dma_offset, XGE_OS_DMA_DIR_TODEVICE); #endif /* increment vaddr_start for the next buffer_append() iteration */ txdl_priv->align_vaddr_start += txdl_priv->align_dma_offset; txdl_priv->align_dma_offset = 0; }
/** * xge_hal_fifo_dtr_buffer_set_aligned - Align transmit buffer and fill * in fifo descriptor. * @channelh: Channel handle. * @dtrh: Descriptor handle. * @frag_idx: Index of the data buffer in the caller's scatter-gather list * (of buffers). * @vaddr: Virtual address of the data buffer. * @dma_pointer: DMA address of the data buffer referenced by @frag_idx. * @size: Size of the data buffer (in bytes). * @misaligned_size: Size (in bytes) of the misaligned portion of the * data buffer. Calculated by the caller, based on the platform/OS/other * specific criteria, which is outside of HAL's domain. See notes below. * * This API is part of the transmit descriptor preparation for posting * (via xge_hal_fifo_dtr_post()). The related "preparation" APIs include * xge_hal_fifo_dtr_mss_set() and xge_hal_fifo_dtr_cksum_set_bits(). * All three APIs fill in the fields of the fifo descriptor, * in accordance with the Xframe specification. * On the PCI-X based systems aligning transmit data typically provides better * transmit performance. The typical alignment granularity: L2 cacheline size. * However, HAL does not make assumptions in terms of the alignment granularity; * this is specified via additional @misaligned_size parameter described above. * Prior to calling xge_hal_fifo_dtr_buffer_set_aligned(), * ULD is supposed to check alignment of a given fragment/buffer. For this HAL * provides a separate xge_hal_check_alignment() API sufficient to cover * most (but not all) possible alignment criteria. * If the buffer appears to be aligned, the ULD calls * xge_hal_fifo_dtr_buffer_set(). * Otherwise, ULD calls xge_hal_fifo_dtr_buffer_set_aligned(). * * Note; This API is a "superset" of xge_hal_fifo_dtr_buffer_set(). In * addition to filling in the specified descriptor it aligns transmit data on * the specified boundary. * Note: Decision on whether to align or not to align a given contiguous * transmit buffer is outside of HAL's domain. To this end ULD can use any * programmable criteria, which can help to 1) boost transmit performance, * and/or 2) provide a workaround for PCI bridge bugs, if any. * * See also: xge_hal_fifo_dtr_buffer_set(), * xge_hal_check_alignment(). * * See also: xge_hal_fifo_dtr_reserve(), xge_hal_fifo_dtr_post(), * xge_hal_fifo_dtr_mss_set(), xge_hal_fifo_dtr_cksum_set_bits() */ __HAL_STATIC_FIFO __HAL_INLINE_FIFO xge_hal_status_e xge_hal_fifo_dtr_buffer_set_aligned(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh, int frag_idx, void *vaddr, dma_addr_t dma_pointer, int size, int misaligned_size) { xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; xge_hal_fifo_txdl_priv_t *txdl_priv; xge_hal_fifo_txd_t *txdp; int remaining_size; ptrdiff_t prev_boff; txdl_priv = __hal_fifo_txdl_priv(dtrh); txdp = (xge_hal_fifo_txd_t *)dtrh + txdl_priv->frags; if (frag_idx != 0) { txdp->control_1 = txdp->control_2 = 0; } /* On some systems buffer size could be zero. * It is the responsibility of ULD and *not HAL* to * detect it and skip it. */ xge_assert(size > 0); xge_assert(frag_idx < txdl_priv->alloc_frags); xge_assert(misaligned_size != 0 && misaligned_size <= fifo->config->alignment_size); remaining_size = size - misaligned_size; xge_assert(remaining_size >= 0); xge_os_memcpy((char*)txdl_priv->align_vaddr_start, vaddr, misaligned_size); if (txdl_priv->align_used_frags >= fifo->config->max_aligned_frags) { return XGE_HAL_ERR_OUT_ALIGNED_FRAGS; } /* setup new buffer */ prev_boff = txdl_priv->align_vaddr_start - txdl_priv->align_vaddr; txdp->buffer_pointer = (u64)txdl_priv->align_dma_addr + prev_boff; txdp->control_1 |= XGE_HAL_TXD_BUFFER0_SIZE(misaligned_size); txdl_priv->bytes_sent += misaligned_size; fifo->channel.stats.total_buffers++; txdl_priv->frags++; txdl_priv->align_used_frags++; txdl_priv->align_vaddr_start += fifo->config->alignment_size; txdl_priv->align_dma_offset = 0; #if defined(XGE_OS_DMA_REQUIRES_SYNC) /* sync new buffer */ xge_os_dma_sync(fifo->channel.pdev, txdl_priv->align_dma_handle, txdp->buffer_pointer, 0, misaligned_size, XGE_OS_DMA_DIR_TODEVICE); #endif if (remaining_size) { xge_assert(frag_idx < txdl_priv->alloc_frags); txdp++; txdp->buffer_pointer = (u64)dma_pointer + misaligned_size; txdp->control_1 = XGE_HAL_TXD_BUFFER0_SIZE(remaining_size); txdl_priv->bytes_sent += remaining_size; txdp->control_2 = 0; fifo->channel.stats.total_buffers++; txdl_priv->frags++; } return XGE_HAL_OK; }
__HAL_STATIC_FIFO __HAL_INLINE_FIFO void __hal_fifo_dtr_post_single(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh, u64 ctrl_1) { xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; xge_hal_fifo_hw_pair_t *hw_pair = fifo->hw_pair; xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)dtrh; xge_hal_fifo_txdl_priv_t *txdl_priv; u64 ctrl; txdp->control_1 |= XGE_HAL_TXD_LIST_OWN_XENA; #ifdef XGE_DEBUG_ASSERT /* make sure Xena overwrites the (illegal) t_code value on completion */ XGE_HAL_SET_TXD_T_CODE(txdp->control_1, XGE_HAL_TXD_T_CODE_UNUSED_5); #endif txdl_priv = __hal_fifo_txdl_priv(dtrh); #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING) /* sync the TxDL to device */ xge_os_dma_sync(fifo->channel.pdev, txdl_priv->dma_handle, txdl_priv->dma_addr, txdl_priv->dma_offset, txdl_priv->frags << 5 /* sizeof(xge_hal_fifo_txd_t) */, XGE_OS_DMA_DIR_TODEVICE); #endif /* write the pointer first */ xge_os_pio_mem_write64(fifo->channel.pdev, fifo->channel.regh1, txdl_priv->dma_addr, &hw_pair->txdl_pointer); /* spec: 0x00 = 1 TxD in the list */ ctrl = XGE_HAL_TX_FIFO_LAST_TXD_NUM(txdl_priv->frags - 1); ctrl |= ctrl_1; ctrl |= fifo->no_snoop_bits; if (txdp->control_1 & XGE_HAL_TXD_LSO_COF_CTRL(XGE_HAL_TXD_TCP_LSO)) { ctrl |= XGE_HAL_TX_FIFO_SPECIAL_FUNC; } /* * according to the XENA spec: * * It is important to note that pointers and list control words are * always written in pairs: in the first write, the host must write a * pointer, and in the second write, it must write the list control * word. Any other access will result in an error. Also, all 16 bytes * of the pointer/control structure must be written, including any * reserved bytes. */ xge_os_wmb(); /* * we want touch work_arr in order with ownership bit set to HW */ __hal_channel_dtr_post(channelh, dtrh); xge_os_pio_mem_write64(fifo->channel.pdev, fifo->channel.regh1, ctrl, &hw_pair->list_control); xge_debug_fifo(XGE_TRACE, "posted txdl 0x"XGE_OS_LLXFMT" ctrl 0x"XGE_OS_LLXFMT" " "into 0x"XGE_OS_LLXFMT"", (unsigned long long)txdl_priv->dma_addr, (unsigned long long)ctrl, (unsigned long long)(ulong_t)&hw_pair->txdl_pointer); #ifdef XGE_HAL_FIFO_DUMP_TXD xge_os_printf(""XGE_OS_LLXFMT":"XGE_OS_LLXFMT":"XGE_OS_LLXFMT":" XGE_OS_LLXFMT" dma "XGE_OS_LLXFMT, txdp->control_1, txdp->control_2, txdp->buffer_pointer, txdp->host_control, txdl_priv->dma_addr); #endif fifo->channel.stats.total_posts++; fifo->channel.usage_cnt++; if (fifo->channel.stats.usage_max < fifo->channel.usage_cnt) fifo->channel.stats.usage_max = fifo->channel.usage_cnt; }