/**
* xge_hal_fifo_dtr_next_completed - Retrieve next completed descriptor.
* @channelh: Channel handle.
* @dtrh: Descriptor handle. Returned by HAL.
* @t_code: Transfer code, as per Xframe User Guide,
* Transmit Descriptor Format.
* Returned by HAL.
*
* Retrieve the _next_ completed descriptor.
* HAL uses channel callback (*xge_hal_channel_callback_f) to notifiy
* upper-layer driver (ULD) of new completed descriptors. After that
* the ULD can use xge_hal_fifo_dtr_next_completed to retrieve the rest
* completions (the very first completion is passed by HAL via
* xge_hal_channel_callback_f).
*
* Implementation-wise, the upper-layer driver is free to call
* xge_hal_fifo_dtr_next_completed either immediately from inside the
* channel callback, or in a deferred fashion and separate (from HAL)
* context.
*
* Non-zero @t_code means failure to process the descriptor.
* The failure could happen, for instance, when the link is
* down, in which case Xframe completes the descriptor because it
* is not able to send the data out.
*
* For details please refer to Xframe User Guide.
*
* Returns: XGE_HAL_OK - success.
* XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS - No completed descriptors
* are currently available for processing.
*
* See also: xge_hal_channel_callback_f{},
* xge_hal_ring_dtr_next_completed().
* Usage: See ex_tx_compl{}.
*/
__HAL_STATIC_FIFO __HAL_INLINE_FIFO xge_hal_status_e
xge_hal_fifo_dtr_next_completed(xge_hal_channel_h channelh,
xge_hal_dtr_h *dtrh, u8 *t_code)
{
xge_hal_fifo_txd_t *txdp;
xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;
#if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING)
xge_hal_fifo_txdl_priv_t *txdl_priv;
#endif
__hal_channel_dtr_try_complete(channelh, dtrh);
txdp = (xge_hal_fifo_txd_t *)*dtrh;
if (txdp == NULL) {
return XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS;
}
#if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING)
txdl_priv = __hal_fifo_txdl_priv(txdp);
/* sync TxDL to read the ownership
*
* Note: 16bytes means Control_1 & Control_2 */
xge_os_dma_sync(fifo->channel.pdev,
txdl_priv->dma_handle,
txdl_priv->dma_addr,
txdl_priv->dma_offset,
16,
XGE_OS_DMA_DIR_FROMDEVICE);
#endif
/* check whether host owns it */
if ( !(txdp->control_1 & XGE_HAL_TXD_LIST_OWN_XENA) ) {
xge_assert(txdp->host_control!=0);
__hal_channel_dtr_complete(channelh);
*t_code = (u8)XGE_HAL_GET_TXD_T_CODE(txdp->control_1);
/* see XGE_HAL_SET_TXD_T_CODE() above.. */
xge_assert(*t_code != XGE_HAL_TXD_T_CODE_UNUSED_5);
if (fifo->channel.usage_cnt > 0)
fifo->channel.usage_cnt--;
return XGE_HAL_OK;
}
/* no more completions */
*dtrh = 0;
return XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS;
}
/**
* xge_hal_stats_hw - Get HW device statistics.
* @devh: HAL device handle.
* @hw_info: Xframe statistic counters. See xge_hal_stats_hw_info_t.
* Returned by HAL.
*
* Get device and HAL statistics. The latter is part of the in-host statistics
* that HAL maintains for _that_ device.
*
* Returns: XGE_HAL_OK - success.
* XGE_HAL_INF_STATS_IS_NOT_READY - Statistics information is not
* currently available.
*
* See also: xge_hal_status_e{}.
*/
xge_hal_status_e
xge_hal_stats_hw(xge_hal_device_h devh, xge_hal_stats_hw_info_t **hw_info)
{
xge_hal_device_t *hldev = (xge_hal_device_t *)devh;
xge_assert(xge_hal_device_check_id(hldev) != XGE_HAL_CARD_TITAN)
if (!hldev->stats.is_initialized ||
!hldev->stats.is_enabled) {
*hw_info = NULL;
return XGE_HAL_INF_STATS_IS_NOT_READY;
}
#if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_STATS_STREAMING)
xge_os_dma_sync(hldev->pdev,
hldev->stats.hw_info_dmah,
hldev->stats.dma_addr,
0,
sizeof(xge_hal_stats_hw_info_t),
XGE_OS_DMA_DIR_FROMDEVICE);
#endif
/*
* update hw counters, taking into account
* the "reset" or "saved"
* values
*/
__hal_stats_update_latest(devh);
/*
* statistics HW bug fixups for Xena and Herc
*/
if (xge_hal_device_check_id(hldev) == XGE_HAL_CARD_XENA ||
xge_hal_device_check_id(hldev) == XGE_HAL_CARD_HERC) {
u64 mcst, bcst;
xge_hal_stats_hw_info_t *hwsta = &hldev->stats.hw_info_latest;
mcst = ((u64)hwsta->rmac_vld_mcst_frms_oflow << 32) |
hwsta->rmac_vld_mcst_frms;
bcst = ((u64)hwsta->rmac_vld_bcst_frms_oflow << 32) |
hwsta->rmac_vld_bcst_frms;
mcst -= bcst;
hwsta->rmac_vld_mcst_frms_oflow = (u32)(mcst >> 32);
hwsta->rmac_vld_mcst_frms = (u32)mcst;
}
static void
__hal_ring_rxdblock_link(xge_hal_mempool_h mempoolh,
xge_hal_ring_t *ring, int from, int to)
{
xge_hal_ring_block_t *to_item, *from_item;
dma_addr_t to_dma, from_dma;
pci_dma_h to_dma_handle, from_dma_handle;
/* get "from" RxD block */
from_item = (xge_hal_ring_block_t *)
__hal_mempool_item((xge_hal_mempool_t *) mempoolh, from);
xge_assert(from_item);
/* get "to" RxD block */
to_item = (xge_hal_ring_block_t *)
__hal_mempool_item((xge_hal_mempool_t *) mempoolh, to);
xge_assert(to_item);
/* return address of the beginning of previous RxD block */
to_dma = __hal_ring_item_dma_addr(mempoolh, to_item, &to_dma_handle);
/* set next pointer for this RxD block to point on
* previous item's DMA start address */
__hal_ring_block_next_pointer_set(from_item, to_dma);
/* return "from" RxD block's DMA start address */
from_dma =
__hal_ring_item_dma_addr(mempoolh, from_item, &from_dma_handle);
#if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING)
/* we must sync "from" RxD block, so hardware will see it */
xge_os_dma_sync(ring->channel.pdev,
from_dma_handle,
from_dma + XGE_HAL_RING_NEXT_BLOCK_POINTER_OFFSET,
__hal_ring_item_dma_offset(mempoolh, from_item) +
XGE_HAL_RING_NEXT_BLOCK_POINTER_OFFSET,
sizeof(u64),
XGE_OS_DMA_DIR_TODEVICE);
#endif
xge_debug_ring(XGE_TRACE, "block%d:0x"XGE_OS_LLXFMT" => block%d:0x"XGE_OS_LLXFMT,
from, (unsigned long long)from_dma, to,
(unsigned long long)to_dma);
}
/**
* xge_hal_fifo_dtr_buffer_finalize - Prepares a descriptor that contains the
* single physically contiguous buffer.
*
* @channelh: Channel handle.
* @dtrh: Descriptor handle.
* @frag_idx: Index of the data buffer in the Txdl list.
*
* This API in conjuction with xge_hal_fifo_dtr_buffer_append() prepares
* a descriptor that consists of a single physically contiguous buffer
* which inturn contains the contents of one or more virtually contiguous
* buffers received from the upper layer.
*
* See Also: xge_hal_fifo_dtr_buffer_append().
*/
__HAL_STATIC_FIFO __HAL_INLINE_FIFO void
xge_hal_fifo_dtr_buffer_finalize(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
int frag_idx)
{
xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;
xge_hal_fifo_txdl_priv_t *txdl_priv;
xge_hal_fifo_txd_t *txdp;
ptrdiff_t prev_boff;
xge_assert(frag_idx < fifo->config->max_frags);
txdl_priv = __hal_fifo_txdl_priv(dtrh);
txdp = (xge_hal_fifo_txd_t *)dtrh + txdl_priv->frags;
if (frag_idx != 0) {
txdp->control_1 = txdp->control_2 = 0;
}
prev_boff = txdl_priv->align_vaddr_start - txdl_priv->align_vaddr;
txdp->buffer_pointer = (u64)txdl_priv->align_dma_addr + prev_boff;
txdp->control_1 |=
XGE_HAL_TXD_BUFFER0_SIZE(txdl_priv->align_dma_offset);
txdl_priv->bytes_sent += (unsigned int)txdl_priv->align_dma_offset;
fifo->channel.stats.total_buffers++;
fifo->channel.stats.copied_buffers++;
txdl_priv->frags++;
txdl_priv->align_used_frags++;
#if defined(XGE_OS_DMA_REQUIRES_SYNC)
/* sync pre-mapped buffer */
xge_os_dma_sync(fifo->channel.pdev,
txdl_priv->align_dma_handle,
txdp->buffer_pointer,
0,
txdl_priv->align_dma_offset,
XGE_OS_DMA_DIR_TODEVICE);
#endif
/* increment vaddr_start for the next buffer_append() iteration */
txdl_priv->align_vaddr_start += txdl_priv->align_dma_offset;
txdl_priv->align_dma_offset = 0;
}
/**
* xge_hal_fifo_dtr_buffer_set_aligned - Align transmit buffer and fill
* in fifo descriptor.
* @channelh: Channel handle.
* @dtrh: Descriptor handle.
* @frag_idx: Index of the data buffer in the caller's scatter-gather list
* (of buffers).
* @vaddr: Virtual address of the data buffer.
* @dma_pointer: DMA address of the data buffer referenced by @frag_idx.
* @size: Size of the data buffer (in bytes).
* @misaligned_size: Size (in bytes) of the misaligned portion of the
* data buffer. Calculated by the caller, based on the platform/OS/other
* specific criteria, which is outside of HAL's domain. See notes below.
*
* This API is part of the transmit descriptor preparation for posting
* (via xge_hal_fifo_dtr_post()). The related "preparation" APIs include
* xge_hal_fifo_dtr_mss_set() and xge_hal_fifo_dtr_cksum_set_bits().
* All three APIs fill in the fields of the fifo descriptor,
* in accordance with the Xframe specification.
* On the PCI-X based systems aligning transmit data typically provides better
* transmit performance. The typical alignment granularity: L2 cacheline size.
* However, HAL does not make assumptions in terms of the alignment granularity;
* this is specified via additional @misaligned_size parameter described above.
* Prior to calling xge_hal_fifo_dtr_buffer_set_aligned(),
* ULD is supposed to check alignment of a given fragment/buffer. For this HAL
* provides a separate xge_hal_check_alignment() API sufficient to cover
* most (but not all) possible alignment criteria.
* If the buffer appears to be aligned, the ULD calls
* xge_hal_fifo_dtr_buffer_set().
* Otherwise, ULD calls xge_hal_fifo_dtr_buffer_set_aligned().
*
* Note; This API is a "superset" of xge_hal_fifo_dtr_buffer_set(). In
* addition to filling in the specified descriptor it aligns transmit data on
* the specified boundary.
* Note: Decision on whether to align or not to align a given contiguous
* transmit buffer is outside of HAL's domain. To this end ULD can use any
* programmable criteria, which can help to 1) boost transmit performance,
* and/or 2) provide a workaround for PCI bridge bugs, if any.
*
* See also: xge_hal_fifo_dtr_buffer_set(),
* xge_hal_check_alignment().
*
* See also: xge_hal_fifo_dtr_reserve(), xge_hal_fifo_dtr_post(),
* xge_hal_fifo_dtr_mss_set(), xge_hal_fifo_dtr_cksum_set_bits()
*/
__HAL_STATIC_FIFO __HAL_INLINE_FIFO xge_hal_status_e
xge_hal_fifo_dtr_buffer_set_aligned(xge_hal_channel_h channelh,
xge_hal_dtr_h dtrh, int frag_idx, void *vaddr,
dma_addr_t dma_pointer, int size, int misaligned_size)
{
xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;
xge_hal_fifo_txdl_priv_t *txdl_priv;
xge_hal_fifo_txd_t *txdp;
int remaining_size;
ptrdiff_t prev_boff;
txdl_priv = __hal_fifo_txdl_priv(dtrh);
txdp = (xge_hal_fifo_txd_t *)dtrh + txdl_priv->frags;
if (frag_idx != 0) {
txdp->control_1 = txdp->control_2 = 0;
}
/* On some systems buffer size could be zero.
* It is the responsibility of ULD and *not HAL* to
* detect it and skip it. */
xge_assert(size > 0);
xge_assert(frag_idx < txdl_priv->alloc_frags);
xge_assert(misaligned_size != 0 &&
misaligned_size <= fifo->config->alignment_size);
remaining_size = size - misaligned_size;
xge_assert(remaining_size >= 0);
xge_os_memcpy((char*)txdl_priv->align_vaddr_start,
vaddr, misaligned_size);
if (txdl_priv->align_used_frags >= fifo->config->max_aligned_frags) {
return XGE_HAL_ERR_OUT_ALIGNED_FRAGS;
}
/* setup new buffer */
prev_boff = txdl_priv->align_vaddr_start - txdl_priv->align_vaddr;
txdp->buffer_pointer = (u64)txdl_priv->align_dma_addr + prev_boff;
txdp->control_1 |= XGE_HAL_TXD_BUFFER0_SIZE(misaligned_size);
txdl_priv->bytes_sent += misaligned_size;
fifo->channel.stats.total_buffers++;
txdl_priv->frags++;
txdl_priv->align_used_frags++;
txdl_priv->align_vaddr_start += fifo->config->alignment_size;
txdl_priv->align_dma_offset = 0;
#if defined(XGE_OS_DMA_REQUIRES_SYNC)
/* sync new buffer */
xge_os_dma_sync(fifo->channel.pdev,
txdl_priv->align_dma_handle,
txdp->buffer_pointer,
0,
misaligned_size,
XGE_OS_DMA_DIR_TODEVICE);
#endif
if (remaining_size) {
xge_assert(frag_idx < txdl_priv->alloc_frags);
txdp++;
txdp->buffer_pointer = (u64)dma_pointer +
misaligned_size;
txdp->control_1 =
XGE_HAL_TXD_BUFFER0_SIZE(remaining_size);
txdl_priv->bytes_sent += remaining_size;
txdp->control_2 = 0;
fifo->channel.stats.total_buffers++;
txdl_priv->frags++;
}
return XGE_HAL_OK;
}
__HAL_STATIC_FIFO __HAL_INLINE_FIFO void
__hal_fifo_dtr_post_single(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
u64 ctrl_1)
{
xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;
xge_hal_fifo_hw_pair_t *hw_pair = fifo->hw_pair;
xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)dtrh;
xge_hal_fifo_txdl_priv_t *txdl_priv;
u64 ctrl;
txdp->control_1 |= XGE_HAL_TXD_LIST_OWN_XENA;
#ifdef XGE_DEBUG_ASSERT
/* make sure Xena overwrites the (illegal) t_code value on completion */
XGE_HAL_SET_TXD_T_CODE(txdp->control_1, XGE_HAL_TXD_T_CODE_UNUSED_5);
#endif
txdl_priv = __hal_fifo_txdl_priv(dtrh);
#if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING)
/* sync the TxDL to device */
xge_os_dma_sync(fifo->channel.pdev,
txdl_priv->dma_handle,
txdl_priv->dma_addr,
txdl_priv->dma_offset,
txdl_priv->frags << 5 /* sizeof(xge_hal_fifo_txd_t) */,
XGE_OS_DMA_DIR_TODEVICE);
#endif
/* write the pointer first */
xge_os_pio_mem_write64(fifo->channel.pdev,
fifo->channel.regh1,
txdl_priv->dma_addr,
&hw_pair->txdl_pointer);
/* spec: 0x00 = 1 TxD in the list */
ctrl = XGE_HAL_TX_FIFO_LAST_TXD_NUM(txdl_priv->frags - 1);
ctrl |= ctrl_1;
ctrl |= fifo->no_snoop_bits;
if (txdp->control_1 & XGE_HAL_TXD_LSO_COF_CTRL(XGE_HAL_TXD_TCP_LSO)) {
ctrl |= XGE_HAL_TX_FIFO_SPECIAL_FUNC;
}
/*
* according to the XENA spec:
*
* It is important to note that pointers and list control words are
* always written in pairs: in the first write, the host must write a
* pointer, and in the second write, it must write the list control
* word. Any other access will result in an error. Also, all 16 bytes
* of the pointer/control structure must be written, including any
* reserved bytes.
*/
xge_os_wmb();
/*
* we want touch work_arr in order with ownership bit set to HW
*/
__hal_channel_dtr_post(channelh, dtrh);
xge_os_pio_mem_write64(fifo->channel.pdev, fifo->channel.regh1,
ctrl, &hw_pair->list_control);
xge_debug_fifo(XGE_TRACE, "posted txdl 0x"XGE_OS_LLXFMT" ctrl 0x"XGE_OS_LLXFMT" "
"into 0x"XGE_OS_LLXFMT"", (unsigned long long)txdl_priv->dma_addr,
(unsigned long long)ctrl,
(unsigned long long)(ulong_t)&hw_pair->txdl_pointer);
#ifdef XGE_HAL_FIFO_DUMP_TXD
xge_os_printf(""XGE_OS_LLXFMT":"XGE_OS_LLXFMT":"XGE_OS_LLXFMT":"
XGE_OS_LLXFMT" dma "XGE_OS_LLXFMT,
txdp->control_1, txdp->control_2, txdp->buffer_pointer,
txdp->host_control, txdl_priv->dma_addr);
#endif
fifo->channel.stats.total_posts++;
fifo->channel.usage_cnt++;
if (fifo->channel.stats.usage_max < fifo->channel.usage_cnt)
fifo->channel.stats.usage_max = fifo->channel.usage_cnt;
}
