/* hfi1_vnic_handle_rx - handle skb receive */
static void hfi1_vnic_handle_rx(struct hfi1_vnic_rx_queue *rxq,
int *work_done, int work_to_do)
{
struct hfi1_vnic_vport_info *vinfo = rxq->vinfo;
struct sk_buff *skb;
int rc;
while (1) {
if (*work_done >= work_to_do)
break;
skb = hfi1_vnic_get_skb(rxq);
if (unlikely(!skb))
break;
rc = hfi1_vnic_decap_skb(rxq, skb);
/* update rx counters */
hfi1_vnic_update_rx_counters(vinfo, rxq->idx, skb, rc);
if (unlikely(rc)) {
dev_kfree_skb_any(skb);
continue;
}
skb_checksum_none_assert(skb);
skb->protocol = eth_type_trans(skb, rxq->netdev);
napi_gro_receive(&rxq->napi, skb);
(*work_done)++;
}
}
static int hisi_femac_rx(struct net_device *dev, int limit)
{
struct hisi_femac_priv *priv = netdev_priv(dev);
struct hisi_femac_queue *rxq = &priv->rxq;
struct sk_buff *skb;
dma_addr_t addr;
u32 rx_pkt_info, pos, len, rx_pkts_num = 0;
pos = rxq->tail;
while (readl(priv->glb_base + GLB_IRQ_RAW) & IRQ_INT_RX_RDY) {
rx_pkt_info = readl(priv->port_base + IQFRM_DES);
len = rx_pkt_info & RX_FRAME_LEN_MASK;
len -= ETH_FCS_LEN;
/* tell hardware we will deal with this packet */
writel(IRQ_INT_RX_RDY, priv->glb_base + GLB_IRQ_RAW);
rx_pkts_num++;
skb = rxq->skb[pos];
if (unlikely(!skb)) {
netdev_err(dev, "rx skb NULL. pos=%d\n", pos);
break;
}
rxq->skb[pos] = NULL;
addr = rxq->dma_phys[pos];
dma_unmap_single(priv->dev, addr, MAX_FRAME_SIZE,
DMA_FROM_DEVICE);
skb_put(skb, len);
if (unlikely(skb->len > MAX_FRAME_SIZE)) {
netdev_err(dev, "rcv len err, len = %d\n", skb->len);
dev->stats.rx_errors++;
dev->stats.rx_length_errors++;
dev_kfree_skb_any(skb);
goto next;
}
skb->protocol = eth_type_trans(skb, dev);
napi_gro_receive(&priv->napi, skb);
dev->stats.rx_packets++;
dev->stats.rx_bytes += skb->len;
next:
pos = (pos + 1) % rxq->num;
if (rx_pkts_num >= limit)
break;
}
rxq->tail = pos;
hisi_femac_rx_refill(priv);
return rx_pkts_num;
}
/* Pass a received packet up through the generic LRO stack
*
* Handles driverlink veto, and passes the fragment up via
* the appropriate LRO method
*/
static void efx_rx_packet_lro(struct efx_channel *channel,
struct efx_rx_buffer *rx_buf,
bool checksummed)
{
struct napi_struct *napi = &channel->napi_str;
gro_result_t gro_result;
/* Pass the skb/page into the LRO engine */
if (rx_buf->page) {
struct page *page = rx_buf->page;
struct sk_buff *skb;
EFX_BUG_ON_PARANOID(rx_buf->skb);
rx_buf->page = NULL;
skb = napi_get_frags(napi);
if (!skb) {
put_page(page);
return;
}
skb_shinfo(skb)->frags[0].page = page;
skb_shinfo(skb)->frags[0].page_offset =
efx_rx_buf_offset(rx_buf);
skb_shinfo(skb)->frags[0].size = rx_buf->len;
skb_shinfo(skb)->nr_frags = 1;
skb->len = rx_buf->len;
skb->data_len = rx_buf->len;
skb->truesize += rx_buf->len;
skb->ip_summed =
checksummed ? CHECKSUM_UNNECESSARY : CHECKSUM_NONE;
skb_record_rx_queue(skb, channel->channel);
gro_result = napi_gro_frags(napi);
} else {
struct sk_buff *skb = rx_buf->skb;
EFX_BUG_ON_PARANOID(!skb);
EFX_BUG_ON_PARANOID(!checksummed);
rx_buf->skb = NULL;
gro_result = napi_gro_receive(napi, skb);
}
if (gro_result == GRO_NORMAL) {
channel->rx_alloc_level += RX_ALLOC_FACTOR_SKB;
} else if (gro_result != GRO_DROP) {
channel->rx_alloc_level += RX_ALLOC_FACTOR_LRO;
channel->irq_mod_score += 2;
}
}
/* Pass a received packet up through the generic LRO stack
*
* Handles driverlink veto, and passes the fragment up via
* the appropriate LRO method
*/
static void efx_rx_packet_lro(struct efx_channel *channel,
struct efx_rx_buffer *rx_buf)
{
struct napi_struct *napi = &channel->napi_str;
/* Pass the skb/page into the LRO engine */
if (rx_buf->page) {
struct sk_buff *skb = napi_get_frags(napi);
if (!skb) {
put_page(rx_buf->page);
goto out;
}
skb_shinfo(skb)->frags[0].page = rx_buf->page;
skb_shinfo(skb)->frags[0].page_offset =
efx_rx_buf_offset(rx_buf);
skb_shinfo(skb)->frags[0].size = rx_buf->len;
skb_shinfo(skb)->nr_frags = 1;
skb->len = rx_buf->len;
skb->data_len = rx_buf->len;
skb->truesize += rx_buf->len;
skb->ip_summed = CHECKSUM_UNNECESSARY;
napi_gro_frags(napi);
out:
EFX_BUG_ON_PARANOID(rx_buf->skb);
rx_buf->page = NULL;
} else {
EFX_BUG_ON_PARANOID(!rx_buf->skb);
napi_gro_receive(napi, rx_buf->skb);
rx_buf->skb = NULL;
}
}
static int xge_rx_poll(struct net_device *ndev, unsigned int budget)
{
struct xge_pdata *pdata = netdev_priv(ndev);
struct device *dev = &pdata->pdev->dev;
struct xge_desc_ring *rx_ring;
struct xge_raw_desc *raw_desc;
struct sk_buff *skb;
dma_addr_t dma_addr;
int processed = 0;
u8 head, rx_error;
int i, ret;
u32 data;
u16 len;
rx_ring = pdata->rx_ring;
head = rx_ring->head;
data = xge_rd_csr(pdata, DMARXSTATUS);
if (!GET_BITS(RXPKTCOUNT, data))
return 0;
for (i = 0; i < budget; i++) {
raw_desc = &rx_ring->raw_desc[head];
if (GET_BITS(E, le64_to_cpu(raw_desc->m0)))
break;
dma_rmb();
skb = rx_ring->pkt_info[head].skb;
rx_ring->pkt_info[head].skb = NULL;
dma_addr = rx_ring->pkt_info[head].dma_addr;
len = GET_BITS(PKT_SIZE, le64_to_cpu(raw_desc->m0));
dma_unmap_single(dev, dma_addr, XGENE_ENET_STD_MTU,
DMA_FROM_DEVICE);
rx_error = GET_BITS(D, le64_to_cpu(raw_desc->m2));
if (unlikely(rx_error)) {
pdata->stats.rx_errors++;
dev_kfree_skb_any(skb);
goto out;
}
skb_put(skb, len);
skb->protocol = eth_type_trans(skb, ndev);
pdata->stats.rx_packets++;
pdata->stats.rx_bytes += len;
napi_gro_receive(&pdata->napi, skb);
out:
ret = xge_refill_buffers(ndev, 1);
xge_wr_csr(pdata, DMARXSTATUS, 1);
xge_wr_csr(pdata, DMARXCTRL, 1);
if (ret)
break;
head = (head + 1) & (XGENE_ENET_NUM_DESC - 1);
processed++;
}
rx_ring->head = head;
return processed;
}
static int bgmac_dma_rx_read(struct bgmac *bgmac, struct bgmac_dma_ring *ring,
int weight)
{
u32 end_slot;
int handled = 0;
end_slot = bgmac_read(bgmac, ring->mmio_base + BGMAC_DMA_RX_STATUS);
end_slot &= BGMAC_DMA_RX_STATDPTR;
end_slot -= ring->index_base;
end_slot &= BGMAC_DMA_RX_STATDPTR;
end_slot /= sizeof(struct bgmac_dma_desc);
while (ring->start != end_slot) {
struct device *dma_dev = bgmac->core->dma_dev;
struct bgmac_slot_info *slot = &ring->slots[ring->start];
struct bgmac_rx_header *rx = slot->buf + BGMAC_RX_BUF_OFFSET;
struct sk_buff *skb;
void *buf = slot->buf;
dma_addr_t dma_addr = slot->dma_addr;
u16 len, flags;
do {
/* Prepare new skb as replacement */
if (bgmac_dma_rx_skb_for_slot(bgmac, slot)) {
bgmac_dma_rx_poison_buf(dma_dev, slot);
break;
}
/* Unmap buffer to make it accessible to the CPU */
dma_unmap_single(dma_dev, dma_addr,
BGMAC_RX_BUF_SIZE, DMA_FROM_DEVICE);
/* Get info from the header */
len = le16_to_cpu(rx->len);
flags = le16_to_cpu(rx->flags);
/* Check for poison and drop or pass the packet */
if (len == 0xdead && flags == 0xbeef) {
bgmac_err(bgmac, "Found poisoned packet at slot %d, DMA issue!\n",
ring->start);
put_page(virt_to_head_page(buf));
break;
}
if (len > BGMAC_RX_ALLOC_SIZE) {
bgmac_err(bgmac, "Found oversized packet at slot %d, DMA issue!\n",
ring->start);
put_page(virt_to_head_page(buf));
break;
}
/* Omit CRC. */
len -= ETH_FCS_LEN;
skb = build_skb(buf, BGMAC_RX_ALLOC_SIZE);
if (unlikely(!skb)) {
bgmac_err(bgmac, "build_skb failed\n");
put_page(virt_to_head_page(buf));
break;
}
skb_put(skb, BGMAC_RX_FRAME_OFFSET +
BGMAC_RX_BUF_OFFSET + len);
skb_pull(skb, BGMAC_RX_FRAME_OFFSET +
BGMAC_RX_BUF_OFFSET);
skb_checksum_none_assert(skb);
skb->protocol = eth_type_trans(skb, bgmac->net_dev);
napi_gro_receive(&bgmac->napi, skb);
handled++;
} while (0);
bgmac_dma_rx_setup_desc(bgmac, ring, ring->start);
if (++ring->start >= BGMAC_RX_RING_SLOTS)
ring->start = 0;
if (handled >= weight) /* Should never be greater */
break;
}
bgmac_dma_rx_update_index(bgmac, ring);
return handled;
}
/** Routine to push packets arriving on Octeon interface upto network layer.
* @param oct_id - octeon device id.
* @param skbuff - skbuff struct to be passed to network layer.
* @param len - size of total data received.
* @param rh - Control header associated with the packet
* @param param - additional control data with the packet
* @param arg - farg registered in droq_ops
*/
static void
liquidio_push_packet(u32 octeon_id __attribute__((unused)),
void *skbuff,
u32 len,
union octeon_rh *rh,
void *param,
void *arg)
{
struct net_device *netdev = (struct net_device *)arg;
struct octeon_droq *droq =
container_of(param, struct octeon_droq, napi);
struct sk_buff *skb = (struct sk_buff *)skbuff;
struct skb_shared_hwtstamps *shhwtstamps;
struct napi_struct *napi = param;
u16 vtag = 0;
u32 r_dh_off;
u64 ns;
if (netdev) {
struct lio *lio = GET_LIO(netdev);
struct octeon_device *oct = lio->oct_dev;
/* Do not proceed if the interface is not in RUNNING state. */
if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) {
recv_buffer_free(skb);
droq->stats.rx_dropped++;
return;
}
skb->dev = netdev;
skb_record_rx_queue(skb, droq->q_no);
if (likely(len > MIN_SKB_SIZE)) {
struct octeon_skb_page_info *pg_info;
unsigned char *va;
pg_info = ((struct octeon_skb_page_info *)(skb->cb));
if (pg_info->page) {
/* For Paged allocation use the frags */
va = page_address(pg_info->page) +
pg_info->page_offset;
memcpy(skb->data, va, MIN_SKB_SIZE);
skb_put(skb, MIN_SKB_SIZE);
skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
pg_info->page,
pg_info->page_offset +
MIN_SKB_SIZE,
len - MIN_SKB_SIZE,
LIO_RXBUFFER_SZ);
}
} else {
struct octeon_skb_page_info *pg_info =
((struct octeon_skb_page_info *)(skb->cb));
skb_copy_to_linear_data(skb, page_address(pg_info->page)
+ pg_info->page_offset, len);
skb_put(skb, len);
put_page(pg_info->page);
}
r_dh_off = (rh->r_dh.len - 1) * BYTES_PER_DHLEN_UNIT;
if (oct->ptp_enable) {
if (rh->r_dh.has_hwtstamp) {
/* timestamp is included from the hardware at
* the beginning of the packet.
*/
if (ifstate_check
(lio,
LIO_IFSTATE_RX_TIMESTAMP_ENABLED)) {
/* Nanoseconds are in the first 64-bits
* of the packet.
*/
memcpy(&ns, (skb->data + r_dh_off),
sizeof(ns));
r_dh_off -= BYTES_PER_DHLEN_UNIT;
shhwtstamps = skb_hwtstamps(skb);
shhwtstamps->hwtstamp =
ns_to_ktime(ns +
lio->ptp_adjust);
}
}
}
if (rh->r_dh.has_hash) {
__be32 *hash_be = (__be32 *)(skb->data + r_dh_off);
u32 hash = be32_to_cpu(*hash_be);
skb_set_hash(skb, hash, PKT_HASH_TYPE_L4);
r_dh_off -= BYTES_PER_DHLEN_UNIT;
}
skb_pull(skb, rh->r_dh.len * BYTES_PER_DHLEN_UNIT);
skb->protocol = eth_type_trans(skb, skb->dev);
if ((netdev->features & NETIF_F_RXCSUM) &&
(((rh->r_dh.encap_on) &&
(rh->r_dh.csum_verified & CNNIC_TUN_CSUM_VERIFIED)) ||
(!(rh->r_dh.encap_on) &&
(rh->r_dh.csum_verified & CNNIC_CSUM_VERIFIED))))
/* checksum has already been verified */
skb->ip_summed = CHECKSUM_UNNECESSARY;
else
skb->ip_summed = CHECKSUM_NONE;
/* Setting Encapsulation field on basis of status received
* from the firmware
*/
if (rh->r_dh.encap_on) {
skb->encapsulation = 1;
skb->csum_level = 1;
droq->stats.rx_vxlan++;
}
/* inbound VLAN tag */
if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
rh->r_dh.vlan) {
u16 priority = rh->r_dh.priority;
u16 vid = rh->r_dh.vlan;
vtag = (priority << VLAN_PRIO_SHIFT) | vid;
__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vtag);
}
napi_gro_receive(napi, skb);
droq->stats.rx_bytes_received += len -
rh->r_dh.len * BYTES_PER_DHLEN_UNIT;
droq->stats.rx_pkts_received++;
} else {
recv_buffer_free(skb);
}
}
static int xlgmac_rx_poll(struct xlgmac_channel *channel, int budget)
{
struct xlgmac_pdata *pdata = channel->pdata;
struct xlgmac_ring *ring = channel->rx_ring;
struct net_device *netdev = pdata->netdev;
unsigned int len, dma_desc_len, max_len;
unsigned int context_next, context;
struct xlgmac_desc_data *desc_data;
struct xlgmac_pkt_info *pkt_info;
unsigned int incomplete, error;
struct xlgmac_hw_ops *hw_ops;
unsigned int received = 0;
struct napi_struct *napi;
struct sk_buff *skb;
int packet_count = 0;
hw_ops = &pdata->hw_ops;
/* Nothing to do if there isn't a Rx ring for this channel */
if (!ring)
return 0;
incomplete = 0;
context_next = 0;
napi = (pdata->per_channel_irq) ? &channel->napi : &pdata->napi;
desc_data = XLGMAC_GET_DESC_DATA(ring, ring->cur);
pkt_info = &ring->pkt_info;
while (packet_count < budget) {
/* First time in loop see if we need to restore state */
if (!received && desc_data->state_saved) {
skb = desc_data->state.skb;
error = desc_data->state.error;
len = desc_data->state.len;
} else {
memset(pkt_info, 0, sizeof(*pkt_info));
skb = NULL;
error = 0;
len = 0;
}
read_again:
desc_data = XLGMAC_GET_DESC_DATA(ring, ring->cur);
if (xlgmac_rx_dirty_desc(ring) > XLGMAC_RX_DESC_MAX_DIRTY)
xlgmac_rx_refresh(channel);
if (hw_ops->dev_read(channel))
break;
received++;
ring->cur++;
incomplete = XLGMAC_GET_REG_BITS(
pkt_info->attributes,
RX_PACKET_ATTRIBUTES_INCOMPLETE_POS,
RX_PACKET_ATTRIBUTES_INCOMPLETE_LEN);
context_next = XLGMAC_GET_REG_BITS(
pkt_info->attributes,
RX_PACKET_ATTRIBUTES_CONTEXT_NEXT_POS,
RX_PACKET_ATTRIBUTES_CONTEXT_NEXT_LEN);
context = XLGMAC_GET_REG_BITS(
pkt_info->attributes,
RX_PACKET_ATTRIBUTES_CONTEXT_POS,
RX_PACKET_ATTRIBUTES_CONTEXT_LEN);
/* Earlier error, just drain the remaining data */
if ((incomplete || context_next) && error)
goto read_again;
if (error || pkt_info->errors) {
if (pkt_info->errors)
netif_err(pdata, rx_err, netdev,
"error in received packet\n");
dev_kfree_skb(skb);
goto next_packet;
}
if (!context) {
/* Length is cumulative, get this descriptor's length */
dma_desc_len = desc_data->rx.len - len;
len += dma_desc_len;
if (dma_desc_len && !skb) {
skb = xlgmac_create_skb(pdata, napi, desc_data,
dma_desc_len);
if (!skb)
error = 1;
} else if (dma_desc_len) {
dma_sync_single_range_for_cpu(
pdata->dev,
desc_data->rx.buf.dma_base,
desc_data->rx.buf.dma_off,
desc_data->rx.buf.dma_len,
DMA_FROM_DEVICE);
skb_add_rx_frag(
skb, skb_shinfo(skb)->nr_frags,
desc_data->rx.buf.pa.pages,
desc_data->rx.buf.pa.pages_offset,
dma_desc_len,
desc_data->rx.buf.dma_len);
desc_data->rx.buf.pa.pages = NULL;
}
}
if (incomplete || context_next)
goto read_again;
if (!skb)
goto next_packet;
/* Be sure we don't exceed the configured MTU */
max_len = netdev->mtu + ETH_HLEN;
if (!(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
(skb->protocol == htons(ETH_P_8021Q)))
max_len += VLAN_HLEN;
if (skb->len > max_len) {
netif_err(pdata, rx_err, netdev,
"packet length exceeds configured MTU\n");
dev_kfree_skb(skb);
goto next_packet;
}
if (netif_msg_pktdata(pdata))
xlgmac_print_pkt(netdev, skb, false);
skb_checksum_none_assert(skb);
if (XLGMAC_GET_REG_BITS(pkt_info->attributes,
RX_PACKET_ATTRIBUTES_CSUM_DONE_POS,
RX_PACKET_ATTRIBUTES_CSUM_DONE_LEN))
skb->ip_summed = CHECKSUM_UNNECESSARY;
if (XLGMAC_GET_REG_BITS(pkt_info->attributes,
RX_PACKET_ATTRIBUTES_VLAN_CTAG_POS,
RX_PACKET_ATTRIBUTES_VLAN_CTAG_LEN)) {
__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
pkt_info->vlan_ctag);
pdata->stats.rx_vlan_packets++;
}
if (XLGMAC_GET_REG_BITS(pkt_info->attributes,
RX_PACKET_ATTRIBUTES_RSS_HASH_POS,
RX_PACKET_ATTRIBUTES_RSS_HASH_LEN))
skb_set_hash(skb, pkt_info->rss_hash,
pkt_info->rss_hash_type);
skb->dev = netdev;
skb->protocol = eth_type_trans(skb, netdev);
skb_record_rx_queue(skb, channel->queue_index);
napi_gro_receive(napi, skb);
next_packet:
packet_count++;
}
/* Check if we need to save state before leaving */
if (received && (incomplete || context_next)) {
desc_data = XLGMAC_GET_DESC_DATA(ring, ring->cur);
desc_data->state_saved = 1;
desc_data->state.skb = skb;
desc_data->state.len = len;
desc_data->state.error = error;
}
XLGMAC_PR("packet_count = %d\n", packet_count);
return packet_count;
}
/*
* netvsc_recv_callback - Callback when we receive a packet from the
* "wire" on the specified device.
*/
int netvsc_recv_callback(struct net_device *net,
struct vmbus_channel *channel,
void *data, u32 len,
const struct ndis_tcp_ip_checksum_info *csum_info,
const struct ndis_pkt_8021q_info *vlan)
{
struct net_device_context *net_device_ctx = netdev_priv(net);
struct netvsc_device *net_device = net_device_ctx->nvdev;
u16 q_idx = channel->offermsg.offer.sub_channel_index;
struct netvsc_channel *nvchan = &net_device->chan_table[q_idx];
struct sk_buff *skb;
struct sk_buff *vf_skb;
struct netvsc_stats *rx_stats;
int ret = 0;
if (!net || net->reg_state != NETREG_REGISTERED)
return NVSP_STAT_FAIL;
if (READ_ONCE(net_device_ctx->vf_inject)) {
atomic_inc(&net_device_ctx->vf_use_cnt);
if (!READ_ONCE(net_device_ctx->vf_inject)) {
/*
* We raced; just move on.
*/
atomic_dec(&net_device_ctx->vf_use_cnt);
goto vf_injection_done;
}
/*
* Inject this packet into the VF inerface.
* On Hyper-V, multicast and brodcast packets
* are only delivered on the synthetic interface
* (after subjecting these to policy filters on
* the host). Deliver these via the VF interface
* in the guest.
*/
vf_skb = netvsc_alloc_recv_skb(net_device_ctx->vf_netdev,
csum_info, vlan, data, len);
if (vf_skb != NULL) {
++net_device_ctx->vf_netdev->stats.rx_packets;
net_device_ctx->vf_netdev->stats.rx_bytes +=
len;
netif_receive_skb(vf_skb);
} else {
++net->stats.rx_dropped;
ret = NVSP_STAT_FAIL;
}
atomic_dec(&net_device_ctx->vf_use_cnt);
return ret;
}
vf_injection_done:
rx_stats = &nvchan->rx_stats;
/* Allocate a skb - TODO direct I/O to pages? */
skb = netvsc_alloc_recv_skb(net, csum_info, vlan, data, len);
if (unlikely(!skb)) {
++net->stats.rx_dropped;
return NVSP_STAT_FAIL;
}
skb_record_rx_queue(skb, q_idx);
u64_stats_update_begin(&rx_stats->syncp);
rx_stats->packets++;
rx_stats->bytes += len;
if (skb->pkt_type == PACKET_BROADCAST)
++rx_stats->broadcast;
else if (skb->pkt_type == PACKET_MULTICAST)
++rx_stats->multicast;
u64_stats_update_end(&rx_stats->syncp);
net->stats.rx_packets++;
net->stats.rx_bytes += len;
napi_gro_receive(&nvchan->napi, skb);
return 0;
}
static inline int
pdma_recv(struct net_device* dev, END_DEVICE* ei_local, int work_todo)
{
struct PDMA_rxdesc *rxd_ring;
struct sk_buff *new_skb, *rx_skb;
int gmac_no = PSE_PORT_GMAC1;
int work_done = 0;
u32 rxd_dma_owner_idx;
u32 rxd_info2, rxd_info4;
#if defined (CONFIG_RAETH_HW_VLAN_RX)
u32 rxd_info3;
#endif
#if defined (CONFIG_RAETH_SPECIAL_TAG)
struct vlan_ethhdr *veth;
#endif
rxd_dma_owner_idx = le32_to_cpu(sysRegRead(RX_CALC_IDX0));
while (work_done < work_todo) {
rxd_dma_owner_idx = (rxd_dma_owner_idx + 1) % NUM_RX_DESC;
rxd_ring = &ei_local->rxd_ring[rxd_dma_owner_idx];
if (!(rxd_ring->rxd_info2 & RX2_DMA_DONE))
break;
/* load completed skb pointer */
rx_skb = ei_local->rxd_buff[rxd_dma_owner_idx];
/* copy RX desc to CPU */
rxd_info2 = rxd_ring->rxd_info2;
#if defined (CONFIG_RAETH_HW_VLAN_RX)
rxd_info3 = rxd_ring->rxd_info3;
#endif
rxd_info4 = rxd_ring->rxd_info4;
#if defined (CONFIG_PSEUDO_SUPPORT)
gmac_no = RX4_DMA_SP(rxd_info4);
#endif
/* We have to check the free memory size is big enough
* before pass the packet to cpu */
new_skb = __dev_alloc_skb(MAX_RX_LENGTH + NET_IP_ALIGN, GFP_ATOMIC);
if (unlikely(new_skb == NULL)) {
#if defined (RAETH_PDMA_V2)
rxd_ring->rxd_info2 = RX2_DMA_SDL0_SET(MAX_RX_LENGTH);
#else
rxd_ring->rxd_info2 = RX2_DMA_LS0;
#endif
/* move CPU pointer to next RXD */
sysRegWrite(RX_CALC_IDX0, cpu_to_le32(rxd_dma_owner_idx));
inc_rx_drop(ei_local, gmac_no);
#if !defined (CONFIG_RAETH_NAPI)
/* mean need reschedule */
work_done = work_todo;
#endif
#if defined (CONFIG_RAETH_DEBUG)
if (net_ratelimit())
printk(KERN_ERR "%s: Failed to alloc new RX skb! (GMAC: %d)\n", RAETH_DEV_NAME, gmac_no);
#endif
break;
}
#if !defined (RAETH_PDMA_V2)
skb_reserve(new_skb, NET_IP_ALIGN);
#endif
/* store new empty skb pointer */
ei_local->rxd_buff[rxd_dma_owner_idx] = new_skb;
/* map new skb to ring (unmap is not required on generic mips mm) */
rxd_ring->rxd_info1 = (u32)dma_map_single(NULL, new_skb->data, MAX_RX_LENGTH, DMA_FROM_DEVICE);
#if defined (RAETH_PDMA_V2)
rxd_ring->rxd_info2 = RX2_DMA_SDL0_SET(MAX_RX_LENGTH);
#else
rxd_ring->rxd_info2 = RX2_DMA_LS0;
#endif
wmb();
/* move CPU pointer to next RXD */
sysRegWrite(RX_CALC_IDX0, cpu_to_le32(rxd_dma_owner_idx));
/* skb processing */
rx_skb->len = RX2_DMA_SDL0_GET(rxd_info2);
#if defined (RAETH_PDMA_V2)
rx_skb->data += NET_IP_ALIGN;
#endif
rx_skb->tail = rx_skb->data + rx_skb->len;
#if defined (CONFIG_RA_HW_NAT) || defined (CONFIG_RA_HW_NAT_MODULE)
FOE_MAGIC_TAG(rx_skb) = FOE_MAGIC_GE;
DO_FILL_FOE_DESC(rx_skb, (rxd_info4 & ~(RX4_DMA_ALG_SET)));
#endif
#if defined (CONFIG_RAETH_CHECKSUM_OFFLOAD)
if (rxd_info4 & RX4_DMA_L4FVLD)
rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
#endif
#if defined (CONFIG_RAETH_HW_VLAN_RX)
if ((rxd_info2 & RX2_DMA_TAG) && rxd_info3) {
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)
__vlan_hwaccel_put_tag(rx_skb, __constant_htons(ETH_P_8021Q), RX3_DMA_VID(rxd_info3));
#else
__vlan_hwaccel_put_tag(rx_skb, RX3_DMA_VID(rxd_info3));
#endif
}
#endif
#if defined (CONFIG_PSEUDO_SUPPORT)
if (gmac_no == PSE_PORT_GMAC2)
rx_skb->protocol = eth_type_trans(rx_skb, ei_local->PseudoDev);
else
#endif
rx_skb->protocol = eth_type_trans(rx_skb, dev);
#if defined (CONFIG_RAETH_SPECIAL_TAG)
#if defined (CONFIG_MT7530_GSW)
#define ESW_TAG_ID 0x00
#else
#define ESW_TAG_ID 0x81
#endif
// port0: 0x8100 => 0x8100 0001
// port1: 0x8101 => 0x8100 0002
// port2: 0x8102 => 0x8100 0003
// port3: 0x8103 => 0x8100 0004
// port4: 0x8104 => 0x8100 0005
// port5: 0x8105 => 0x8100 0006
veth = vlan_eth_hdr(rx_skb);
if ((veth->h_vlan_proto & 0xFF) == ESW_TAG_ID) {
veth->h_vlan_TCI = htons((((veth->h_vlan_proto >> 8) & 0xF) + 1));
veth->h_vlan_proto = __constant_htons(ETH_P_8021Q);
rx_skb->protocol = veth->h_vlan_proto;
}
#endif
/* ra_sw_nat_hook_rx return 1 --> continue
* ra_sw_nat_hook_rx return 0 --> FWD & without netif_rx
*/
#if defined (CONFIG_RA_HW_NAT) || defined (CONFIG_RA_HW_NAT_MODULE)
if((ra_sw_nat_hook_rx == NULL) ||
(ra_sw_nat_hook_rx != NULL && ra_sw_nat_hook_rx(rx_skb)))
#endif
{
#if defined (CONFIG_RAETH_NAPI)
#if defined (CONFIG_RAETH_NAPI_GRO)
if (rx_skb->ip_summed == CHECKSUM_UNNECESSARY)
napi_gro_receive(&ei_local->napi, rx_skb);
else
#endif
netif_receive_skb(rx_skb);
#else
netif_rx(rx_skb);
#endif
}
work_done++;
}
/*
* netvsc_recv_callback - Callback when we receive a packet from the
* "wire" on the specified device.
*/
int netvsc_recv_callback(struct net_device *net,
struct vmbus_channel *channel,
void *data, u32 len,
const struct ndis_tcp_ip_checksum_info *csum_info,
const struct ndis_pkt_8021q_info *vlan)
{
struct net_device_context *net_device_ctx = netdev_priv(net);
struct netvsc_device *net_device;
u16 q_idx = channel->offermsg.offer.sub_channel_index;
struct netvsc_channel *nvchan;
struct net_device *vf_netdev;
struct sk_buff *skb;
struct netvsc_stats *rx_stats;
if (net->reg_state != NETREG_REGISTERED)
return NVSP_STAT_FAIL;
/*
* If necessary, inject this packet into the VF interface.
* On Hyper-V, multicast and brodcast packets are only delivered
* to the synthetic interface (after subjecting these to
* policy filters on the host). Deliver these via the VF
* interface in the guest.
*/
rcu_read_lock();
net_device = rcu_dereference(net_device_ctx->nvdev);
if (unlikely(!net_device))
goto drop;
nvchan = &net_device->chan_table[q_idx];
vf_netdev = rcu_dereference(net_device_ctx->vf_netdev);
if (vf_netdev && (vf_netdev->flags & IFF_UP))
net = vf_netdev;
/* Allocate a skb - TODO direct I/O to pages? */
skb = netvsc_alloc_recv_skb(net, &nvchan->napi,
csum_info, vlan, data, len);
if (unlikely(!skb)) {
drop:
++net->stats.rx_dropped;
rcu_read_unlock();
return NVSP_STAT_FAIL;
}
if (net != vf_netdev)
skb_record_rx_queue(skb, q_idx);
/*
* Even if injecting the packet, record the statistics
* on the synthetic device because modifying the VF device
* statistics will not work correctly.
*/
rx_stats = &nvchan->rx_stats;
u64_stats_update_begin(&rx_stats->syncp);
rx_stats->packets++;
rx_stats->bytes += len;
if (skb->pkt_type == PACKET_BROADCAST)
++rx_stats->broadcast;
else if (skb->pkt_type == PACKET_MULTICAST)
++rx_stats->multicast;
u64_stats_update_end(&rx_stats->syncp);
napi_gro_receive(&nvchan->napi, skb);
rcu_read_unlock();
return 0;
}
static int sprdwl_rx_handler(struct napi_struct *napi, int budget)
{
struct sprdwl_priv *priv = container_of(napi, struct sprdwl_priv, napi);
struct sblock blk;
struct sk_buff *skb;
int ret, work_done;
u16 decryp_data_len = 0;
struct wlan_sblock_recv_data *data;
uint32_t length = 0;
#ifdef CONFIG_SPRDWL_FW_ZEROCOPY
u8 offset = 0;
#endif
for (work_done = 0; work_done < budget; work_done++) {
ret = sblock_receive(WLAN_CP_ID, WLAN_SBLOCK_CH, &blk, 0);
if (ret) {
dev_dbg(&priv->ndev->dev, "no more sblock to read\n");
break;
}
#ifdef CONFIG_SPRDWL_FW_ZEROCOPY
offset = *(u8 *)blk.addr;
length = blk.length - 2 - offset;
#else
length = blk.length;
#endif
/*16 bytes align */
skb = dev_alloc_skb(length + NET_IP_ALIGN);
if (!skb) {
dev_err(&priv->ndev->dev,
"Failed to allocate skbuff!\n");
priv->ndev->stats.rx_dropped++;
goto rx_failed;
}
#ifdef CONFIG_SPRDWL_FW_ZEROCOPY
data = (struct wlan_sblock_recv_data *)(blk.addr + 2 + offset);
#else
data = (struct wlan_sblock_recv_data *)blk.addr;
#endif
if (data->is_encrypted == 1) {
if (priv->connect_status == SPRDWL_CONNECTED &&
priv->cipher_type == SPRDWL_CIPHER_WAPI &&
priv->key_len[GROUP][priv->key_index[GROUP]] != 0 &&
priv->
key_len[PAIRWISE][priv->key_index[PAIRWISE]] != 0) {
u8 snap_header[6] = { 0xaa, 0xaa, 0x03,
0x00, 0x00, 0x00
};
skb_reserve(skb, NET_IP_ALIGN);
decryp_data_len = wlan_rx_wapi_decryption(priv,
(u8 *)&data->u2.encrypt,
data->u1.encrypt.header_len,
(length -
sizeof(data->is_encrypted) -
sizeof(data->u1) -
data->u1.encrypt.header_len),
(skb->data + 12));
if (decryp_data_len == 0) {
dev_err(&priv->ndev->dev,
"Failed to decrypt WAPI data!\n");
priv->ndev->stats.rx_dropped++;
dev_kfree_skb(skb);
goto rx_failed;
}
if (memcmp((skb->data + 12), snap_header,
sizeof(snap_header)) == 0) {
skb_reserve(skb, 6);
/* copy the eth address from eth header,
* but not copy eth type
*/
memcpy(skb->data,
data->u2.encrypt.
mac_header.addr1, 6);
memcpy(skb->data + 6,
data->u2.encrypt.
mac_header.addr2, 6);
skb_put(skb, (decryp_data_len + 6));
} else {
/* copy eth header */
memcpy(skb->data,
data->u2.encrypt.
mac_header.addr3, 6);
memcpy(skb->data + 6,
data->u2.encrypt.
mac_header.addr2, 6);
skb_put(skb, (decryp_data_len + 12));
}
} else {
dev_err(&priv->ndev->dev,
"wrong encryption data!\n");
priv->ndev->stats.rx_dropped++;
dev_kfree_skb(skb);
goto rx_failed;
}
} else if (data->is_encrypted == 0) {
skb_reserve(skb, NET_IP_ALIGN);
/* dec the first encrypt byte */
memcpy(skb->data, (u8 *)&data->u2,
(length - sizeof(data->is_encrypted) -
sizeof(data->u1)));
skb_put(skb,
(length - sizeof(data->is_encrypted) -
sizeof(data->u1)));
} else {
dev_err(&priv->ndev->dev,
"wrong data fromat recieved!\n");
priv->ndev->stats.rx_dropped++;
dev_kfree_skb(skb);
goto rx_failed;
}
#ifdef DUMP_RECEIVE_PACKET
print_hex_dump(KERN_DEBUG, "receive packet: ",
DUMP_PREFIX_OFFSET, 16, 1, skb->data, skb->len,
0);
#endif
skb->dev = priv->ndev;
skb->protocol = eth_type_trans(skb, priv->ndev);
/* CHECKSUM_UNNECESSARY not supported by our hardware */
/* skb->ip_summed = CHECKSUM_UNNECESSARY; */
priv->ndev->stats.rx_packets++;
priv->ndev->stats.rx_bytes += skb->len;
napi_gro_receive(napi, skb);
rx_failed:
ret = sblock_release(WLAN_CP_ID, WLAN_SBLOCK_CH, &blk);
if (ret)
dev_err(&priv->ndev->dev,
"Failed to release sblock (%d)!\n", ret);
}
if (work_done < budget)
napi_complete(napi);
return work_done;
}
static int hip04_rx_poll(struct napi_struct *napi, int budget)
{
struct hip04_priv *priv = container_of(napi, struct hip04_priv, napi);
struct net_device *ndev = priv->ndev;
struct net_device_stats *stats = &ndev->stats;
unsigned int cnt = hip04_recv_cnt(priv);
struct rx_desc *desc;
struct sk_buff *skb;
unsigned char *buf;
bool last = false;
dma_addr_t phys;
int rx = 0;
int tx_remaining;
u16 len;
u32 err;
while (cnt && !last) {
buf = priv->rx_buf[priv->rx_head];
skb = build_skb(buf, priv->rx_buf_size);
if (unlikely(!skb)) {
net_dbg_ratelimited("build_skb failed\n");
goto refill;
}
dma_unmap_single(&ndev->dev, priv->rx_phys[priv->rx_head],
RX_BUF_SIZE, DMA_FROM_DEVICE);
priv->rx_phys[priv->rx_head] = 0;
desc = (struct rx_desc *)skb->data;
len = be16_to_cpu(desc->pkt_len);
err = be32_to_cpu(desc->pkt_err);
if (0 == len) {
dev_kfree_skb_any(skb);
last = true;
} else if ((err & RX_PKT_ERR) || (len >= GMAC_MAX_PKT_LEN)) {
dev_kfree_skb_any(skb);
stats->rx_dropped++;
stats->rx_errors++;
} else {
skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
skb_put(skb, len);
skb->protocol = eth_type_trans(skb, ndev);
napi_gro_receive(&priv->napi, skb);
stats->rx_packets++;
stats->rx_bytes += len;
rx++;
}
refill:
buf = netdev_alloc_frag(priv->rx_buf_size);
if (!buf)
goto done;
phys = dma_map_single(&ndev->dev, buf,
RX_BUF_SIZE, DMA_FROM_DEVICE);
if (dma_mapping_error(&ndev->dev, phys))
goto done;
priv->rx_buf[priv->rx_head] = buf;
priv->rx_phys[priv->rx_head] = phys;
hip04_set_recv_desc(priv, phys);
priv->rx_head = RX_NEXT(priv->rx_head);
if (rx >= budget)
goto done;
if (--cnt == 0)
cnt = hip04_recv_cnt(priv);
}
if (!(priv->reg_inten & RCV_INT)) {
/* enable rx interrupt */
priv->reg_inten |= RCV_INT;
writel_relaxed(priv->reg_inten, priv->base + PPE_INTEN);
}
napi_complete(napi);
done:
/* clean up tx descriptors and start a new timer if necessary */
tx_remaining = hip04_tx_reclaim(ndev, false);
if (rx < budget && tx_remaining)
hip04_start_tx_timer(priv);
return rx;
}
static int stmmac_rx(struct stmmac_priv *priv, int limit)
{
unsigned int rxsize = priv->dma_rx_size;
unsigned int entry = priv->cur_rx % rxsize;
unsigned int next_entry;
unsigned int count = 0;
struct dma_desc *p = priv->dma_rx + entry;
struct dma_desc *p_next;
#ifdef STMMAC_RX_DEBUG
if (netif_msg_hw(priv)) {
pr_debug(">>> stmmac_rx: descriptor ring:\n");
display_ring(priv->dma_rx, rxsize);
}
#endif
count = 0;
while (!priv->hw->desc->get_rx_owner(p)) {
int status;
if (count >= limit)
break;
count++;
next_entry = (++priv->cur_rx) % rxsize;
p_next = priv->dma_rx + next_entry;
prefetch(p_next);
/* read the status of the incoming frame */
status = (priv->hw->desc->rx_status(&priv->dev->stats,
&priv->xstats, p));
if (unlikely(status == discard_frame))
priv->dev->stats.rx_errors++;
else {
struct sk_buff *skb;
int frame_len;
frame_len = priv->hw->desc->get_rx_frame_len(p);
/* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
* Type frames (LLC/LLC-SNAP) */
if (unlikely(status != llc_snap))
frame_len -= ETH_FCS_LEN;
#ifdef STMMAC_RX_DEBUG
if (frame_len > ETH_FRAME_LEN)
pr_debug("\tRX frame size %d, COE status: %d\n",
frame_len, status);
if (netif_msg_hw(priv))
pr_debug("\tdesc: %p [entry %d] buff=0x%x\n",
p, entry, p->des2);
#endif
skb = priv->rx_skbuff[entry];
if (unlikely(!skb)) {
pr_err("%s: Inconsistent Rx descriptor chain\n",
priv->dev->name);
priv->dev->stats.rx_dropped++;
break;
}
prefetch(skb->data - NET_IP_ALIGN);
priv->rx_skbuff[entry] = NULL;
skb_put(skb, frame_len);
dma_unmap_single(priv->device,
priv->rx_skbuff_dma[entry],
priv->dma_buf_sz, DMA_FROM_DEVICE);
#ifdef STMMAC_RX_DEBUG
if (netif_msg_pktdata(priv)) {
pr_info(" frame received (%dbytes)", frame_len);
print_pkt(skb->data, frame_len);
}
#endif
skb->protocol = eth_type_trans(skb, priv->dev);
if (unlikely(status == csum_none)) {
/* always for the old mac 10/100 */
skb_checksum_none_assert(skb);
netif_receive_skb(skb);
} else {
skb->ip_summed = CHECKSUM_UNNECESSARY;
napi_gro_receive(&priv->napi, skb);
}
priv->dev->stats.rx_packets++;
priv->dev->stats.rx_bytes += frame_len;
}
entry = next_entry;
p = p_next; /* use prefetched values */
}
stmmac_rx_refill(priv);
priv->xstats.rx_pkt_n += count;
return count;
}
/* During a receive, the cur_rx points to the current incoming buffer.
* When we update through the ring, if the next incoming buffer has
* not been given to the system, we just set the empty indicator,
* effectively tossing the packet.
*/
static int
fec_enet_rx(struct net_device *ndev, int budget)
{
struct fec_enet_private *fep = netdev_priv(ndev);
const struct platform_device_id *id_entry =
platform_get_device_id(fep->pdev);
struct bufdesc *bdp;
unsigned short status;
struct sk_buff *skb;
ushort pkt_len;
__u8 *data;
int pkt_received = 0;
#ifdef CONFIG_M532x
flush_cache_all();
#endif
/* First, grab all of the stats for the incoming packet.
* These get messed up if we get called due to a busy condition.
*/
bdp = fep->cur_rx;
while (!((status = bdp->cbd_sc) & BD_ENET_RX_EMPTY)) {
if (pkt_received >= budget)
break;
pkt_received++;
/* Since we have allocated space to hold a complete frame,
* the last indicator should be set.
*/
if ((status & BD_ENET_RX_LAST) == 0)
printk("FEC ENET: rcv is not +last\n");
if (!fep->opened)
goto rx_processing_done;
/* Check for errors. */
if (status & (BD_ENET_RX_LG | BD_ENET_RX_SH | BD_ENET_RX_NO |
BD_ENET_RX_CR | BD_ENET_RX_OV)) {
ndev->stats.rx_errors++;
if (status & (BD_ENET_RX_LG | BD_ENET_RX_SH)) {
/* Frame too long or too short. */
ndev->stats.rx_length_errors++;
}
if (status & BD_ENET_RX_NO) /* Frame alignment */
ndev->stats.rx_frame_errors++;
if (status & BD_ENET_RX_CR) /* CRC Error */
ndev->stats.rx_crc_errors++;
if (status & BD_ENET_RX_OV) /* FIFO overrun */
ndev->stats.rx_fifo_errors++;
}
/* Report late collisions as a frame error.
* On this error, the BD is closed, but we don't know what we
* have in the buffer. So, just drop this frame on the floor.
*/
if (status & BD_ENET_RX_CL) {
ndev->stats.rx_errors++;
ndev->stats.rx_frame_errors++;
goto rx_processing_done;
}
/* Process the incoming frame. */
ndev->stats.rx_packets++;
pkt_len = bdp->cbd_datlen;
ndev->stats.rx_bytes += pkt_len;
data = (__u8*)__va(bdp->cbd_bufaddr);
dma_unmap_single(&fep->pdev->dev, bdp->cbd_bufaddr,
FEC_ENET_TX_FRSIZE, DMA_FROM_DEVICE);
if (id_entry->driver_data & FEC_QUIRK_SWAP_FRAME)
swap_buffer(data, pkt_len);
/* This does 16 byte alignment, exactly what we need.
* The packet length includes FCS, but we don't want to
* include that when passing upstream as it messes up
* bridging applications.
*/
skb = netdev_alloc_skb(ndev, pkt_len - 4 + NET_IP_ALIGN);
if (unlikely(!skb)) {
printk("%s: Memory squeeze, dropping packet.\n",
ndev->name);
ndev->stats.rx_dropped++;
} else {
skb_reserve(skb, NET_IP_ALIGN);
skb_put(skb, pkt_len - 4); /* Make room */
skb_copy_to_linear_data(skb, data, pkt_len - 4);
skb->protocol = eth_type_trans(skb, ndev);
/* Get receive timestamp from the skb */
if (fep->hwts_rx_en && fep->bufdesc_ex) {
struct skb_shared_hwtstamps *shhwtstamps =
skb_hwtstamps(skb);
unsigned long flags;
struct bufdesc_ex *ebdp =
(struct bufdesc_ex *)bdp;
memset(shhwtstamps, 0, sizeof(*shhwtstamps));
spin_lock_irqsave(&fep->tmreg_lock, flags);
shhwtstamps->hwtstamp = ns_to_ktime(
timecounter_cyc2time(&fep->tc, ebdp->ts));
spin_unlock_irqrestore(&fep->tmreg_lock, flags);
}
if (!skb_defer_rx_timestamp(skb))
napi_gro_receive(&fep->napi, skb);
}
bdp->cbd_bufaddr = dma_map_single(&fep->pdev->dev, data,
FEC_ENET_TX_FRSIZE, DMA_FROM_DEVICE);
rx_processing_done:
/* Clear the status flags for this buffer */
status &= ~BD_ENET_RX_STATS;
/* Mark the buffer empty */
status |= BD_ENET_RX_EMPTY;
bdp->cbd_sc = status;
if (fep->bufdesc_ex) {
struct bufdesc_ex *ebdp = (struct bufdesc_ex *)bdp;
ebdp->cbd_esc = BD_ENET_RX_INT;
ebdp->cbd_prot = 0;
ebdp->cbd_bdu = 0;
}
/* Update BD pointer to next entry */
if (status & BD_ENET_RX_WRAP)
bdp = fep->rx_bd_base;
else
bdp = fec_enet_get_nextdesc(bdp, fep->bufdesc_ex);
/* Doing this here will keep the FEC running while we process
* incoming frames. On a heavily loaded network, we should be
* able to keep up at the expense of system resources.
*/
writel(0, fep->hwp + FEC_R_DES_ACTIVE);
}
fep->cur_rx = bdp;
return pkt_received;
}
/**
* nfp_net_rx() - receive up to @budget packets on @rx_ring
* @rx_ring: RX ring to receive from
* @budget: NAPI budget
*
* Note, this function is separated out from the napi poll function to
* more cleanly separate packet receive code from other bookkeeping
* functions performed in the napi poll function.
*
* There are differences between the NFP-3200 firmware and the
* NFP-6000 firmware. The NFP-3200 firmware uses a dedicated RX queue
* to indicate that new packets have arrived. The NFP-6000 does not
* have this queue and uses the DD bit in the RX descriptor. This
* method cannot be used on the NFP-3200 as it causes a race
* condition: The RX ring write pointer on the NFP-3200 is updated
* after packets (and descriptors) have been DMAed. If the DD bit is
* used and subsequently the read pointer is updated this may lead to
* the RX queue to underflow (if the firmware has not yet update the
* write pointer). Therefore we use slightly ugly conditional code
* below to handle the differences. We may, in the future update the
* NFP-3200 firmware to behave the same as the firmware on the
* NFP-6000.
*
* Return: Number of packets received.
*/
static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget)
{
struct nfp_net_r_vector *r_vec = rx_ring->r_vec;
struct nfp_net *nn = r_vec->nfp_net;
unsigned int data_len, meta_len;
int avail = 0, pkts_polled = 0;
struct sk_buff *skb, *new_skb;
struct nfp_net_rx_desc *rxd;
dma_addr_t new_dma_addr;
u32 qcp_wr_p;
int idx;
if (nn->is_nfp3200) {
/* Work out how many packets arrived */
qcp_wr_p = nfp_qcp_wr_ptr_read(rx_ring->qcp_rx);
idx = rx_ring->rd_p % rx_ring->cnt;
if (qcp_wr_p == idx)
/* No new packets */
return 0;
if (qcp_wr_p > idx)
avail = qcp_wr_p - idx;
else
avail = qcp_wr_p + rx_ring->cnt - idx;
} else {
avail = budget + 1;
}
while (avail > 0 && pkts_polled < budget) {
idx = rx_ring->rd_p % rx_ring->cnt;
rxd = &rx_ring->rxds[idx];
if (!(rxd->rxd.meta_len_dd & PCIE_DESC_RX_DD)) {
if (nn->is_nfp3200)
nn_dbg(nn, "RX descriptor not valid (DD)%d:%u rxd[0]=%#x rxd[1]=%#x\n",
rx_ring->idx, idx,
rxd->vals[0], rxd->vals[1]);
break;
}
/* Memory barrier to ensure that we won't do other reads
* before the DD bit.
*/
dma_rmb();
rx_ring->rd_p++;
pkts_polled++;
avail--;
skb = rx_ring->rxbufs[idx].skb;
new_skb = nfp_net_rx_alloc_one(rx_ring, &new_dma_addr,
nn->fl_bufsz);
if (!new_skb) {
nfp_net_rx_give_one(rx_ring, rx_ring->rxbufs[idx].skb,
rx_ring->rxbufs[idx].dma_addr);
u64_stats_update_begin(&r_vec->rx_sync);
r_vec->rx_drops++;
u64_stats_update_end(&r_vec->rx_sync);
continue;
}
dma_unmap_single(&nn->pdev->dev,
rx_ring->rxbufs[idx].dma_addr,
nn->fl_bufsz, DMA_FROM_DEVICE);
nfp_net_rx_give_one(rx_ring, new_skb, new_dma_addr);
/* < meta_len >
* <-- [rx_offset] -->
* ---------------------------------------------------------
* | [XX] | metadata | packet | XXXX |
* ---------------------------------------------------------
* <---------------- data_len --------------->
*
* The rx_offset is fixed for all packets, the meta_len can vary
* on a packet by packet basis. If rx_offset is set to zero
* (_RX_OFFSET_DYNAMIC) metadata starts at the beginning of the
* buffer and is immediately followed by the packet (no [XX]).
*/
meta_len = rxd->rxd.meta_len_dd & PCIE_DESC_RX_META_LEN_MASK;
data_len = le16_to_cpu(rxd->rxd.data_len);
if (nn->rx_offset == NFP_NET_CFG_RX_OFFSET_DYNAMIC)
skb_reserve(skb, meta_len);
else
skb_reserve(skb, nn->rx_offset);
skb_put(skb, data_len - meta_len);
nfp_net_set_hash(nn->netdev, skb, rxd);
/* Pad small frames to minimum */
if (skb_put_padto(skb, 60))
break;
/* Stats update */
u64_stats_update_begin(&r_vec->rx_sync);
r_vec->rx_pkts++;
r_vec->rx_bytes += skb->len;
u64_stats_update_end(&r_vec->rx_sync);
skb_record_rx_queue(skb, rx_ring->idx);
skb->protocol = eth_type_trans(skb, nn->netdev);
nfp_net_rx_csum(nn, r_vec, rxd, skb);
if (rxd->rxd.flags & PCIE_DESC_RX_VLAN)
__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
le16_to_cpu(rxd->rxd.vlan));
napi_gro_receive(&rx_ring->r_vec->napi, skb);
}
if (nn->is_nfp3200)
nfp_qcp_rd_ptr_add(rx_ring->qcp_rx, pkts_polled);
return pkts_polled;
}
static int ibmveth_poll(struct napi_struct *napi, int budget)
{
struct ibmveth_adapter *adapter =
container_of(napi, struct ibmveth_adapter, napi);
struct net_device *netdev = adapter->netdev;
int frames_processed = 0;
unsigned long lpar_rc;
struct iphdr *iph;
restart_poll:
while (frames_processed < budget) {
if (!ibmveth_rxq_pending_buffer(adapter))
break;
smp_rmb();
if (!ibmveth_rxq_buffer_valid(adapter)) {
wmb(); /* suggested by larson1 */
adapter->rx_invalid_buffer++;
netdev_dbg(netdev, "recycling invalid buffer\n");
ibmveth_rxq_recycle_buffer(adapter);
} else {
struct sk_buff *skb, *new_skb;
int length = ibmveth_rxq_frame_length(adapter);
int offset = ibmveth_rxq_frame_offset(adapter);
int csum_good = ibmveth_rxq_csum_good(adapter);
skb = ibmveth_rxq_get_buffer(adapter);
new_skb = NULL;
if (length < rx_copybreak)
new_skb = netdev_alloc_skb(netdev, length);
if (new_skb) {
skb_copy_to_linear_data(new_skb,
skb->data + offset,
length);
if (rx_flush)
ibmveth_flush_buffer(skb->data,
length + offset);
if (!ibmveth_rxq_recycle_buffer(adapter))
kfree_skb(skb);
skb = new_skb;
} else {
ibmveth_rxq_harvest_buffer(adapter);
skb_reserve(skb, offset);
}
skb_put(skb, length);
skb->protocol = eth_type_trans(skb, netdev);
if (csum_good) {
skb->ip_summed = CHECKSUM_UNNECESSARY;
if (be16_to_cpu(skb->protocol) == ETH_P_IP) {
iph = (struct iphdr *)skb->data;
/* If the IP checksum is not offloaded and if the packet
* is large send, the checksum must be rebuilt.
*/
if (iph->check == 0xffff) {
iph->check = 0;
iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
adapter->rx_large_packets++;
}
}
}
napi_gro_receive(napi, skb); /* send it up */
netdev->stats.rx_packets++;
netdev->stats.rx_bytes += length;
frames_processed++;
}
}
ibmveth_replenish_task(adapter);
if (frames_processed < budget) {
napi_complete(napi);
/* We think we are done - reenable interrupts,
* then check once more to make sure we are done.
*/
lpar_rc = h_vio_signal(adapter->vdev->unit_address,
VIO_IRQ_ENABLE);
BUG_ON(lpar_rc != H_SUCCESS);
if (ibmveth_rxq_pending_buffer(adapter) &&
napi_reschedule(napi)) {
lpar_rc = h_vio_signal(adapter->vdev->unit_address,
VIO_IRQ_DISABLE);
goto restart_poll;
}
}
return frames_processed;
}
static int qtnf_pcie_pearl_rx_poll(struct napi_struct *napi, int budget)
{
struct qtnf_bus *bus = container_of(napi, struct qtnf_bus, mux_napi);
struct qtnf_pcie_pearl_state *ps = get_bus_priv(bus);
struct qtnf_pcie_bus_priv *priv = &ps->base;
struct net_device *ndev = NULL;
struct sk_buff *skb = NULL;
int processed = 0;
struct qtnf_pearl_rx_bd *rxbd;
dma_addr_t skb_paddr;
int consume;
u32 descw;
u32 psize;
u16 r_idx;
u16 w_idx;
int ret;
while (processed < budget) {
if (!qtnf_rx_data_ready(ps))
goto rx_out;
r_idx = priv->rx_bd_r_index;
rxbd = &ps->rx_bd_vbase[r_idx];
descw = le32_to_cpu(rxbd->info);
skb = priv->rx_skb[r_idx];
psize = QTN_GET_LEN(descw);
consume = 1;
if (!(descw & QTN_TXDONE_MASK)) {
pr_warn("skip invalid rxbd[%d]\n", r_idx);
consume = 0;
}
if (!skb) {
pr_warn("skip missing rx_skb[%d]\n", r_idx);
consume = 0;
}
if (skb && (skb_tailroom(skb) < psize)) {
pr_err("skip packet with invalid length: %u > %u\n",
psize, skb_tailroom(skb));
consume = 0;
}
if (skb) {
skb_paddr = QTN_HOST_ADDR(le32_to_cpu(rxbd->addr_h),
le32_to_cpu(rxbd->addr));
pci_unmap_single(priv->pdev, skb_paddr, SKB_BUF_SIZE,
PCI_DMA_FROMDEVICE);
}
if (consume) {
skb_put(skb, psize);
ndev = qtnf_classify_skb(bus, skb);
if (likely(ndev)) {
qtnf_update_rx_stats(ndev, skb);
skb->protocol = eth_type_trans(skb, ndev);
napi_gro_receive(napi, skb);
} else {
pr_debug("drop untagged skb\n");
bus->mux_dev.stats.rx_dropped++;
dev_kfree_skb_any(skb);
}
} else {
if (skb) {
bus->mux_dev.stats.rx_dropped++;
dev_kfree_skb_any(skb);
}
}
priv->rx_skb[r_idx] = NULL;
if (++r_idx >= priv->rx_bd_num)
r_idx = 0;
priv->rx_bd_r_index = r_idx;
/* repalce processed buffer by a new one */
w_idx = priv->rx_bd_w_index;
while (CIRC_SPACE(priv->rx_bd_w_index, priv->rx_bd_r_index,
priv->rx_bd_num) > 0) {
if (++w_idx >= priv->rx_bd_num)
w_idx = 0;
ret = pearl_skb2rbd_attach(ps, w_idx);
if (ret) {
pr_err("failed to allocate new rx_skb[%d]\n",
w_idx);
break;
}
}
processed++;
}
rx_out:
if (processed < budget) {
napi_complete(napi);
qtnf_en_rxdone_irq(ps);
}
return processed;
}
gro_result_t vlan_gro_receive(struct napi_struct *napi, struct vlan_group *grp,
unsigned int vlan_tci, struct sk_buff *skb)
{
__vlan_hwaccel_put_tag(skb, vlan_tci);
return napi_gro_receive(napi, skb);
}
