/* Azure hosts don't support non-TCP port numbers in hashing yet. We compute
* hash for non-TCP traffic with only IP numbers.
*/
static inline u32 netvsc_get_hash(struct sk_buff *skb, struct sock *sk)
{
struct flow_keys flow;
u32 hash;
static u32 hashrnd __read_mostly;
net_get_random_once(&hashrnd, sizeof(hashrnd));
if (!skb_flow_dissect_flow_keys(skb, &flow, 0))
return 0;
if (flow.basic.ip_proto == IPPROTO_TCP) {
return skb_get_hash(skb);
} else {
if (flow.basic.n_proto == htons(ETH_P_IP))
hash = jhash2((u32 *)&flow.addrs.v4addrs, 2, hashrnd);
else if (flow.basic.n_proto == htons(ETH_P_IPV6))
hash = jhash2((u32 *)&flow.addrs.v6addrs, 8, hashrnd);
else
hash = 0;
skb_set_hash(skb, hash, PKT_HASH_TYPE_L3);
}
return hash;
}
/**
* nfp_net_set_hash() - Set SKB hash data
* @netdev: adapter's net_device structure
* @skb: SKB to set the hash data on
* @rxd: RX descriptor
*
* The RSS hash and hash-type are pre-pended to the packet data.
* Extract and decode it and set the skb fields.
*/
static void nfp_net_set_hash(struct net_device *netdev, struct sk_buff *skb,
struct nfp_net_rx_desc *rxd)
{
struct nfp_net_rx_hash *rx_hash;
if (!(rxd->rxd.flags & PCIE_DESC_RX_RSS) ||
!(netdev->features & NETIF_F_RXHASH))
return;
rx_hash = (struct nfp_net_rx_hash *)(skb->data - sizeof(*rx_hash));
switch (be32_to_cpu(rx_hash->hash_type)) {
case NFP_NET_RSS_IPV4:
case NFP_NET_RSS_IPV6:
case NFP_NET_RSS_IPV6_EX:
skb_set_hash(skb, be32_to_cpu(rx_hash->hash), PKT_HASH_TYPE_L3);
break;
default:
skb_set_hash(skb, be32_to_cpu(rx_hash->hash), PKT_HASH_TYPE_L4);
break;
}
}
/* Pass a received packet up through GRO. GRO can handle pages
* regardless of checksum state and skbs with a good checksum.
*/
static void
efx_rx_packet_gro(struct efx_channel *channel, struct efx_rx_buffer *rx_buf,
unsigned int n_frags, u8 *eh)
{
struct napi_struct *napi = &channel->napi_str;
gro_result_t gro_result;
struct efx_nic *efx = channel->efx;
struct sk_buff *skb;
skb = napi_get_frags(napi);
if (unlikely(!skb)) {
struct efx_rx_queue *rx_queue;
rx_queue = efx_channel_get_rx_queue(channel);
efx_free_rx_buffers(rx_queue, rx_buf, n_frags);
return;
}
if (efx->net_dev->features & NETIF_F_RXHASH)
skb_set_hash(skb, efx_rx_buf_hash(efx, eh),
PKT_HASH_TYPE_L3);
skb->ip_summed = ((rx_buf->flags & EFX_RX_PKT_CSUMMED) ?
CHECKSUM_UNNECESSARY : CHECKSUM_NONE);
for (;;) {
skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
rx_buf->page, rx_buf->page_offset,
rx_buf->len);
rx_buf->page = NULL;
skb->len += rx_buf->len;
if (skb_shinfo(skb)->nr_frags == n_frags)
break;
rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf);
}
skb->data_len = skb->len;
skb->truesize += n_frags * efx->rx_buffer_truesize;
skb_record_rx_queue(skb, channel->rx_queue.core_index);
skb_mark_napi_id(skb, &channel->napi_str);
gro_result = napi_gro_frags(napi);
if (gro_result != GRO_DROP)
channel->irq_mod_score += 2;
}
static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb)
{
struct net_device_context *net_device_ctx = netdev_priv(ndev);
struct hv_device *hdev = net_device_ctx->device_ctx;
struct netvsc_device *nvsc_dev = hv_get_drvdata(hdev);
u32 hash;
u16 q_idx = 0;
if (nvsc_dev == NULL || ndev->real_num_tx_queues <= 1)
return 0;
if (netvsc_set_hash(&hash, skb)) {
q_idx = nvsc_dev->send_table[hash % VRSS_SEND_TAB_SIZE] %
ndev->real_num_tx_queues;
skb_set_hash(skb, hash, 0);
}
return q_idx;
}
/** Routine to push packets arriving on Octeon interface upto network layer.
* @param oct_id - octeon device id.
* @param skbuff - skbuff struct to be passed to network layer.
* @param len - size of total data received.
* @param rh - Control header associated with the packet
* @param param - additional control data with the packet
* @param arg - farg registered in droq_ops
*/
static void
liquidio_push_packet(u32 octeon_id __attribute__((unused)),
void *skbuff,
u32 len,
union octeon_rh *rh,
void *param,
void *arg)
{
struct net_device *netdev = (struct net_device *)arg;
struct octeon_droq *droq =
container_of(param, struct octeon_droq, napi);
struct sk_buff *skb = (struct sk_buff *)skbuff;
struct skb_shared_hwtstamps *shhwtstamps;
struct napi_struct *napi = param;
u16 vtag = 0;
u32 r_dh_off;
u64 ns;
if (netdev) {
struct lio *lio = GET_LIO(netdev);
struct octeon_device *oct = lio->oct_dev;
/* Do not proceed if the interface is not in RUNNING state. */
if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) {
recv_buffer_free(skb);
droq->stats.rx_dropped++;
return;
}
skb->dev = netdev;
skb_record_rx_queue(skb, droq->q_no);
if (likely(len > MIN_SKB_SIZE)) {
struct octeon_skb_page_info *pg_info;
unsigned char *va;
pg_info = ((struct octeon_skb_page_info *)(skb->cb));
if (pg_info->page) {
/* For Paged allocation use the frags */
va = page_address(pg_info->page) +
pg_info->page_offset;
memcpy(skb->data, va, MIN_SKB_SIZE);
skb_put(skb, MIN_SKB_SIZE);
skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
pg_info->page,
pg_info->page_offset +
MIN_SKB_SIZE,
len - MIN_SKB_SIZE,
LIO_RXBUFFER_SZ);
}
} else {
struct octeon_skb_page_info *pg_info =
((struct octeon_skb_page_info *)(skb->cb));
skb_copy_to_linear_data(skb, page_address(pg_info->page)
+ pg_info->page_offset, len);
skb_put(skb, len);
put_page(pg_info->page);
}
r_dh_off = (rh->r_dh.len - 1) * BYTES_PER_DHLEN_UNIT;
if (oct->ptp_enable) {
if (rh->r_dh.has_hwtstamp) {
/* timestamp is included from the hardware at
* the beginning of the packet.
*/
if (ifstate_check
(lio,
LIO_IFSTATE_RX_TIMESTAMP_ENABLED)) {
/* Nanoseconds are in the first 64-bits
* of the packet.
*/
memcpy(&ns, (skb->data + r_dh_off),
sizeof(ns));
r_dh_off -= BYTES_PER_DHLEN_UNIT;
shhwtstamps = skb_hwtstamps(skb);
shhwtstamps->hwtstamp =
ns_to_ktime(ns +
lio->ptp_adjust);
}
}
}
if (rh->r_dh.has_hash) {
__be32 *hash_be = (__be32 *)(skb->data + r_dh_off);
u32 hash = be32_to_cpu(*hash_be);
skb_set_hash(skb, hash, PKT_HASH_TYPE_L4);
r_dh_off -= BYTES_PER_DHLEN_UNIT;
}
skb_pull(skb, rh->r_dh.len * BYTES_PER_DHLEN_UNIT);
skb->protocol = eth_type_trans(skb, skb->dev);
if ((netdev->features & NETIF_F_RXCSUM) &&
(((rh->r_dh.encap_on) &&
(rh->r_dh.csum_verified & CNNIC_TUN_CSUM_VERIFIED)) ||
(!(rh->r_dh.encap_on) &&
(rh->r_dh.csum_verified & CNNIC_CSUM_VERIFIED))))
/* checksum has already been verified */
skb->ip_summed = CHECKSUM_UNNECESSARY;
else
skb->ip_summed = CHECKSUM_NONE;
/* Setting Encapsulation field on basis of status received
* from the firmware
*/
if (rh->r_dh.encap_on) {
skb->encapsulation = 1;
skb->csum_level = 1;
droq->stats.rx_vxlan++;
}
/* inbound VLAN tag */
if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
rh->r_dh.vlan) {
u16 priority = rh->r_dh.priority;
u16 vid = rh->r_dh.vlan;
vtag = (priority << VLAN_PRIO_SHIFT) | vid;
__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vtag);
}
napi_gro_receive(napi, skb);
droq->stats.rx_bytes_received += len -
rh->r_dh.len * BYTES_PER_DHLEN_UNIT;
droq->stats.rx_pkts_received++;
} else {
recv_buffer_free(skb);
}
}
static int xlgmac_rx_poll(struct xlgmac_channel *channel, int budget)
{
struct xlgmac_pdata *pdata = channel->pdata;
struct xlgmac_ring *ring = channel->rx_ring;
struct net_device *netdev = pdata->netdev;
unsigned int len, dma_desc_len, max_len;
unsigned int context_next, context;
struct xlgmac_desc_data *desc_data;
struct xlgmac_pkt_info *pkt_info;
unsigned int incomplete, error;
struct xlgmac_hw_ops *hw_ops;
unsigned int received = 0;
struct napi_struct *napi;
struct sk_buff *skb;
int packet_count = 0;
hw_ops = &pdata->hw_ops;
/* Nothing to do if there isn't a Rx ring for this channel */
if (!ring)
return 0;
incomplete = 0;
context_next = 0;
napi = (pdata->per_channel_irq) ? &channel->napi : &pdata->napi;
desc_data = XLGMAC_GET_DESC_DATA(ring, ring->cur);
pkt_info = &ring->pkt_info;
while (packet_count < budget) {
/* First time in loop see if we need to restore state */
if (!received && desc_data->state_saved) {
skb = desc_data->state.skb;
error = desc_data->state.error;
len = desc_data->state.len;
} else {
memset(pkt_info, 0, sizeof(*pkt_info));
skb = NULL;
error = 0;
len = 0;
}
read_again:
desc_data = XLGMAC_GET_DESC_DATA(ring, ring->cur);
if (xlgmac_rx_dirty_desc(ring) > XLGMAC_RX_DESC_MAX_DIRTY)
xlgmac_rx_refresh(channel);
if (hw_ops->dev_read(channel))
break;
received++;
ring->cur++;
incomplete = XLGMAC_GET_REG_BITS(
pkt_info->attributes,
RX_PACKET_ATTRIBUTES_INCOMPLETE_POS,
RX_PACKET_ATTRIBUTES_INCOMPLETE_LEN);
context_next = XLGMAC_GET_REG_BITS(
pkt_info->attributes,
RX_PACKET_ATTRIBUTES_CONTEXT_NEXT_POS,
RX_PACKET_ATTRIBUTES_CONTEXT_NEXT_LEN);
context = XLGMAC_GET_REG_BITS(
pkt_info->attributes,
RX_PACKET_ATTRIBUTES_CONTEXT_POS,
RX_PACKET_ATTRIBUTES_CONTEXT_LEN);
/* Earlier error, just drain the remaining data */
if ((incomplete || context_next) && error)
goto read_again;
if (error || pkt_info->errors) {
if (pkt_info->errors)
netif_err(pdata, rx_err, netdev,
"error in received packet\n");
dev_kfree_skb(skb);
goto next_packet;
}
if (!context) {
/* Length is cumulative, get this descriptor's length */
dma_desc_len = desc_data->rx.len - len;
len += dma_desc_len;
if (dma_desc_len && !skb) {
skb = xlgmac_create_skb(pdata, napi, desc_data,
dma_desc_len);
if (!skb)
error = 1;
} else if (dma_desc_len) {
dma_sync_single_range_for_cpu(
pdata->dev,
desc_data->rx.buf.dma_base,
desc_data->rx.buf.dma_off,
desc_data->rx.buf.dma_len,
DMA_FROM_DEVICE);
skb_add_rx_frag(
skb, skb_shinfo(skb)->nr_frags,
desc_data->rx.buf.pa.pages,
desc_data->rx.buf.pa.pages_offset,
dma_desc_len,
desc_data->rx.buf.dma_len);
desc_data->rx.buf.pa.pages = NULL;
}
}
if (incomplete || context_next)
goto read_again;
if (!skb)
goto next_packet;
/* Be sure we don't exceed the configured MTU */
max_len = netdev->mtu + ETH_HLEN;
if (!(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
(skb->protocol == htons(ETH_P_8021Q)))
max_len += VLAN_HLEN;
if (skb->len > max_len) {
netif_err(pdata, rx_err, netdev,
"packet length exceeds configured MTU\n");
dev_kfree_skb(skb);
goto next_packet;
}
if (netif_msg_pktdata(pdata))
xlgmac_print_pkt(netdev, skb, false);
skb_checksum_none_assert(skb);
if (XLGMAC_GET_REG_BITS(pkt_info->attributes,
RX_PACKET_ATTRIBUTES_CSUM_DONE_POS,
RX_PACKET_ATTRIBUTES_CSUM_DONE_LEN))
skb->ip_summed = CHECKSUM_UNNECESSARY;
if (XLGMAC_GET_REG_BITS(pkt_info->attributes,
RX_PACKET_ATTRIBUTES_VLAN_CTAG_POS,
RX_PACKET_ATTRIBUTES_VLAN_CTAG_LEN)) {
__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
pkt_info->vlan_ctag);
pdata->stats.rx_vlan_packets++;
}
if (XLGMAC_GET_REG_BITS(pkt_info->attributes,
RX_PACKET_ATTRIBUTES_RSS_HASH_POS,
RX_PACKET_ATTRIBUTES_RSS_HASH_LEN))
skb_set_hash(skb, pkt_info->rss_hash,
pkt_info->rss_hash_type);
skb->dev = netdev;
skb->protocol = eth_type_trans(skb, netdev);
skb_record_rx_queue(skb, channel->queue_index);
napi_gro_receive(napi, skb);
next_packet:
packet_count++;
}
/* Check if we need to save state before leaving */
if (received && (incomplete || context_next)) {
desc_data = XLGMAC_GET_DESC_DATA(ring, ring->cur);
desc_data->state_saved = 1;
desc_data->state.skb = skb;
desc_data->state.len = len;
desc_data->state.error = error;
}
XLGMAC_PR("packet_count = %d\n", packet_count);
return packet_count;
}