/* ARGSUSED */
static void
vmxnet3s_rx_hwcksum(vmxnet3s_softc_t *dp, mblk_t *mp,
vmxnet3s_gendesc_t *compdesc)
{
uint32_t flags = 0;
if (!compdesc->rcd.cnc) {
if (compdesc->rcd.v4 && compdesc->rcd.ipc) {
flags |= HCK_IPV4_HDRCKSUM;
if ((compdesc->rcd.tcp || compdesc->rcd.udp) &&
compdesc->rcd.tuc)
flags |= HCK_FULLCKSUM | HCK_FULLCKSUM_OK;
}
(void) hcksum_assoc(mp, NULL, NULL, 0, 0, 0, 0, flags, 0);
}
}
/*
* e1000g_receive - main receive routine
*
* This routine will process packets received in an interrupt
*/
mblk_t *
e1000g_receive(e1000g_rx_ring_t *rx_ring, mblk_t **tail, uint_t sz)
{
struct e1000_hw *hw;
mblk_t *nmp;
mblk_t *ret_mp;
mblk_t *ret_nmp;
struct e1000_rx_desc *current_desc;
struct e1000_rx_desc *last_desc;
p_rx_sw_packet_t packet;
p_rx_sw_packet_t newpkt;
uint16_t length;
uint32_t pkt_count;
uint32_t desc_count;
boolean_t accept_frame;
boolean_t end_of_packet;
boolean_t need_copy;
struct e1000g *Adapter;
dma_buffer_t *rx_buf;
uint16_t cksumflags;
uint_t chain_sz = 0;
e1000g_rx_data_t *rx_data;
uint32_t max_size;
uint32_t min_size;
ret_mp = NULL;
ret_nmp = NULL;
pkt_count = 0;
desc_count = 0;
cksumflags = 0;
Adapter = rx_ring->adapter;
rx_data = rx_ring->rx_data;
hw = &Adapter->shared;
/* Sync the Rx descriptor DMA buffers */
(void) ddi_dma_sync(rx_data->rbd_dma_handle,
0, 0, DDI_DMA_SYNC_FORKERNEL);
if (e1000g_check_dma_handle(rx_data->rbd_dma_handle) != DDI_FM_OK) {
ddi_fm_service_impact(Adapter->dip, DDI_SERVICE_DEGRADED);
Adapter->e1000g_state |= E1000G_ERROR;
return (NULL);
}
current_desc = rx_data->rbd_next;
if (!(current_desc->status & E1000_RXD_STAT_DD)) {
/*
* don't send anything up. just clear the RFD
*/
E1000G_DEBUG_STAT(rx_ring->stat_none);
return (NULL);
}
max_size = Adapter->max_frame_size - ETHERFCSL - VLAN_TAGSZ;
min_size = ETHERMIN;
/*
* Loop through the receive descriptors starting at the last known
* descriptor owned by the hardware that begins a packet.
*/
while ((current_desc->status & E1000_RXD_STAT_DD) &&
(pkt_count < Adapter->rx_limit_onintr) &&
((sz == E1000G_CHAIN_NO_LIMIT) || (chain_sz <= sz))) {
desc_count++;
/*
* Now this can happen in Jumbo frame situation.
*/
if (current_desc->status & E1000_RXD_STAT_EOP) {
/* packet has EOP set */
end_of_packet = B_TRUE;
} else {
/*
* If this received buffer does not have the
* End-Of-Packet bit set, the received packet
* will consume multiple buffers. We won't send this
* packet upstack till we get all the related buffers.
*/
end_of_packet = B_FALSE;
}
/*
* Get a pointer to the actual receive buffer
* The mp->b_rptr is mapped to The CurrentDescriptor
* Buffer Address.
*/
packet =
(p_rx_sw_packet_t)QUEUE_GET_HEAD(&rx_data->recv_list);
ASSERT(packet != NULL);
rx_buf = packet->rx_buf;
length = current_desc->length;
#ifdef __sparc
if (packet->dma_type == USE_DVMA)
dvma_sync(rx_buf->dma_handle, 0,
DDI_DMA_SYNC_FORKERNEL);
else
(void) ddi_dma_sync(rx_buf->dma_handle,
E1000G_IPALIGNROOM, length,
DDI_DMA_SYNC_FORKERNEL);
#else
(void) ddi_dma_sync(rx_buf->dma_handle,
E1000G_IPALIGNROOM, length,
DDI_DMA_SYNC_FORKERNEL);
#endif
if (e1000g_check_dma_handle(
rx_buf->dma_handle) != DDI_FM_OK) {
ddi_fm_service_impact(Adapter->dip,
DDI_SERVICE_DEGRADED);
Adapter->e1000g_state |= E1000G_ERROR;
goto rx_drop;
}
accept_frame = (current_desc->errors == 0) ||
((current_desc->errors &
(E1000_RXD_ERR_TCPE | E1000_RXD_ERR_IPE)) != 0);
if (hw->mac.type == e1000_82543) {
unsigned char last_byte;
last_byte =
*((unsigned char *)rx_buf->address + length - 1);
if (TBI_ACCEPT(hw,
current_desc->status, current_desc->errors,
current_desc->length, last_byte,
Adapter->min_frame_size, Adapter->max_frame_size)) {
e1000_tbi_adjust_stats(Adapter,
length, hw->mac.addr);
length--;
accept_frame = B_TRUE;
} else if (e1000_tbi_sbp_enabled_82543(hw) &&
(current_desc->errors == E1000_RXD_ERR_CE)) {
accept_frame = B_TRUE;
}
}
/*
* Indicate the packet to the NOS if it was good.
* Normally, hardware will discard bad packets for us.
* Check for the packet to be a valid Ethernet packet
*/
if (!accept_frame) {
/*
* error in incoming packet, either the packet is not a
* ethernet size packet, or the packet has an error. In
* either case, the packet will simply be discarded.
*/
E1000G_DEBUGLOG_0(Adapter, E1000G_INFO_LEVEL,
"Process Receive Interrupts: Error in Packet\n");
E1000G_STAT(rx_ring->stat_error);
/*
* Returning here as we are done here. There is
* no point in waiting for while loop to elapse
* and the things which were done. More efficient
* and less error prone...
*/
goto rx_drop;
}
/*
* If the Ethernet CRC is not stripped by the hardware,
* we need to strip it before sending it up to the stack.
*/
if (end_of_packet && !Adapter->strip_crc) {
if (length > ETHERFCSL) {
length -= ETHERFCSL;
} else {
/*
* If the fragment is smaller than the CRC,
* drop this fragment, do the processing of
* the end of the packet.
*/
ASSERT(rx_data->rx_mblk_tail != NULL);
rx_data->rx_mblk_tail->b_wptr -=
ETHERFCSL - length;
rx_data->rx_mblk_len -=
ETHERFCSL - length;
QUEUE_POP_HEAD(&rx_data->recv_list);
goto rx_end_of_packet;
}
}
need_copy = B_TRUE;
if (length <= Adapter->rx_bcopy_thresh)
goto rx_copy;
/*
* Get the pre-constructed mblk that was associated
* to the receive data buffer.
*/
if (packet->mp == NULL) {
packet->mp = desballoc((unsigned char *)
rx_buf->address, length,
BPRI_MED, &packet->free_rtn);
}
if (packet->mp != NULL) {
/*
* We have two sets of buffer pool. One associated with
* the Rxdescriptors and other a freelist buffer pool.
* Each time we get a good packet, Try to get a buffer
* from the freelist pool using e1000g_get_buf. If we
* get free buffer, then replace the descriptor buffer
* address with the free buffer we just got, and pass
* the pre-constructed mblk upstack. (note no copying)
*
* If we failed to get a free buffer, then try to
* allocate a new buffer(mp) and copy the recv buffer
* content to our newly allocated buffer(mp). Don't
* disturb the desriptor buffer address. (note copying)
*/
newpkt = e1000g_get_buf(rx_data);
if (newpkt != NULL) {
/*
* Get the mblk associated to the data,
* and strip it off the sw packet.
*/
nmp = packet->mp;
packet->mp = NULL;
atomic_inc_32(&packet->ref_cnt);
/*
* Now replace old buffer with the new
* one we got from free list
* Both the RxSwPacket as well as the
* Receive Buffer Descriptor will now
* point to this new packet.
*/
packet = newpkt;
current_desc->buffer_addr =
newpkt->rx_buf->dma_address;
need_copy = B_FALSE;
} else {
/* EMPTY */
E1000G_DEBUG_STAT(rx_ring->stat_no_freepkt);
}
}
rx_copy:
if (need_copy) {
/*
* No buffers available on free list,
* bcopy the data from the buffer and
* keep the original buffer. Dont want to
* do this.. Yack but no other way
*/
if ((nmp = allocb(length + E1000G_IPALIGNROOM,
BPRI_MED)) == NULL) {
/*
* The system has no buffers available
* to send up the incoming packet, hence
* the packet will have to be processed
* when there're more buffers available.
*/
E1000G_STAT(rx_ring->stat_allocb_fail);
goto rx_drop;
}
nmp->b_rptr += E1000G_IPALIGNROOM;
nmp->b_wptr += E1000G_IPALIGNROOM;
/*
* The free list did not have any buffers
* available, so, the received packet will
* have to be copied into a mp and the original
* buffer will have to be retained for future
* packet reception.
*/
bcopy(rx_buf->address, nmp->b_wptr, length);
}
/*
* The rx_sw_packet MUST be popped off the
* RxSwPacketList before either a putnext or freemsg
* is done on the mp that has now been created by the
* desballoc. If not, it is possible that the free
* routine will get called from the interrupt context
* and try to put this packet on the free list
*/
(p_rx_sw_packet_t)QUEUE_POP_HEAD(&rx_data->recv_list);
ASSERT(nmp != NULL);
nmp->b_wptr += length;
if (rx_data->rx_mblk == NULL) {
/*
* TCP/UDP checksum offload and
* IP checksum offload
*/
if (!(current_desc->status & E1000_RXD_STAT_IXSM)) {
/*
* Check TCP/UDP checksum
*/
if ((current_desc->status &
E1000_RXD_STAT_TCPCS) &&
!(current_desc->errors &
E1000_RXD_ERR_TCPE))
cksumflags |= HCK_FULLCKSUM |
HCK_FULLCKSUM_OK;
/*
* Check IP Checksum
*/
if ((current_desc->status &
E1000_RXD_STAT_IPCS) &&
!(current_desc->errors &
E1000_RXD_ERR_IPE))
cksumflags |= HCK_IPV4_HDRCKSUM;
}
}
/*
* We need to maintain our packet chain in the global
* Adapter structure, for the Rx processing can end
* with a fragment that has no EOP set.
*/
if (rx_data->rx_mblk == NULL) {
/* Get the head of the message chain */
rx_data->rx_mblk = nmp;
rx_data->rx_mblk_tail = nmp;
rx_data->rx_mblk_len = length;
} else { /* Not the first packet */
/* Continue adding buffers */
rx_data->rx_mblk_tail->b_cont = nmp;
rx_data->rx_mblk_tail = nmp;
rx_data->rx_mblk_len += length;
}
ASSERT(rx_data->rx_mblk != NULL);
ASSERT(rx_data->rx_mblk_tail != NULL);
ASSERT(rx_data->rx_mblk_tail->b_cont == NULL);
/*
* Now this MP is ready to travel upwards but some more
* fragments are coming.
* We will send packet upwards as soon as we get EOP
* set on the packet.
*/
if (!end_of_packet) {
/*
* continue to get the next descriptor,
* Tail would be advanced at the end
*/
goto rx_next_desc;
}
rx_end_of_packet:
if (E1000G_IS_VLAN_PACKET(rx_data->rx_mblk->b_rptr))
max_size = Adapter->max_frame_size - ETHERFCSL;
if ((rx_data->rx_mblk_len > max_size) ||
(rx_data->rx_mblk_len < min_size)) {
E1000G_STAT(rx_ring->stat_size_error);
goto rx_drop;
}
/*
* Found packet with EOP
* Process the last fragment.
*/
if (cksumflags != 0) {
(void) hcksum_assoc(rx_data->rx_mblk,
NULL, NULL, 0, 0, 0, 0, cksumflags, 0);
cksumflags = 0;
}
/*
* Count packets that span multi-descriptors
*/
E1000G_DEBUG_STAT_COND(rx_ring->stat_multi_desc,
(rx_data->rx_mblk->b_cont != NULL));
/*
* Append to list to send upstream
*/
if (ret_mp == NULL) {
ret_mp = ret_nmp = rx_data->rx_mblk;
} else {
ret_nmp->b_next = rx_data->rx_mblk;
ret_nmp = rx_data->rx_mblk;
}
ret_nmp->b_next = NULL;
*tail = ret_nmp;
chain_sz += length;
rx_data->rx_mblk = NULL;
rx_data->rx_mblk_tail = NULL;
rx_data->rx_mblk_len = 0;
pkt_count++;
rx_next_desc:
/*
* Zero out the receive descriptors status
*/
current_desc->status = 0;
if (current_desc == rx_data->rbd_last)
rx_data->rbd_next = rx_data->rbd_first;
else
rx_data->rbd_next++;
last_desc = current_desc;
current_desc = rx_data->rbd_next;
/*
* Put the buffer that we just indicated back
* at the end of our list
*/
QUEUE_PUSH_TAIL(&rx_data->recv_list,
&packet->Link);
} /* while loop */
/* Sync the Rx descriptor DMA buffers */
(void) ddi_dma_sync(rx_data->rbd_dma_handle,
0, 0, DDI_DMA_SYNC_FORDEV);
/*
* Advance the E1000's Receive Queue #0 "Tail Pointer".
*/
E1000_WRITE_REG(hw, E1000_RDT(0),
(uint32_t)(last_desc - rx_data->rbd_first));
if (e1000g_check_acc_handle(Adapter->osdep.reg_handle) != DDI_FM_OK) {
ddi_fm_service_impact(Adapter->dip, DDI_SERVICE_DEGRADED);
Adapter->e1000g_state |= E1000G_ERROR;
}
Adapter->rx_pkt_cnt = pkt_count;
return (ret_mp);
rx_drop:
/*
* Zero out the receive descriptors status
*/
current_desc->status = 0;
/* Sync the Rx descriptor DMA buffers */
(void) ddi_dma_sync(rx_data->rbd_dma_handle,
0, 0, DDI_DMA_SYNC_FORDEV);
if (current_desc == rx_data->rbd_last)
rx_data->rbd_next = rx_data->rbd_first;
else
rx_data->rbd_next++;
last_desc = current_desc;
(p_rx_sw_packet_t)QUEUE_POP_HEAD(&rx_data->recv_list);
QUEUE_PUSH_TAIL(&rx_data->recv_list, &packet->Link);
/*
* Reclaim all old buffers already allocated during
* Jumbo receives.....for incomplete reception
*/
if (rx_data->rx_mblk != NULL) {
freemsg(rx_data->rx_mblk);
rx_data->rx_mblk = NULL;
rx_data->rx_mblk_tail = NULL;
rx_data->rx_mblk_len = 0;
}
/*
* Advance the E1000's Receive Queue #0 "Tail Pointer".
*/
E1000_WRITE_REG(hw, E1000_RDT(0),
(uint32_t)(last_desc - rx_data->rbd_first));
if (e1000g_check_acc_handle(Adapter->osdep.reg_handle) != DDI_FM_OK) {
ddi_fm_service_impact(Adapter->dip, DDI_SERVICE_DEGRADED);
Adapter->e1000g_state |= E1000G_ERROR;
}
return (ret_mp);
}
/*
* DL_UINTDATA_REQ
*/
void
proto_unitdata_req(dld_str_t *dsp, mblk_t *mp)
{
queue_t *q = dsp->ds_wq;
dl_unitdata_req_t *dlp = (dl_unitdata_req_t *)mp->b_rptr;
off_t off;
size_t len, size;
const uint8_t *addr;
uint16_t sap;
uint_t addr_length;
mblk_t *bp, *payload;
uint32_t start, stuff, end, value, flags;
t_uscalar_t dl_err;
uint_t max_sdu;
if (MBLKL(mp) < sizeof (dl_unitdata_req_t) || mp->b_cont == NULL) {
dlerrorack(q, mp, DL_UNITDATA_REQ, DL_BADPRIM, 0);
return;
}
mutex_enter(&dsp->ds_lock);
if (dsp->ds_dlstate != DL_IDLE) {
mutex_exit(&dsp->ds_lock);
dlerrorack(q, mp, DL_UNITDATA_REQ, DL_OUTSTATE, 0);
return;
}
DLD_DATATHR_INC(dsp);
mutex_exit(&dsp->ds_lock);
addr_length = dsp->ds_mip->mi_addr_length;
off = dlp->dl_dest_addr_offset;
len = dlp->dl_dest_addr_length;
if (!MBLKIN(mp, off, len) || !IS_P2ALIGNED(off, sizeof (uint16_t))) {
dl_err = DL_BADPRIM;
goto failed;
}
if (len != addr_length + sizeof (uint16_t)) {
dl_err = DL_BADADDR;
goto failed;
}
addr = mp->b_rptr + off;
sap = *(uint16_t *)(mp->b_rptr + off + addr_length);
/*
* Check the length of the packet and the block types.
*/
size = 0;
payload = mp->b_cont;
for (bp = payload; bp != NULL; bp = bp->b_cont) {
if (DB_TYPE(bp) != M_DATA)
goto baddata;
size += MBLKL(bp);
}
mac_sdu_get(dsp->ds_mh, NULL, &max_sdu);
if (size > max_sdu)
goto baddata;
/*
* Build a packet header.
*/
if ((bp = dls_header(dsp, addr, sap, dlp->dl_priority.dl_max,
&payload)) == NULL) {
dl_err = DL_BADADDR;
goto failed;
}
/*
* We no longer need the M_PROTO header, so free it.
*/
freeb(mp);
/*
* Transfer the checksum offload information if it is present.
*/
hcksum_retrieve(payload, NULL, NULL, &start, &stuff, &end, &value,
&flags);
(void) hcksum_assoc(bp, NULL, NULL, start, stuff, end, value, flags, 0);
/*
* Link the payload onto the new header.
*/
ASSERT(bp->b_cont == NULL);
bp->b_cont = payload;
/*
* No lock can be held across modules and putnext()'s,
* which can happen here with the call from DLD_TX().
*/
if (DLD_TX(dsp, bp, 0, 0) != NULL) {
/* flow-controlled */
DLD_SETQFULL(dsp);
}
DLD_DATATHR_DCR(dsp);
return;
failed:
dlerrorack(q, mp, DL_UNITDATA_REQ, dl_err, 0);
DLD_DATATHR_DCR(dsp);
return;
baddata:
dluderrorind(q, mp, (void *)addr, len, DL_BADDATA, 0);
DLD_DATATHR_DCR(dsp);
}
