/*
* function to charge a given rq with buffers from a pool's free list
*
* dev - software handle to the device
* rq - pointer to the RQ to charge
* nbufs - numbers of buffers to be charged
*
* return number of rqe's charges.
*/
static inline int
oce_rq_charge(struct oce_rq *rq, uint32_t nbufs, boolean_t repost)
{
struct oce_nic_rqe *rqe;
oce_rq_bdesc_t *rqbd;
oce_rq_bdesc_t **shadow_rq;
int cnt;
int cur_index;
oce_ring_buffer_t *ring;
shadow_rq = rq->shadow_ring;
ring = rq->ring;
cur_index = ring->cidx;
for (cnt = 0; cnt < nbufs; cnt++) {
if (!repost) {
rqbd = oce_rqb_alloc(rq);
} else {
/* just repost the buffers from shadow ring */
rqbd = shadow_rq[cur_index];
cur_index = GET_Q_NEXT(cur_index, 1, ring->num_items);
}
/* fill the rqes */
rqe = RING_GET_PRODUCER_ITEM_VA(rq->ring,
struct oce_nic_rqe);
rqe->u0.s.frag_pa_lo = rqbd->frag_addr.dw.addr_lo;
rqe->u0.s.frag_pa_hi = rqbd->frag_addr.dw.addr_hi;
shadow_rq[rq->ring->pidx] = rqbd;
DW_SWAP(u32ptr(rqe), sizeof (struct oce_nic_rqe));
RING_PUT(rq->ring, 1);
}
return (cnt);
} /* oce_rq_charge */
/*
* function to stop the RX
*
* rq - pointer to RQ structure
*
* return none
*/
void
oce_clean_rq(struct oce_rq *rq)
{
uint16_t num_cqe = 0;
struct oce_cq *cq;
struct oce_dev *dev;
struct oce_nic_rx_cqe *cqe;
int32_t ti = 0;
dev = rq->parent;
cq = rq->cq;
cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe);
/* dequeue till you reach an invalid cqe */
for (ti = 0; ti < DEFAULT_DRAIN_TIME; ti++) {
while (RQ_CQE_VALID(cqe)) {
DW_SWAP(u32ptr(cqe), sizeof (struct oce_nic_rx_cqe));
oce_rx_drop_pkt(rq, cqe);
atomic_add_32(&rq->buf_avail,
-(cqe->u0.s.num_fragments & 0x7));
oce_arm_cq(dev, cq->cq_id, 1, B_TRUE);
RQ_CQE_INVALIDATE(cqe);
RING_GET(cq->ring, 1);
cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring,
struct oce_nic_rx_cqe);
num_cqe++;
}
OCE_MSDELAY(1);
}
} /* oce_clean_rq */
boolean_t
oce_m_getcap(void *arg, mac_capab_t cap, void *data)
{
struct oce_dev *dev = arg;
boolean_t ret = B_TRUE;
switch (cap) {
case MAC_CAPAB_HCKSUM: {
uint32_t *csum_flags = u32ptr(data);
*csum_flags = HCKSUM_ENABLE |
HCKSUM_INET_FULL_V4 |
HCKSUM_IPHDRCKSUM;
break;
}
case MAC_CAPAB_LSO: {
mac_capab_lso_t *mcap_lso = (mac_capab_lso_t *)data;
if (dev->lso_capable) {
mcap_lso->lso_flags = LSO_TX_BASIC_TCP_IPV4;
mcap_lso->lso_basic_tcp_ipv4.lso_max = OCE_LSO_MAX_SIZE;
} else {
ret = B_FALSE;
}
break;
}
default:
ret = B_FALSE;
break;
}
return (ret);
} /* oce_m_getcap */
/* Add frame links at the end of the snapshot. */
static MSize snapshot_framelinks(jit_State *J, IRRef2 *map)
{
cTValue *frame = J->L->base - 1;
cTValue *lim = J->L->base - J->baseslot;
MSize f = 0;
map[f++] = u32ptr(J->pc);
while (frame > lim) {
if (frame_islua(frame)) {
map[f++] = u32ptr(frame_pc(frame));
frame = frame_prevl(frame);
} else if (frame_ispcall(frame)) {
map[f++] = (uint32_t)frame_ftsz(frame);
frame = frame_prevd(frame);
} else if (frame_iscont(frame)) {
map[f++] = (uint32_t)frame_ftsz(frame);
map[f++] = u32ptr(frame_contpc(frame));
frame = frame_prevd(frame);
} else {
lua_assert(0);
}
}
return f;
}
static inline int
oce_process_tx_compl(struct oce_wq *wq, boolean_t rearm)
{
struct oce_nic_tx_cqe *cqe;
uint16_t num_cqe = 0;
struct oce_cq *cq;
oce_wqe_desc_t *wqed;
int wqe_freed = 0;
struct oce_dev *dev;
cq = wq->cq;
dev = wq->parent;
(void) ddi_dma_sync(cq->ring->dbuf->dma_handle, 0, 0,
DDI_DMA_SYNC_FORKERNEL);
mutex_enter(&wq->txc_lock);
cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
while (WQ_CQE_VALID(cqe)) {
DW_SWAP(u32ptr(cqe), sizeof (struct oce_nic_tx_cqe));
/* update stats */
if (cqe->u0.s.status != 0) {
atomic_inc_32(&dev->tx_errors);
}
/* complete the WQEs */
wqed = OCE_LIST_REM_HEAD(&wq->wqe_desc_list);
wqe_freed = wqed->wqe_cnt;
oce_free_wqed(wq, wqed);
RING_GET(wq->ring, wqe_freed);
atomic_add_32(&wq->wq_free, wqe_freed);
/* clear the valid bit and progress cqe */
WQ_CQE_INVALIDATE(cqe);
RING_GET(cq->ring, 1);
cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring,
struct oce_nic_tx_cqe);
num_cqe++;
} /* for all valid CQE */
mutex_exit(&wq->txc_lock);
if (num_cqe)
oce_arm_cq(wq->parent, cq->cq_id, num_cqe, rearm);
return (num_cqe);
} /* oce_process_tx_completion */
/*
* function to process a Recieve queue
*
* arg - pointer to the RQ to charge
*
* return number of cqes processed
*/
uint16_t
oce_drain_rq_cq(void *arg)
{
struct oce_nic_rx_cqe *cqe;
struct oce_rq *rq;
mblk_t *mp = NULL;
mblk_t *mblk_head;
mblk_t **mblk_tail;
uint16_t num_cqe = 0;
struct oce_cq *cq;
struct oce_dev *dev;
int32_t frag_cnt;
uint32_t nbufs = 0;
rq = (struct oce_rq *)arg;
dev = rq->parent;
cq = rq->cq;
mblk_head = NULL;
mblk_tail = &mblk_head;
cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe);
(void) DBUF_SYNC(cq->ring->dbuf, DDI_DMA_SYNC_FORKERNEL);
/* dequeue till you reach an invalid cqe */
while (RQ_CQE_VALID(cqe)) {
DW_SWAP(u32ptr(cqe), sizeof (struct oce_nic_rx_cqe));
frag_cnt = cqe->u0.s.num_fragments & 0x7;
/* if insufficient buffers to charge then do copy */
if ((cqe->u0.s.pkt_size < dev->rx_bcopy_limit) ||
(oce_atomic_reserve(&rq->rqb_free, frag_cnt) < 0)) {
mp = oce_rx_bcopy(dev, rq, cqe);
} else {
mp = oce_rx(dev, rq, cqe);
if (mp == NULL) {
atomic_add_32(&rq->rqb_free, frag_cnt);
mp = oce_rx_bcopy(dev, rq, cqe);
}
}
if (mp != NULL) {
if (cqe->u0.s.vlan_tag_present) {
oce_rx_insert_tag(mp, cqe->u0.s.vlan_tag);
}
oce_set_rx_oflags(mp, cqe);
*mblk_tail = mp;
mblk_tail = &mp->b_next;
} else {
(void) oce_rq_charge(rq, frag_cnt, B_TRUE);
}
RING_GET(rq->ring, frag_cnt);
rq->buf_avail -= frag_cnt;
nbufs += frag_cnt;
oce_rq_post_buffer(rq, frag_cnt);
RQ_CQE_INVALIDATE(cqe);
RING_GET(cq->ring, 1);
cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring,
struct oce_nic_rx_cqe);
num_cqe++;
/* process max ring size */
if (num_cqe > dev->rx_pkt_per_intr) {
break;
}
} /* for all valid CQEs */
if (mblk_head) {
mac_rx(dev->mac_handle, NULL, mblk_head);
}
oce_arm_cq(dev, cq->cq_id, num_cqe, B_TRUE);
return (num_cqe);
} /* oce_drain_rq_cq */
/*
* function to xmit Single packet over the wire
*
* wq - pointer to WQ
* mp - Pointer to packet chain
*
* return pointer to the packet
*/
mblk_t *
oce_send_packet(struct oce_wq *wq, mblk_t *mp)
{
struct oce_nic_hdr_wqe *wqeh;
struct oce_dev *dev;
struct ether_header *eh;
struct ether_vlan_header *evh;
int32_t num_wqes;
uint16_t etype;
uint32_t ip_offset;
uint32_t csum_flags = 0;
boolean_t use_copy = B_FALSE;
boolean_t tagged = B_FALSE;
uint16_t vlan_tag;
uint32_t reg_value = 0;
oce_wqe_desc_t *wqed = NULL;
mblk_t *nmp = NULL;
mblk_t *tmp = NULL;
uint32_t pkt_len = 0;
int num_mblks = 0;
int ret = 0;
uint32_t mss = 0;
uint32_t flags = 0;
int len = 0;
/* retrieve the adap priv struct ptr */
dev = wq->parent;
/* check if we have enough free slots */
if (wq->wq_free < dev->tx_reclaim_threshold) {
(void) oce_process_tx_compl(wq, B_FALSE);
}
if (wq->wq_free < OCE_MAX_TX_HDL) {
return (mp);
}
/* check if we should copy */
for (tmp = mp; tmp != NULL; tmp = tmp->b_cont) {
pkt_len += MBLKL(tmp);
num_mblks++;
}
if (pkt_len == 0 || num_mblks == 0) {
freemsg(mp);
return (NULL);
}
/* retrieve LSO information */
mac_lso_get(mp, &mss, &flags);
/* get the offload flags */
mac_hcksum_get(mp, NULL, NULL, NULL, NULL, &csum_flags);
/* restrict the mapped segment to wat we support */
if (num_mblks > OCE_MAX_TX_HDL) {
nmp = msgpullup(mp, -1);
if (nmp == NULL) {
atomic_inc_32(&wq->pkt_drops);
freemsg(mp);
return (NULL);
}
/* Reset it to new collapsed mp */
freemsg(mp);
mp = nmp;
}
/* Get the packet descriptor for Tx */
wqed = kmem_cache_alloc(wq->wqed_cache, KM_NOSLEEP);
if (wqed == NULL) {
atomic_inc_32(&wq->pkt_drops);
freemsg(mp);
return (NULL);
}
eh = (struct ether_header *)(void *)mp->b_rptr;
if (ntohs(eh->ether_type) == VLAN_TPID) {
evh = (struct ether_vlan_header *)(void *)mp->b_rptr;
tagged = B_TRUE;
etype = ntohs(evh->ether_type);
ip_offset = sizeof (struct ether_vlan_header);
pkt_len -= VTAG_SIZE;
vlan_tag = ntohs(evh->ether_tci);
oce_remove_vtag(mp);
} else {
etype = ntohs(eh->ether_type);
ip_offset = sizeof (struct ether_header);
}
/* Save the WQ pointer */
wqed->wq = wq;
wqed->frag_idx = 1; /* index zero is always header */
wqed->frag_cnt = 0;
wqed->nhdl = 0;
wqed->mp = NULL;
OCE_LIST_LINK_INIT(&wqed->link);
/* If entire packet is less than the copy limit just do copy */
if (pkt_len < dev->tx_bcopy_limit) {
use_copy = B_TRUE;
ret = oce_bcopy_wqe(wq, wqed, mp, pkt_len);
} else {
/* copy or dma map the individual fragments */
for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) {
len = MBLKL(nmp);
if (len == 0) {
continue;
}
if (len < dev->tx_bcopy_limit) {
ret = oce_bcopy_wqe(wq, wqed, nmp, len);
} else {
ret = oce_map_wqe(wq, wqed, nmp, len);
}
if (ret != 0)
break;
}
}
/*
* Any failure other than insufficient Q entries
* drop the packet
*/
if (ret != 0) {
oce_free_wqed(wq, wqed);
atomic_inc_32(&wq->pkt_drops);
freemsg(mp);
return (NULL);
}
wqeh = (struct oce_nic_hdr_wqe *)&wqed->frag[0];
bzero(wqeh, sizeof (struct oce_nic_hdr_wqe));
/* fill rest of wqe header fields based on packet */
if (flags & HW_LSO) {
wqeh->u0.s.lso = B_TRUE;
wqeh->u0.s.lso_mss = mss;
}
if (csum_flags & HCK_FULLCKSUM) {
uint8_t *proto;
if (etype == ETHERTYPE_IP) {
proto = (uint8_t *)(void *)
(mp->b_rptr + ip_offset);
if (proto[9] == 6)
/* IPPROTO_TCP */
wqeh->u0.s.tcpcs = B_TRUE;
else if (proto[9] == 17)
/* IPPROTO_UDP */
wqeh->u0.s.udpcs = B_TRUE;
}
}
if (csum_flags & HCK_IPV4_HDRCKSUM)
wqeh->u0.s.ipcs = B_TRUE;
if (tagged) {
wqeh->u0.s.vlan = B_TRUE;
wqeh->u0.s.vlan_tag = vlan_tag;
}
wqeh->u0.s.complete = B_TRUE;
wqeh->u0.s.event = B_TRUE;
wqeh->u0.s.crc = B_TRUE;
wqeh->u0.s.total_length = pkt_len;
num_wqes = wqed->frag_cnt + 1;
/* h/w expects even no. of WQEs */
if (num_wqes & 0x1) {
bzero(&wqed->frag[num_wqes], sizeof (struct oce_nic_frag_wqe));
num_wqes++;
}
wqed->wqe_cnt = (uint16_t)num_wqes;
wqeh->u0.s.num_wqe = num_wqes;
DW_SWAP(u32ptr(&wqed->frag[0]), (wqed->wqe_cnt * NIC_WQE_SIZE));
mutex_enter(&wq->tx_lock);
if (num_wqes > wq->wq_free) {
atomic_inc_32(&wq->tx_deferd);
mutex_exit(&wq->tx_lock);
goto wqe_fail;
}
atomic_add_32(&wq->wq_free, -num_wqes);
/* fill the wq for adapter */
oce_fill_ring_descs(wq, wqed);
/* Set the mp pointer in the wqe descriptor */
if (use_copy == B_FALSE) {
wqed->mp = mp;
}
/* Add the packet desc to list to be retrieved during cmpl */
OCE_LIST_INSERT_TAIL(&wq->wqe_desc_list, wqed);
(void) ddi_dma_sync(wq->ring->dbuf->dma_handle, 0, 0,
DDI_DMA_SYNC_FORDEV);
/* ring tx doorbell */
reg_value = (num_wqes << 16) | wq->wq_id;
/* Ring the door bell */
OCE_DB_WRITE32(dev, PD_TXULP_DB, reg_value);
mutex_exit(&wq->tx_lock);
if (oce_fm_check_acc_handle(dev, dev->db_handle) != DDI_FM_OK) {
ddi_fm_service_impact(dev->dip, DDI_SERVICE_DEGRADED);
}
/* free mp if copied or packet chain collapsed */
if (use_copy == B_TRUE) {
freemsg(mp);
}
return (NULL);
wqe_fail:
if (tagged) {
oce_insert_vtag(mp, vlan_tag);
}
oce_free_wqed(wq, wqed);
return (mp);
} /* oce_send_packet */
