int
octeon_eth_recv_mbuf(struct octeon_eth_softc *sc, uint64_t *work,
struct mbuf **rm)
{
struct mbuf *m;
void (*ext_free)(caddr_t, u_int, void *);
void *ext_buf;
size_t ext_size;
void *data;
uint64_t word1 = work[1];
uint64_t word2 = work[2];
uint64_t word3 = work[3];
MGETHDR(m, M_NOWAIT, MT_DATA);
if (m == NULL)
return 1;
OCTEON_ETH_KASSERT(m != NULL);
if ((word2 & PIP_WQE_WORD2_IP_BUFS) == 0) {
/* Dynamic short */
ext_free = octeon_eth_buf_ext_free_m;
ext_buf = &work[4];
ext_size = 96;
data = &work[4 + sc->sc_ip_offset / sizeof(uint64_t)];
} else {
vaddr_t addr;
vaddr_t start_buffer;
addr = PHYS_TO_XKPHYS(word3 & PIP_WQE_WORD3_ADDR, CCA_CACHED);
start_buffer = addr & ~(2048 - 1);
ext_free = octeon_eth_buf_ext_free_ext;
ext_buf = (void *)start_buffer;
ext_size = 2048;
data = (void *)addr;
}
MEXTADD(m, ext_buf, ext_size, 0, ext_free, work);
OCTEON_ETH_KASSERT(ISSET(m->m_flags, M_EXT));
m->m_data = data;
m->m_len = m->m_pkthdr.len = (word1 & PIP_WQE_WORD1_LEN) >> 48;
#if 0
/*
* not readonly buffer
*/
m->m_flags |= M_EXT_RW;
#endif
*rm = m;
OCTEON_ETH_KASSERT(*rm != NULL);
return 0;
}
void
octeon_eth_send_queue_flush_fetch(struct octeon_eth_softc *sc)
{
#ifndef OCTEON_ETH_DEBUG
if (!sc->sc_prefetch)
return;
#endif
OCTEON_ETH_KASSERT(sc->sc_prefetch == 1);
sc->sc_hard_done_cnt = cn30xxfau_op_inc_read_8(&sc->sc_fau_done);
OCTEON_ETH_KASSERT(sc->sc_hard_done_cnt <= 0);
sc->sc_prefetch = 0;
}
int
octeon_eth_send_makecmd_gbuf(struct octeon_eth_softc *sc, struct mbuf *m0,
uint64_t *gbuf, int *rsegs)
{
struct mbuf *m;
int segs = 0;
uint32_t laddr, rlen, nlen;
for (m = m0; m != NULL; m = m->m_next) {
if (__predict_false(m->m_len == 0))
continue;
#if 0
OCTEON_ETH_KASSERT(((uint32_t)m->m_data & (PAGE_SIZE - 1))
== (kvtophys((vaddr_t)m->m_data) & (PAGE_SIZE - 1)));
#endif
/*
* aligned 4k
*/
laddr = (uintptr_t)m->m_data & (PAGE_SIZE - 1);
if (laddr + m->m_len > PAGE_SIZE) {
/* XXX */
rlen = PAGE_SIZE - laddr;
nlen = m->m_len - rlen;
*(gbuf + segs) = octeon_eth_send_makecmd_w1(rlen,
KVTOPHYS(m->m_data));
segs++;
if (segs > 63) {
return 1;
}
/* XXX */
} else {
rlen = 0;
nlen = m->m_len;
}
*(gbuf + segs) = octeon_eth_send_makecmd_w1(nlen,
KVTOPHYS((caddr_t)m->m_data + rlen));
segs++;
if (segs > 63) {
return 1;
}
}
OCTEON_ETH_KASSERT(m == NULL);
*rsegs = segs;
return 0;
}
void
octeon_eth_send_queue_del(struct octeon_eth_softc *sc, struct mbuf **rm,
uint64_t **rgbuf)
{
struct mbuf *m;
m = ml_dequeue(&sc->sc_sendq);
OCTEON_ETH_KASSERT(m != NULL);
*rm = m;
*rgbuf = m->m_pkthdr.ph_cookie;
if (m->m_ext.ext_free != NULL) {
sc->sc_ext_callback_cnt--;
OCTEON_ETH_KASSERT(sc->sc_ext_callback_cnt >= 0);
}
}
void
octeon_eth_send_queue_flush_prefetch(struct octeon_eth_softc *sc)
{
OCTEON_ETH_KASSERT(sc->sc_prefetch == 0);
cn30xxfau_op_inc_fetch_8(&sc->sc_fau_done, 0);
sc->sc_prefetch = 1;
}
void
octeon_eth_send_queue_flush_sync(struct octeon_eth_softc *sc)
{
if (sc->sc_flush == 0)
return;
OCTEON_ETH_KASSERT(sc->sc_flush > 0);
/* XXX */
cn30xxfau_op_inc_read_8(&sc->sc_fau_done);
sc->sc_soft_req_cnt -= sc->sc_flush;
OCTEON_ETH_KASSERT(sc->sc_soft_req_cnt >= 0);
/* XXX */
sc->sc_flush = 0;
}
int
octeon_eth_recv(struct octeon_eth_softc *sc, uint64_t *work)
{
struct ifnet *ifp;
struct mbuf_list ml = MBUF_LIST_INITIALIZER();
struct mbuf *m;
uint64_t word2;
OCTEON_ETH_KASSERT(sc != NULL);
OCTEON_ETH_KASSERT(work != NULL);
word2 = work[2];
ifp = &sc->sc_arpcom.ac_if;
OCTEON_ETH_KASSERT(ifp != NULL);
if (!(ifp->if_flags & IFF_RUNNING))
goto drop;
if (__predict_false(octeon_eth_recv_check(sc, word2) != 0)) {
ifp->if_ierrors++;
goto drop;
}
if (__predict_false(octeon_eth_recv_mbuf(sc, work, &m) != 0)) {
ifp->if_ierrors++;
goto drop;
}
/* work[0] .. work[3] may not be valid any more */
OCTEON_ETH_KASSERT(m != NULL);
cn30xxipd_offload(word2, &m->m_pkthdr.csum_flags);
ml_enqueue(&ml, m);
if_input(ifp, &ml);
return 0;
drop:
octeon_eth_buf_free_work(sc, work, word2);
return 1;
}
void
octeon_eth_send_queue_add(struct octeon_eth_softc *sc, struct mbuf *m,
uint64_t *gbuf)
{
OCTEON_ETH_KASSERT(m->m_flags & M_PKTHDR);
m->m_pkthdr.ph_cookie = gbuf;
ml_enqueue(&sc->sc_sendq, m);
if (m->m_ext.ext_free != NULL)
sc->sc_ext_callback_cnt++;
}
void
octeon_eth_recv_intr(void *data, uint64_t *work)
{
struct octeon_eth_softc *sc;
int port;
OCTEON_ETH_KASSERT(work != NULL);
port = (work[1] & PIP_WQE_WORD1_IPRT) >> 42;
OCTEON_ETH_KASSERT(port < GMX_PORT_NUNITS);
sc = octeon_eth_gsc[port];
OCTEON_ETH_KASSERT(sc != NULL);
OCTEON_ETH_KASSERT(port == sc->sc_port);
/* XXX process all work queue entries anyway */
(void)octeon_eth_recv(sc, work);
}
int
octeon_eth_send_cmd(struct octeon_eth_softc *sc, uint64_t pko_cmd_w0,
uint64_t pko_cmd_w1)
{
uint64_t *cmdptr;
int result = 0;
cmdptr = (uint64_t *)PHYS_TO_XKPHYS(sc->sc_cmdptr.cmdptr, CCA_CACHED);
cmdptr += sc->sc_cmdptr.cmdptr_idx;
OCTEON_ETH_KASSERT(cmdptr != NULL);
*cmdptr++ = pko_cmd_w0;
*cmdptr++ = pko_cmd_w1;
OCTEON_ETH_KASSERT(sc->sc_cmdptr.cmdptr_idx + 2 <= FPA_COMMAND_BUFFER_POOL_NWORDS - 1);
if (sc->sc_cmdptr.cmdptr_idx + 2 == FPA_COMMAND_BUFFER_POOL_NWORDS - 1) {
paddr_t buf;
buf = cn30xxfpa_buf_get_paddr(octeon_eth_fb_cmd);
if (buf == 0) {
log(LOG_WARNING,
"%s: cannot allocate command buffer from free pool allocator\n",
sc->sc_dev.dv_xname);
result = 1;
goto done;
}
*cmdptr++ = buf;
sc->sc_cmdptr.cmdptr = (uint64_t)buf;
sc->sc_cmdptr.cmdptr_idx = 0;
} else {
sc->sc_cmdptr.cmdptr_idx += 2;
}
cn30xxpko_op_doorbell_write(sc->sc_port, sc->sc_port, 2);
done:
return result;
}
void
octeon_eth_send_queue_flush(struct octeon_eth_softc *sc)
{
const int64_t sent_count = sc->sc_hard_done_cnt;
int i;
OCTEON_ETH_KASSERT(sc->sc_flush == 0);
OCTEON_ETH_KASSERT(sent_count <= 0);
for (i = 0; i < 0 - sent_count; i++) {
struct mbuf *m;
uint64_t *gbuf;
octeon_eth_send_queue_del(sc, &m, &gbuf);
cn30xxfpa_buf_put_paddr(octeon_eth_fb_sg, XKPHYS_TO_PHYS(gbuf));
m_freem(m);
}
cn30xxfau_op_inc_fetch_8(&sc->sc_fau_done, i);
sc->sc_flush = i;
}
int
octeon_eth_recv_mbuf(struct octeon_eth_softc *sc, uint64_t *work,
struct mbuf **rm)
{
struct mbuf *m;
void (*ext_free)(caddr_t, u_int, void *);
void *ext_buf;
size_t ext_size;
caddr_t data;
uint64_t word1 = work[1];
uint64_t word2 = work[2];
uint64_t word3 = work[3];
MGETHDR(m, M_NOWAIT, MT_DATA);
if (m == NULL)
return 1;
OCTEON_ETH_KASSERT(m != NULL);
if ((word2 & PIP_WQE_WORD2_IP_BUFS) == 0) {
/* Dynamic short */
ext_free = octeon_eth_buf_ext_free_m;
ext_buf = &work[4];
ext_size = 96;
/*
* If the packet is IP, the hardware has padded it so that the
* IP source address starts on the next 64-bit word boundary.
*/
data = (caddr_t)&work[4] + ETHER_ALIGN;
if (!ISSET(word2, PIP_WQE_WORD2_IP_NI) &&
!ISSET(word2, PIP_WQE_WORD2_IP_V6))
data += 4;
} else {
vaddr_t addr;
vaddr_t start_buffer;
addr = PHYS_TO_XKPHYS(word3 & PIP_WQE_WORD3_ADDR, CCA_CACHED);
start_buffer = addr & ~(2048 - 1);
ext_free = octeon_eth_buf_ext_free_ext;
ext_buf = (void *)start_buffer;
ext_size = 2048;
data = (void *)addr;
}
MEXTADD(m, ext_buf, ext_size, 0, ext_free, work);
OCTEON_ETH_KASSERT(ISSET(m->m_flags, M_EXT));
m->m_data = data;
m->m_len = m->m_pkthdr.len = (word1 & PIP_WQE_WORD1_LEN) >> 48;
#if 0
/*
* not readonly buffer
*/
m->m_flags |= M_EXT_RW;
#endif
*rm = m;
OCTEON_ETH_KASSERT(*rm != NULL);
return 0;
}
int
octeon_eth_recv(struct octeon_eth_softc *sc, uint64_t *work)
{
int result = 0;
struct ifnet *ifp;
struct mbuf_list ml = MBUF_LIST_INITIALIZER();
struct mbuf *m;
uint64_t word2;
/* XXX */
/*
* performance tuning
* presend iobdma request
*/
if (sc->sc_soft_req_cnt > sc->sc_soft_req_thresh) {
octeon_eth_send_queue_flush_prefetch(sc);
}
/* XXX */
OCTEON_ETH_KASSERT(sc != NULL);
OCTEON_ETH_KASSERT(work != NULL);
word2 = work[2];
ifp = &sc->sc_arpcom.ac_if;
OCTEON_ETH_KASSERT(ifp != NULL);
if (__predict_false(octeon_eth_recv_check(sc, word2) != 0)) {
ifp->if_ierrors++;
result = 1;
octeon_eth_buf_free_work(sc, work, word2);
goto drop;
}
if (__predict_false(octeon_eth_recv_mbuf(sc, work, &m) != 0)) {
ifp->if_ierrors++;
result = 1;
octeon_eth_buf_free_work(sc, work, word2);
goto drop;
}
/* work[0] .. work[3] may not be valid any more */
OCTEON_ETH_KASSERT(m != NULL);
cn30xxipd_offload(word2, m->m_data, &m->m_pkthdr.csum_flags);
/* XXX */
if (sc->sc_soft_req_cnt > sc->sc_soft_req_thresh) {
octeon_eth_send_queue_flush_fetch(sc);
octeon_eth_send_queue_flush(sc);
}
/* XXX */
if (sc->sc_flush)
octeon_eth_send_queue_flush_sync(sc);
ml_enqueue(&ml, m);
if_input(ifp, &ml);
return 0;
drop:
/* XXX */
if (sc->sc_soft_req_cnt > sc->sc_soft_req_thresh) {
octeon_eth_send_queue_flush_fetch(sc);
}
/* XXX */
return result;
}
