static inline void *
mk_set_tcb_field_ulp(struct ulp_txpkt *ulpmc, struct toepcb *toep,
uint64_t word, uint64_t mask, uint64_t val)
{
struct ulptx_idata *ulpsc;
struct cpl_set_tcb_field_core *req;
ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
ulpmc->len = htobe32(howmany(LEN__SET_TCB_FIELD_ULP, 16));
ulpsc = (struct ulptx_idata *)(ulpmc + 1);
ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
ulpsc->len = htobe32(sizeof(*req));
req = (struct cpl_set_tcb_field_core *)(ulpsc + 1);
OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_SET_TCB_FIELD, toep->tid));
req->reply_ctrl = htobe16(V_NO_REPLY(1) |
V_QUEUENO(toep->ofld_rxq->iq.abs_id));
req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(0));
req->mask = htobe64(mask);
req->val = htobe64(val);
ulpsc = (struct ulptx_idata *)(req + 1);
if (LEN__SET_TCB_FIELD_ULP % 16) {
ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
ulpsc->len = htobe32(0);
return (ulpsc + 1);
}
return (ulpsc);
}
static inline void *
mk_rx_data_ack_ulp(struct ulp_txpkt *ulpmc, struct toepcb *toep)
{
struct ulptx_idata *ulpsc;
struct cpl_rx_data_ack_core *req;
ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
ulpmc->len = htobe32(howmany(LEN__RX_DATA_ACK_ULP, 16));
ulpsc = (struct ulptx_idata *)(ulpmc + 1);
ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
ulpsc->len = htobe32(sizeof(*req));
req = (struct cpl_rx_data_ack_core *)(ulpsc + 1);
OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tid));
req->credit_dack = htobe32(F_RX_MODULATE_RX);
ulpsc = (struct ulptx_idata *)(req + 1);
if (LEN__RX_DATA_ACK_ULP % 16) {
ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
ulpsc->len = htobe32(0);
return (ulpsc + 1);
}
return (ulpsc);
}
static inline void
ulp_mem_io_set_hdr(struct adapter *sc, int tid, struct ulp_mem_io *req,
unsigned int wr_len, unsigned int dlen,
unsigned int pm_addr)
{
struct ulptx_idata *idata = (struct ulptx_idata *)(req + 1);
INIT_ULPTX_WR(req, wr_len, 0, 0);
req->cmd = cpu_to_be32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) |
V_ULP_MEMIO_ORDER(is_t4(sc)) |
V_T5_ULP_MEMIO_IMM(is_t5(sc)));
req->dlen = htonl(V_ULP_MEMIO_DATA_LEN(dlen >> 5));
req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16)
| V_FW_WR_FLOWID(tid));
req->lock_addr = htonl(V_ULP_MEMIO_ADDR(pm_addr >> 5));
idata->cmd_more = htonl(V_ULPTX_CMD(ULP_TX_SC_IMM));
idata->len = htonl(dlen);
}
/**
* Build a CPL_ABORT_RPL message as payload of a ULP_TX_PKT command.
*/
static void mk_abort_rpl_ulp(struct cpl_abort_rpl *abort_rpl,
unsigned int tid)
{
struct ulp_txpkt *txpkt = (struct ulp_txpkt *)abort_rpl;
struct ulptx_idata *sc = (struct ulptx_idata *)(txpkt + 1);
txpkt->cmd_dest = cpu_to_be32(V_ULPTX_CMD(ULP_TX_PKT) |
V_ULP_TXPKT_DEST(0));
txpkt->len = cpu_to_be32(DIV_ROUND_UP(sizeof(*abort_rpl), 16));
sc->cmd_more = cpu_to_be32(V_ULPTX_CMD(ULP_TX_SC_IMM));
sc->len = cpu_to_be32(sizeof(*abort_rpl) -
sizeof(struct work_request_hdr));
OPCODE_TID(abort_rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, tid));
abort_rpl->rsvd0 = cpu_to_be32(0);
abort_rpl->rsvd1 = 0;
abort_rpl->cmd = CPL_ABORT_NO_RST;
sc = (struct ulptx_idata *)(abort_rpl + 1);
sc->cmd_more = cpu_to_be32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
sc->len = cpu_to_be32(0);
}
/**
* Build a CPL_SET_TCB_FIELD message as payload of a ULP_TX_PKT command.
*/
static inline void mk_set_tcb_field_ulp(struct filter_entry *f,
struct cpl_set_tcb_field *req,
unsigned int word,
u64 mask, u64 val, u8 cookie,
int no_reply)
{
struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req;
struct ulptx_idata *sc = (struct ulptx_idata *)(txpkt + 1);
txpkt->cmd_dest = cpu_to_be32(V_ULPTX_CMD(ULP_TX_PKT) |
V_ULP_TXPKT_DEST(0));
txpkt->len = cpu_to_be32(DIV_ROUND_UP(sizeof(*req), 16));
sc->cmd_more = cpu_to_be32(V_ULPTX_CMD(ULP_TX_SC_IMM));
sc->len = cpu_to_be32(sizeof(*req) - sizeof(struct work_request_hdr));
OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_SET_TCB_FIELD, f->tid));
req->reply_ctrl = cpu_to_be16(V_NO_REPLY(no_reply) | V_REPLY_CHAN(0) |
V_QUEUENO(0));
req->word_cookie = cpu_to_be16(V_WORD(word) | V_COOKIE(cookie));
req->mask = cpu_to_be64(mask);
req->val = cpu_to_be64(val);
sc = (struct ulptx_idata *)(req + 1);
sc->cmd_more = cpu_to_be32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
sc->len = cpu_to_be32(0);
}
/*
* Generate a DSGL from a starting mbuf. The total number of segments and the
* maximum segments in any one mbuf are provided.
*/
static void
write_tx_sgl(void *dst, struct mbuf *start, struct mbuf *stop, int nsegs, int n)
{
struct mbuf *m;
struct ulptx_sgl *usgl = dst;
int i, j, rc;
struct sglist sg;
struct sglist_seg segs[n];
KASSERT(nsegs > 0, ("%s: nsegs 0", __func__));
sglist_init(&sg, n, segs);
usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
V_ULPTX_NSGE(nsegs));
i = -1;
for (m = start; m != stop; m = m->m_next) {
rc = sglist_append(&sg, mtod(m, void *), m->m_len);
if (__predict_false(rc != 0))
panic("%s: sglist_append %d", __func__, rc);
for (j = 0; j < sg.sg_nseg; i++, j++) {
if (i < 0) {
usgl->len0 = htobe32(segs[j].ss_len);
usgl->addr0 = htobe64(segs[j].ss_paddr);
} else {
usgl->sge[i / 2].len[i & 1] =
htobe32(segs[j].ss_len);
usgl->sge[i / 2].addr[i & 1] =
htobe64(segs[j].ss_paddr);
}
#ifdef INVARIANTS
nsegs--;
#endif
}
sglist_reset(&sg);
}
if (i & 1)
usgl->sge[i / 2].len[1] = htobe32(0);
KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, stop %p",
__func__, nsegs, start, stop));
}
static int
write_page_pods(struct adapter *sc, struct toepcb *toep, struct ddp_buffer *db)
{
struct wrqe *wr;
struct ulp_mem_io *ulpmc;
struct ulptx_idata *ulpsc;
struct pagepod *ppod;
int i, j, k, n, chunk, len, ddp_pgsz, idx;
u_int ppod_addr;
uint32_t cmd;
cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE));
if (is_t4(sc))
cmd |= htobe32(F_ULP_MEMIO_ORDER);
else
cmd |= htobe32(F_T5_ULP_MEMIO_IMM);
ddp_pgsz = t4_ddp_pgsz[G_PPOD_PGSZ(db->tag)];
ppod_addr = db->ppod_addr;
for (i = 0; i < db->nppods; ppod_addr += chunk) {
/* How many page pods are we writing in this cycle */
n = min(db->nppods - i, NUM_ULP_TX_SC_IMM_PPODS);
chunk = PPOD_SZ(n);
len = roundup2(sizeof(*ulpmc) + sizeof(*ulpsc) + chunk, 16);
wr = alloc_wrqe(len, toep->ctrlq);
if (wr == NULL)
return (ENOMEM); /* ok to just bail out */
ulpmc = wrtod(wr);
INIT_ULPTX_WR(ulpmc, len, 0, 0);
ulpmc->cmd = cmd;
ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32));
ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16));
ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5));
ulpsc = (struct ulptx_idata *)(ulpmc + 1);
ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
ulpsc->len = htobe32(chunk);
ppod = (struct pagepod *)(ulpsc + 1);
for (j = 0; j < n; i++, j++, ppod++) {
ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID |
V_PPOD_TID(toep->tid) | db->tag);
ppod->len_offset = htobe64(V_PPOD_LEN(db->len) |
V_PPOD_OFST(db->offset));
ppod->rsvd = 0;
idx = i * PPOD_PAGES * (ddp_pgsz / PAGE_SIZE);
for (k = 0; k < nitems(ppod->addr); k++) {
if (idx < db->npages) {
ppod->addr[k] =
htobe64(db->pages[idx]->phys_addr);
idx += ddp_pgsz / PAGE_SIZE;
} else
ppod->addr[k] = 0;
#if 0
CTR5(KTR_CXGBE,
"%s: tid %d ppod[%d]->addr[%d] = %p",
__func__, toep->tid, i, k,
htobe64(ppod->addr[k]));
#endif
}
}
t4_wrq_tx(sc, wr);
}
return (0);
}
