static inline void
write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen,
unsigned int plen, uint8_t credits, int more_to_come)
{
struct fw_ofld_tx_data_wr *txwr = dst;
int shove = !more_to_come;
int compl = 1;
/*
* We always request completion notifications from the firmware. The
* only exception is when we know we'll get more data to send shortly
* and that we'll have some tx credits remaining to transmit that data.
*/
if (more_to_come && toep->tx_credits - credits >= MIN_OFLD_TX_CREDITS)
compl = 0;
txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) |
V_FW_WR_COMPL(compl) | V_FW_WR_IMMDLEN(immdlen));
txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) |
V_FW_WR_LEN16(credits));
txwr->tunnel_to_proxy =
htobe32(V_FW_OFLD_TX_DATA_WR_ULPMODE(toep->ulp_mode) |
V_FW_OFLD_TX_DATA_WR_URGENT(0) | /* XXX */
V_FW_OFLD_TX_DATA_WR_SHOVE(shove));
txwr->plen = htobe32(plen);
}
static inline void
write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen,
unsigned int plen, uint8_t credits, int shove, int ulp_submode, int txalign)
{
struct fw_ofld_tx_data_wr *txwr = dst;
txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) |
V_FW_WR_IMMDLEN(immdlen));
txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) |
V_FW_WR_LEN16(credits));
txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(toep->ulp_mode) |
V_TX_ULP_SUBMODE(ulp_submode) | V_TX_URG(0) | V_TX_SHOVE(shove));
txwr->plen = htobe32(plen);
if (txalign > 0) {
struct tcpcb *tp = intotcpcb(toep->inp);
if (plen < 2 * tp->t_maxseg || is_10G_port(toep->vi->pi))
txwr->lsodisable_to_flags |=
htobe32(F_FW_OFLD_TX_DATA_WR_LSODISABLE);
else
txwr->lsodisable_to_flags |=
htobe32(F_FW_OFLD_TX_DATA_WR_ALIGNPLD |
(tp->t_flags & TF_NODELAY ? 0 :
F_FW_OFLD_TX_DATA_WR_ALIGNPLDSHOVE));
}
}
static inline void
write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen,
unsigned int plen, uint8_t credits, int shove)
{
struct fw_ofld_tx_data_wr *txwr = dst;
txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) |
V_FW_WR_IMMDLEN(immdlen));
txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) |
V_FW_WR_LEN16(credits));
txwr->lsodisable_to_proxy =
htobe32(V_FW_OFLD_TX_DATA_WR_ULPMODE(toep->ulp_mode) |
V_FW_OFLD_TX_DATA_WR_URGENT(0) | /* XXX */
V_FW_OFLD_TX_DATA_WR_SHOVE(shove));
txwr->plen = htobe32(plen);
}
static inline void
write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen,
unsigned int plen, uint8_t credits, int shove, int ulp_mode, int txalign)
{
struct fw_ofld_tx_data_wr *txwr = dst;
unsigned int wr_ulp_mode;
txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) |
V_FW_WR_IMMDLEN(immdlen));
txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) |
V_FW_WR_LEN16(credits));
/* for iscsi, the mode & submode setting is per-packet */
if (toep->ulp_mode == ULP_MODE_ISCSI)
wr_ulp_mode = V_FW_OFLD_TX_DATA_WR_ULPMODE(ulp_mode >> 4) |
V_FW_OFLD_TX_DATA_WR_ULPSUBMODE(ulp_mode & 3);
else
static inline void
ulp_mem_io_set_hdr(struct adapter *sc, int tid, struct ulp_mem_io *req,
unsigned int wr_len, unsigned int dlen,
unsigned int pm_addr)
{
struct ulptx_idata *idata = (struct ulptx_idata *)(req + 1);
INIT_ULPTX_WR(req, wr_len, 0, 0);
req->cmd = cpu_to_be32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) |
V_ULP_MEMIO_ORDER(is_t4(sc)) |
V_T5_ULP_MEMIO_IMM(is_t5(sc)));
req->dlen = htonl(V_ULP_MEMIO_DATA_LEN(dlen >> 5));
req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16)
| V_FW_WR_FLOWID(tid));
req->lock_addr = htonl(V_ULP_MEMIO_ADDR(pm_addr >> 5));
idata->cmd_more = htonl(V_ULPTX_CMD(ULP_TX_SC_IMM));
idata->len = htonl(dlen);
}
/*
* Close a connection by sending a CPL_CLOSE_CON_REQ message.
*/
static int
close_conn(struct adapter *sc, struct toepcb *toep)
{
struct wrqe *wr;
struct cpl_close_con_req *req;
unsigned int tid = toep->tid;
CTR3(KTR_CXGBE, "%s: tid %u%s", __func__, toep->tid,
toep->flags & TPF_FIN_SENT ? ", IGNORED" : "");
if (toep->flags & TPF_FIN_SENT)
return (0);
KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
("%s: flowc_wr not sent for tid %u.", __func__, tid));
wr = alloc_wrqe(sizeof(*req), toep->ofld_txq);
if (wr == NULL) {
/* XXX */
panic("%s: allocation failure.", __func__);
}
req = wrtod(wr);
req->wr.wr_hi = htonl(V_FW_WR_OP(FW_TP_WR) |
V_FW_WR_IMMDLEN(sizeof(*req) - sizeof(req->wr)));
req->wr.wr_mid = htonl(V_FW_WR_LEN16(howmany(sizeof(*req), 16)) |
V_FW_WR_FLOWID(tid));
req->wr.wr_lo = cpu_to_be64(0);
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
req->rsvd = 0;
toep->flags |= TPF_FIN_SENT;
toep->flags &= ~TPF_SEND_FIN;
t4_l2t_send(sc, wr, toep->l2te);
return (0);
}
void
send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp)
{
struct wrqe *wr;
struct fw_flowc_wr *flowc;
unsigned int nparams = ftxp ? 8 : 6, flowclen;
struct port_info *pi = toep->port;
struct adapter *sc = pi->adapter;
unsigned int pfvf = G_FW_VIID_PFN(pi->viid) << S_FW_VIID_PFN;
struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
KASSERT(!(toep->flags & TPF_FLOWC_WR_SENT),
("%s: flowc for tid %u sent already", __func__, toep->tid));
CTR2(KTR_CXGBE, "%s: tid %u", __func__, toep->tid);
flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval);
wr = alloc_wrqe(roundup(flowclen, 16), toep->ofld_txq);
if (wr == NULL) {
/* XXX */
panic("%s: allocation failure.", __func__);
}
flowc = wrtod(wr);
memset(flowc, 0, wr->wr_len);
flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
V_FW_FLOWC_WR_NPARAMS(nparams));
flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) |
V_FW_WR_FLOWID(toep->tid));
flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
flowc->mnemval[0].val = htobe32(pfvf);
flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
flowc->mnemval[1].val = htobe32(pi->tx_chan);
flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
flowc->mnemval[2].val = htobe32(pi->tx_chan);
flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
flowc->mnemval[3].val = htobe32(toep->ofld_rxq->iq.abs_id);
if (ftxp) {
uint32_t sndbuf = min(ftxp->snd_space, sc->tt.sndbuf);
flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT;
flowc->mnemval[4].val = htobe32(ftxp->snd_nxt);
flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT;
flowc->mnemval[5].val = htobe32(ftxp->rcv_nxt);
flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF;
flowc->mnemval[6].val = htobe32(sndbuf);
flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS;
flowc->mnemval[7].val = htobe32(ftxp->mss);
} else {
flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDBUF;
flowc->mnemval[4].val = htobe32(512);
flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_MSS;
flowc->mnemval[5].val = htobe32(512);
}
txsd->tx_credits = howmany(flowclen, 16);
txsd->plen = 0;
KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0,
("%s: not enough credits (%d)", __func__, toep->tx_credits));
toep->tx_credits -= txsd->tx_credits;
if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
toep->txsd_pidx = 0;
toep->txsd_avail--;
toep->flags |= TPF_FLOWC_WR_SENT;
t4_wrq_tx(sc, wr);
}