/*
* Options2 for active open.
*/
static uint32_t
calc_opt2a(struct socket *so, struct toepcb *toep)
{
struct tcpcb *tp = so_sototcpcb(so);
struct port_info *pi = toep->port;
struct adapter *sc = pi->adapter;
uint32_t opt2 = 0;
if (tp->t_flags & TF_SACK_PERMIT)
opt2 |= F_SACK_EN;
if (tp->t_flags & TF_REQ_TSTMP)
opt2 |= F_TSTAMPS_EN;
if (tp->t_flags & TF_REQ_SCALE)
opt2 |= F_WND_SCALE_EN;
if (V_tcp_do_ecn)
opt2 |= F_CCTRL_ECN;
opt2 |= V_TX_QUEUE(sc->params.tp.tx_modq[pi->tx_chan]);
opt2 |= F_RX_COALESCE_VALID | V_RX_COALESCE(M_RX_COALESCE);
opt2 |= F_RSS_QUEUE_VALID | V_RSS_QUEUE(toep->ofld_rxq->iq.abs_id);
#ifdef USE_DDP_RX_FLOW_CONTROL
if (toep->ulp_mode == ULP_MODE_TCPDDP)
opt2 |= F_RX_FC_VALID | F_RX_FC_DDP;
#endif
return (htobe32(opt2));
}
/*
* Options2 for active open.
*/
static uint32_t
calc_opt2a(struct socket *so, struct toepcb *toep)
{
struct tcpcb *tp = so_sototcpcb(so);
struct port_info *pi = toep->port;
struct adapter *sc = pi->adapter;
uint32_t opt2;
opt2 = V_TX_QUEUE(sc->params.tp.tx_modq[pi->tx_chan]) |
F_RSS_QUEUE_VALID | V_RSS_QUEUE(toep->ofld_rxq->iq.abs_id);
if (tp->t_flags & TF_SACK_PERMIT)
opt2 |= F_SACK_EN;
if (tp->t_flags & TF_REQ_TSTMP)
opt2 |= F_TSTAMPS_EN;
if (tp->t_flags & TF_REQ_SCALE)
opt2 |= F_WND_SCALE_EN;
if (V_tcp_do_ecn)
opt2 |= F_CCTRL_ECN;
/* RX_COALESCE is always a valid value (M_RX_COALESCE). */
if (is_t4(sc))
opt2 |= F_RX_COALESCE_VALID;
else {
opt2 |= F_T5_OPT_2_VALID;
opt2 |= F_CONG_CNTRL_VALID; /* OPT_2_ISS really, for T5 */
}
if (sc->tt.rx_coalesce)
opt2 |= V_RX_COALESCE(M_RX_COALESCE);
#ifdef USE_DDP_RX_FLOW_CONTROL
if (toep->ulp_mode == ULP_MODE_TCPDDP)
opt2 |= F_RX_FC_VALID | F_RX_FC_DDP;
#endif
return (htobe32(opt2));
}
/*
* Options2 for active open.
*/
static uint32_t
calc_opt2a(struct socket *so, struct toepcb *toep,
const struct offload_settings *s)
{
struct tcpcb *tp = so_sototcpcb(so);
struct port_info *pi = toep->vi->pi;
struct adapter *sc = pi->adapter;
uint32_t opt2 = 0;
/*
* rx flow control, rx coalesce, congestion control, and tx pace are all
* explicitly set by the driver. On T5+ the ISS is also set by the
* driver to the value picked by the kernel.
*/
if (is_t4(sc)) {
opt2 |= F_RX_FC_VALID | F_RX_COALESCE_VALID;
opt2 |= F_CONG_CNTRL_VALID | F_PACE_VALID;
} else {
opt2 |= F_T5_OPT_2_VALID; /* all 4 valid */
opt2 |= F_T5_ISS; /* ISS provided in CPL */
}
if (s->sack > 0 || (s->sack < 0 && (tp->t_flags & TF_SACK_PERMIT)))
opt2 |= F_SACK_EN;
if (s->tstamp > 0 || (s->tstamp < 0 && (tp->t_flags & TF_REQ_TSTMP)))
opt2 |= F_TSTAMPS_EN;
if (tp->t_flags & TF_REQ_SCALE)
opt2 |= F_WND_SCALE_EN;
if (s->ecn > 0 || (s->ecn < 0 && V_tcp_do_ecn == 1))
opt2 |= F_CCTRL_ECN;
/* XXX: F_RX_CHANNEL for multiple rx c-chan support goes here. */
opt2 |= V_TX_QUEUE(sc->params.tp.tx_modq[pi->tx_chan]);
/* These defaults are subject to ULP specific fixups later. */
opt2 |= V_RX_FC_DDP(0) | V_RX_FC_DISABLE(0);
opt2 |= V_PACE(0);
if (s->cong_algo >= 0)
opt2 |= V_CONG_CNTRL(s->cong_algo);
else if (sc->tt.cong_algorithm >= 0)
opt2 |= V_CONG_CNTRL(sc->tt.cong_algorithm & M_CONG_CNTRL);
else {
struct cc_algo *cc = CC_ALGO(tp);
if (strcasecmp(cc->name, "reno") == 0)
opt2 |= V_CONG_CNTRL(CONG_ALG_RENO);
else if (strcasecmp(cc->name, "tahoe") == 0)
opt2 |= V_CONG_CNTRL(CONG_ALG_TAHOE);
if (strcasecmp(cc->name, "newreno") == 0)
opt2 |= V_CONG_CNTRL(CONG_ALG_NEWRENO);
if (strcasecmp(cc->name, "highspeed") == 0)
opt2 |= V_CONG_CNTRL(CONG_ALG_HIGHSPEED);
else {
/*
* Use newreno in case the algorithm selected by the
* host stack is not supported by the hardware.
*/
opt2 |= V_CONG_CNTRL(CONG_ALG_NEWRENO);
}
}
if (s->rx_coalesce > 0 || (s->rx_coalesce < 0 && sc->tt.rx_coalesce))
opt2 |= V_RX_COALESCE(M_RX_COALESCE);
/* Note that ofld_rxq is already set according to s->rxq. */
opt2 |= F_RSS_QUEUE_VALID;
opt2 |= V_RSS_QUEUE(toep->ofld_rxq->iq.abs_id);
#ifdef USE_DDP_RX_FLOW_CONTROL
if (toep->ulp_mode == ULP_MODE_TCPDDP)
opt2 |= F_RX_FC_DDP;
#endif
if (toep->ulp_mode == ULP_MODE_TLS) {
opt2 &= ~V_RX_COALESCE(M_RX_COALESCE);
opt2 |= F_RX_FC_DISABLE;
}
return (htobe32(opt2));
}
static int
handle_ddp(struct socket *so, struct uio *uio, int flags, int error)
{
struct sockbuf *sb = &so->so_rcv;
struct tcpcb *tp = so_sototcpcb(so);
struct toepcb *toep = tp->t_toe;
struct adapter *sc = td_adapter(toep->td);
vm_page_t *pages;
int npages, db_idx, rc, buf_flag;
struct ddp_buffer *db;
struct wrqe *wr;
uint64_t ddp_flags;
SOCKBUF_LOCK_ASSERT(sb);
#if 0
if (sbused(sb) + sc->tt.ddp_thres > uio->uio_resid) {
CTR4(KTR_CXGBE, "%s: sb_cc %d, threshold %d, resid %d",
__func__, sbused(sb), sc->tt.ddp_thres, uio->uio_resid);
}
#endif
/* XXX: too eager to disable DDP, could handle NBIO better than this. */
if (sbused(sb) >= uio->uio_resid || uio->uio_resid < sc->tt.ddp_thres ||
uio->uio_resid > MAX_DDP_BUFFER_SIZE || uio->uio_iovcnt > 1 ||
so->so_state & SS_NBIO || flags & (MSG_DONTWAIT | MSG_NBIO) ||
error || so->so_error || sb->sb_state & SBS_CANTRCVMORE)
goto no_ddp;
/*
* Fault in and then hold the pages of the uio buffers. We'll wire them
* a bit later if everything else works out.
*/
SOCKBUF_UNLOCK(sb);
if (hold_uio(uio, &pages, &npages) != 0) {
SOCKBUF_LOCK(sb);
goto no_ddp;
}
SOCKBUF_LOCK(sb);
if (__predict_false(so->so_error || sb->sb_state & SBS_CANTRCVMORE)) {
vm_page_unhold_pages(pages, npages);
free(pages, M_CXGBE);
goto no_ddp;
}
/*
* Figure out which one of the two DDP buffers to use this time.
*/
db_idx = select_ddp_buffer(sc, toep, pages, npages,
(uintptr_t)uio->uio_iov->iov_base & PAGE_MASK, uio->uio_resid);
pages = NULL; /* handed off to select_ddp_buffer */
if (db_idx < 0)
goto no_ddp;
db = toep->db[db_idx];
buf_flag = db_idx == 0 ? DDP_BUF0_ACTIVE : DDP_BUF1_ACTIVE;
/*
* Build the compound work request that tells the chip where to DMA the
* payload.
*/
ddp_flags = select_ddp_flags(so, flags, db_idx);
wr = mk_update_tcb_for_ddp(sc, toep, db_idx, sbused(sb), ddp_flags);
if (wr == NULL) {
/*
* Just unhold the pages. The DDP buffer's software state is
* left as-is in the toep. The page pods were written
* successfully and we may have an opportunity to use it in the
* future.
*/
vm_page_unhold_pages(db->pages, db->npages);
goto no_ddp;
}
/* Wire (and then unhold) the pages, and give the chip the go-ahead. */
wire_ddp_buffer(db);
t4_wrq_tx(sc, wr);
sb->sb_flags &= ~SB_DDP_INDICATE;
toep->ddp_flags |= buf_flag;
/*
* Wait for the DDP operation to complete and then unwire the pages.
* The return code from the sbwait will be the final return code of this
* function. But we do need to wait for DDP no matter what.
*/
rc = sbwait(sb);
while (toep->ddp_flags & buf_flag) {
/* XXXGL: shouldn't here be sbwait() call? */
sb->sb_flags |= SB_WAIT;
msleep(&sb->sb_acc, &sb->sb_mtx, PSOCK , "sbwait", 0);
}
unwire_ddp_buffer(db);
return (rc);
no_ddp:
disable_ddp(sc, toep);
discourage_ddp(toep);
sb->sb_flags &= ~SB_DDP_INDICATE;
return (0);
}
