static void sdp_poll_tx_timeout(unsigned long data) { struct sdp_sock *ssk = (struct sdp_sock *)data; struct sock *sk = sk_ssk(ssk); u32 inflight, wc_processed; sdp_prf1(sk_ssk(ssk), NULL, "TX timeout: inflight=%d, head=%d tail=%d", (u32) tx_ring_posted(ssk), ring_head(ssk->tx_ring), ring_tail(ssk->tx_ring)); /* Only process if the socket is not in use */ bh_lock_sock(sk); if (sock_owned_by_user(sk)) { sdp_prf(sk_ssk(ssk), NULL, "TX comp: socket is busy"); if (sdp_tx_handler_select(ssk) && sk->sk_state != TCP_CLOSE && likely(ssk->qp_active)) { sdp_prf1(sk, NULL, "schedule a timer"); mod_timer(&ssk->tx_ring.timer, jiffies + SDP_TX_POLL_TIMEOUT); } SDPSTATS_COUNTER_INC(tx_poll_busy); goto out; } if (unlikely(!ssk->qp || sk->sk_state == TCP_CLOSE)) { SDPSTATS_COUNTER_INC(tx_poll_no_op); goto out; } wc_processed = sdp_process_tx_cq(ssk); if (!wc_processed) SDPSTATS_COUNTER_INC(tx_poll_miss); else { sdp_post_sends(ssk, GFP_ATOMIC); SDPSTATS_COUNTER_INC(tx_poll_hit); } inflight = (u32) tx_ring_posted(ssk); sdp_prf1(sk_ssk(ssk), NULL, "finished tx proccessing. inflight = %d", tx_ring_posted(ssk)); /* If there are still packets in flight and the timer has not already * been scheduled by the Tx routine then schedule it here to guarantee * completion processing of these packets */ if (inflight && likely(ssk->qp_active)) mod_timer(&ssk->tx_ring.timer, jiffies + SDP_TX_POLL_TIMEOUT); out: if (ssk->tx_ring.rdma_inflight && ssk->tx_ring.rdma_inflight->busy) { sdp_prf1(sk, NULL, "RDMA is inflight - arming irq"); sdp_arm_tx_cq(sk); } bh_unlock_sock(sk); }
static void sdp_poll_tx(struct sdp_sock *ssk) { struct socket *sk = ssk->socket; u32 inflight, wc_processed; sdp_prf1(ssk->socket, NULL, "TX timeout: inflight=%d, head=%d tail=%d", (u32) tx_ring_posted(ssk), ring_head(ssk->tx_ring), ring_tail(ssk->tx_ring)); if (unlikely(ssk->state == TCPS_CLOSED)) { sdp_warn(sk, "Socket is closed\n"); goto out; } wc_processed = sdp_process_tx_cq(ssk); if (!wc_processed) SDPSTATS_COUNTER_INC(tx_poll_miss); else SDPSTATS_COUNTER_INC(tx_poll_hit); inflight = (u32) tx_ring_posted(ssk); sdp_prf1(ssk->socket, NULL, "finished tx proccessing. inflight = %d", inflight); /* If there are still packets in flight and the timer has not already * been scheduled by the Tx routine then schedule it here to guarantee * completion processing of these packets */ if (inflight) callout_reset(&ssk->tx_ring.timer, SDP_TX_POLL_TIMEOUT, sdp_poll_tx_timeout, ssk); out: #ifdef SDP_ZCOPY if (ssk->tx_ring.rdma_inflight && ssk->tx_ring.rdma_inflight->busy) { sdp_prf1(sk, NULL, "RDMA is inflight - arming irq"); sdp_arm_tx_cq(ssk); } #endif return; }
static void sdp_tx_irq(struct ib_cq *cq, void *cq_context) { struct sdp_sock *ssk; ssk = cq_context; sdp_prf1(ssk->socket, NULL, "tx irq"); sdp_dbg_data(ssk->socket, "Got tx comp interrupt\n"); SDPSTATS_COUNTER_INC(tx_int_count); SDP_WLOCK(ssk); sdp_poll_tx(ssk); SDP_WUNLOCK(ssk); }
static void sdp_rx_irq(struct ib_cq *cq, void *cq_context) { struct socket *sk = cq_context; struct sdp_sock *ssk = sdp_sk(sk); if (cq != ssk->rx_ring.cq) { sdp_dbg(sk, "cq = %p, ssk->cq = %p\n", cq, ssk->rx_ring.cq); return; } SDPSTATS_COUNTER_INC(rx_int_count); sdp_prf(sk, NULL, "rx irq"); sdp_process_rx(ssk); }
static void sdp_tx_irq(struct ib_cq *cq, void *cq_context) { struct sock *sk = cq_context; struct sdp_sock *ssk = sdp_sk(sk); sdp_prf1(sk, NULL, "tx irq"); sdp_dbg_data(sk, "Got tx comp interrupt\n"); SDPSTATS_COUNTER_INC(tx_int_count); ssk->tx_compl_pending = 1; if (sdp_tx_handler_select(ssk) && likely(ssk->qp_active && sk->sk_state != TCP_CLOSE)) { sdp_prf1(sk, NULL, "poll and post from tasklet"); mod_timer(&ssk->tx_ring.timer, jiffies + SDP_TX_POLL_TIMEOUT); tasklet_schedule(&ssk->tx_ring.tasklet); } }
void sdp_nagle_timeout(unsigned long data) { struct sdp_sock *ssk = (struct sdp_sock *)data; struct sock *sk = sk_ssk(ssk); SDPSTATS_COUNTER_INC(nagle_timer); sdp_dbg_data(sk, "last_unacked = %u\n", ssk->nagle_last_unacked); if (!ssk->nagle_last_unacked) goto out2; /* Only process if the socket is not in use */ bh_lock_sock(sk); if (sock_owned_by_user(sk)) { sdp_dbg_data(sk, "socket is busy - will try later\n"); goto out; } if (sk->sk_state == TCP_CLOSE) { bh_unlock_sock(sk); return; } ssk->nagle_last_unacked = 0; sdp_post_sends(ssk, GFP_ATOMIC); if (sdp_sk_sleep(sk) && waitqueue_active(sdp_sk_sleep(sk))) sk_stream_write_space(sk); out: bh_unlock_sock(sk); out2: if (sk->sk_send_head && ssk->qp_active) { /* If has pending sends - rearm */ mod_timer(&ssk->nagle_timer, jiffies + SDP_NAGLE_TIMEOUT); } }
void sdp_post_send(struct sdp_sock *ssk, struct sk_buff *skb) { struct sdp_buf *tx_req; struct sdp_bsdh *h = (struct sdp_bsdh *)skb_transport_header(skb); unsigned long mseq = ring_head(ssk->tx_ring); int i, rc, frags; u64 addr; struct ib_device *dev; struct ib_send_wr *bad_wr; struct ib_sge ibsge[SDP_MAX_SEND_SGES]; struct ib_sge *sge = ibsge; struct ib_send_wr tx_wr = { NULL }; u32 send_flags = IB_SEND_SIGNALED; SDPSTATS_COUNTER_MID_INC(post_send, h->mid); SDPSTATS_HIST(send_size, skb->len); if (!ssk->qp_active) goto err; ssk->tx_packets++; if (h->mid != SDP_MID_SRCAVAIL && h->mid != SDP_MID_DATA && h->mid != SDP_MID_SRCAVAIL_CANCEL) { struct sock *sk = sk_ssk(ssk); sk->sk_wmem_queued += skb->truesize; sk_mem_charge(sk, skb->truesize); } if (unlikely(h->mid == SDP_MID_SRCAVAIL)) { struct tx_srcavail_state *tx_sa = TX_SRCAVAIL_STATE(skb); if (ssk->tx_sa != tx_sa) { sdp_dbg_data(sk_ssk(ssk), "SrcAvail cancelled " "before being sent!\n"); SDP_WARN_ON(1); sk_wmem_free_skb(sk_ssk(ssk), skb); return; } TX_SRCAVAIL_STATE(skb)->mseq = mseq; } if (unlikely(SDP_SKB_CB(skb)->flags & TCPHDR_URG)) h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; h->bufs = htons(rx_ring_posted(ssk)); h->len = htonl(skb->len); h->mseq = htonl(mseq); h->mseq_ack = htonl(mseq_ack(ssk)); sdp_prf(sk_ssk(ssk), skb, "TX: %s bufs: %d mseq:%ld ack:%d c: %d", mid2str(h->mid), rx_ring_posted(ssk), mseq, ntohl(h->mseq_ack), tx_credits(ssk)); SDP_DUMP_PACKET(sk_ssk(ssk), "TX", skb, h); tx_req = &ssk->tx_ring.buffer[mseq & (SDP_TX_SIZE - 1)]; tx_req->skb = skb; dev = ssk->ib_device; if (skb->len <= ssk->inline_thresh && !skb_shinfo(skb)->nr_frags) { SDPSTATS_COUNTER_INC(inline_sends); sge->addr = (u64) skb->data; sge->length = skb->len; sge->lkey = 0; frags = 0; tx_req->mapping[0] = 0; /* Nothing to be cleaned up by sdp_cleanup_sdp_buf() */ send_flags |= IB_SEND_INLINE; } else { addr = ib_dma_map_single(dev, skb->data, skb->len - skb->data_len, DMA_TO_DEVICE); tx_req->mapping[0] = addr; /* TODO: proper error handling */ BUG_ON(ib_dma_mapping_error(dev, addr)); sge->addr = addr; sge->length = skb->len - skb->data_len; sge->lkey = ssk->sdp_dev->mr->lkey; frags = skb_shinfo(skb)->nr_frags; for (i = 0; i < frags; ++i) { ++sge; addr = ib_dma_map_page(dev, skb_shinfo(skb)->frags[i].page.p, skb_shinfo(skb)->frags[i].page_offset, skb_shinfo(skb)->frags[i].size, DMA_TO_DEVICE); BUG_ON(ib_dma_mapping_error(dev, addr)); tx_req->mapping[i + 1] = addr; sge->addr = addr; sge->length = skb_shinfo(skb)->frags[i].size; sge->lkey = ssk->sdp_dev->mr->lkey; } } tx_wr.next = NULL; tx_wr.wr_id = ring_head(ssk->tx_ring) | SDP_OP_SEND; tx_wr.sg_list = ibsge; tx_wr.num_sge = frags + 1; tx_wr.opcode = IB_WR_SEND; tx_wr.send_flags = send_flags; if (unlikely(SDP_SKB_CB(skb)->flags & TCPHDR_URG)) tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &tx_wr, &bad_wr); if (unlikely(rc)) { sdp_dbg(sk_ssk(ssk), "ib_post_send failed with status %d.\n", rc); sdp_cleanup_sdp_buf(ssk, tx_req, skb->len - skb->data_len, DMA_TO_DEVICE); sdp_set_error(sk_ssk(ssk), -ECONNRESET); goto err; } atomic_inc(&ssk->tx_ring.head); atomic_dec(&ssk->tx_ring.credits); atomic_set(&ssk->remote_credits, rx_ring_posted(ssk)); return; err: sk_wmem_free_skb(sk_ssk(ssk), skb); }
int sdp_post_sends(struct sdp_sock *ssk, gfp_t gfp) { /* TODO: nonagle? */ struct sk_buff *skb; int post_count = 0; struct sock *sk = sk_ssk(ssk); if (unlikely(!ssk->id)) { if (sk->sk_send_head) { sdp_dbg(sk, "Send on socket without cmid ECONNRESET\n"); /* TODO: flush send queue? */ sdp_reset(sk); } return -ECONNRESET; } again: if (sdp_tx_ring_slots_left(ssk) < SDP_TX_SIZE / 2) sdp_xmit_poll(ssk, 1); /* Run out of credits, check if got a credit update */ if (unlikely(tx_credits(ssk) <= SDP_MIN_TX_CREDITS)) { sdp_poll_rx_cq(ssk); if (unlikely(sdp_should_rearm(sk) || !posts_handler(ssk))) sdp_arm_rx_cq(sk); } if (unlikely((ssk->sa_post_rdma_rd_compl || ssk->sa_post_sendsm) && tx_credits(ssk) < SDP_MIN_TX_CREDITS)) { sdp_dbg_data(sk, "Run out of credits, can't abort SrcAvail. " "RdmaRdCompl: %d SendSm: %d\n", ssk->sa_post_rdma_rd_compl, ssk->sa_post_sendsm); } if (ssk->sa_post_rdma_rd_compl && tx_credits(ssk) >= SDP_MIN_TX_CREDITS) { int unreported = ssk->sa_post_rdma_rd_compl; skb = sdp_alloc_skb_rdmardcompl(sk, unreported, gfp); if (!skb) goto no_mem; sdp_post_send(ssk, skb); post_count++; ssk->sa_post_rdma_rd_compl = 0; } if (ssk->sa_post_sendsm && tx_credits(ssk) >= SDP_MIN_TX_CREDITS) { skb = sdp_alloc_skb_sendsm(sk, gfp); if (unlikely(!skb)) goto no_mem; sdp_post_send(ssk, skb); ssk->sa_post_sendsm = 0; post_count++; } if (ssk->recv_request && ring_tail(ssk->rx_ring) >= SDP_MIN_TX_CREDITS && tx_credits(ssk) >= SDP_MIN_TX_CREDITS && sdp_tx_ring_slots_left(ssk)) { skb = sdp_alloc_skb_chrcvbuf_ack(sk, ssk->recv_frags * PAGE_SIZE, gfp); if (!skb) goto no_mem; ssk->recv_request = 0; sdp_post_send(ssk, skb); post_count++; } if (tx_credits(ssk) <= SDP_MIN_TX_CREDITS && sdp_tx_ring_slots_left(ssk) && sk->sk_send_head && sdp_nagle_off(ssk, sk->sk_send_head)) { SDPSTATS_COUNTER_INC(send_miss_no_credits); } while (tx_credits(ssk) > SDP_MIN_TX_CREDITS && sdp_tx_ring_slots_left(ssk) && (skb = sk->sk_send_head) && sdp_nagle_off(ssk, skb)) { update_send_head(sk, skb); __skb_dequeue(&sk->sk_write_queue); sdp_post_send(ssk, skb); post_count++; } if (credit_update_needed(ssk) && likely((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { skb = sdp_alloc_skb_data(sk, 0, gfp); if (!skb) goto no_mem; sk->sk_wmem_queued += skb->truesize; sk_mem_charge(sk, skb->truesize); sdp_post_send(ssk, skb); SDPSTATS_COUNTER_INC(post_send_credits); post_count++; } /* send DisConn if needed * Do not send DisConn if there is only 1 credit. Compliance with CA4-82 * If one credit is available, an implementation shall only send SDP * messages that provide additional credits and also do not contain ULP * payload. */ if (unlikely(ssk->sdp_disconnect) && !sk->sk_send_head && tx_credits(ssk) >= SDP_MIN_TX_CREDITS) { skb = sdp_alloc_skb_disconnect(sk, gfp); if (!skb) goto no_mem; ssk->sdp_disconnect = 0; sdp_post_send(ssk, skb); post_count++; } if (!sdp_tx_ring_slots_left(ssk) || post_count) { if (sdp_xmit_poll(ssk, 1)) goto again; } no_mem: return post_count; }
void sdp_post_sends(struct sdp_sock *ssk, int wait) { struct mbuf *mb; int post_count = 0; struct socket *sk; int low; sk = ssk->socket; if (unlikely(!ssk->id)) { if (sk->so_snd.sb_sndptr) { sdp_dbg(ssk->socket, "Send on socket without cmid ECONNRESET.\n"); sdp_notify(ssk, ECONNRESET); } return; } again: if (sdp_tx_ring_slots_left(ssk) < SDP_TX_SIZE / 2) sdp_xmit_poll(ssk, 1); if (ssk->recv_request && ring_tail(ssk->rx_ring) >= ssk->recv_request_head && tx_credits(ssk) >= SDP_MIN_TX_CREDITS && sdp_tx_ring_slots_left(ssk)) { mb = sdp_alloc_mb_chrcvbuf_ack(sk, ssk->recv_bytes - SDP_HEAD_SIZE, wait); if (mb == NULL) goto allocfail; ssk->recv_request = 0; sdp_post_send(ssk, mb); post_count++; } if (tx_credits(ssk) <= SDP_MIN_TX_CREDITS && sdp_tx_ring_slots_left(ssk) && sk->so_snd.sb_sndptr && sdp_nagle_off(ssk, sk->so_snd.sb_sndptr)) { SDPSTATS_COUNTER_INC(send_miss_no_credits); } while (tx_credits(ssk) > SDP_MIN_TX_CREDITS && sdp_tx_ring_slots_left(ssk) && (mb = sk->so_snd.sb_sndptr) && sdp_nagle_off(ssk, mb)) { struct mbuf *n; SOCKBUF_LOCK(&sk->so_snd); sk->so_snd.sb_sndptr = mb->m_nextpkt; sk->so_snd.sb_mb = mb->m_nextpkt; mb->m_nextpkt = NULL; SB_EMPTY_FIXUP(&sk->so_snd); for (n = mb; n != NULL; n = n->m_next) sbfree(&sk->so_snd, n); SOCKBUF_UNLOCK(&sk->so_snd); sdp_post_send(ssk, mb); post_count++; } if (credit_update_needed(ssk) && ssk->state >= TCPS_ESTABLISHED && ssk->state < TCPS_FIN_WAIT_2) { mb = sdp_alloc_mb_data(ssk->socket, wait); if (mb == NULL) goto allocfail; sdp_post_send(ssk, mb); SDPSTATS_COUNTER_INC(post_send_credits); post_count++; } /* send DisConn if needed * Do not send DisConn if there is only 1 credit. Compliance with CA4-82 * If one credit is available, an implementation shall only send SDP * messages that provide additional credits and also do not contain ULP * payload. */ if ((ssk->flags & SDP_NEEDFIN) && !sk->so_snd.sb_sndptr && tx_credits(ssk) > 1) { mb = sdp_alloc_mb_disconnect(sk, wait); if (mb == NULL) goto allocfail; ssk->flags &= ~SDP_NEEDFIN; sdp_post_send(ssk, mb); post_count++; } low = (sdp_tx_ring_slots_left(ssk) <= SDP_MIN_TX_CREDITS); if (post_count || low) { if (low) sdp_arm_tx_cq(ssk); if (sdp_xmit_poll(ssk, low)) goto again; } return; allocfail: ssk->nagle_last_unacked = -1; callout_reset(&ssk->nagle_timer, 1, sdp_nagle_timeout, ssk); return; }
static int sdp_post_recv(struct sdp_sock *ssk) { struct sdp_buf *rx_req; int i, rc; u64 addr; struct ib_device *dev; struct ib_recv_wr rx_wr = { NULL }; struct ib_sge ibsge[SDP_MAX_RECV_SGES]; struct ib_sge *sge = ibsge; struct ib_recv_wr *bad_wr; struct mbuf *mb, *m; struct sdp_bsdh *h; int id = ring_head(ssk->rx_ring); /* Now, allocate and repost recv */ sdp_prf(ssk->socket, mb, "Posting mb"); mb = m_getm2(NULL, ssk->recv_bytes, M_NOWAIT, MT_DATA, M_PKTHDR); if (mb == NULL) { /* Retry so we can't stall out with no memory. */ if (!rx_ring_posted(ssk)) queue_work(rx_comp_wq, &ssk->rx_comp_work); return -1; } for (m = mb; m != NULL; m = m->m_next) { m->m_len = (m->m_flags & M_EXT) ? m->m_ext.ext_size : ((m->m_flags & M_PKTHDR) ? MHLEN : MLEN); mb->m_pkthdr.len += m->m_len; } h = mtod(mb, struct sdp_bsdh *); rx_req = ssk->rx_ring.buffer + (id & (SDP_RX_SIZE - 1)); rx_req->mb = mb; dev = ssk->ib_device; for (i = 0; mb != NULL; i++, mb = mb->m_next, sge++) { addr = ib_dma_map_single(dev, mb->m_data, mb->m_len, DMA_TO_DEVICE); /* TODO: proper error handling */ BUG_ON(ib_dma_mapping_error(dev, addr)); BUG_ON(i >= SDP_MAX_RECV_SGES); rx_req->mapping[i] = addr; sge->addr = addr; sge->length = mb->m_len; sge->lkey = ssk->sdp_dev->mr->lkey; } rx_wr.next = NULL; rx_wr.wr_id = id | SDP_OP_RECV; rx_wr.sg_list = ibsge; rx_wr.num_sge = i; rc = ib_post_recv(ssk->qp, &rx_wr, &bad_wr); if (unlikely(rc)) { sdp_warn(ssk->socket, "ib_post_recv failed. status %d\n", rc); sdp_cleanup_sdp_buf(ssk, rx_req, DMA_FROM_DEVICE); m_freem(mb); sdp_notify(ssk, ECONNRESET); return -1; } atomic_inc(&ssk->rx_ring.head); SDPSTATS_COUNTER_INC(post_recv); return 0; }