void
sdp_post_send(struct sdp_sock *ssk, struct mbuf *mb)
{
struct sdp_buf *tx_req;
struct sdp_bsdh *h;
unsigned long mseq;
struct ib_device *dev;
struct ib_send_wr *bad_wr;
struct ib_sge ibsge[SDP_MAX_SEND_SGES];
struct ib_sge *sge;
struct ib_send_wr tx_wr = { NULL };
int i, rc;
u64 addr;
SDPSTATS_COUNTER_MID_INC(post_send, h->mid);
SDPSTATS_HIST(send_size, mb->len);
if (!ssk->qp_active) {
m_freem(mb);
return;
}
mseq = ring_head(ssk->tx_ring);
h = mtod(mb, struct sdp_bsdh *);
ssk->tx_packets++;
ssk->tx_bytes += mb->m_pkthdr.len;
#ifdef SDP_ZCOPY
if (unlikely(h->mid == SDP_MID_SRCAVAIL)) {
struct tx_srcavail_state *tx_sa = TX_SRCAVAIL_STATE(mb);
if (ssk->tx_sa != tx_sa) {
sdp_dbg_data(ssk->socket, "SrcAvail cancelled "
"before being sent!\n");
WARN_ON(1);
m_freem(mb);
return;
}
TX_SRCAVAIL_STATE(mb)->mseq = mseq;
}
#endif
if (unlikely(mb->m_flags & M_URG))
h->flags = SDP_OOB_PRES | SDP_OOB_PEND;
else
h->flags = 0;
mb->m_flags |= M_RDONLY; /* Don't allow compression once sent. */
h->bufs = htons(rx_ring_posted(ssk));
h->len = htonl(mb->m_pkthdr.len);
h->mseq = htonl(mseq);
h->mseq_ack = htonl(mseq_ack(ssk));
sdp_prf1(ssk->socket, mb, "TX: %s bufs: %d mseq:%ld ack:%d",
mid2str(h->mid), rx_ring_posted(ssk), mseq,
ntohl(h->mseq_ack));
SDP_DUMP_PACKET(ssk->socket, "TX", mb, h);
tx_req = &ssk->tx_ring.buffer[mseq & (SDP_TX_SIZE - 1)];
tx_req->mb = mb;
dev = ssk->ib_device;
sge = &ibsge[0];
for (i = 0; mb != NULL; i++, mb = mb->m_next, sge++) {
addr = ib_dma_map_single(dev, mb->m_data, mb->m_len,
DMA_TO_DEVICE);
/* TODO: proper error handling */
BUG_ON(ib_dma_mapping_error(dev, addr));
BUG_ON(i >= SDP_MAX_SEND_SGES);
tx_req->mapping[i] = addr;
sge->addr = addr;
sge->length = mb->m_len;
sge->lkey = ssk->sdp_dev->mr->lkey;
}
tx_wr.next = NULL;
tx_wr.wr_id = mseq | SDP_OP_SEND;
tx_wr.sg_list = ibsge;
tx_wr.num_sge = i;
tx_wr.opcode = IB_WR_SEND;
tx_wr.send_flags = IB_SEND_SIGNALED;
if (unlikely(tx_req->mb->m_flags & M_URG))
tx_wr.send_flags |= IB_SEND_SOLICITED;
rc = ib_post_send(ssk->qp, &tx_wr, &bad_wr);
if (unlikely(rc)) {
sdp_dbg(ssk->socket,
"ib_post_send failed with status %d.\n", rc);
sdp_cleanup_sdp_buf(ssk, tx_req, DMA_TO_DEVICE);
sdp_notify(ssk, ECONNRESET);
m_freem(tx_req->mb);
return;
}
atomic_inc(&ssk->tx_ring.head);
atomic_dec(&ssk->tx_ring.credits);
atomic_set(&ssk->remote_credits, rx_ring_posted(ssk));
return;
}
static int
sdp_process_rx_mb(struct sdp_sock *ssk, struct mbuf *mb)
{
struct socket *sk;
struct sdp_bsdh *h;
unsigned long mseq_ack;
int credits_before;
h = mtod(mb, struct sdp_bsdh *);
sk = ssk->socket;
/*
* If another thread is in so_pcbfree this may be partially torn
* down but no further synchronization is required as the destroying
* thread will wait for receive to shutdown before discarding the
* socket.
*/
if (sk == NULL) {
m_freem(mb);
return 0;
}
SDPSTATS_HIST_LINEAR(credits_before_update, tx_credits(ssk));
mseq_ack = ntohl(h->mseq_ack);
credits_before = tx_credits(ssk);
atomic_set(&ssk->tx_ring.credits, mseq_ack - ring_head(ssk->tx_ring) +
1 + ntohs(h->bufs));
if (mseq_ack >= ssk->nagle_last_unacked)
ssk->nagle_last_unacked = 0;
sdp_prf1(ssk->socket, mb, "RX %s +%d c:%d->%d mseq:%d ack:%d\n",
mid2str(h->mid), ntohs(h->bufs), credits_before,
tx_credits(ssk), ntohl(h->mseq), ntohl(h->mseq_ack));
if (unlikely(h->mid == SDP_MID_DATA &&
mb->m_pkthdr.len == SDP_HEAD_SIZE)) {
/* Credit update is valid even after RCV_SHUTDOWN */
m_freem(mb);
return 0;
}
if ((h->mid != SDP_MID_DATA && h->mid != SDP_MID_SRCAVAIL) ||
TCPS_HAVERCVDFIN(ssk->state)) {
sdp_prf(sk, NULL, "Control mb - queing to control queue");
#ifdef SDP_ZCOPY
if (h->mid == SDP_MID_SRCAVAIL_CANCEL) {
sdp_dbg_data(sk, "Got SrcAvailCancel. "
"seq: 0x%d seq_ack: 0x%d\n",
ntohl(h->mseq), ntohl(h->mseq_ack));
ssk->srcavail_cancel_mseq = ntohl(h->mseq);
}
if (h->mid == SDP_MID_RDMARDCOMPL) {
struct sdp_rrch *rrch = (struct sdp_rrch *)(h+1);
sdp_dbg_data(sk, "RdmaRdCompl message arrived\n");
sdp_handle_rdma_read_compl(ssk, ntohl(h->mseq_ack),
ntohl(rrch->len));
}
#endif
mb->m_nextpkt = NULL;
if (ssk->rx_ctl_tail)
ssk->rx_ctl_tail->m_nextpkt = mb;
else
ssk->rx_ctl_q = mb;
ssk->rx_ctl_tail = mb;
return 0;
}
sdp_prf1(sk, NULL, "queueing %s mb\n", mid2str(h->mid));
mb = sdp_sock_queue_rcv_mb(sk, mb);
return 0;
}
