uint8_t add_tcp_data(struct rte_mbuf *mbuf, unsigned char *data, uint8_t len)
{
//char *src = (char *)rte_pktmbuf_prepend (mbuf, len);
if((MBUF_BUFFER_LEN - rte_pktmbuf_headroom(mbuf) - rte_pktmbuf_tailroom(mbuf)) < len) {
logger(LOG_TCP, CRITICAL, "critical overflow **** buffer touching its limit.\n");
logger(LOG_TCP, CRITICAL, "total ava %d, head %d, tail %d, len %d, \n", MBUF_BUFFER_LEN, rte_pktmbuf_headroom(mbuf), rte_pktmbuf_tailroom(mbuf), len);
}
if(MBUF_BUFFER_LEN) {
//char *src = rte_pktmbuf_mtod(mbuf, char *);
// at starting we have everything in tail. by append we move tail and get the space for our new data. Do not use mtod to put data. call append and then fill data.
char *src = rte_pktmbuf_append (mbuf, len); // always pad the option to make total size multiple of 4.
if(src == NULL) {
assert(0);
}
memcpy(src, data, len);
if(len > 0) {
FILE *fp = fopen(DATA_SEND_DEBUG_FILE, "a");
data[len] = '\0'; // only for debuuging.
fprintf(fp, "added %s to mbuf\n", src);
fclose(fp);
}
}
else{
assert(0);
}
return len;
}
char *rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
{
void *tail;
struct rte_mbuf *m_last;
utl_rte_pktmbuf_check(m);
m_last = rte_pktmbuf_lastseg(m);
if (len > rte_pktmbuf_tailroom(m_last))
return NULL;
tail = (char*) m_last->buf_addr + m_last->data_len;
m_last->data_len = (uint16_t)(m_last->data_len + len);
m->pkt_len = (m->pkt_len + len);
return (char*) tail;
}
/*
* test data manipulation in mbuf
*/
static int
test_one_pktmbuf(void)
{
struct rte_mbuf *m = NULL;
char *data, *data2, *hdr;
unsigned i;
printf("Test pktmbuf API\n");
/* alloc a mbuf */
m = rte_pktmbuf_alloc(pktmbuf_pool);
if (m == NULL)
GOTO_FAIL("Cannot allocate mbuf");
if (rte_pktmbuf_pkt_len(m) != 0)
GOTO_FAIL("Bad length");
rte_pktmbuf_dump(m, 0);
/* append data */
data = rte_pktmbuf_append(m, MBUF_TEST_DATA_LEN);
if (data == NULL)
GOTO_FAIL("Cannot append data");
if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN)
GOTO_FAIL("Bad pkt length");
if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN)
GOTO_FAIL("Bad data length");
memset(data, 0x66, rte_pktmbuf_pkt_len(m));
if (!rte_pktmbuf_is_contiguous(m))
GOTO_FAIL("Buffer should be continuous");
rte_pktmbuf_dump(m, MBUF_TEST_DATA_LEN);
rte_pktmbuf_dump(m, 2*MBUF_TEST_DATA_LEN);
/* this append should fail */
data2 = rte_pktmbuf_append(m, (uint16_t)(rte_pktmbuf_tailroom(m) + 1));
if (data2 != NULL)
GOTO_FAIL("Append should not succeed");
/* append some more data */
data2 = rte_pktmbuf_append(m, MBUF_TEST_DATA_LEN2);
if (data2 == NULL)
GOTO_FAIL("Cannot append data");
if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_DATA_LEN2)
GOTO_FAIL("Bad pkt length");
if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_DATA_LEN2)
GOTO_FAIL("Bad data length");
if (!rte_pktmbuf_is_contiguous(m))
GOTO_FAIL("Buffer should be continuous");
/* trim data at the end of mbuf */
if (rte_pktmbuf_trim(m, MBUF_TEST_DATA_LEN2) < 0)
GOTO_FAIL("Cannot trim data");
if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN)
GOTO_FAIL("Bad pkt length");
if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN)
GOTO_FAIL("Bad data length");
if (!rte_pktmbuf_is_contiguous(m))
GOTO_FAIL("Buffer should be continuous");
/* this trim should fail */
if (rte_pktmbuf_trim(m, (uint16_t)(rte_pktmbuf_data_len(m) + 1)) == 0)
GOTO_FAIL("trim should not succeed");
/* prepend one header */
hdr = rte_pktmbuf_prepend(m, MBUF_TEST_HDR1_LEN);
if (hdr == NULL)
GOTO_FAIL("Cannot prepend");
if (data - hdr != MBUF_TEST_HDR1_LEN)
GOTO_FAIL("Prepend failed");
if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_HDR1_LEN)
GOTO_FAIL("Bad pkt length");
if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_HDR1_LEN)
GOTO_FAIL("Bad data length");
if (!rte_pktmbuf_is_contiguous(m))
GOTO_FAIL("Buffer should be continuous");
memset(hdr, 0x55, MBUF_TEST_HDR1_LEN);
/* prepend another header */
hdr = rte_pktmbuf_prepend(m, MBUF_TEST_HDR2_LEN);
if (hdr == NULL)
GOTO_FAIL("Cannot prepend");
if (data - hdr != MBUF_TEST_ALL_HDRS_LEN)
GOTO_FAIL("Prepend failed");
if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_ALL_HDRS_LEN)
GOTO_FAIL("Bad pkt length");
if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_ALL_HDRS_LEN)
GOTO_FAIL("Bad data length");
if (!rte_pktmbuf_is_contiguous(m))
GOTO_FAIL("Buffer should be continuous");
memset(hdr, 0x55, MBUF_TEST_HDR2_LEN);
rte_mbuf_sanity_check(m, RTE_MBUF_PKT, 1);
rte_mbuf_sanity_check(m, RTE_MBUF_PKT, 0);
rte_pktmbuf_dump(m, 0);
/* this prepend should fail */
hdr = rte_pktmbuf_prepend(m, (uint16_t)(rte_pktmbuf_headroom(m) + 1));
if (hdr != NULL)
GOTO_FAIL("prepend should not succeed");
/* remove data at beginning of mbuf (adj) */
if (data != rte_pktmbuf_adj(m, MBUF_TEST_ALL_HDRS_LEN))
GOTO_FAIL("rte_pktmbuf_adj failed");
if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN)
GOTO_FAIL("Bad pkt length");
if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN)
GOTO_FAIL("Bad data length");
if (!rte_pktmbuf_is_contiguous(m))
GOTO_FAIL("Buffer should be continuous");
/* this adj should fail */
if (rte_pktmbuf_adj(m, (uint16_t)(rte_pktmbuf_data_len(m) + 1)) != NULL)
GOTO_FAIL("rte_pktmbuf_adj should not succeed");
/* check data */
if (!rte_pktmbuf_is_contiguous(m))
GOTO_FAIL("Buffer should be continuous");
for (i=0; i<MBUF_TEST_DATA_LEN; i++) {
if (data[i] != 0x66)
GOTO_FAIL("Data corrupted at offset %u", i);
}
/* free mbuf */
rte_pktmbuf_free(m);
m = NULL;
return 0;
fail:
if (m)
rte_pktmbuf_free(m);
return -1;
}
/**
* DPDK callback for RX with scattered packets support.
*
* @param dpdk_rxq
* Generic pointer to RX queue structure.
* @param[out] pkts
* Array to store received packets.
* @param pkts_n
* Maximum number of packets in array.
*
* @return
* Number of packets successfully received (<= pkts_n).
*/
uint16_t
mlx5_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
{
struct rxq *rxq = (struct rxq *)dpdk_rxq;
struct rxq_elt_sp (*elts)[rxq->elts_n] = rxq->elts.sp;
const unsigned int elts_n = rxq->elts_n;
unsigned int elts_head = rxq->elts_head;
unsigned int i;
unsigned int pkts_ret = 0;
int ret;
if (unlikely(!rxq->sp))
return mlx5_rx_burst(dpdk_rxq, pkts, pkts_n);
if (unlikely(elts == NULL)) /* See RTE_DEV_CMD_SET_MTU. */
return 0;
for (i = 0; (i != pkts_n); ++i) {
struct rxq_elt_sp *elt = &(*elts)[elts_head];
unsigned int len;
unsigned int pkt_buf_len;
struct rte_mbuf *pkt_buf = NULL; /* Buffer returned in pkts. */
struct rte_mbuf **pkt_buf_next = &pkt_buf;
unsigned int seg_headroom = RTE_PKTMBUF_HEADROOM;
unsigned int j = 0;
uint32_t flags;
uint16_t vlan_tci;
/* Sanity checks. */
assert(elts_head < rxq->elts_n);
assert(rxq->elts_head < rxq->elts_n);
ret = rxq->poll(rxq->cq, NULL, NULL, &flags, &vlan_tci);
if (unlikely(ret < 0)) {
struct ibv_wc wc;
int wcs_n;
DEBUG("rxq=%p, poll_length() failed (ret=%d)",
(void *)rxq, ret);
/* ibv_poll_cq() must be used in case of failure. */
wcs_n = ibv_poll_cq(rxq->cq, 1, &wc);
if (unlikely(wcs_n == 0))
break;
if (unlikely(wcs_n < 0)) {
DEBUG("rxq=%p, ibv_poll_cq() failed (wcs_n=%d)",
(void *)rxq, wcs_n);
break;
}
assert(wcs_n == 1);
if (unlikely(wc.status != IBV_WC_SUCCESS)) {
/* Whatever, just repost the offending WR. */
DEBUG("rxq=%p, wr_id=%" PRIu64 ": bad work"
" completion status (%d): %s",
(void *)rxq, wc.wr_id, wc.status,
ibv_wc_status_str(wc.status));
#ifdef MLX5_PMD_SOFT_COUNTERS
/* Increment dropped packets counter. */
++rxq->stats.idropped;
#endif
goto repost;
}
ret = wc.byte_len;
}
if (ret == 0)
break;
assert(ret >= (rxq->crc_present << 2));
len = ret - (rxq->crc_present << 2);
pkt_buf_len = len;
/*
* Replace spent segments with new ones, concatenate and
* return them as pkt_buf.
*/
while (1) {
struct ibv_sge *sge = &elt->sges[j];
struct rte_mbuf *seg = elt->bufs[j];
struct rte_mbuf *rep;
unsigned int seg_tailroom;
assert(seg != NULL);
/*
* Fetch initial bytes of packet descriptor into a
* cacheline while allocating rep.
*/
rte_prefetch0(seg);
rep = __rte_mbuf_raw_alloc(rxq->mp);
if (unlikely(rep == NULL)) {
/*
* Unable to allocate a replacement mbuf,
* repost WR.
*/
DEBUG("rxq=%p: can't allocate a new mbuf",
(void *)rxq);
if (pkt_buf != NULL) {
*pkt_buf_next = NULL;
rte_pktmbuf_free(pkt_buf);
}
/* Increment out of memory counters. */
++rxq->stats.rx_nombuf;
++rxq->priv->dev->data->rx_mbuf_alloc_failed;
goto repost;
}
#ifndef NDEBUG
/* Poison user-modifiable fields in rep. */
NEXT(rep) = (void *)((uintptr_t)-1);
SET_DATA_OFF(rep, 0xdead);
DATA_LEN(rep) = 0xd00d;
PKT_LEN(rep) = 0xdeadd00d;
NB_SEGS(rep) = 0x2a;
PORT(rep) = 0x2a;
rep->ol_flags = -1;
#endif
assert(rep->buf_len == seg->buf_len);
assert(rep->buf_len == rxq->mb_len);
/* Reconfigure sge to use rep instead of seg. */
assert(sge->lkey == rxq->mr->lkey);
sge->addr = ((uintptr_t)rep->buf_addr + seg_headroom);
elt->bufs[j] = rep;
++j;
/* Update pkt_buf if it's the first segment, or link
* seg to the previous one and update pkt_buf_next. */
*pkt_buf_next = seg;
pkt_buf_next = &NEXT(seg);
/* Update seg information. */
seg_tailroom = (seg->buf_len - seg_headroom);
assert(sge->length == seg_tailroom);
SET_DATA_OFF(seg, seg_headroom);
if (likely(len <= seg_tailroom)) {
/* Last segment. */
DATA_LEN(seg) = len;
PKT_LEN(seg) = len;
/* Sanity check. */
assert(rte_pktmbuf_headroom(seg) ==
seg_headroom);
assert(rte_pktmbuf_tailroom(seg) ==
(seg_tailroom - len));
break;
}
DATA_LEN(seg) = seg_tailroom;
PKT_LEN(seg) = seg_tailroom;
/* Sanity check. */
assert(rte_pktmbuf_headroom(seg) == seg_headroom);
assert(rte_pktmbuf_tailroom(seg) == 0);
/* Fix len and clear headroom for next segments. */
len -= seg_tailroom;
seg_headroom = 0;
}
/* Update head and tail segments. */
*pkt_buf_next = NULL;
assert(pkt_buf != NULL);
assert(j != 0);
NB_SEGS(pkt_buf) = j;
PORT(pkt_buf) = rxq->port_id;
PKT_LEN(pkt_buf) = pkt_buf_len;
if (rxq->csum | rxq->csum_l2tun | rxq->vlan_strip) {
pkt_buf->packet_type = rxq_cq_to_pkt_type(flags);
pkt_buf->ol_flags = rxq_cq_to_ol_flags(rxq, flags);
#ifdef HAVE_EXP_DEVICE_ATTR_VLAN_OFFLOADS
if (flags & IBV_EXP_CQ_RX_CVLAN_STRIPPED_V1) {
pkt_buf->ol_flags |= PKT_RX_VLAN_PKT;
pkt_buf->vlan_tci = vlan_tci;
}
#endif /* HAVE_EXP_DEVICE_ATTR_VLAN_OFFLOADS */
}
/* Return packet. */
*(pkts++) = pkt_buf;
++pkts_ret;
#ifdef MLX5_PMD_SOFT_COUNTERS
/* Increment bytes counter. */
rxq->stats.ibytes += pkt_buf_len;
#endif
repost:
ret = rxq->recv(rxq->wq, elt->sges, RTE_DIM(elt->sges));
if (unlikely(ret)) {
/* Inability to repost WRs is fatal. */
DEBUG("%p: recv_sg_list(): failed (ret=%d)",
(void *)rxq->priv,
ret);
abort();
}
if (++elts_head >= elts_n)
elts_head = 0;
continue;
}
if (unlikely(i == 0))
return 0;
rxq->elts_head = elts_head;
#ifdef MLX5_PMD_SOFT_COUNTERS
/* Increment packets counter. */
rxq->stats.ipackets += pkts_ret;
#endif
return pkts_ret;
}
