void
tcp_pcap_add(struct tcphdr *th, struct mbuf *m, struct mbufq *queue)
{
struct mbuf *n = NULL, *mhead;
KASSERT(th, ("%s: called with th == NULL", __func__));
KASSERT(m, ("%s: called with m == NULL", __func__));
KASSERT(queue, ("%s: called with queue == NULL", __func__));
/* We only care about data packets. */
while (m && m->m_type != MT_DATA)
m = m->m_next;
/* We only need to do something if we still have an mbuf. */
if (!m)
return;
/* If we are not saving mbufs, return now. */
if (queue->mq_maxlen == 0)
return;
/*
* Check to see if we will need to recycle mbufs.
*
* If we need to get rid of mbufs to stay below
* our packet count, try to reuse the mbuf. Once
* we already have a new mbuf (n), then we can
* simply free subsequent mbufs.
*
* Note that most of the logic in here is to deal
* with the reuse. If we are fine with constant
* mbuf allocs/deallocs, we could ditch this logic.
* But, it only seems to make sense to reuse
* mbufs we already have.
*/
while (mbufq_full(queue)) {
mhead = mbufq_dequeue(queue);
if (n) {
tcp_pcap_m_freem(mhead);
}
else {
/*
* If this held an external cluster, try to
* detach the cluster. But, if we held the
* last reference, go through the normal
* free-ing process.
*/
if (mhead->m_flags & M_EXT) {
switch (mhead->m_ext.ext_type) {
case EXT_SFBUF:
/* Don't mess around with these. */
tcp_pcap_m_freem(mhead);
continue;
default:
if (atomic_fetchadd_int(
mhead->m_ext.ext_cnt, -1) == 1)
{
/*
* We held the last reference
* on this cluster. Restore
* the reference count and put
* it back in the pool.
*/
*(mhead->m_ext.ext_cnt) = 1;
tcp_pcap_m_freem(mhead);
continue;
}
/*
* We were able to cleanly free the
* reference.
*/
atomic_subtract_int(
&tcp_pcap_clusters_referenced_cur,
1);
tcp_pcap_alloc_reuse_ext++;
break;
}
}
else {
tcp_pcap_alloc_reuse_mbuf++;
}
n = mhead;
tcp_pcap_m_freem(n->m_next);
m_init(n, NULL, 0, M_NOWAIT, MT_DATA, 0);
}
}
/* Check to see if we need to get a new mbuf. */
if (!n) {
if (!(n = m_get(M_NOWAIT, MT_DATA)))
return;
tcp_pcap_alloc_new_mbuf++;
}
/*
* What are we dealing with? If a cluster, attach it. Otherwise,
* try to copy the data from the beginning of the mbuf to the
* end of data. (There may be data between the start of the data
* area and the current data pointer. We want to get this, because
* it may contain header information that is useful.)
* In cases where that isn't possible, settle for what we can
* get.
*/
if ((m->m_flags & M_EXT) && tcp_pcap_take_cluster_reference()) {
n->m_data = m->m_data;
n->m_len = m->m_len;
mb_dupcl(n, m);
}
else if (((m->m_data + m->m_len) - M_START(m)) <= M_SIZE(n)) {
/*
* At this point, n is guaranteed to be a normal mbuf
* with no cluster and no packet header. Because the
* logic in this code block requires this, the assert
* is here to catch any instances where someone
* changes the logic to invalidate that assumption.
*/
KASSERT((n->m_flags & (M_EXT | M_PKTHDR)) == 0,
("%s: Unexpected flags (%#x) for mbuf",
__func__, n->m_flags));
n->m_data = n->m_dat + M_LEADINGSPACE_NOWRITE(m);
n->m_len = m->m_len;
bcopy(M_START(m), n->m_dat,
m->m_len + M_LEADINGSPACE_NOWRITE(m));
}
else {
/*
* This is the case where we need to "settle for what
* we can get". The most probable way to this code
* path is that we've already taken references to the
* maximum number of mbuf clusters we can, and the data
* is too long to fit in an mbuf's internal storage.
* Try for a "best fit".
*/
tcp_pcap_copy_bestfit(th, m, n);
/* Don't try to get additional data. */
goto add_to_queue;
}
if (m->m_next) {
n->m_next = m_copym(m->m_next, 0, M_COPYALL, M_NOWAIT);
tcp_pcap_adj_cluster_reference(n->m_next, 1);
}
add_to_queue:
/* Add the new mbuf to the list. */
if (mbufq_enqueue(queue, n)) {
/* This shouldn't happen. If INVARIANTS is defined, panic. */
KASSERT(0, ("%s: mbufq was unexpectedly full!", __func__));
tcp_pcap_m_freem(n);
}
}
/*
* Sanity checks on mbuf (chain) for use in KASSERT() and general
* debugging.
* Returns 0 or panics when bad and 1 on all tests passed.
* Sanitize, 0 to run M_SANITY_ACTION, 1 to garble things so they
* blow up later.
*/
int
m_sanity(struct mbuf *m0, int sanitize)
{
struct mbuf *m;
caddr_t a, b;
int pktlen = 0;
#ifdef INVARIANTS
#define M_SANITY_ACTION(s) panic("mbuf %p: " s, m)
#else
#define M_SANITY_ACTION(s) printf("mbuf %p: " s, m)
#endif
for (m = m0; m != NULL; m = m->m_next) {
/*
* Basic pointer checks. If any of these fails then some
* unrelated kernel memory before or after us is trashed.
* No way to recover from that.
*/
a = M_START(m);
b = a + M_SIZE(m);
if ((caddr_t)m->m_data < a)
M_SANITY_ACTION("m_data outside mbuf data range left");
if ((caddr_t)m->m_data > b)
M_SANITY_ACTION("m_data outside mbuf data range right");
if ((caddr_t)m->m_data + m->m_len > b)
M_SANITY_ACTION("m_data + m_len exeeds mbuf space");
/* m->m_nextpkt may only be set on first mbuf in chain. */
if (m != m0 && m->m_nextpkt != NULL) {
if (sanitize) {
m_freem(m->m_nextpkt);
m->m_nextpkt = (struct mbuf *)0xDEADC0DE;
} else
M_SANITY_ACTION("m->m_nextpkt on in-chain mbuf");
}
/* packet length (not mbuf length!) calculation */
if (m0->m_flags & M_PKTHDR)
pktlen += m->m_len;
/* m_tags may only be attached to first mbuf in chain. */
if (m != m0 && m->m_flags & M_PKTHDR &&
!SLIST_EMPTY(&m->m_pkthdr.tags)) {
if (sanitize) {
m_tag_delete_chain(m, NULL);
/* put in 0xDEADC0DE perhaps? */
} else
M_SANITY_ACTION("m_tags on in-chain mbuf");
}
/* M_PKTHDR may only be set on first mbuf in chain */
if (m != m0 && m->m_flags & M_PKTHDR) {
if (sanitize) {
bzero(&m->m_pkthdr, sizeof(m->m_pkthdr));
m->m_flags &= ~M_PKTHDR;
/* put in 0xDEADCODE and leave hdr flag in */
} else
M_SANITY_ACTION("M_PKTHDR on in-chain mbuf");
}
}
m = m0;
if (pktlen && pktlen != m->m_pkthdr.len) {
if (sanitize)
m->m_pkthdr.len = 0;
else
M_SANITY_ACTION("m_pkthdr.len != mbuf chain length");
}
return 1;
#undef M_SANITY_ACTION
}
/*
* Copy data from m to n, where n cannot fit all the data we might
* want from m.
*
* Prioritize data like this:
* 1. TCP header
* 2. IP header
* 3. Data
*/
static void
tcp_pcap_copy_bestfit(struct tcphdr *th, struct mbuf *m, struct mbuf *n)
{
struct mbuf *m_cur = m;
int bytes_to_copy=0, trailing_data, skip=0, tcp_off;
/* Below, we assume these will be non-NULL. */
KASSERT(th, ("%s: called with th == NULL", __func__));
KASSERT(m, ("%s: called with m == NULL", __func__));
KASSERT(n, ("%s: called with n == NULL", __func__));
/* We assume this initialization occurred elsewhere. */
KASSERT(n->m_len == 0, ("%s: called with n->m_len=%d (expected 0)",
__func__, n->m_len));
KASSERT(n->m_data == M_START(n),
("%s: called with n->m_data != M_START(n)", __func__));
/*
* Calculate the size of the TCP header. We use this often
* enough that it is worth just calculating at the start.
*/
tcp_off = th->th_off << 2;
/* Trim off leading empty mbufs. */
while (m && m->m_len == 0)
m = m->m_next;
if (m) {
m_cur = m;
}
else {
/*
* No data? Highly unusual. We would expect to at
* least see a TCP header in the mbuf.
* As we have a pointer to the TCP header, I guess
* we should just copy that. (???)
*/
fallback:
bytes_to_copy = tcp_off;
if (bytes_to_copy > M_SIZE(n))
bytes_to_copy = M_SIZE(n);
bcopy(th, n->m_data, bytes_to_copy);
n->m_len = bytes_to_copy;
return;
}
/*
* Find TCP header. Record the total number of bytes up to,
* and including, the TCP header.
*/
while (m_cur) {
if ((caddr_t) th >= (caddr_t) m_cur->m_data &&
(caddr_t) th < (caddr_t) (m_cur->m_data + m_cur->m_len))
break;
bytes_to_copy += m_cur->m_len;
m_cur = m_cur->m_next;
}
if (m_cur)
bytes_to_copy += (caddr_t) th - (caddr_t) m_cur->m_data;
else
goto fallback;
bytes_to_copy += tcp_off;
/*
* If we already want to copy more bytes than we can hold
* in the destination mbuf, skip leading bytes and copy
* what we can.
*
* Otherwise, consider trailing data.
*/
if (bytes_to_copy > M_SIZE(n)) {
skip = bytes_to_copy - M_SIZE(n);
bytes_to_copy = M_SIZE(n);
}
else {
/*
* Determine how much trailing data is in the chain.
* We start with the length of this mbuf (the one
* containing th) and subtract the size of the TCP
* header (tcp_off) and the size of the data prior
* to th (th - m_cur->m_data).
*
* This *should not* be negative, as the TCP code
* should put the whole TCP header in a single
* mbuf. But, it isn't a problem if it is. We will
* simple work off our negative balance as we look
* at subsequent mbufs.
*/
trailing_data = m_cur->m_len - tcp_off;
trailing_data -= (caddr_t) th - (caddr_t) m_cur->m_data;
m_cur = m_cur->m_next;
while (m_cur) {
trailing_data += m_cur->m_len;
m_cur = m_cur->m_next;
}
if ((bytes_to_copy + trailing_data) > M_SIZE(n))
bytes_to_copy = M_SIZE(n);
else
bytes_to_copy += trailing_data;
}
m_copydata(m, skip, bytes_to_copy, n->m_data);
n->m_len = bytes_to_copy;
}
