/*
* Copy an entire packet, including header (which must be present).
* An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'.
* Note that the copy is read-only, because clusters are not copied,
* only their reference counts are incremented.
* Preserve alignment of the first mbuf so if the creator has left
* some room at the beginning (e.g. for inserting protocol headers)
* the copies still have the room available.
*/
struct mbuf *
m_copypacket(struct mbuf *m, int how)
{
struct mbuf *top, *n, *o;
MBUF_CHECKSLEEP(how);
n = m_get(how, m->m_type);
top = n;
if (n == NULL)
goto nospace;
if (!m_dup_pkthdr(n, m, how))
goto nospace;
n->m_len = m->m_len;
if (m->m_flags & M_EXT) {
n->m_data = m->m_data;
mb_dupcl(n, m);
} else {
n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat );
bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
}
m = m->m_next;
while (m) {
o = m_get(how, m->m_type);
if (o == NULL)
goto nospace;
n->m_next = o;
n = n->m_next;
n->m_len = m->m_len;
if (m->m_flags & M_EXT) {
n->m_data = m->m_data;
mb_dupcl(n, m);
} else {
bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
}
m = m->m_next;
}
return top;
nospace:
m_freem(top);
return (NULL);
}
/*
* Partition an mbuf chain in two pieces, returning the tail --
* all but the first len0 bytes. In case of failure, it returns NULL and
* attempts to restore the chain to its original state.
*
* Note that the resulting mbufs might be read-only, because the new
* mbuf can end up sharing an mbuf cluster with the original mbuf if
* the "breaking point" happens to lie within a cluster mbuf. Use the
* M_WRITABLE() macro to check for this case.
*/
struct mbuf *
m_split(struct mbuf *m0, int len0, int wait)
{
struct mbuf *m, *n;
u_int len = len0, remain;
MBUF_CHECKSLEEP(wait);
for (m = m0; m && len > m->m_len; m = m->m_next)
len -= m->m_len;
if (m == NULL)
return (NULL);
remain = m->m_len - len;
if (m0->m_flags & M_PKTHDR && remain == 0) {
n = m_gethdr(wait, m0->m_type);
if (n == NULL)
return (NULL);
n->m_next = m->m_next;
m->m_next = NULL;
n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
n->m_pkthdr.len = m0->m_pkthdr.len - len0;
m0->m_pkthdr.len = len0;
return (n);
} else if (m0->m_flags & M_PKTHDR) {
n = m_gethdr(wait, m0->m_type);
if (n == NULL)
return (NULL);
n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
n->m_pkthdr.len = m0->m_pkthdr.len - len0;
m0->m_pkthdr.len = len0;
if (m->m_flags & M_EXT)
goto extpacket;
if (remain > MHLEN) {
/* m can't be the lead packet */
M_ALIGN(n, 0);
n->m_next = m_split(m, len, wait);
if (n->m_next == NULL) {
(void) m_free(n);
return (NULL);
} else {
n->m_len = 0;
return (n);
}
} else
M_ALIGN(n, remain);
} else if (remain == 0) {
n = m->m_next;
m->m_next = NULL;
return (n);
} else {
n = m_get(wait, m->m_type);
if (n == NULL)
return (NULL);
M_ALIGN(n, remain);
}
extpacket:
if (m->m_flags & M_EXT) {
n->m_data = m->m_data + len;
mb_dupcl(n, m);
} else {
bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
}
n->m_len = remain;
m->m_len = len;
n->m_next = m->m_next;
m->m_next = NULL;
return (n);
}
/*
* Make a copy of an mbuf chain starting "off0" bytes from the beginning,
* continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
* The wait parameter is a choice of M_WAITOK/M_NOWAIT from caller.
* Note that the copy is read-only, because clusters are not copied,
* only their reference counts are incremented.
*/
struct mbuf *
m_copym(struct mbuf *m, int off0, int len, int wait)
{
struct mbuf *n, **np;
int off = off0;
struct mbuf *top;
int copyhdr = 0;
KASSERT(off >= 0, ("m_copym, negative off %d", off));
KASSERT(len >= 0, ("m_copym, negative len %d", len));
MBUF_CHECKSLEEP(wait);
if (off == 0 && m->m_flags & M_PKTHDR)
copyhdr = 1;
while (off > 0) {
KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain"));
if (off < m->m_len)
break;
off -= m->m_len;
m = m->m_next;
}
np = ⊤
top = NULL;
while (len > 0) {
if (m == NULL) {
KASSERT(len == M_COPYALL,
("m_copym, length > size of mbuf chain"));
break;
}
if (copyhdr)
n = m_gethdr(wait, m->m_type);
else
n = m_get(wait, m->m_type);
*np = n;
if (n == NULL)
goto nospace;
if (copyhdr) {
if (!m_dup_pkthdr(n, m, wait))
goto nospace;
if (len == M_COPYALL)
n->m_pkthdr.len -= off0;
else
n->m_pkthdr.len = len;
copyhdr = 0;
}
n->m_len = min(len, m->m_len - off);
if (m->m_flags & M_EXT) {
n->m_data = m->m_data + off;
mb_dupcl(n, m);
} else
bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
(u_int)n->m_len);
if (len != M_COPYALL)
len -= n->m_len;
off = 0;
m = m->m_next;
np = &n->m_next;
}
return (top);
nospace:
m_freem(top);
return (NULL);
}
void
tcp_pcap_add(struct tcphdr *th, struct mbuf *m, struct mbufq *queue)
{
struct mbuf *n = NULL, *mhead;
KASSERT(th, ("%s: called with th == NULL", __func__));
KASSERT(m, ("%s: called with m == NULL", __func__));
KASSERT(queue, ("%s: called with queue == NULL", __func__));
/* We only care about data packets. */
while (m && m->m_type != MT_DATA)
m = m->m_next;
/* We only need to do something if we still have an mbuf. */
if (!m)
return;
/* If we are not saving mbufs, return now. */
if (queue->mq_maxlen == 0)
return;
/*
* Check to see if we will need to recycle mbufs.
*
* If we need to get rid of mbufs to stay below
* our packet count, try to reuse the mbuf. Once
* we already have a new mbuf (n), then we can
* simply free subsequent mbufs.
*
* Note that most of the logic in here is to deal
* with the reuse. If we are fine with constant
* mbuf allocs/deallocs, we could ditch this logic.
* But, it only seems to make sense to reuse
* mbufs we already have.
*/
while (mbufq_full(queue)) {
mhead = mbufq_dequeue(queue);
if (n) {
tcp_pcap_m_freem(mhead);
}
else {
/*
* If this held an external cluster, try to
* detach the cluster. But, if we held the
* last reference, go through the normal
* free-ing process.
*/
if (mhead->m_flags & M_EXT) {
switch (mhead->m_ext.ext_type) {
case EXT_SFBUF:
/* Don't mess around with these. */
tcp_pcap_m_freem(mhead);
continue;
default:
if (atomic_fetchadd_int(
mhead->m_ext.ext_cnt, -1) == 1)
{
/*
* We held the last reference
* on this cluster. Restore
* the reference count and put
* it back in the pool.
*/
*(mhead->m_ext.ext_cnt) = 1;
tcp_pcap_m_freem(mhead);
continue;
}
/*
* We were able to cleanly free the
* reference.
*/
atomic_subtract_int(
&tcp_pcap_clusters_referenced_cur,
1);
tcp_pcap_alloc_reuse_ext++;
break;
}
}
else {
tcp_pcap_alloc_reuse_mbuf++;
}
n = mhead;
tcp_pcap_m_freem(n->m_next);
m_init(n, NULL, 0, M_NOWAIT, MT_DATA, 0);
}
}
/* Check to see if we need to get a new mbuf. */
if (!n) {
if (!(n = m_get(M_NOWAIT, MT_DATA)))
return;
tcp_pcap_alloc_new_mbuf++;
}
/*
* What are we dealing with? If a cluster, attach it. Otherwise,
* try to copy the data from the beginning of the mbuf to the
* end of data. (There may be data between the start of the data
* area and the current data pointer. We want to get this, because
* it may contain header information that is useful.)
* In cases where that isn't possible, settle for what we can
* get.
*/
if ((m->m_flags & M_EXT) && tcp_pcap_take_cluster_reference()) {
n->m_data = m->m_data;
n->m_len = m->m_len;
mb_dupcl(n, m);
}
else if (((m->m_data + m->m_len) - M_START(m)) <= M_SIZE(n)) {
/*
* At this point, n is guaranteed to be a normal mbuf
* with no cluster and no packet header. Because the
* logic in this code block requires this, the assert
* is here to catch any instances where someone
* changes the logic to invalidate that assumption.
*/
KASSERT((n->m_flags & (M_EXT | M_PKTHDR)) == 0,
("%s: Unexpected flags (%#x) for mbuf",
__func__, n->m_flags));
n->m_data = n->m_dat + M_LEADINGSPACE_NOWRITE(m);
n->m_len = m->m_len;
bcopy(M_START(m), n->m_dat,
m->m_len + M_LEADINGSPACE_NOWRITE(m));
}
else {
/*
* This is the case where we need to "settle for what
* we can get". The most probable way to this code
* path is that we've already taken references to the
* maximum number of mbuf clusters we can, and the data
* is too long to fit in an mbuf's internal storage.
* Try for a "best fit".
*/
tcp_pcap_copy_bestfit(th, m, n);
/* Don't try to get additional data. */
goto add_to_queue;
}
if (m->m_next) {
n->m_next = m_copym(m->m_next, 0, M_COPYALL, M_NOWAIT);
tcp_pcap_adj_cluster_reference(n->m_next, 1);
}
add_to_queue:
/* Add the new mbuf to the list. */
if (mbufq_enqueue(queue, n)) {
/* This shouldn't happen. If INVARIANTS is defined, panic. */
KASSERT(0, ("%s: mbufq was unexpectedly full!", __func__));
tcp_pcap_m_freem(n);
}
}