/* * Attempt to merge two adjacent blocks (mh below mhnext). */ static void __malloc_trymerge(struct mheader *mh, struct mheader *mhnext) { struct mheader *mhnextnext; if (mh->mh_nextblock != mhnext->mh_prevblock) { errx(1, "free: Heap corrupt (%p and %p inconsistent)", mh, mhnext); } if (mh->mh_inuse || mhnext->mh_inuse) { /* can't merge */ return; } mhnextnext = M_NEXT(mhnext); mh->mh_nextblock = M_MKFIELD(MBLOCKSIZE + M_SIZE(mh) + MBLOCKSIZE + M_SIZE(mhnext)); if (mhnextnext != (struct mheader *)__heaptop) { mhnextnext->mh_prevblock = mh->mh_nextblock; } /* Deadbeef out the memory used by the now-obsolete header */ __malloc_deadbeef(mhnext, sizeof(struct mheader)); }
/** * Free memory. Merely a wrapper for the case that we * want to keep track of allocations. * * @param ptr the pointer to free * @param filename where in the code was the call to GNUNET_free * @param linenumber where in the code was the call to GNUNET_free */ void GNUNET_xfree_ (void *ptr, const char *filename, int linenumber) { GNUNET_assert_at (NULL != ptr, filename, linenumber); #ifdef W32_MEM_LIMIT ptr = &((size_t *) ptr)[-1]; mem_used -= *((size_t *) ptr); #endif #if defined(M_SIZE) #if ENABLE_POISONING { const uint64_t baadfood = GNUNET_ntohll (0xBAADF00DBAADF00DLL); uint64_t *base = ptr; size_t s = M_SIZE (ptr); size_t i; for (i=0;i<s/8;i++) base[i] = baadfood; GNUNET_memcpy (&base[s/8], &baadfood, s % 8); } #endif #endif free (ptr); }
/* * The actual free() implementation. */ void free(void *x) { struct mheader *mh, *mhnext, *mhprev; if (x==NULL) { /* safest practice */ return; } /* Consistency check. */ if (__heapbase==0 || __heaptop==0 || __heapbase > __heaptop) { warnx("free: Internal error - local data corrupt"); errx(1, "free: heapbase 0x%lx; heaptop 0x%lx", (unsigned long) __heapbase, (unsigned long) __heaptop); } /* Don't allow freeing pointers that aren't on the heap. */ if ((uintptr_t)x < __heapbase || (uintptr_t)x >= __heaptop) { errx(1, "free: Invalid pointer %p freed (out of range)", x); } #ifdef MALLOCDEBUG warnx("free: about to free %p", x); __malloc_dump(); #endif mh = ((struct mheader *)x)-1; if (!M_OK(mh)) { errx(1, "free: Invalid pointer %p freed (corrupt header)", x); } if (!mh->mh_inuse) { errx(1, "free: Invalid pointer %p freed (already free)", x); } /* mark it free */ mh->mh_inuse = 0; /* wipe it */ __malloc_deadbeef(M_DATA(mh), M_SIZE(mh)); /* Try merging with the block above (but not if we're at the top) */ mhnext = M_NEXT(mh); if (mhnext != (struct mheader *)__heaptop) { __malloc_trymerge(mh, mhnext); } /* Try merging with the block below (but not if we're at the bottom) */ if (mh != (struct mheader *)__heapbase) { mhprev = M_PREV(mh); __malloc_trymerge(mhprev, mh); } #ifdef MALLOCDEBUG warnx("free: freed %p", x); __malloc_dump(); #endif }
/* * Allocate a given length worth of mbufs and/or clusters (whatever fits * best) and return a pointer to the top of the allocated chain. If an * existing mbuf chain is provided, then we will append the new chain * to the existing one but still return the top of the newly allocated * chain. */ struct mbuf * m_getm2(struct mbuf *m, int len, int how, short type, int flags) { struct mbuf *mb, *nm = NULL, *mtail = NULL; KASSERT(len >= 0, ("%s: len is < 0", __func__)); /* Validate flags. */ flags &= (M_PKTHDR | M_EOR); /* Packet header mbuf must be first in chain. */ if ((flags & M_PKTHDR) && m != NULL) flags &= ~M_PKTHDR; /* Loop and append maximum sized mbufs to the chain tail. */ while (len > 0) { if (len > MCLBYTES) mb = m_getjcl(how, type, (flags & M_PKTHDR), MJUMPAGESIZE); else if (len >= MINCLSIZE) mb = m_getcl(how, type, (flags & M_PKTHDR)); else if (flags & M_PKTHDR) mb = m_gethdr(how, type); else mb = m_get(how, type); /* Fail the whole operation if one mbuf can't be allocated. */ if (mb == NULL) { if (nm != NULL) m_freem(nm); return (NULL); } /* Book keeping. */ len -= M_SIZE(mb); if (mtail != NULL) mtail->m_next = mb; else nm = mb; mtail = mb; flags &= ~M_PKTHDR; /* Only valid on the first mbuf. */ } if (flags & M_EOR) mtail->m_flags |= M_EOR; /* Only valid on the last mbuf. */ /* If mbuf was supplied, append new chain to the end of it. */ if (m != NULL) { for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next) ; mtail->m_next = nm; mtail->m_flags &= ~M_EOR; } else m = nm; return (m); }
/* * Make a new (free) block from the block passed in, leaving size * bytes for data in the current block. size must be a multiple of * MBLOCKSIZE. * * Only split if the excess space is at least twice the blocksize - * one blocksize to hold a header and one for data. */ static void __malloc_split(struct mheader *mh, size_t size) { struct mheader *mhnext, *mhnew; size_t oldsize; if (size % MBLOCKSIZE != 0) { errx(1, "malloc: Internal error (size %lu passed to split)", (unsigned long) size); } if (M_SIZE(mh) - size < 2*MBLOCKSIZE) { /* no room */ return; } mhnext = M_NEXT(mh); oldsize = M_SIZE(mh); mh->mh_nextblock = M_MKFIELD(size + MBLOCKSIZE); mhnew = M_NEXT(mh); if (mhnew==mhnext) { errx(1, "malloc: Internal error (split screwed up?)"); } mhnew->mh_prevblock = M_MKFIELD(size + MBLOCKSIZE); mhnew->mh_pad = 0; mhnew->mh_magic1 = MMAGIC; mhnew->mh_nextblock = M_MKFIELD(oldsize - size); mhnew->mh_inuse = 0; mhnew->mh_magic2 = MMAGIC; if (mhnext != (struct mheader *) __heaptop) { mhnext->mh_prevblock = mhnew->mh_nextblock; } }
/* * Debugging print function to iterate and dump the entire heap. */ static void __malloc_dump(void) { struct mheader *mh; uintptr_t i; size_t rightprevblock; warnx("heap: ************************************************"); rightprevblock = 0; for (i=__heapbase; i<__heaptop; i += M_NEXTOFF(mh)) { mh = (struct mheader *) i; if (!M_OK(mh)) { errx(1, "malloc: Heap corrupt; header at 0x%lx" " has bad magic bits", (unsigned long) i); } if (mh->mh_prevblock != rightprevblock) { errx(1, "malloc: Heap corrupt; header at 0x%lx" " has bad previous-block size %lu " "(should be %lu)", (unsigned long) i, (unsigned long) mh->mh_prevblock << MBLOCKSHIFT, (unsigned long) rightprevblock << MBLOCKSHIFT); } rightprevblock = mh->mh_nextblock; warnx("heap: 0x%lx 0x%-6lx (next: 0x%lx) %s", (unsigned long) i + MBLOCKSIZE, (unsigned long) M_SIZE(mh), (unsigned long) (i+M_NEXTOFF(mh)), mh->mh_inuse ? "INUSE" : "FREE"); } if (i!=__heaptop) { errx(1, "malloc: Heap corrupt; ran off end"); } warnx("heap: ************************************************"); }
/* * Lesser-used path for M_PREPEND: * allocate new mbuf to prepend to chain, * copy junk along. */ struct mbuf * m_prepend(struct mbuf *m, int len, int how) { struct mbuf *mn; if (m->m_flags & M_PKTHDR) mn = m_gethdr(how, m->m_type); else mn = m_get(how, m->m_type); if (mn == NULL) { m_freem(m); return (NULL); } if (m->m_flags & M_PKTHDR) m_move_pkthdr(mn, m); mn->m_next = m; m = mn; if (len < M_SIZE(m)) M_ALIGN(m, len); m->m_len = len; return (m); }
/* * Sanity checks on mbuf (chain) for use in KASSERT() and general * debugging. * Returns 0 or panics when bad and 1 on all tests passed. * Sanitize, 0 to run M_SANITY_ACTION, 1 to garble things so they * blow up later. */ int m_sanity(struct mbuf *m0, int sanitize) { struct mbuf *m; caddr_t a, b; int pktlen = 0; #ifdef INVARIANTS #define M_SANITY_ACTION(s) panic("mbuf %p: " s, m) #else #define M_SANITY_ACTION(s) printf("mbuf %p: " s, m) #endif for (m = m0; m != NULL; m = m->m_next) { /* * Basic pointer checks. If any of these fails then some * unrelated kernel memory before or after us is trashed. * No way to recover from that. */ a = M_START(m); b = a + M_SIZE(m); if ((caddr_t)m->m_data < a) M_SANITY_ACTION("m_data outside mbuf data range left"); if ((caddr_t)m->m_data > b) M_SANITY_ACTION("m_data outside mbuf data range right"); if ((caddr_t)m->m_data + m->m_len > b) M_SANITY_ACTION("m_data + m_len exeeds mbuf space"); /* m->m_nextpkt may only be set on first mbuf in chain. */ if (m != m0 && m->m_nextpkt != NULL) { if (sanitize) { m_freem(m->m_nextpkt); m->m_nextpkt = (struct mbuf *)0xDEADC0DE; } else M_SANITY_ACTION("m->m_nextpkt on in-chain mbuf"); } /* packet length (not mbuf length!) calculation */ if (m0->m_flags & M_PKTHDR) pktlen += m->m_len; /* m_tags may only be attached to first mbuf in chain. */ if (m != m0 && m->m_flags & M_PKTHDR && !SLIST_EMPTY(&m->m_pkthdr.tags)) { if (sanitize) { m_tag_delete_chain(m, NULL); /* put in 0xDEADC0DE perhaps? */ } else M_SANITY_ACTION("m_tags on in-chain mbuf"); } /* M_PKTHDR may only be set on first mbuf in chain */ if (m != m0 && m->m_flags & M_PKTHDR) { if (sanitize) { bzero(&m->m_pkthdr, sizeof(m->m_pkthdr)); m->m_flags &= ~M_PKTHDR; /* put in 0xDEADCODE and leave hdr flag in */ } else M_SANITY_ACTION("M_PKTHDR on in-chain mbuf"); } } m = m0; if (pktlen && pktlen != m->m_pkthdr.len) { if (sanitize) m->m_pkthdr.len = 0; else M_SANITY_ACTION("m_pkthdr.len != mbuf chain length"); } return 1; #undef M_SANITY_ACTION }
void tcp_pcap_add(struct tcphdr *th, struct mbuf *m, struct mbufq *queue) { struct mbuf *n = NULL, *mhead; KASSERT(th, ("%s: called with th == NULL", __func__)); KASSERT(m, ("%s: called with m == NULL", __func__)); KASSERT(queue, ("%s: called with queue == NULL", __func__)); /* We only care about data packets. */ while (m && m->m_type != MT_DATA) m = m->m_next; /* We only need to do something if we still have an mbuf. */ if (!m) return; /* If we are not saving mbufs, return now. */ if (queue->mq_maxlen == 0) return; /* * Check to see if we will need to recycle mbufs. * * If we need to get rid of mbufs to stay below * our packet count, try to reuse the mbuf. Once * we already have a new mbuf (n), then we can * simply free subsequent mbufs. * * Note that most of the logic in here is to deal * with the reuse. If we are fine with constant * mbuf allocs/deallocs, we could ditch this logic. * But, it only seems to make sense to reuse * mbufs we already have. */ while (mbufq_full(queue)) { mhead = mbufq_dequeue(queue); if (n) { tcp_pcap_m_freem(mhead); } else { /* * If this held an external cluster, try to * detach the cluster. But, if we held the * last reference, go through the normal * free-ing process. */ if (mhead->m_flags & M_EXT) { switch (mhead->m_ext.ext_type) { case EXT_SFBUF: /* Don't mess around with these. */ tcp_pcap_m_freem(mhead); continue; default: if (atomic_fetchadd_int( mhead->m_ext.ext_cnt, -1) == 1) { /* * We held the last reference * on this cluster. Restore * the reference count and put * it back in the pool. */ *(mhead->m_ext.ext_cnt) = 1; tcp_pcap_m_freem(mhead); continue; } /* * We were able to cleanly free the * reference. */ atomic_subtract_int( &tcp_pcap_clusters_referenced_cur, 1); tcp_pcap_alloc_reuse_ext++; break; } } else { tcp_pcap_alloc_reuse_mbuf++; } n = mhead; tcp_pcap_m_freem(n->m_next); m_init(n, NULL, 0, M_NOWAIT, MT_DATA, 0); } } /* Check to see if we need to get a new mbuf. */ if (!n) { if (!(n = m_get(M_NOWAIT, MT_DATA))) return; tcp_pcap_alloc_new_mbuf++; } /* * What are we dealing with? If a cluster, attach it. Otherwise, * try to copy the data from the beginning of the mbuf to the * end of data. (There may be data between the start of the data * area and the current data pointer. We want to get this, because * it may contain header information that is useful.) * In cases where that isn't possible, settle for what we can * get. */ if ((m->m_flags & M_EXT) && tcp_pcap_take_cluster_reference()) { n->m_data = m->m_data; n->m_len = m->m_len; mb_dupcl(n, m); } else if (((m->m_data + m->m_len) - M_START(m)) <= M_SIZE(n)) { /* * At this point, n is guaranteed to be a normal mbuf * with no cluster and no packet header. Because the * logic in this code block requires this, the assert * is here to catch any instances where someone * changes the logic to invalidate that assumption. */ KASSERT((n->m_flags & (M_EXT | M_PKTHDR)) == 0, ("%s: Unexpected flags (%#x) for mbuf", __func__, n->m_flags)); n->m_data = n->m_dat + M_LEADINGSPACE_NOWRITE(m); n->m_len = m->m_len; bcopy(M_START(m), n->m_dat, m->m_len + M_LEADINGSPACE_NOWRITE(m)); } else { /* * This is the case where we need to "settle for what * we can get". The most probable way to this code * path is that we've already taken references to the * maximum number of mbuf clusters we can, and the data * is too long to fit in an mbuf's internal storage. * Try for a "best fit". */ tcp_pcap_copy_bestfit(th, m, n); /* Don't try to get additional data. */ goto add_to_queue; } if (m->m_next) { n->m_next = m_copym(m->m_next, 0, M_COPYALL, M_NOWAIT); tcp_pcap_adj_cluster_reference(n->m_next, 1); } add_to_queue: /* Add the new mbuf to the list. */ if (mbufq_enqueue(queue, n)) { /* This shouldn't happen. If INVARIANTS is defined, panic. */ KASSERT(0, ("%s: mbufq was unexpectedly full!", __func__)); tcp_pcap_m_freem(n); } }
/* * Copy data from m to n, where n cannot fit all the data we might * want from m. * * Prioritize data like this: * 1. TCP header * 2. IP header * 3. Data */ static void tcp_pcap_copy_bestfit(struct tcphdr *th, struct mbuf *m, struct mbuf *n) { struct mbuf *m_cur = m; int bytes_to_copy=0, trailing_data, skip=0, tcp_off; /* Below, we assume these will be non-NULL. */ KASSERT(th, ("%s: called with th == NULL", __func__)); KASSERT(m, ("%s: called with m == NULL", __func__)); KASSERT(n, ("%s: called with n == NULL", __func__)); /* We assume this initialization occurred elsewhere. */ KASSERT(n->m_len == 0, ("%s: called with n->m_len=%d (expected 0)", __func__, n->m_len)); KASSERT(n->m_data == M_START(n), ("%s: called with n->m_data != M_START(n)", __func__)); /* * Calculate the size of the TCP header. We use this often * enough that it is worth just calculating at the start. */ tcp_off = th->th_off << 2; /* Trim off leading empty mbufs. */ while (m && m->m_len == 0) m = m->m_next; if (m) { m_cur = m; } else { /* * No data? Highly unusual. We would expect to at * least see a TCP header in the mbuf. * As we have a pointer to the TCP header, I guess * we should just copy that. (???) */ fallback: bytes_to_copy = tcp_off; if (bytes_to_copy > M_SIZE(n)) bytes_to_copy = M_SIZE(n); bcopy(th, n->m_data, bytes_to_copy); n->m_len = bytes_to_copy; return; } /* * Find TCP header. Record the total number of bytes up to, * and including, the TCP header. */ while (m_cur) { if ((caddr_t) th >= (caddr_t) m_cur->m_data && (caddr_t) th < (caddr_t) (m_cur->m_data + m_cur->m_len)) break; bytes_to_copy += m_cur->m_len; m_cur = m_cur->m_next; } if (m_cur) bytes_to_copy += (caddr_t) th - (caddr_t) m_cur->m_data; else goto fallback; bytes_to_copy += tcp_off; /* * If we already want to copy more bytes than we can hold * in the destination mbuf, skip leading bytes and copy * what we can. * * Otherwise, consider trailing data. */ if (bytes_to_copy > M_SIZE(n)) { skip = bytes_to_copy - M_SIZE(n); bytes_to_copy = M_SIZE(n); } else { /* * Determine how much trailing data is in the chain. * We start with the length of this mbuf (the one * containing th) and subtract the size of the TCP * header (tcp_off) and the size of the data prior * to th (th - m_cur->m_data). * * This *should not* be negative, as the TCP code * should put the whole TCP header in a single * mbuf. But, it isn't a problem if it is. We will * simple work off our negative balance as we look * at subsequent mbufs. */ trailing_data = m_cur->m_len - tcp_off; trailing_data -= (caddr_t) th - (caddr_t) m_cur->m_data; m_cur = m_cur->m_next; while (m_cur) { trailing_data += m_cur->m_len; m_cur = m_cur->m_next; } if ((bytes_to_copy + trailing_data) > M_SIZE(n)) bytes_to_copy = M_SIZE(n); else bytes_to_copy += trailing_data; } m_copydata(m, skip, bytes_to_copy, n->m_data); n->m_len = bytes_to_copy; }
/* * malloc itself. */ void * malloc(size_t size) { struct mheader *mh; uintptr_t i; size_t rightprevblock; if (__heapbase==0) { __malloc_init(); } if (__heapbase==0 || __heaptop==0 || __heapbase > __heaptop) { warnx("malloc: Internal error - local data corrupt"); errx(1, "malloc: heapbase 0x%lx; heaptop 0x%lx", (unsigned long) __heapbase, (unsigned long) __heaptop); } #ifdef MALLOCDEBUG warnx("malloc: about to allocate %lu (0x%lx) bytes", (unsigned long) size, (unsigned long) size); __malloc_dump(); #endif /* Round size up to an integral number of blocks. */ size = ((size + MBLOCKSIZE - 1) & ~(size_t)(MBLOCKSIZE-1)); /* * First-fit search algorithm for available blocks. * Check to make sure the next/previous sizes all agree. */ rightprevblock = 0; for (i=__heapbase; i<__heaptop; i += M_NEXTOFF(mh)) { mh = (struct mheader *) i; if (!M_OK(mh)) { errx(1, "malloc: Heap corrupt; header at 0x%lx" " has bad magic bits", (unsigned long) i); } if (mh->mh_prevblock != rightprevblock) { errx(1, "malloc: Heap corrupt; header at 0x%lx" " has bad previous-block size %lu " "(should be %lu)", (unsigned long) i, (unsigned long) mh->mh_prevblock << MBLOCKSHIFT, (unsigned long) rightprevblock << MBLOCKSHIFT); } rightprevblock = mh->mh_nextblock; /* Can't allocate a block that's in use. */ if (mh->mh_inuse) { continue; } /* Can't allocate a block that isn't big enough. */ if (M_SIZE(mh) < size) { continue; } /* Try splitting block. */ __malloc_split(mh, size); /* * Now, allocate. */ mh->mh_inuse = 1; #ifdef MALLOCDEBUG warnx("malloc: allocating at %p", M_DATA(mh)); __malloc_dump(); #endif return M_DATA(mh); } if (i!=__heaptop) { errx(1, "malloc: Heap corrupt; ran off end"); } /* * Didn't find anything. Expand the heap. */ mh = __malloc_sbrk(size + MBLOCKSIZE); if (mh == NULL) { return NULL; } mh->mh_prevblock = rightprevblock; mh->mh_magic1 = MMAGIC; mh->mh_magic2 = MMAGIC; mh->mh_pad = 0; mh->mh_inuse = 1; mh->mh_nextblock = M_MKFIELD(size + MBLOCKSIZE); #ifdef MALLOCDEBUG warnx("malloc: allocating at %p", M_DATA(mh)); __malloc_dump(); #endif return M_DATA(mh); }