static void *
__malloc_from_heap (size_t size, struct heap_free_area **heap
#ifdef HEAP_USE_LOCKING
, malloc_mutex_t *heap_lock
#endif
)
{
void *mem;
MALLOC_DEBUG (1, "malloc: %d bytes", size);
/* Include extra space to record the size of the allocated block. */
size += MALLOC_HEADER_SIZE;
__heap_lock (heap_lock);
/* First try to get memory that's already in our heap. */
mem = __heap_alloc (heap, &size);
__heap_unlock (heap_lock);
if (unlikely (! mem))
/* We couldn't allocate from the heap, so grab some more
from the system, add it to the heap, and try again. */
{
/* If we're trying to allocate a block bigger than the default
MALLOC_HEAP_EXTEND_SIZE, make sure we get enough to hold it. */
void *block;
size_t block_size
= (size < MALLOC_HEAP_EXTEND_SIZE
? MALLOC_HEAP_EXTEND_SIZE
: MALLOC_ROUND_UP_TO_PAGE_SIZE (size));
/* Allocate the new heap block. */
#ifdef MALLOC_USE_SBRK
__malloc_lock_sbrk ();
/* Use sbrk we can, as it's faster than mmap, and guarantees
contiguous allocation. */
block = sbrk (block_size);
if (likely (block != (void *)-1))
{
/* Because sbrk can return results of arbitrary
alignment, align the result to a MALLOC_ALIGNMENT boundary. */
long aligned_block = MALLOC_ROUND_UP ((long)block, MALLOC_ALIGNMENT);
if (block != (void *)aligned_block)
/* Have to adjust. We should only have to actually do this
the first time (after which we will have aligned the brk
correctly). */
{
/* Move the brk to reflect the alignment; our next allocation
should start on exactly the right alignment. */
sbrk (aligned_block - (long)block);
block = (void *)aligned_block;
}
}
__malloc_unlock_sbrk ();
#else /* !MALLOC_USE_SBRK */
/* Otherwise, use mmap. */
#ifdef __ARCH_USE_MMU__
block = mmap ((void *)0, block_size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
#else
block = mmap ((void *)0, block_size, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, 0, 0);
#endif
#endif /* MALLOC_USE_SBRK */
if (likely (block != (void *)-1))
{
#if !defined(MALLOC_USE_SBRK) && defined(__UCLIBC_UCLINUX_BROKEN_MUNMAP__)
struct malloc_mmb *mmb, *prev_mmb, *new_mmb;
#endif
MALLOC_DEBUG (1, "adding system memory to heap: 0x%lx - 0x%lx (%d bytes)",
(long)block, (long)block + block_size, block_size);
/* Get back the heap lock. */
__heap_lock (heap_lock);
/* Put BLOCK into the heap. */
__heap_free (heap, block, block_size);
MALLOC_DEBUG_INDENT (-1);
/* Try again to allocate. */
mem = __heap_alloc (heap, &size);
#if !defined(MALLOC_USE_SBRK) && defined(__UCLIBC_UCLINUX_BROKEN_MUNMAP__)
/* Insert a record of BLOCK in sorted order into the
__malloc_mmapped_blocks list. */
new_mmb = malloc_from_heap (sizeof *new_mmb, &__malloc_mmb_heap, &__malloc_mmb_heap_lock);
for (prev_mmb = 0, mmb = __malloc_mmapped_blocks;
mmb;
prev_mmb = mmb, mmb = mmb->next)
if (block < mmb->mem)
break;
new_mmb->next = mmb;
new_mmb->mem = block;
new_mmb->size = block_size;
if (prev_mmb)
prev_mmb->next = new_mmb;
else
__malloc_mmapped_blocks = new_mmb;
MALLOC_MMB_DEBUG (0, "new mmb at 0x%x: 0x%x[%d]",
(unsigned)new_mmb,
(unsigned)new_mmb->mem, block_size);
#endif /* !MALLOC_USE_SBRK && __UCLIBC_UCLINUX_BROKEN_MUNMAP__ */
__heap_unlock (heap_lock);
}
}
if (likely (mem))
/* Record the size of the block and get the user address. */
{
mem = MALLOC_SETUP (mem, size);
MALLOC_DEBUG (-1, "malloc: returning 0x%lx (base:0x%lx, total_size:%ld)",
(long)mem, (long)MALLOC_BASE(mem), (long)MALLOC_SIZE(mem));
}
else
MALLOC_DEBUG (-1, "malloc: returning 0");
return mem;
}
static void
__free_to_heap (void *mem, struct heap_free_area **heap
#ifdef HEAP_USE_LOCKING
, __UCLIBC_MUTEX_TYPE *heap_lock
#endif
)
{
size_t size;
struct heap_free_area *fa;
/* Check for special cases. */
if (unlikely (! mem))
return;
/* Normal free. */
MALLOC_DEBUG (1, "free: 0x%lx (base = 0x%lx, total_size = %d)",
(long)mem, (long)MALLOC_BASE (mem), MALLOC_SIZE (mem));
size = MALLOC_SIZE (mem);
mem = MALLOC_BASE (mem);
__heap_lock (heap_lock);
/* Put MEM back in the heap, and get the free-area it was placed in. */
fa = __heap_free (heap, mem, size);
/* See if the free-area FA has grown big enough that it should be
unmapped. */
if (HEAP_FREE_AREA_SIZE (fa) < MALLOC_UNMAP_THRESHOLD)
/* Nope, nothing left to do, just release the lock. */
__heap_unlock (heap_lock);
else
/* Yup, try to unmap FA. */
{
unsigned long start = (unsigned long)HEAP_FREE_AREA_START (fa);
unsigned long end = (unsigned long)HEAP_FREE_AREA_END (fa);
#ifndef MALLOC_USE_SBRK
# ifdef __UCLIBC_UCLINUX_BROKEN_MUNMAP__
struct malloc_mmb *mmb, *prev_mmb;
unsigned long mmb_start, mmb_end;
# else /* !__UCLIBC_UCLINUX_BROKEN_MUNMAP__ */
unsigned long unmap_start, unmap_end;
# endif /* __UCLIBC_UCLINUX_BROKEN_MUNMAP__ */
#endif /* !MALLOC_USE_SBRK */
#ifdef MALLOC_USE_SBRK
/* Get the sbrk lock so that the two possible calls to sbrk below
are guaranteed to be contiguous. */
__malloc_lock_sbrk ();
/* When using sbrk, we only shrink the heap from the end. It would
be possible to allow _both_ -- shrinking via sbrk when possible,
and otherwise shrinking via munmap, but this results in holes in
memory that prevent the brk from every growing back down; since
we only ever grow the heap via sbrk, this tends to produce a
continuously growing brk (though the actual memory is unmapped),
which could eventually run out of address space. Note that
`sbrk(0)' shouldn't normally do a system call, so this test is
reasonably cheap. */
if ((void *)end != sbrk (0))
{
MALLOC_DEBUG (-1, "not unmapping: 0x%lx - 0x%lx (%ld bytes)",
start, end, end - start);
__malloc_unlock_sbrk ();
__heap_unlock (heap_lock);
return;
}
#endif
MALLOC_DEBUG (0, "unmapping: 0x%lx - 0x%lx (%ld bytes)",
start, end, end - start);
/* Remove FA from the heap. */
__heap_delete (heap, fa);
if (__heap_is_empty (heap))
/* We want to avoid the heap from losing all memory, so reserve
a bit. This test is only a heuristic -- the existance of
another free area, even if it's smaller than
MALLOC_MIN_SIZE, will cause us not to reserve anything. */
{
/* Put the reserved memory back in the heap; we assume that
MALLOC_UNMAP_THRESHOLD is greater than MALLOC_MIN_SIZE, so
we use the latter unconditionally here. */
__heap_free (heap, (void *)start, MALLOC_MIN_SIZE);
start += MALLOC_MIN_SIZE;
}
#ifdef MALLOC_USE_SBRK
/* Release the heap lock; we're still holding the sbrk lock. */
__heap_unlock (heap_lock);
/* Lower the brk. */
sbrk (start - end);
/* Release the sbrk lock too; now we hold no locks. */
__malloc_unlock_sbrk ();
#else /* !MALLOC_USE_SBRK */
# ifdef __UCLIBC_UCLINUX_BROKEN_MUNMAP__
/* Using the uClinux broken munmap, we have to only munmap blocks
exactly as we got them from mmap, so scan through our list of
mmapped blocks, and return them in order. */
MALLOC_MMB_DEBUG (1, "walking mmb list for region 0x%x[%d]...",
start, end - start);
prev_mmb = 0;
mmb = __malloc_mmapped_blocks;
while (mmb
&& ((mmb_end = (mmb_start = (unsigned long)mmb->mem) + mmb->size)
<= end))
{
MALLOC_MMB_DEBUG (1, "considering mmb at 0x%x: 0x%x[%d]",
(unsigned)mmb, mmb_start, mmb_end - mmb_start);
if (mmb_start >= start
/* If the space between START and MMB_START is non-zero, but
too small to return to the heap, we can't unmap MMB. */
&& (start == mmb_start
|| mmb_start - start > HEAP_MIN_FREE_AREA_SIZE))
{
struct malloc_mmb *next_mmb = mmb->next;
if (mmb_end != end && mmb_end + HEAP_MIN_FREE_AREA_SIZE > end)
/* There's too little space left at the end to deallocate
this block, so give up. */
break;
MALLOC_MMB_DEBUG (1, "unmapping mmb at 0x%x: 0x%x[%d]",
(unsigned)mmb, mmb_start, mmb_end - mmb_start);
if (mmb_start != start)
/* We're going to unmap a part of the heap that begins after
start, so put the intervening region back into the heap. */
{
MALLOC_MMB_DEBUG (0, "putting intervening region back into heap: 0x%x[%d]",
start, mmb_start - start);
__heap_free (heap, (void *)start, mmb_start - start);
}
MALLOC_MMB_DEBUG_INDENT (-1);
/* Unlink MMB from the list. */
if (prev_mmb)
prev_mmb->next = next_mmb;
else
__malloc_mmapped_blocks = next_mmb;
/* Start searching again from the end of this block. */
start = mmb_end;
/* Release the descriptor block we used. */
free_to_heap (mmb, &__malloc_mmb_heap, &__malloc_mmb_heap_lock);
/* We have to unlock the heap before we recurse to free the mmb
descriptor, because we might be unmapping from the mmb
heap. */
__heap_unlock (heap_lock);
/* Do the actual munmap. */
munmap ((void *)mmb_start, mmb_end - mmb_start);
__heap_lock (heap_lock);
# ifdef __UCLIBC_HAS_THREADS__
/* In a multi-threaded program, it's possible that PREV_MMB has
been invalidated by another thread when we released the
heap lock to do the munmap system call, so just start over
from the beginning of the list. It sucks, but oh well;
it's probably not worth the bother to do better. */
prev_mmb = 0;
mmb = __malloc_mmapped_blocks;
# else
mmb = next_mmb;
# endif
}
else
{
prev_mmb = mmb;
mmb = mmb->next;
}
MALLOC_MMB_DEBUG_INDENT (-1);
}
if (start != end)
/* Hmm, well there's something we couldn't unmap, so put it back
into the heap. */
{
MALLOC_MMB_DEBUG (0, "putting tail region back into heap: 0x%x[%d]",
start, end - start);
__heap_free (heap, (void *)start, end - start);
}
/* Finally release the lock for good. */
__heap_unlock (heap_lock);
MALLOC_MMB_DEBUG_INDENT (-1);
# else /* !__UCLIBC_UCLINUX_BROKEN_MUNMAP__ */
/* MEM/LEN may not be page-aligned, so we have to page-align them,
and return any left-over bits on the end to the heap. */
unmap_start = MALLOC_ROUND_UP_TO_PAGE_SIZE (start);
unmap_end = MALLOC_ROUND_DOWN_TO_PAGE_SIZE (end);
/* We have to be careful that any left-over bits are large enough to
return. Note that we _don't check_ to make sure there's room to
grow/shrink the start/end by another page, we just assume that
the unmap threshold is high enough so that this is always safe
(i.e., it should probably be at least 3 pages). */
if (unmap_start > start)
{
if (unmap_start - start < HEAP_MIN_FREE_AREA_SIZE)
unmap_start += MALLOC_PAGE_SIZE;
__heap_free (heap, (void *)start, unmap_start - start);
}
if (end > unmap_end)
{
if (end - unmap_end < HEAP_MIN_FREE_AREA_SIZE)
unmap_end -= MALLOC_PAGE_SIZE;
__heap_free (heap, (void *)unmap_end, end - unmap_end);
}
/* Release the heap lock before we do the system call. */
__heap_unlock (heap_lock);
if (unmap_end > unmap_start)
/* Finally, actually unmap the memory. */
munmap ((void *)unmap_start, unmap_end - unmap_start);
# endif /* __UCLIBC_UCLINUX_BROKEN_MUNMAP__ */
#endif /* MALLOC_USE_SBRK */
}
MALLOC_DEBUG_INDENT (-1);
}
void *
memalign (size_t alignment, size_t size)
{
void *mem, *base;
unsigned long tot_addr, tot_end_addr, addr, end_addr;
struct heap_free_area **heap = &__malloc_heap;
/* Make SIZE something we like. */
size = HEAP_ADJUST_SIZE (size);
/* Use malloc to do the initial allocation, since it deals with getting
system memory. We over-allocate enough to be sure that we'll get
enough memory to hold a properly aligned block of size SIZE,
_somewhere_ in the result. */
mem = malloc (size + 2 * alignment);
if (! mem)
/* Allocation failed, we can't do anything. */
return 0;
if (alignment < MALLOC_ALIGNMENT)
return mem;
/* Remember the base-address, of the allocation, although we normally
use the user-address for calculations, since that's where the
alignment matters. */
base = MALLOC_BASE (mem);
/* The bounds of the initial allocation. */
tot_addr = (unsigned long)mem;
tot_end_addr = (unsigned long)base + MALLOC_SIZE (mem);
/* Find a likely place inside MEM with the right alignment. */
addr = MALLOC_ROUND_UP (tot_addr, alignment);
/* Unless TOT_ADDR was already aligned correctly, we need to return the
initial part of MEM to the heap. */
if (addr != tot_addr)
{
size_t init_size = addr - tot_addr;
/* Ensure that memory returned to the heap is large enough. */
if (init_size < HEAP_MIN_SIZE)
{
addr = MALLOC_ROUND_UP (tot_addr + HEAP_MIN_SIZE, alignment);
init_size = addr - tot_addr;
}
__heap_free (heap, base, init_size);
/* Remember that we've freed the initial part of MEM. */
base += init_size;
}
/* Return the end part of MEM to the heap, unless it's too small. */
end_addr = addr + size;
if (end_addr + MALLOC_REALLOC_MIN_FREE_SIZE < tot_end_addr)
__heap_free (heap, (void *)end_addr, tot_end_addr - end_addr);
else
/* We didn't free the end, so include it in the size. */
end_addr = tot_end_addr;
return MALLOC_SETUP (base, end_addr - (unsigned long)base);
}
void *
realloc (void *mem, size_t new_size)
{
size_t size;
char *base_mem;
/* Check for special cases. */
if (! new_size)
{
free (mem);
return malloc (new_size);
}
if (! mem)
return malloc (new_size);
/* Normal realloc. */
base_mem = MALLOC_BASE (mem);
size = MALLOC_SIZE (mem);
/* Include extra space to record the size of the allocated block.
Also make sure that we're dealing in a multiple of the heap
allocation unit (SIZE is already guaranteed to be so).*/
new_size = HEAP_ADJUST_SIZE (new_size + MALLOC_HEADER_SIZE);
if (new_size < sizeof (struct heap_free_area))
/* Because we sometimes must use a freed block to hold a free-area node,
we must make sure that every allocated block can hold one. */
new_size = HEAP_ADJUST_SIZE (sizeof (struct heap_free_area));
MALLOC_DEBUG (1, "realloc: 0x%lx, %d (base = 0x%lx, total_size = %d)",
(long)mem, new_size, (long)base_mem, size);
if (new_size > size)
/* Grow the block. */
{
size_t extra = new_size - size;
__heap_lock (&__malloc_heap_lock);
extra = __heap_alloc_at (&__malloc_heap, base_mem + size, extra);
__heap_unlock (&__malloc_heap_lock);
if (extra)
/* Record the changed size. */
MALLOC_SET_SIZE (base_mem, size + extra);
else
/* Our attempts to extend MEM in place failed, just
allocate-and-copy. */
{
void *new_mem = malloc (new_size - MALLOC_HEADER_SIZE);
if (new_mem)
{
memcpy (new_mem, mem, size - MALLOC_HEADER_SIZE);
free (mem);
}
mem = new_mem;
}
}
else if (new_size + MALLOC_REALLOC_MIN_FREE_SIZE <= size)
/* Shrink the block. */
{
__heap_lock (&__malloc_heap_lock);
__heap_free (&__malloc_heap, base_mem + new_size, size - new_size);
__heap_unlock (&__malloc_heap_lock);
MALLOC_SET_SIZE (base_mem, new_size);
}
if (mem)
MALLOC_DEBUG (-1, "realloc: returning 0x%lx (base:0x%lx, total_size:%d)",
(long)mem, (long)MALLOC_BASE(mem), (long)MALLOC_SIZE(mem));
else
MALLOC_DEBUG (-1, "realloc: returning 0");
return mem;
}