static void sanity_check_bigalloc(struct big_allocation *b)
{
#ifndef NDEBUG
if (BIGALLOC_IN_USE(b))
{
assert(pageindex[PAGENUM(((char*)(b)->begin)-1)] != ((b) - &big_allocations[0]));
assert(pageindex[PAGENUM((b)->end)] != ((b) - &big_allocations[0]));
/* Check that our depth is 1 + our parent's depth */
if (b->parent)
{
assert(bigalloc_depth(b) == 1 + bigalloc_depth(b->parent));
}
/* Check that old children all have the same depth as each other. */
if (b->first_child)
{
unsigned first_child_depth = bigalloc_depth(b->first_child);
for (struct big_allocation *child = b->first_child->next_sib; child; child = child->next_sib)
{
assert(bigalloc_depth(child) == first_child_depth);
}
}
}
#endif
}
static boolean
getmoreblocks(int sz)
{
void *page;
int bpp;
int ind;
struct freeblock *e;
struct pageinfo *pi;
int i;
FPRINTF((stderr, "getmoreblocks(%d)\n", sz));
if (sz < 1 || sz > PAGETHRESHHOLD)
return FALSE;
FPRINTF((stderr, "getmoreblocks(%d), size is valid\n", sz));
page = allocpages(1); /* get a page */
if (page == NULL)
return FALSE;
FPRINTF((stderr, "getmoreblocks(%d), got new page\n", sz));
/* fill out pageinfo for this page */
bpp = (BYTESPERPAGE - sizeof (struct pageinfo)) / sz;
ind = sz / MINBLOCK;
e = (struct freeblock *)((char *)page + sizeof (struct pageinfo));
pi = (struct pageinfo *)page;
pi->blocksize = sz;
pi->numblocks = bpp;
pi->block = e;
pi->link = g.blockmap[ind];
g.blockmap[ind] = pi;
bpp--;
for (i = 0; i < bpp; i++, e = e->link)
e->link = (struct freeblock *)((char *)e + sz);
e->link = NULL;
#ifdef DO_GC
CLEARBITMAP(pi->refbits);
ind = PAGENUM(pi);
if (ind >= MAXPAGES)
crash("getmoreblocks(): out of space for new pages");
SETBIT(g.ourpages, ind);
#endif
return TRUE;
}
void __auxv_allocator_notify_init_stack_mapping_sequence(struct big_allocation *b)
{
if (!auxv_array_start) __auxv_allocator_init();
if (!p_argcount) abort();
void *begin = b->begin;
void *end = b->end;
if (our_bigalloc)
{
/* We've been here before. Adjust the lower bound of the stack,
* which is the *minimum* of the *begins*. */
if ((char*) our_bigalloc->begin > (char*) begin) our_bigalloc->begin = begin;
/* We also adjust the upper bound of the stack. The reason is a giant
* HACK. After the "[stack]" region which is rwx, there may be a separate rw-
* region which contains the asciiz but is a separate /proc line. When we are
* first called, we are reading from /proc (on Linux anyway)
* and so we haven't seen the /proc line for that yet. So it would be premature
* to expand ourselves into that space. But now that we're being called the second
* time, it's fair game. FIXME: will the next /proc-processing iteration
* clobber this hard work? */
if (asciiz_end > (const char *) our_bigalloc->end)
{
const char *new_end = RELF_ROUND_UP_PTR_(asciiz_end, PAGE_SIZE);
unsigned pi = pageindex[PAGENUM(asciiz_end)];
_Bool success;
if (pi)
{
_Bool success = __liballocs_truncate_bigalloc_at_beginning(
&big_allocations[pi], new_end);
assert(success);
}
success = __liballocs_extend_bigalloc(our_bigalloc, new_end);
assert(success);
}
return;
}
__top_of_initial_stack = end; /* i.e. the highest address */
our_bigalloc = __liballocs_new_bigalloc(
begin,
(char*) end - (char*) begin,
(struct meta_info) {
.what = DATA_PTR,
.un = {
opaque_data: {
.data_ptr = NULL,
.free_func = NULL
}
}
},
void *
malloc(size_t sz)
{
int nsz;
int ind, i;
struct pageinfo *ppi;
struct pageinfo *pi;
void *ptr;
if (!initialized)
init();
if (sz >= PAGETHRESHHOLD - sizeof (struct pageinfo))
{
/* block is bigger than threshhold so allocate whole pages */
int pgs = (sz + sizeof (struct pageinfo)+BYTESPERPAGE - 1) /
BYTESPERPAGE;
void *base = allocpages(pgs);
FPRINTF((stderr, "malloc(%d), large block\n", sz));
if (base == NULL)
return NULL;
pi = (struct pageinfo *)base;
pi->blocksize = -1;
pi->numblocks = 0;
pi->count = pgs;
pi->link = NULL;
base = (char *)base + sizeof (struct pageinfo);
if (g.clearblocks)
memset(base, 0, pgs * BYTESPERPAGE - sizeof (struct pageinfo));
#ifdef DO_GC
CLEARBITMAP(pi->refbits);
ind = PAGENUM(pi);
if (ind + pgs >= MAXPAGES)
crash("malloc(): out of space for new pages");
SETBIT(g.ourpages, ind);
/* all following pages are marked as part of a big block for GC */
for (ind++, i = 1; i < pgs; i++, ind++)
CLEARBIT(g.ourpages, ind);
#endif
return base;
}
/* search page list for a free block of our size */
nsz = g.sizemap[sz];
if (nsz < sz)
nsz = sz;
FPRINTF((stderr, "malloc(%d), small block, sz = %d\n", sz, nsz));
ind = nsz / MINBLOCK;
ppi = NULL;
pi = g.blockmap[ind];
/* search pages with blocks of the given size */
for (; pi != NULL && pi->block == NULL; pi = pi->link)
ppi = pi;
if (pi == NULL)
{
if (!getmoreblocks(nsz))
{
FPRINTF((stderr, "malloc(%d), no more small blocks\n", sz));
return NULL;
}
/* this works since getmoreblocks always sticks new pages */
/* at the front of the list */
pi = g.blockmap[ind];
ppi = NULL;
}
if (ppi != NULL)
{
/* move page with free blocks to the head of the list so we save
time on the next call for this block size. */
ppi->link = pi->link;
pi->link = g.blockmap[ind];
g.blockmap[ind] = pi;
}
ptr = pi->block;
pi->block = pi->block->link;
pi->count++;
if (g.clearblocks)
memset(ptr, 0, nsz);
FPRINTF((stderr, "malloc(%d), nsz=%d small block=%p\n", sz, nsz, ptr));
return (void *)ptr;
}
static void
freepages(void *ptr, int numpgs)
{
struct pageinfo *fpi;
void *ptrend;
struct pageinfo *prev = NULL;
struct pageinfo *freepage = g.freepagelist;
FPRINTF((stderr, "freepages(ptr=%p, numpgs=%d)\n", ptr, numpgs));
/* insert pages into the free list in sorted order by address. */
fpi = GETPAGEINFO(ptr);
if (fpi != ptr)
crash("freepages(): fpi != ptr");
if (g.clearfree)
memset(ptr, 0, numpgs * BYTESPERPAGE);
#ifdef DO_GC
{
int i;
int pn = PAGENUM(ptr);
struct pageinfo *p = fpi;
for (i = 0; i < numpgs; i++, pn++)
{
if (pn >= MAXPAGES)
crash("freepages(): out of space for freepages");
SETBIT(g.ourpages, pn);
p->blocksize = 0;
p = (struct pageinfo *)((char *)p + BYTESPERPAGE);
}
}
#endif
g.numfreepages += numpgs;
ptrend = ((char *)ptr) + numpgs * BYTESPERPAGE;
for (; freepage != NULL; (prev = freepage), freepage = freepage->link)
{
if ((char *)freepage + freepage->count * BYTESPERPAGE == (char *)ptr)
{
struct pageinfo *nextpage = freepage->link;
freepage->count += numpgs;
if (nextpage && (char *)nextpage ==
(char *)freepage + freepage->count * BYTESPERPAGE)
{
freepage->count += nextpage->count;
freepage->link = nextpage->link;
}
return;
}
if (ptrend == freepage)
{
fpi->link = freepage->link;
fpi->count = numpgs + freepage->count;
if (prev == NULL)
g.freepagelist = fpi;
else
prev->link = fpi;
return;
}
if ((char *)ptr < (char *)freepage)
break;
}
fpi->link = freepage;
fpi->count = numpgs;
fpi->blocksize = 0;
if (prev == NULL)
g.freepagelist = fpi;
else
prev->link = fpi;
return;
}
void
gcdebug(FILE *fp)
{
int i, numpgs;
struct pageinfo *freepage, *page;
if (fp == NULL) /* so we can call this easily from a
debugger */
fp = stderr;
fprintf(fp, "gcdebug()...\n");
fprintf(fp, " g.pages=%d g.pagecount=%d g.numregions=%d\n",
g.pages, g.pagecount, g.numregions);
for (i = 0; i < g.numregions; i++)
if (g.regions[i].start != NULL)
fprintf(fp, " regions[%d] start=%p end=%p\n",
i, g.regions[i].start, g.regions[i].end);
fprintf(fp, " g.heaphigh=%p g.heaplow=%p g.numfreepages=%d\n",
g.heaphigh, g.heaplow, g.numfreepages);
fprintf(fp, " freepage...\n");
freepage = g.freepagelist;
for (i = 1; freepage != NULL; i++, freepage = freepage->link)
fprintf(fp, " freepage[%d] = %p, count=%d\n", i,
freepage, freepage->count);
fprintf(fp, " allocated memory...\n");
numpgs = PAGENUM(g.heaphigh);
page = (struct pageinfo *)g.heaplow;
/* release unused memory */
for (i = PAGENUM(page); i < numpgs; i++,
page = (struct pageinfo *)((char *)page + BYTESPERPAGE))
{
if (i >= MAXPAGES)
crash("gcdebug(): out of space for new pages");
if (!GETBIT(g.ourpages, i))
continue;
if (page->blocksize == -1)
{
fprintf(fp, " block=%p numpages=%d refbits[0]=%ld\n",
page, page->count, page->refbits[0]);
}
else if (page->blocksize > 0)
{
int j;
fprintf(fp, " page=%p blocksize=%d numblocks=%d used=%d\n",
page, page->blocksize, page->numblocks, page->count);
fprintf(fp, " block=%p link=%p\n",
page->block, page->link);
fprintf(fp, " refbits=%p", page->refbits);
for (j = 0; j < GC_WORDS / 2; j++)
fprintf(fp, "0x%08lX ", page->refbits[j]);
fprintf(fp, "\n ");
for (; j < GC_WORDS; j++)
fprintf(fp, "0x%08lX ", page->refbits[j]);
fprintf(fp, "\n");
}
}
/****
if (g.snapshot)
gcdumpsnapstack(fp);
****/
fflush(fp);
}
/* garbage collect using mark & sweep */
void
garbagecollect()
{
static jmp_buf jmp;
long stacktop; /* to get the top of stack */
int i, numpgs;
struct pageinfo *page;
size_t totfree;
int arenasz, expandcnt;
if (g.ingc || g.heaphigh == NULL)
return;
g.ingc = TRUE;
/* ref bits should already be cleared since we clear them upon return
to avoid doing an extra pass over all of memory */
/* this does not go in our "gc" struct as we really do want it
searched */
memset(jmp, 0, sizeof jmp);
setjmp(jmp);
totfree = 0;
/* set ref bits of referenced blocks */
gcmarkregion(STACKSTART, STACKEND);
for (i = 0; i < g.numregions; i++)
{
if (g.regions[i].start == NULL)
continue;
gcmarkregion(g.regions[i].start, g.regions[i].end);
}
numpgs = PAGENUM(g.heaphigh);
page = g.heaplow;
/* release unused memory */
for (i = PAGENUM(page); i < numpgs; i++,
page = (struct pageinfo *)((char *)page + BYTESPERPAGE))
{
if (i >= MAXPAGES)
crash("garbagecollect(): out of space for new pages");
if (!GETBIT(g.ourpages, i))
continue;
if (page->blocksize == -1)
{
if (page->refbits[0])
page->refbits[0] = 0;
else
freepages(page, page->count);
}
else if (page->blocksize > 0)
{
int j;
int freed = 0;
page->block = NULL;
for (j = 0; j < page->numblocks; j++)
if (!GETBIT(page->refbits, j))
{
struct freeblock *e = (struct freeblock *)
((char *)page + sizeof (struct pageinfo)+
page->blocksize * j);
if (g.clearfree)
memset(e, 0, page->blocksize);
e->link = page->block;
page->block = e;
freed++;
}
CLEARBITMAP(page->refbits);
if (freed == page->numblocks)
freeblockspage(page);
else
{
page->count = page->numblocks - freed;
totfree += freed * page->blocksize;
}
}
}
/* rebuildfreelist(); */
g.ingc = FALSE;
totfree += g.numfreepages * BYTESPERPAGE;
arenasz = ((char *)g.heaphigh - (char *)g.heaplow) / BYTESPERPAGE;
expandcnt = arenasz * g.percent;
expandcnt /= 100;
if (g.freepagelist == NULL)
{
/* if garbage collection didn't yield a page then allow */
/* heap to grow by g.page or g.percent which ever is larger. */
g.pagecount = (g.pages > expandcnt ? g.pages : expandcnt);
}
else
{
/* if garbage collection yielded less than g.percent of the heap */
/* then give the heap some extra growing room. */
expandcnt -= totfree / BYTESPERPAGE;
expandcnt -= g.numfreepages;
if (g.pagecount < expandcnt)
g.pagecount = expandcnt;
if (g.pagecount <= 0)
g.pagecount = 16;
}
return;
}
static void
gcmarkregion(void *start, void *end)
{
#define STACK 128
void *startstk[STACK];
void *endstk[STACK];
char *low = (char *)g.heaplow;
char *high = (char *)g.heaphigh - sizeof (char *);
int tos = 1;
startstk[0] = start;
endstk[0] = end;
while (tos-- > 0)
{
#ifdef PTRMASK
char *st = (char *)(((uLong)startstk[tos]) & PTRMASK);
char *e = (char *)(((uLong)endstk[tos] - sizeof (char *)) & PTRMASK);
#else
char *st = (char *)startstk[tos];
char *e = (char *)endstk[tos] - sizeof (char *);
#endif
char *s;
for (s = st; s <= e; s += PTRBUMP)
{
char *p = *(char **)s;
int num;
struct pageinfo *pi;
void *pp;
size_t bb;
boolean big = FALSE;
if (p < low || p > high)
continue;
pi = GETPAGEINFO(p);
num = PAGENUM(pi);
if (num >= MAXPAGES)/* bogus block? */
continue;
while (!GETBIT(g.ourpages, num) && num >= 0)
{
num--;
pi = (struct pageinfo *)((char *)pi - BYTESPERPAGE);
big = TRUE;
}
if (num < 0) /* bogus block? */
continue;
if (pi->blocksize == 0) /* hit in free block */
{
/* pi->history |= 1; */ /* flag the hit */
continue;
}
if (pi->blocksize == -1) /* big block */
{
pp = (char *)pi + sizeof (struct pageinfo);
if (g.skipinterior && pp != p)
continue;
if (pi->refbits[0]) /* already marked and walked */
continue;
pi->refbits[0] = 1;
bb = pi->count * BYTESPERPAGE - sizeof (struct pageinfo);
FPRINTF((stdout, "gcmarkregion: st=%p e=%p s=%p"
" p=%p pp=%p bb=%d\n",
st, e, s, p, pp, bb));
}
else if (big) /* bogus block */
continue;
else /* regular block */
{
num = BLOCKNUM(pi, p);
/* if the last byte of this block is outside a page, it is
bogus */
if (num >= pi->numblocks)
{
/* pi->history |= 1; */ /* flag the hit */
continue;
}
bb = pi->blocksize;
pp = (char *)pi + num * bb + sizeof (struct pageinfo);
if (g.skipinterior && pp != p)
continue;
if (GETBIT(pi->refbits, num))
/* already marked and walked */
continue;
SETBIT(pi->refbits, num);
}
if ((char *)pp < low || (char *)pp > high)
crash("gcmarkregion(): block pointer pp not in heap");
if ((char *)pp + bb < low || (char *)pp + bb > (char *)g.heaphigh)
crash("gcmarkregion(): end of block pointer pp not in heap");
if (tos >= STACK)
gcmarkregion(pp, pp + bb);
else
{
startstk[tos] = pp;
endstk[tos] = pp + bb;
tos++;
}
}
}
}
