static void sanity_check_bigalloc(struct big_allocation *b) { #ifndef NDEBUG if (BIGALLOC_IN_USE(b)) { assert(pageindex[PAGENUM(((char*)(b)->begin)-1)] != ((b) - &big_allocations[0])); assert(pageindex[PAGENUM((b)->end)] != ((b) - &big_allocations[0])); /* Check that our depth is 1 + our parent's depth */ if (b->parent) { assert(bigalloc_depth(b) == 1 + bigalloc_depth(b->parent)); } /* Check that old children all have the same depth as each other. */ if (b->first_child) { unsigned first_child_depth = bigalloc_depth(b->first_child); for (struct big_allocation *child = b->first_child->next_sib; child; child = child->next_sib) { assert(bigalloc_depth(child) == first_child_depth); } } } #endif }
static boolean getmoreblocks(int sz) { void *page; int bpp; int ind; struct freeblock *e; struct pageinfo *pi; int i; FPRINTF((stderr, "getmoreblocks(%d)\n", sz)); if (sz < 1 || sz > PAGETHRESHHOLD) return FALSE; FPRINTF((stderr, "getmoreblocks(%d), size is valid\n", sz)); page = allocpages(1); /* get a page */ if (page == NULL) return FALSE; FPRINTF((stderr, "getmoreblocks(%d), got new page\n", sz)); /* fill out pageinfo for this page */ bpp = (BYTESPERPAGE - sizeof (struct pageinfo)) / sz; ind = sz / MINBLOCK; e = (struct freeblock *)((char *)page + sizeof (struct pageinfo)); pi = (struct pageinfo *)page; pi->blocksize = sz; pi->numblocks = bpp; pi->block = e; pi->link = g.blockmap[ind]; g.blockmap[ind] = pi; bpp--; for (i = 0; i < bpp; i++, e = e->link) e->link = (struct freeblock *)((char *)e + sz); e->link = NULL; #ifdef DO_GC CLEARBITMAP(pi->refbits); ind = PAGENUM(pi); if (ind >= MAXPAGES) crash("getmoreblocks(): out of space for new pages"); SETBIT(g.ourpages, ind); #endif return TRUE; }
void __auxv_allocator_notify_init_stack_mapping_sequence(struct big_allocation *b) { if (!auxv_array_start) __auxv_allocator_init(); if (!p_argcount) abort(); void *begin = b->begin; void *end = b->end; if (our_bigalloc) { /* We've been here before. Adjust the lower bound of the stack, * which is the *minimum* of the *begins*. */ if ((char*) our_bigalloc->begin > (char*) begin) our_bigalloc->begin = begin; /* We also adjust the upper bound of the stack. The reason is a giant * HACK. After the "[stack]" region which is rwx, there may be a separate rw- * region which contains the asciiz but is a separate /proc line. When we are * first called, we are reading from /proc (on Linux anyway) * and so we haven't seen the /proc line for that yet. So it would be premature * to expand ourselves into that space. But now that we're being called the second * time, it's fair game. FIXME: will the next /proc-processing iteration * clobber this hard work? */ if (asciiz_end > (const char *) our_bigalloc->end) { const char *new_end = RELF_ROUND_UP_PTR_(asciiz_end, PAGE_SIZE); unsigned pi = pageindex[PAGENUM(asciiz_end)]; _Bool success; if (pi) { _Bool success = __liballocs_truncate_bigalloc_at_beginning( &big_allocations[pi], new_end); assert(success); } success = __liballocs_extend_bigalloc(our_bigalloc, new_end); assert(success); } return; } __top_of_initial_stack = end; /* i.e. the highest address */ our_bigalloc = __liballocs_new_bigalloc( begin, (char*) end - (char*) begin, (struct meta_info) { .what = DATA_PTR, .un = { opaque_data: { .data_ptr = NULL, .free_func = NULL } } },
void * malloc(size_t sz) { int nsz; int ind, i; struct pageinfo *ppi; struct pageinfo *pi; void *ptr; if (!initialized) init(); if (sz >= PAGETHRESHHOLD - sizeof (struct pageinfo)) { /* block is bigger than threshhold so allocate whole pages */ int pgs = (sz + sizeof (struct pageinfo)+BYTESPERPAGE - 1) / BYTESPERPAGE; void *base = allocpages(pgs); FPRINTF((stderr, "malloc(%d), large block\n", sz)); if (base == NULL) return NULL; pi = (struct pageinfo *)base; pi->blocksize = -1; pi->numblocks = 0; pi->count = pgs; pi->link = NULL; base = (char *)base + sizeof (struct pageinfo); if (g.clearblocks) memset(base, 0, pgs * BYTESPERPAGE - sizeof (struct pageinfo)); #ifdef DO_GC CLEARBITMAP(pi->refbits); ind = PAGENUM(pi); if (ind + pgs >= MAXPAGES) crash("malloc(): out of space for new pages"); SETBIT(g.ourpages, ind); /* all following pages are marked as part of a big block for GC */ for (ind++, i = 1; i < pgs; i++, ind++) CLEARBIT(g.ourpages, ind); #endif return base; } /* search page list for a free block of our size */ nsz = g.sizemap[sz]; if (nsz < sz) nsz = sz; FPRINTF((stderr, "malloc(%d), small block, sz = %d\n", sz, nsz)); ind = nsz / MINBLOCK; ppi = NULL; pi = g.blockmap[ind]; /* search pages with blocks of the given size */ for (; pi != NULL && pi->block == NULL; pi = pi->link) ppi = pi; if (pi == NULL) { if (!getmoreblocks(nsz)) { FPRINTF((stderr, "malloc(%d), no more small blocks\n", sz)); return NULL; } /* this works since getmoreblocks always sticks new pages */ /* at the front of the list */ pi = g.blockmap[ind]; ppi = NULL; } if (ppi != NULL) { /* move page with free blocks to the head of the list so we save time on the next call for this block size. */ ppi->link = pi->link; pi->link = g.blockmap[ind]; g.blockmap[ind] = pi; } ptr = pi->block; pi->block = pi->block->link; pi->count++; if (g.clearblocks) memset(ptr, 0, nsz); FPRINTF((stderr, "malloc(%d), nsz=%d small block=%p\n", sz, nsz, ptr)); return (void *)ptr; }
static void freepages(void *ptr, int numpgs) { struct pageinfo *fpi; void *ptrend; struct pageinfo *prev = NULL; struct pageinfo *freepage = g.freepagelist; FPRINTF((stderr, "freepages(ptr=%p, numpgs=%d)\n", ptr, numpgs)); /* insert pages into the free list in sorted order by address. */ fpi = GETPAGEINFO(ptr); if (fpi != ptr) crash("freepages(): fpi != ptr"); if (g.clearfree) memset(ptr, 0, numpgs * BYTESPERPAGE); #ifdef DO_GC { int i; int pn = PAGENUM(ptr); struct pageinfo *p = fpi; for (i = 0; i < numpgs; i++, pn++) { if (pn >= MAXPAGES) crash("freepages(): out of space for freepages"); SETBIT(g.ourpages, pn); p->blocksize = 0; p = (struct pageinfo *)((char *)p + BYTESPERPAGE); } } #endif g.numfreepages += numpgs; ptrend = ((char *)ptr) + numpgs * BYTESPERPAGE; for (; freepage != NULL; (prev = freepage), freepage = freepage->link) { if ((char *)freepage + freepage->count * BYTESPERPAGE == (char *)ptr) { struct pageinfo *nextpage = freepage->link; freepage->count += numpgs; if (nextpage && (char *)nextpage == (char *)freepage + freepage->count * BYTESPERPAGE) { freepage->count += nextpage->count; freepage->link = nextpage->link; } return; } if (ptrend == freepage) { fpi->link = freepage->link; fpi->count = numpgs + freepage->count; if (prev == NULL) g.freepagelist = fpi; else prev->link = fpi; return; } if ((char *)ptr < (char *)freepage) break; } fpi->link = freepage; fpi->count = numpgs; fpi->blocksize = 0; if (prev == NULL) g.freepagelist = fpi; else prev->link = fpi; return; }
void gcdebug(FILE *fp) { int i, numpgs; struct pageinfo *freepage, *page; if (fp == NULL) /* so we can call this easily from a debugger */ fp = stderr; fprintf(fp, "gcdebug()...\n"); fprintf(fp, " g.pages=%d g.pagecount=%d g.numregions=%d\n", g.pages, g.pagecount, g.numregions); for (i = 0; i < g.numregions; i++) if (g.regions[i].start != NULL) fprintf(fp, " regions[%d] start=%p end=%p\n", i, g.regions[i].start, g.regions[i].end); fprintf(fp, " g.heaphigh=%p g.heaplow=%p g.numfreepages=%d\n", g.heaphigh, g.heaplow, g.numfreepages); fprintf(fp, " freepage...\n"); freepage = g.freepagelist; for (i = 1; freepage != NULL; i++, freepage = freepage->link) fprintf(fp, " freepage[%d] = %p, count=%d\n", i, freepage, freepage->count); fprintf(fp, " allocated memory...\n"); numpgs = PAGENUM(g.heaphigh); page = (struct pageinfo *)g.heaplow; /* release unused memory */ for (i = PAGENUM(page); i < numpgs; i++, page = (struct pageinfo *)((char *)page + BYTESPERPAGE)) { if (i >= MAXPAGES) crash("gcdebug(): out of space for new pages"); if (!GETBIT(g.ourpages, i)) continue; if (page->blocksize == -1) { fprintf(fp, " block=%p numpages=%d refbits[0]=%ld\n", page, page->count, page->refbits[0]); } else if (page->blocksize > 0) { int j; fprintf(fp, " page=%p blocksize=%d numblocks=%d used=%d\n", page, page->blocksize, page->numblocks, page->count); fprintf(fp, " block=%p link=%p\n", page->block, page->link); fprintf(fp, " refbits=%p", page->refbits); for (j = 0; j < GC_WORDS / 2; j++) fprintf(fp, "0x%08lX ", page->refbits[j]); fprintf(fp, "\n "); for (; j < GC_WORDS; j++) fprintf(fp, "0x%08lX ", page->refbits[j]); fprintf(fp, "\n"); } } /**** if (g.snapshot) gcdumpsnapstack(fp); ****/ fflush(fp); }
/* garbage collect using mark & sweep */ void garbagecollect() { static jmp_buf jmp; long stacktop; /* to get the top of stack */ int i, numpgs; struct pageinfo *page; size_t totfree; int arenasz, expandcnt; if (g.ingc || g.heaphigh == NULL) return; g.ingc = TRUE; /* ref bits should already be cleared since we clear them upon return to avoid doing an extra pass over all of memory */ /* this does not go in our "gc" struct as we really do want it searched */ memset(jmp, 0, sizeof jmp); setjmp(jmp); totfree = 0; /* set ref bits of referenced blocks */ gcmarkregion(STACKSTART, STACKEND); for (i = 0; i < g.numregions; i++) { if (g.regions[i].start == NULL) continue; gcmarkregion(g.regions[i].start, g.regions[i].end); } numpgs = PAGENUM(g.heaphigh); page = g.heaplow; /* release unused memory */ for (i = PAGENUM(page); i < numpgs; i++, page = (struct pageinfo *)((char *)page + BYTESPERPAGE)) { if (i >= MAXPAGES) crash("garbagecollect(): out of space for new pages"); if (!GETBIT(g.ourpages, i)) continue; if (page->blocksize == -1) { if (page->refbits[0]) page->refbits[0] = 0; else freepages(page, page->count); } else if (page->blocksize > 0) { int j; int freed = 0; page->block = NULL; for (j = 0; j < page->numblocks; j++) if (!GETBIT(page->refbits, j)) { struct freeblock *e = (struct freeblock *) ((char *)page + sizeof (struct pageinfo)+ page->blocksize * j); if (g.clearfree) memset(e, 0, page->blocksize); e->link = page->block; page->block = e; freed++; } CLEARBITMAP(page->refbits); if (freed == page->numblocks) freeblockspage(page); else { page->count = page->numblocks - freed; totfree += freed * page->blocksize; } } } /* rebuildfreelist(); */ g.ingc = FALSE; totfree += g.numfreepages * BYTESPERPAGE; arenasz = ((char *)g.heaphigh - (char *)g.heaplow) / BYTESPERPAGE; expandcnt = arenasz * g.percent; expandcnt /= 100; if (g.freepagelist == NULL) { /* if garbage collection didn't yield a page then allow */ /* heap to grow by g.page or g.percent which ever is larger. */ g.pagecount = (g.pages > expandcnt ? g.pages : expandcnt); } else { /* if garbage collection yielded less than g.percent of the heap */ /* then give the heap some extra growing room. */ expandcnt -= totfree / BYTESPERPAGE; expandcnt -= g.numfreepages; if (g.pagecount < expandcnt) g.pagecount = expandcnt; if (g.pagecount <= 0) g.pagecount = 16; } return; }
static void gcmarkregion(void *start, void *end) { #define STACK 128 void *startstk[STACK]; void *endstk[STACK]; char *low = (char *)g.heaplow; char *high = (char *)g.heaphigh - sizeof (char *); int tos = 1; startstk[0] = start; endstk[0] = end; while (tos-- > 0) { #ifdef PTRMASK char *st = (char *)(((uLong)startstk[tos]) & PTRMASK); char *e = (char *)(((uLong)endstk[tos] - sizeof (char *)) & PTRMASK); #else char *st = (char *)startstk[tos]; char *e = (char *)endstk[tos] - sizeof (char *); #endif char *s; for (s = st; s <= e; s += PTRBUMP) { char *p = *(char **)s; int num; struct pageinfo *pi; void *pp; size_t bb; boolean big = FALSE; if (p < low || p > high) continue; pi = GETPAGEINFO(p); num = PAGENUM(pi); if (num >= MAXPAGES)/* bogus block? */ continue; while (!GETBIT(g.ourpages, num) && num >= 0) { num--; pi = (struct pageinfo *)((char *)pi - BYTESPERPAGE); big = TRUE; } if (num < 0) /* bogus block? */ continue; if (pi->blocksize == 0) /* hit in free block */ { /* pi->history |= 1; */ /* flag the hit */ continue; } if (pi->blocksize == -1) /* big block */ { pp = (char *)pi + sizeof (struct pageinfo); if (g.skipinterior && pp != p) continue; if (pi->refbits[0]) /* already marked and walked */ continue; pi->refbits[0] = 1; bb = pi->count * BYTESPERPAGE - sizeof (struct pageinfo); FPRINTF((stdout, "gcmarkregion: st=%p e=%p s=%p" " p=%p pp=%p bb=%d\n", st, e, s, p, pp, bb)); } else if (big) /* bogus block */ continue; else /* regular block */ { num = BLOCKNUM(pi, p); /* if the last byte of this block is outside a page, it is bogus */ if (num >= pi->numblocks) { /* pi->history |= 1; */ /* flag the hit */ continue; } bb = pi->blocksize; pp = (char *)pi + num * bb + sizeof (struct pageinfo); if (g.skipinterior && pp != p) continue; if (GETBIT(pi->refbits, num)) /* already marked and walked */ continue; SETBIT(pi->refbits, num); } if ((char *)pp < low || (char *)pp > high) crash("gcmarkregion(): block pointer pp not in heap"); if ((char *)pp + bb < low || (char *)pp + bb > (char *)g.heaphigh) crash("gcmarkregion(): end of block pointer pp not in heap"); if (tos >= STACK) gcmarkregion(pp, pp + bb); else { startstk[tos] = pp; endstk[tos] = pp + bb; tos++; } } } }