DISPATCH_NOINLINE
static void
_dispatch_alloc_maybe_madvise_page(dispatch_continuation_t c)
{
void *page = madvisable_page_base_for_continuation(c);
if (!page) {
// page can't be madvised; maybe it contains non-continuations
return;
}
// Are all the continuations in this page unallocated?
volatile bitmap_t *page_bitmaps;
get_maps_and_indices_for_continuation((dispatch_continuation_t)page, NULL,
NULL, (bitmap_t **)&page_bitmaps, NULL);
unsigned int i;
for (i = 0; i < BITMAPS_PER_PAGE; i++) {
if (page_bitmaps[i] != 0) {
return;
}
}
// They are all unallocated, so we could madvise the page. Try to
// take ownership of them all.
int last_locked = 0;
do {
if (!os_atomic_cmpxchg(&page_bitmaps[last_locked], BITMAP_C(0),
BITMAP_ALL_ONES, relaxed)) {
// We didn't get one; since there is a cont allocated in
// the page, we can't madvise. Give up and unlock all.
goto unlock;
}
} while (++last_locked < (signed)BITMAPS_PER_PAGE);
#if DISPATCH_DEBUG
//fprintf(stderr, "%s: madvised page %p for cont %p (next = %p), "
// "[%u+1]=%u bitmaps at %p\n", __func__, page, c, c->do_next,
// last_locked-1, BITMAPS_PER_PAGE, &page_bitmaps[0]);
// Scribble to expose use-after-free bugs
// madvise (syscall) flushes these stores
memset(page, DISPATCH_ALLOCATOR_SCRIBBLE, DISPATCH_ALLOCATOR_PAGE_SIZE);
#endif
(void)dispatch_assume_zero(madvise(page, DISPATCH_ALLOCATOR_PAGE_SIZE,
MADV_FREE));
unlock:
while (last_locked > 1) {
page_bitmaps[--last_locked] = BITMAP_C(0);
}
if (last_locked) {
os_atomic_store(&page_bitmaps[0], BITMAP_C(0), relaxed);
}
return;
}
DISPATCH_ALLOC_NOINLINE
static void
_dispatch_alloc_continuation_free(dispatch_continuation_t c)
{
bitmap_t *b, *s;
unsigned int b_idx, idx;
get_maps_and_indices_for_continuation(c, &s, &b_idx, &b, &idx);
bool bitmap_now_empty = bitmap_clear_bit(b, idx, CLEAR_EXCLUSIVELY);
if (slowpath(s)) {
(void)bitmap_clear_bit(s, b_idx, CLEAR_NONEXCLUSIVELY);
}
// We only try to madvise(2) pages outside of the first page.
// (Allocations in the first page do not have a supermap entry.)
if (slowpath(bitmap_now_empty) && slowpath(s)) {
return _dispatch_alloc_maybe_madvise_page(c);
}
}
