DISPATCH_NOINLINE static void _dispatch_alloc_maybe_madvise_page(dispatch_continuation_t c) { void *page = madvisable_page_base_for_continuation(c); if (!page) { // page can't be madvised; maybe it contains non-continuations return; } // Are all the continuations in this page unallocated? volatile bitmap_t *page_bitmaps; get_maps_and_indices_for_continuation((dispatch_continuation_t)page, NULL, NULL, (bitmap_t **)&page_bitmaps, NULL); unsigned int i; for (i = 0; i < BITMAPS_PER_PAGE; i++) { if (page_bitmaps[i] != 0) { return; } } // They are all unallocated, so we could madvise the page. Try to // take ownership of them all. int last_locked = 0; do { if (!os_atomic_cmpxchg(&page_bitmaps[last_locked], BITMAP_C(0), BITMAP_ALL_ONES, relaxed)) { // We didn't get one; since there is a cont allocated in // the page, we can't madvise. Give up and unlock all. goto unlock; } } while (++last_locked < (signed)BITMAPS_PER_PAGE); #if DISPATCH_DEBUG //fprintf(stderr, "%s: madvised page %p for cont %p (next = %p), " // "[%u+1]=%u bitmaps at %p\n", __func__, page, c, c->do_next, // last_locked-1, BITMAPS_PER_PAGE, &page_bitmaps[0]); // Scribble to expose use-after-free bugs // madvise (syscall) flushes these stores memset(page, DISPATCH_ALLOCATOR_SCRIBBLE, DISPATCH_ALLOCATOR_PAGE_SIZE); #endif (void)dispatch_assume_zero(madvise(page, DISPATCH_ALLOCATOR_PAGE_SIZE, MADV_FREE)); unlock: while (last_locked > 1) { page_bitmaps[--last_locked] = BITMAP_C(0); } if (last_locked) { os_atomic_store(&page_bitmaps[0], BITMAP_C(0), relaxed); } return; }
DISPATCH_ALWAYS_INLINE_NDEBUG static void mark_bitmap_as_full_if_still_full(volatile bitmap_t *supermap, unsigned int bitmap_index, volatile bitmap_t *bitmap) { #if DISPATCH_DEBUG dispatch_assert(bitmap_index < BITMAPS_PER_SUPERMAP); #endif const bitmap_t mask = BITMAP_C(1) << bitmap_index; bitmap_t s, s_new, s_masked; if (!bitmap_is_full(*bitmap)) { return; } s_new = *supermap; for (;;) { // No barriers because supermaps are only advisory, they // don't protect access to other memory. s = s_new; s_masked = s | mask; if (dispatch_atomic_cmpxchgvw(supermap, s, s_masked, &s_new, relaxed) || !bitmap_is_full(*bitmap)) { return; } } }
// Return true if this bit was the last in the bitmap, and it is now all zeroes DISPATCH_ALWAYS_INLINE_NDEBUG static bool bitmap_clear_bit(volatile bitmap_t *bitmap, unsigned int index, bool exclusively) { #if DISPATCH_DEBUG dispatch_assert(index < CONTINUATIONS_PER_BITMAP); #endif const bitmap_t mask = BITMAP_C(1) << index; bitmap_t b; if (exclusively == CLEAR_EXCLUSIVELY) { if (slowpath((*bitmap & mask) == 0)) { DISPATCH_CRASH("Corruption: failed to clear bit exclusively"); } } // and-and-fetch b = dispatch_atomic_and(bitmap, ~mask, release); return b == 0; }