void
_dispatch_dispose(dispatch_object_t dou)
{
if (slowpath(dou._do->do_next != DISPATCH_OBJECT_LISTLESS)) {
DISPATCH_CRASH("Release while enqueued");
}
dx_dispose(dou._do);
return _dispatch_dealloc(dou);
}
// Base address of page, or NULL if this page shouldn't be madvise()d
DISPATCH_ALWAYS_INLINE_NDEBUG DISPATCH_CONST
static void *
madvisable_page_base_for_continuation(dispatch_continuation_t c)
{
if (fastpath(continuation_is_in_first_page(c))) {
return NULL;
}
void *page_base = (void *)((uintptr_t)c & ~(uintptr_t)PAGE_MASK);
#if DISPATCH_DEBUG
struct dispatch_magazine_s *m = magazine_for_continuation(c);
if (slowpath(page_base < (void *)&m->conts)) {
DISPATCH_CRASH("madvisable continuation too low");
}
if (slowpath(page_base > (void *)&m->conts[SUPERMAPS_PER_MAGAZINE-1]
[BITMAPS_PER_SUPERMAP-1][CONTINUATIONS_PER_BITMAP-1])) {
DISPATCH_CRASH("madvisable continuation too high");
}
#endif
return page_base;
}
DISPATCH_NOINLINE
void
_os_object_release_internal(_os_object_t obj)
{
int ref_cnt = obj->os_obj_ref_cnt;
if (slowpath(ref_cnt == _OS_OBJECT_GLOBAL_REFCNT)) {
return; // global object
}
ref_cnt = dispatch_atomic_dec2o(obj, os_obj_ref_cnt, relaxed);
if (fastpath(ref_cnt >= 0)) {
return;
}
if (slowpath(ref_cnt < -1)) {
DISPATCH_CRASH("Over-release of an object");
}
#if DISPATCH_DEBUG
if (slowpath(obj->os_obj_xref_cnt >= 0)) {
DISPATCH_CRASH("Release while external references exist");
}
#endif
return _os_object_dispose(obj);
}
DISPATCH_NOINLINE
_os_object_t
_os_object_retain_internal(_os_object_t obj)
{
int ref_cnt = obj->os_obj_ref_cnt;
if (slowpath(ref_cnt == _OS_OBJECT_GLOBAL_REFCNT)) {
return obj; // global object
}
ref_cnt = dispatch_atomic_inc2o(obj, os_obj_ref_cnt, relaxed);
if (slowpath(ref_cnt <= 0)) {
DISPATCH_CRASH("Resurrection of an object");
}
return obj;
}
void
dispatch_main(void)
{
#if HAVE_PTHREAD_MAIN_NP
if (pthread_main_np()) {
#endif
_dispatch_program_is_probably_callback_driven = true;
pthread_exit(NULL);
DISPATCH_CRASH("pthread_exit() returned");
#if HAVE_PTHREAD_MAIN_NP
}
DISPATCH_CLIENT_CRASH("dispatch_main() must be called on the main thread");
#endif
}
// Return true if this bit was the last in the bitmap, and it is now all zeroes
DISPATCH_ALWAYS_INLINE_NDEBUG
static bool
bitmap_clear_bit(volatile bitmap_t *bitmap, unsigned int index,
bool exclusively)
{
#if DISPATCH_DEBUG
dispatch_assert(index < CONTINUATIONS_PER_BITMAP);
#endif
const bitmap_t mask = BITMAP_C(1) << index;
bitmap_t b;
if (exclusively == CLEAR_EXCLUSIVELY) {
if (slowpath((*bitmap & mask) == 0)) {
DISPATCH_CRASH("Corruption: failed to clear bit exclusively");
}
}
// and-and-fetch
b = dispatch_atomic_and(bitmap, ~mask, release);
return b == 0;
}