void dispatch_resume(dispatch_object_t dou) { DISPATCH_OBJECT_TFB(_dispatch_objc_resume, dou); // Global objects cannot be suspended or resumed. This also has the // side effect of saturating the suspend count of an object and // guarding against resuming due to overflow. if (slowpath(dou._do->do_ref_cnt == DISPATCH_OBJECT_GLOBAL_REFCNT) || slowpath(dx_type(dou._do) == DISPATCH_QUEUE_ROOT_TYPE)) { return; } // Check the previous value of the suspend count. If the previous // value was a single suspend interval, the object should be resumed. // If the previous value was less than the suspend interval, the object // has been over-resumed. unsigned int suspend_cnt = dispatch_atomic_sub_orig2o(dou._do, do_suspend_cnt, DISPATCH_OBJECT_SUSPEND_INTERVAL, relaxed); if (fastpath(suspend_cnt > DISPATCH_OBJECT_SUSPEND_INTERVAL)) { // Balancing the retain() done in suspend() for rdar://8181908 return _dispatch_release(dou._do); } if (fastpath(suspend_cnt == DISPATCH_OBJECT_SUSPEND_INTERVAL)) { return _dispatch_resume_slow(dou); } DISPATCH_CLIENT_CRASH("Over-resume of an object"); }
void _dispatch_semaphore_dispose(dispatch_object_t dou) { dispatch_semaphore_t dsema = dou._dsema; if (dsema->dsema_value < dsema->dsema_orig) { DISPATCH_CLIENT_CRASH( "Semaphore/group object deallocated while in use"); } #if USE_MACH_SEM kern_return_t kr; if (dsema->dsema_port) { kr = semaphore_destroy(mach_task_self(), dsema->dsema_port); DISPATCH_SEMAPHORE_VERIFY_KR(kr); } if (dsema->dsema_waiter_port) { kr = semaphore_destroy(mach_task_self(), dsema->dsema_waiter_port); DISPATCH_SEMAPHORE_VERIFY_KR(kr); } #elif USE_POSIX_SEM int ret = sem_destroy(&dsema->dsema_sem); DISPATCH_SEMAPHORE_VERIFY_RET(ret); #endif }
void dispatch_group_leave(dispatch_group_t dg) { dispatch_semaphore_t dsema = (dispatch_semaphore_t)dg; dispatch_atomic_release_barrier(); long value = dispatch_atomic_inc2o(dsema, dsema_value); if (slowpath(value == LONG_MIN)) { DISPATCH_CLIENT_CRASH("Unbalanced call to dispatch_group_leave()"); } if (slowpath(value == dsema->dsema_orig)) { (void)_dispatch_group_wake(dsema); } }
long dispatch_semaphore_signal(dispatch_semaphore_t dsema) { dispatch_atomic_release_barrier(); long value = dispatch_atomic_inc2o(dsema, dsema_value); if (fastpath(value > 0)) { return 0; } if (slowpath(value == LONG_MIN)) { DISPATCH_CLIENT_CRASH("Unbalanced call to dispatch_semaphore_signal()"); } return _dispatch_semaphore_signal_slow(dsema); }
static void _voucher_insert(voucher_t v) { mach_voucher_t kv = v->v_ipc_kvoucher; if (!kv) return; _vouchers_lock_lock(); if (slowpath(_TAILQ_IS_ENQUEUED(v, v_list))) { _dispatch_voucher_debug("corruption", v); DISPATCH_CLIENT_CRASH(v->v_list.tqe_prev, "Voucher corruption"); } TAILQ_INSERT_TAIL(_vouchers_head(kv), v, v_list); _vouchers_lock_unlock(); }
void dispatch_main(void) { #if HAVE_PTHREAD_MAIN_NP if (pthread_main_np()) { #endif _dispatch_program_is_probably_callback_driven = true; pthread_exit(NULL); DISPATCH_CRASH("pthread_exit() returned"); #if HAVE_PTHREAD_MAIN_NP } DISPATCH_CLIENT_CRASH("dispatch_main() must be called on the main thread"); #endif }
void _dispatch_xref_dispose(dispatch_object_t dou) { if (slowpath(DISPATCH_OBJECT_SUSPENDED(dou._do))) { // Arguments for and against this assert are within 6705399 DISPATCH_CLIENT_CRASH("Release of a suspended object"); } #if !USE_OBJC if (dx_type(dou._do) == DISPATCH_SOURCE_KEVENT_TYPE) { _dispatch_source_xref_dispose(dou._ds); } else if (dou._dq->do_vtable == DISPATCH_VTABLE(queue_runloop)) { _dispatch_runloop_queue_xref_dispose(dou._dq); } return _dispatch_release(dou._os_obj); #endif }
static void _voucher_remove(voucher_t v) { mach_voucher_t kv = v->v_ipc_kvoucher; if (!_TAILQ_IS_ENQUEUED(v, v_list)) return; _vouchers_lock_lock(); if (slowpath(!kv)) { _dispatch_voucher_debug("corruption", v); DISPATCH_CLIENT_CRASH(0, "Voucher corruption"); } // check for resurrection race with _voucher_find_and_retain if (os_atomic_load2o(v, os_obj_xref_cnt, ordered) < 0 && _TAILQ_IS_ENQUEUED(v, v_list)) { TAILQ_REMOVE(_vouchers_head(kv), v, v_list); _TAILQ_MARK_NOT_ENQUEUED(v, v_list); v->v_list.tqe_next = (void*)~0ull; } _vouchers_lock_unlock(); }
DISPATCH_NOINLINE void _dispatch_source_xref_release(dispatch_source_t ds) { #ifndef DISPATCH_NO_LEGACY if (ds->ds_is_legacy) { if (!(ds->ds_timer.flags & DISPATCH_TIMER_TYPE_MASK == DISPATCH_TIMER_ONESHOT)) { dispatch_source_cancel(ds); } // Clients often leave sources suspended at the last release dispatch_atomic_and(&ds->do_suspend_cnt, DISPATCH_OBJECT_SUSPEND_LOCK); } else #endif if (slowpath(DISPATCH_OBJECT_SUSPENDED(ds))) { // Arguments for and against this assert are within 6705399 DISPATCH_CLIENT_CRASH("Release of a suspended object"); } _dispatch_wakeup(as_do(ds)); _dispatch_release(as_do(ds)); }
// Return true if this bit was the last in the bitmap, and it is now all zeroes DISPATCH_ALWAYS_INLINE_NDEBUG static bool bitmap_clear_bit(volatile bitmap_t *bitmap, unsigned int index, bool exclusively) { #if DISPATCH_DEBUG dispatch_assert(index < CONTINUATIONS_PER_BITMAP); #endif const bitmap_t mask = BITMAP_C(1) << index; bitmap_t b; if (exclusively == CLEAR_EXCLUSIVELY) { if (slowpath((*bitmap & mask) == 0)) { DISPATCH_CLIENT_CRASH(*bitmap, "Corruption: failed to clear bit exclusively"); } } // and-and-fetch b = os_atomic_and(bitmap, ~mask, release); return b == 0; }
DISPATCH_NOINLINE static void _dispatch_alloc_try_create_heap(dispatch_heap_t *heap_ptr) { #if HAVE_MACH kern_return_t kr; mach_vm_size_t vm_size = MAGAZINES_PER_HEAP * BYTES_PER_MAGAZINE; mach_vm_offset_t vm_mask = ~MAGAZINE_MASK; mach_vm_address_t vm_addr = vm_page_size; while (slowpath(kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size, vm_mask, VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_MEMORY_LIBDISPATCH), MEMORY_OBJECT_NULL, 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT))) { if (kr != KERN_NO_SPACE) { (void)dispatch_assume_zero(kr); DISPATCH_CLIENT_CRASH("Could not allocate heap"); } _dispatch_temporary_resource_shortage(); vm_addr = vm_page_size; } uintptr_t aligned_region = (uintptr_t)vm_addr; #else // HAVE_MACH const size_t region_sz = (1 + MAGAZINES_PER_HEAP) * BYTES_PER_MAGAZINE; void *region_p; while (!dispatch_assume((region_p = mmap(NULL, region_sz, PROT_READ|PROT_WRITE, MAP_ANON | MAP_PRIVATE, VM_MAKE_TAG(VM_MEMORY_LIBDISPATCH), 0)) != MAP_FAILED)) { _dispatch_temporary_resource_shortage(); } uintptr_t region = (uintptr_t)region_p; uintptr_t region_end = region + region_sz; uintptr_t aligned_region, aligned_region_end; uintptr_t bottom_slop_len, top_slop_len; // Realign if needed; find the slop at top/bottom to unmap if ((region & ~(MAGAZINE_MASK)) == 0) { bottom_slop_len = 0; aligned_region = region; aligned_region_end = region_end - BYTES_PER_MAGAZINE; top_slop_len = BYTES_PER_MAGAZINE; } else { aligned_region = (region & MAGAZINE_MASK) + BYTES_PER_MAGAZINE; aligned_region_end = aligned_region + (MAGAZINES_PER_HEAP * BYTES_PER_MAGAZINE); bottom_slop_len = aligned_region - region; top_slop_len = BYTES_PER_MAGAZINE - bottom_slop_len; } #if DISPATCH_DEBUG // Double-check our math. dispatch_assert(aligned_region % PAGE_SIZE == 0); dispatch_assert(aligned_region_end % PAGE_SIZE == 0); dispatch_assert(aligned_region_end > aligned_region); dispatch_assert(top_slop_len % PAGE_SIZE == 0); dispatch_assert(bottom_slop_len % PAGE_SIZE == 0); dispatch_assert(aligned_region_end + top_slop_len == region_end); dispatch_assert(region + bottom_slop_len == aligned_region); dispatch_assert(region_sz == bottom_slop_len + top_slop_len + MAGAZINES_PER_HEAP * BYTES_PER_MAGAZINE); if (bottom_slop_len) { (void)dispatch_assume_zero(mprotect((void *)region, bottom_slop_len, PROT_NONE)); } if (top_slop_len) { (void)dispatch_assume_zero(mprotect((void *)aligned_region_end, top_slop_len, PROT_NONE)); } #else if (bottom_slop_len) { (void)dispatch_assume_zero(munmap((void *)region, bottom_slop_len)); } if (top_slop_len) { (void)dispatch_assume_zero(munmap((void *)aligned_region_end, top_slop_len)); } #endif // DISPATCH_DEBUG #endif // HAVE_MACH if (!dispatch_atomic_cmpxchg(heap_ptr, NULL, (void *)aligned_region, relaxed)) { // If we lost the race to link in the new region, unmap the whole thing. #if DISPATCH_DEBUG (void)dispatch_assume_zero(mprotect((void *)aligned_region, MAGAZINES_PER_HEAP * BYTES_PER_MAGAZINE, PROT_NONE)); #else (void)dispatch_assume_zero(munmap((void *)aligned_region, MAGAZINES_PER_HEAP * BYTES_PER_MAGAZINE)); #endif } }