DISPATCH_NOINLINE
static dispatch_continuation_t
_dispatch_alloc_continuation_from_heap_slow(void)
{
dispatch_heap_t *heap = &_dispatch_main_heap;
dispatch_continuation_t cont;
for (;;) {
if (!fastpath(*heap)) {
_dispatch_alloc_try_create_heap(heap);
}
cont = _dispatch_alloc_continuation_from_heap(*heap);
if (fastpath(cont)) {
return cont;
}
// If we have tuned our parameters right, 99.999% of apps should
// never reach this point! The ones that do have gone off the rails...
//
// Magazine is full? Onto the next heap!
// We tried 'stealing' from other CPUs' magazines. The net effect
// was worse performance from more wasted search time and more
// cache contention.
// rdar://11378331
// Future optimization: start at the page we last used, start
// in the *zone* we last used. But this would only improve deeply
// pathological cases like dispatch_starfish
heap = &(*heap)->header.dh_next;
}
}
void
dispatch_resume(dispatch_object_t dou)
{
DISPATCH_OBJECT_TFB(_dispatch_objc_resume, dou);
// Global objects cannot be suspended or resumed. This also has the
// side effect of saturating the suspend count of an object and
// guarding against resuming due to overflow.
if (slowpath(dou._do->do_ref_cnt == DISPATCH_OBJECT_GLOBAL_REFCNT) ||
slowpath(dx_type(dou._do) == DISPATCH_QUEUE_ROOT_TYPE)) {
return;
}
// Check the previous value of the suspend count. If the previous
// value was a single suspend interval, the object should be resumed.
// If the previous value was less than the suspend interval, the object
// has been over-resumed.
unsigned int suspend_cnt = dispatch_atomic_sub_orig2o(dou._do,
do_suspend_cnt, DISPATCH_OBJECT_SUSPEND_INTERVAL, relaxed);
if (fastpath(suspend_cnt > DISPATCH_OBJECT_SUSPEND_INTERVAL)) {
// Balancing the retain() done in suspend() for rdar://8181908
return _dispatch_release(dou._do);
}
if (fastpath(suspend_cnt == DISPATCH_OBJECT_SUSPEND_INTERVAL)) {
return _dispatch_resume_slow(dou);
}
DISPATCH_CLIENT_CRASH("Over-resume of an object");
}
DISPATCH_ALLOC_NOINLINE
static dispatch_continuation_t
_dispatch_alloc_continuation_alloc(void)
{
dispatch_continuation_t cont;
if (fastpath(_dispatch_main_heap)) {
// Start looking in the same page where we found a continuation
// last time.
bitmap_t *last = last_found_page();
if (fastpath(last)) {
unsigned int i;
for (i = 0; i < BITMAPS_PER_PAGE; i++) {
bitmap_t *cur = last + i;
unsigned int index = bitmap_set_first_unset_bit(cur);
if (fastpath(index != NO_BITS_WERE_UNSET)) {
bitmap_t *supermap;
unsigned int bindex;
get_cont_and_indices_for_bitmap_and_index(cur,
index, &cont, &supermap, &bindex);
mark_bitmap_as_full_if_still_full(supermap, bindex,
cur);
return cont;
}
}
}
cont = _dispatch_alloc_continuation_from_heap(_dispatch_main_heap);
if (fastpath(cont)) {
return cont;
}
}
return _dispatch_alloc_continuation_from_heap_slow();
}
DISPATCH_ALWAYS_INLINE_NDEBUG
static void
get_maps_and_indices_for_continuation(dispatch_continuation_t c,
bitmap_t **supermap_out, unsigned int *bitmap_index_out,
bitmap_t **bitmap_out, unsigned int *index_out)
{
unsigned int cindex, sindex, index, mindex;
padded_continuation *p = (padded_continuation *)c;
struct dispatch_magazine_s *m = magazine_for_continuation(c);
#if PACK_FIRST_PAGE_WITH_CONTINUATIONS
if (fastpath(continuation_is_in_first_page(c))) {
cindex = (unsigned int)(p - m->fp_conts);
index = cindex % CONTINUATIONS_PER_BITMAP;
mindex = cindex / CONTINUATIONS_PER_BITMAP;
if (fastpath(supermap_out)) *supermap_out = NULL;
if (fastpath(bitmap_index_out)) *bitmap_index_out = mindex;
if (fastpath(bitmap_out)) *bitmap_out = &m->fp_maps[mindex];
if (fastpath(index_out)) *index_out = index;
return;
}
#endif // PACK_FIRST_PAGE_WITH_CONTINUATIONS
cindex = (unsigned int)(p - (padded_continuation *)m->conts);
sindex = cindex / (BITMAPS_PER_SUPERMAP * CONTINUATIONS_PER_BITMAP);
mindex = (cindex / CONTINUATIONS_PER_BITMAP) % BITMAPS_PER_SUPERMAP;
index = cindex % CONTINUATIONS_PER_BITMAP;
if (fastpath(supermap_out)) *supermap_out = &m->supermaps[sindex];
if (fastpath(bitmap_index_out)) *bitmap_index_out = mindex;
if (fastpath(bitmap_out)) *bitmap_out = &m->maps[sindex][mindex];
if (fastpath(index_out)) *index_out = index;
}
DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_apply_invoke(void *ctxt)
{
dispatch_apply_t da = (dispatch_apply_t)ctxt;
size_t const iter = da->da_iterations;
typeof(da->da_func) const func = da->da_func;
void *const da_ctxt = da->da_ctxt;
size_t idx, done = 0;
_dispatch_workitem_dec(); // this unit executes many items
// Make nested dispatch_apply fall into serial case rdar://problem/9294578
_dispatch_thread_setspecific(dispatch_apply_key, (void*)~0ul);
// Striding is the responsibility of the caller.
while (fastpath((idx = dispatch_atomic_inc2o(da, da_index) - 1) < iter)) {
_dispatch_client_callout2(da_ctxt, idx, func);
_dispatch_workitem_inc();
done++;
}
_dispatch_thread_setspecific(dispatch_apply_key, NULL);
dispatch_atomic_release_barrier();
// The thread that finished the last workitem wakes up the (possibly waiting)
// thread that called dispatch_apply. They could be one and the same.
if (done && (dispatch_atomic_add2o(da, da_done, done) == iter)) {
_dispatch_thread_semaphore_signal(da->da_sema);
}
if (dispatch_atomic_dec2o(da, da_thr_cnt) == 0) {
_dispatch_continuation_free((dispatch_continuation_t)da);
}
}
void kThread::postSwitchFunc(uThread* nextuThread, void* args = nullptr) {
kThread* ck = kThread::currentKT;
//mainUT does not need to be managed here
if (fastpath(ck->currentUT != kThread::currentKT->mainUT)) {
switch (ck->currentUT->state) {
case uThread::State::TERMINATED:
ck->currentUT->destory(false);
break;
case uThread::State::YIELD:
ck->currentUT->resume();
;
break;
case uThread::State::MIGRATE:
ck->currentUT->resume();
break;
case uThread::State::WAITING: {
//function and the argument should be set for pss
assert(postSuspendFunc != nullptr);
postSuspendFunc((void*)ck->currentUT, args);
break;
}
default:
break;
}
}
//Change the current thread to the next
ck->currentUT = nextuThread;
nextuThread->state = uThread::State::RUNNING;
}
ssize_t IOPoller::_Poll(int timeout){
assert(epoll_fd);
int32_t n, mode;
struct epoll_event *ev;
//TODO: dedicated thread always blocks but others should not
n = epoll_wait(epoll_fd, events, MAXEVENTS, timeout);
if (n == -1) {
if (errno != EINTR) {
return (-1);
}
return (0);
}
//timeout
if(n == 0 ) return 0;
for(int i = 0; i < n; i++) {
ev = &events[i];
if(slowpath(ev->events == 0))
continue;
mode = 0;
if(ev->events & (EPOLLIN|EPOLLRDHUP|EPOLLHUP|EPOLLERR))
mode |= IOHandler::UT_IOREAD;
if(ev->events & (EPOLLOUT|EPOLLHUP|EPOLLERR))
mode |= IOHandler::UT_IOWRITE;
if(fastpath(mode))
ioh.PollReady( *(PollData*) ev->data.ptr , mode);
}
return n;
}
DISPATCH_NOINLINE
void
dispatch_group_notify_f(dispatch_group_t dg, dispatch_queue_t dq, void *ctxt,
void (*func)(void *))
{
dispatch_semaphore_t dsema = (dispatch_semaphore_t)dg;
struct dispatch_sema_notify_s *dsn, *prev;
// FIXME -- this should be updated to use the continuation cache
while (!(dsn = calloc(1, sizeof(*dsn)))) {
sleep(1);
}
dsn->dsn_queue = dq;
dsn->dsn_ctxt = ctxt;
dsn->dsn_func = func;
_dispatch_retain(dq);
dispatch_atomic_store_barrier();
prev = dispatch_atomic_xchg2o(dsema, dsema_notify_tail, dsn);
if (fastpath(prev)) {
prev->dsn_next = dsn;
} else {
_dispatch_retain(dg);
(void)dispatch_atomic_xchg2o(dsema, dsema_notify_head, dsn);
if (dsema->dsema_value == dsema->dsema_orig) {
_dispatch_group_wake(dsema);
}
}
}
DISPATCH_NOINLINE
static long
_dispatch_group_wake(dispatch_semaphore_t dsema)
{
struct dispatch_sema_notify_s *next, *head, *tail = NULL;
long rval;
head = dispatch_atomic_xchg2o(dsema, dsema_notify_head, NULL);
if (head) {
// snapshot before anything is notified/woken <rdar://problem/8554546>
tail = dispatch_atomic_xchg2o(dsema, dsema_notify_tail, NULL);
}
rval = dispatch_atomic_xchg2o(dsema, dsema_group_waiters, 0);
if (rval) {
// wake group waiters
#if USE_MACH_SEM
_dispatch_semaphore_create_port(&dsema->dsema_waiter_port);
do {
kern_return_t kr = semaphore_signal(dsema->dsema_waiter_port);
DISPATCH_SEMAPHORE_VERIFY_KR(kr);
} while (--rval);
#elif USE_POSIX_SEM
do {
int ret = sem_post(&dsema->dsema_sem);
DISPATCH_SEMAPHORE_VERIFY_RET(ret);
} while (--rval);
#endif
}
if (head) {
// async group notify blocks
do {
dispatch_async_f(head->dsn_queue, head->dsn_ctxt, head->dsn_func);
_dispatch_release(head->dsn_queue);
next = fastpath(head->dsn_next);
if (!next && head != tail) {
while (!(next = fastpath(head->dsn_next))) {
_dispatch_hardware_pause();
}
}
free(head);
} while ((head = next));
_dispatch_release(dsema);
}
return 0;
}
long
dispatch_semaphore_wait(dispatch_semaphore_t dsema, dispatch_time_t timeout)
{
long value = dispatch_atomic_dec2o(dsema, dsema_value);
dispatch_atomic_acquire_barrier();
if (fastpath(value >= 0)) {
return 0;
}
return _dispatch_semaphore_wait_slow(dsema, timeout);
}
static dispatch_continuation_t
_dispatch_malloc_continuation_alloc(void)
{
dispatch_continuation_t dc;
while (!(dc = fastpath(calloc(1,
ROUND_UP_TO_CACHELINE_SIZE(sizeof(*dc)))))) {
_dispatch_temporary_resource_shortage();
}
return dc;
}
DISPATCH_ALWAYS_INLINE_NDEBUG
static void
get_cont_and_indices_for_bitmap_and_index(bitmap_t *bitmap,
unsigned int index, dispatch_continuation_t *continuation_out,
bitmap_t **supermap_out, unsigned int *bitmap_index_out)
{
// m_for_c wants a continuation not a bitmap, but it works because it
// just masks off the bottom bits of the address.
struct dispatch_magazine_s *m = magazine_for_continuation((void *)bitmap);
unsigned int mindex = (unsigned int)(bitmap - m->maps[0]);
unsigned int bindex = mindex % BITMAPS_PER_SUPERMAP;
unsigned int sindex = mindex / BITMAPS_PER_SUPERMAP;
dispatch_assert(&m->maps[sindex][bindex] == bitmap);
if (fastpath(continuation_out)) {
*continuation_out = continuation_address(m, sindex, bindex, index);
}
if (fastpath(supermap_out)) *supermap_out = supermap_address(m, sindex);
if (fastpath(bitmap_index_out)) *bitmap_index_out = bindex;
}
long
dispatch_semaphore_signal(dispatch_semaphore_t dsema)
{
dispatch_atomic_release_barrier();
long value = dispatch_atomic_inc2o(dsema, dsema_value);
if (fastpath(value > 0)) {
return 0;
}
if (slowpath(value == LONG_MIN)) {
DISPATCH_CLIENT_CRASH("Unbalanced call to dispatch_semaphore_signal()");
}
return _dispatch_semaphore_signal_slow(dsema);
}
DISPATCH_NOINLINE
void
_os_object_release(_os_object_t obj)
{
int xref_cnt = _os_object_xrefcnt_dec(obj);
if (fastpath(xref_cnt >= 0)) {
return;
}
if (slowpath(xref_cnt < -1)) {
_OS_OBJECT_CLIENT_CRASH("Over-release of an object");
}
return _os_object_xref_dispose(obj);
}
DISPATCH_ALWAYS_INLINE_NDEBUG
static dispatch_continuation_t
alloc_continuation_from_first_page(struct dispatch_magazine_s *magazine)
{
unsigned int i, index, continuation_index;
// TODO: unroll if this is hot?
for (i = 0; i < FULL_BITMAPS_IN_FIRST_PAGE; i++) {
index = bitmap_set_first_unset_bit(&magazine->fp_maps[i]);
if (fastpath(index != NO_BITS_WERE_UNSET)) goto found;
}
if (REMAINDERED_CONTINUATIONS_IN_FIRST_PAGE) {
index = bitmap_set_first_unset_bit_upto_index(&magazine->fp_maps[i],
REMAINDERED_CONTINUATIONS_IN_FIRST_PAGE - 1);
if (fastpath(index != NO_BITS_WERE_UNSET)) goto found;
}
return NULL;
found:
continuation_index = (i * CONTINUATIONS_PER_BITMAP) + index;
return (dispatch_continuation_t)&magazine->fp_conts[continuation_index];
}
DISPATCH_NOINLINE
void
_os_object_release(_os_object_t obj)
{
int xref_cnt = obj->os_obj_xref_cnt;
if (slowpath(xref_cnt == _OS_OBJECT_GLOBAL_REFCNT)) {
return; // global object
}
xref_cnt = dispatch_atomic_dec2o(obj, os_obj_xref_cnt, relaxed);
if (fastpath(xref_cnt >= 0)) {
return;
}
if (slowpath(xref_cnt < -1)) {
_OS_OBJECT_CLIENT_CRASH("Over-release of an object");
}
return _os_object_xref_dispose(obj);
}
// Base address of page, or NULL if this page shouldn't be madvise()d
DISPATCH_ALWAYS_INLINE_NDEBUG DISPATCH_CONST
static void *
madvisable_page_base_for_continuation(dispatch_continuation_t c)
{
if (fastpath(continuation_is_in_first_page(c))) {
return NULL;
}
void *page_base = (void *)((uintptr_t)c & ~(uintptr_t)PAGE_MASK);
#if DISPATCH_DEBUG
struct dispatch_magazine_s *m = magazine_for_continuation(c);
if (slowpath(page_base < (void *)&m->conts)) {
DISPATCH_CRASH("madvisable continuation too low");
}
if (slowpath(page_base > (void *)&m->conts[SUPERMAPS_PER_MAGAZINE-1]
[BITMAPS_PER_SUPERMAP-1][CONTINUATIONS_PER_BITMAP-1])) {
DISPATCH_CRASH("madvisable continuation too high");
}
#endif
return page_base;
}
static void
_dispatch_apply2(void *_ctxt)
{
struct dispatch_apply_s *da = _ctxt;
size_t const iter = da->da_iterations;
dispatch_function_apply_t func = da->da_func;
void *const ctxt = da->da_ctxt;
size_t idx;
_dispatch_workitem_dec(); // this unit executes many items
// Striding is the responsibility of the caller.
while (fastpath((idx = dispatch_atomic_inc((intptr_t*)&da->da_index) - 1) < iter)) {
func(ctxt, idx);
_dispatch_workitem_inc();
}
if (dispatch_atomic_dec((intptr_t*)&da->da_thr_cnt) == 0) {
dispatch_semaphore_signal(da->da_sema);
}
}
DISPATCH_NOINLINE
static dispatch_continuation_t
_dispatch_alloc_continuation_from_heap(dispatch_heap_t heap)
{
dispatch_continuation_t cont;
unsigned int cpu_number = _dispatch_cpu_number();
#ifdef DISPATCH_DEBUG
dispatch_assert(cpu_number < NUM_CPU);
#endif
#if PACK_FIRST_PAGE_WITH_CONTINUATIONS
// First try the continuations in the first page for this CPU
cont = alloc_continuation_from_first_page(&(heap[cpu_number]));
if (fastpath(cont)) {
return cont;
}
#endif
// Next, try the rest of the magazine for this CPU
cont = alloc_continuation_from_magazine(&(heap[cpu_number]));
return cont;
}
DISPATCH_NOINLINE
void
_os_object_release_internal(_os_object_t obj)
{
int ref_cnt = obj->os_obj_ref_cnt;
if (slowpath(ref_cnt == _OS_OBJECT_GLOBAL_REFCNT)) {
return; // global object
}
ref_cnt = dispatch_atomic_dec2o(obj, os_obj_ref_cnt, relaxed);
if (fastpath(ref_cnt >= 0)) {
return;
}
if (slowpath(ref_cnt < -1)) {
DISPATCH_CRASH("Over-release of an object");
}
#if DISPATCH_DEBUG
if (slowpath(obj->os_obj_xref_cnt >= 0)) {
DISPATCH_CRASH("Release while external references exist");
}
#endif
return _os_object_dispose(obj);
}
