DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_apply_invoke(void *ctxt)
{
dispatch_apply_t da = (dispatch_apply_t)ctxt;
size_t const iter = da->da_iterations;
typeof(da->da_func) const func = da->da_func;
void *const da_ctxt = da->da_ctxt;
size_t idx, done = 0;
_dispatch_workitem_dec(); // this unit executes many items
// Make nested dispatch_apply fall into serial case rdar://problem/9294578
_dispatch_thread_setspecific(dispatch_apply_key, (void*)~0ul);
// Striding is the responsibility of the caller.
while (fastpath((idx = dispatch_atomic_inc2o(da, da_index) - 1) < iter)) {
_dispatch_client_callout2(da_ctxt, idx, func);
_dispatch_workitem_inc();
done++;
}
_dispatch_thread_setspecific(dispatch_apply_key, NULL);
dispatch_atomic_release_barrier();
// The thread that finished the last workitem wakes up the (possibly waiting)
// thread that called dispatch_apply. They could be one and the same.
if (done && (dispatch_atomic_add2o(da, da_done, done) == iter)) {
_dispatch_thread_semaphore_signal(da->da_sema);
}
if (dispatch_atomic_dec2o(da, da_thr_cnt) == 0) {
_dispatch_continuation_free((dispatch_continuation_t)da);
}
}
static void
_dispatch_apply_serial(void *context)
{
struct dispatch_apply_s *da = context;
size_t idx = 0;
_dispatch_workitem_dec(); // this unit executes many items
do {
da->da_func(da->da_ctxt, idx);
_dispatch_workitem_inc();
} while (++idx < da->da_iterations);
}
static void
_dispatch_apply_serial(void *ctxt)
{
dispatch_apply_t da = (dispatch_apply_t)ctxt;
size_t idx = 0;
_dispatch_workitem_dec(); // this unit executes many items
do {
_dispatch_client_callout2(da->da_ctxt, idx, da->da_func);
_dispatch_workitem_inc();
} while (++idx < da->da_iterations);
_dispatch_continuation_free((dispatch_continuation_t)da);
}
static void
_dispatch_apply2(void *_ctxt)
{
struct dispatch_apply_s *da = _ctxt;
size_t const iter = da->da_iterations;
dispatch_function_apply_t func = da->da_func;
void *const ctxt = da->da_ctxt;
size_t idx;
_dispatch_workitem_dec(); // this unit executes many items
// Striding is the responsibility of the caller.
while (fastpath((idx = dispatch_atomic_inc((intptr_t*)&da->da_index) - 1) < iter)) {
func(ctxt, idx);
_dispatch_workitem_inc();
}
if (dispatch_atomic_dec((intptr_t*)&da->da_thr_cnt) == 0) {
dispatch_semaphore_signal(da->da_sema);
}
}
DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_apply_f2(dispatch_queue_t dq, dispatch_apply_t da,
dispatch_function_t func)
{
uint32_t i = 0;
dispatch_continuation_t head = NULL, tail = NULL;
// The current thread does not need a continuation
uint32_t continuation_cnt = da->da_thr_cnt - 1;
dispatch_assert(continuation_cnt);
for (i = 0; i < continuation_cnt; i++) {
dispatch_continuation_t next = _dispatch_continuation_alloc();
next->do_vtable = (void *)DISPATCH_OBJ_ASYNC_BIT;
next->dc_func = func;
next->dc_ctxt = da;
next->do_next = head;
head = next;
if (!tail) {
tail = next;
}
}
_dispatch_thread_semaphore_t sema = _dispatch_get_thread_semaphore();
da->da_sema = sema;
_dispatch_queue_push_list(dq, head, tail, continuation_cnt);
// Call the first element directly
_dispatch_apply2(da);
_dispatch_workitem_inc();
_dispatch_thread_semaphore_wait(sema);
_dispatch_put_thread_semaphore(sema);
}