static ERTS_INLINE void async_reply(ErtsAsync *a, ErtsThrQPrepEnQ_t *prep_enq)
{
#if ERTS_USE_ASYNC_READY_Q
ErtsAsyncReadyQ *arq;
#if ERTS_ASYNC_PRINT_JOB
erts_fprintf(stderr, "=>> %ld\n", a->async_id);
#endif
arq = async_ready_q(a->sched_id);
#if ERTS_USE_ASYNC_READY_ENQ_MTX
erts_mtx_lock(&arq->x.data.enq_mtx);
#endif
erts_thr_q_enqueue_prepared(&arq->thr_q, (void *) a, prep_enq);
#if ERTS_USE_ASYNC_READY_ENQ_MTX
erts_mtx_unlock(&arq->x.data.enq_mtx);
#endif
#else /* ERTS_USE_ASYNC_READY_Q */
call_async_ready(a);
erts_free(ERTS_ALC_T_ASYNC, (void *) a);
#endif /* ERTS_USE_ASYNC_READY_Q */
}
static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q)
{
#ifdef USE_VM_PROBES
int len;
#endif
if (is_internal_port(a->port)) {
#if ERTS_USE_ASYNC_READY_Q
ErtsAsyncReadyQ *arq = async_ready_q(a->sched_id);
a->q.prep_enq = erts_thr_q_prepare_enqueue(&arq->thr_q);
#endif
/* make sure the driver will stay around */
if (a->hndl)
erts_ddll_reference_referenced_driver(a->hndl);
}
#if ERTS_ASYNC_PRINT_JOB
erts_fprintf(stderr, "-> %ld\n", a->async_id);
#endif
erts_thr_q_enqueue(&q->thr_q, a);
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(aio_pool_add)) {
DTRACE_CHARBUF(port_str, 16);
erts_snprintf(port_str, sizeof(port_str), "%T", a->port);
/* DTRACE TODO: Get the queue length from erts_thr_q_enqueue() ? */
len = -1;
DTRACE2(aio_pool_add, port_str, len);
}
gcc_optimizer_hack++;
#endif
}
void *
erts_get_async_ready_queue(Uint sched_id)
{
return (void *) async ? async_ready_q(sched_id) : NULL;
}
void
erts_init_async(void)
{
async = NULL;
if (erts_async_max_threads > 0) {
#if ERTS_USE_ASYNC_READY_Q
ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT;
#endif
erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER;
char *ptr, thr_name[16];
size_t tot_size = 0;
int i;
tot_size += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncData));
tot_size += sizeof(ErtsAlgndAsyncQ)*erts_async_max_threads;
#if ERTS_USE_ASYNC_READY_Q
tot_size += sizeof(ErtsAlgndAsyncReadyQ)*erts_no_schedulers;
#endif
ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_ASYNC_DATA,
tot_size);
async = (ErtsAsyncData *) ptr;
ptr += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncData));
async->init.data.no_initialized = 0;
erts_mtx_init(&async->init.data.mtx, "async_init_mtx", NIL,
ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER);
erts_cnd_init(&async->init.data.cnd);
erts_atomic_init_nob(&async->init.data.id, 0);
async->queue = (ErtsAlgndAsyncQ *) ptr;
ptr += sizeof(ErtsAlgndAsyncQ)*erts_async_max_threads;
#if ERTS_USE_ASYNC_READY_Q
qinit.live.queue = ERTS_THR_Q_LIVE_LONG;
qinit.live.objects = ERTS_THR_Q_LIVE_SHORT;
qinit.notify = erts_notify_check_async_ready_queue;
async->ready_queue = (ErtsAlgndAsyncReadyQ *) ptr;
ptr += sizeof(ErtsAlgndAsyncReadyQ)*erts_no_schedulers;
for (i = 1; i <= erts_no_schedulers; i++) {
ErtsAsyncReadyQ *arq = async_ready_q(i);
#if ERTS_USE_ASYNC_READY_ENQ_MTX
erts_mtx_init(&arq->x.data.enq_mtx, "async_enq_mtx", make_small(i),
ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER);
#endif
erts_thr_q_finalize_dequeue_state_init(&arq->fin_deq);
qinit.arg = (void *) (SWord) i;
erts_thr_q_initialize(&arq->thr_q, &qinit);
}
#endif
/* Create async threads... */
thr_opts.detached = 0;
thr_opts.suggested_stack_size
= erts_async_thread_suggested_stack_size;
thr_opts.name = thr_name;
for (i = 0; i < erts_async_max_threads; i++) {
ErtsAsyncQ *aq = async_q(i);
erts_snprintf(thr_opts.name, 16, "async_%d", i+1);
erts_thr_create(&aq->thr_id, async_main, (void*) aq, &thr_opts);
}
/* Wait for async threads to initialize... */
erts_mtx_lock(&async->init.data.mtx);
while (async->init.data.no_initialized != erts_async_max_threads)
erts_cnd_wait(&async->init.data.cnd, &async->init.data.mtx);
erts_mtx_unlock(&async->init.data.mtx);
erts_mtx_destroy(&async->init.data.mtx);
erts_cnd_destroy(&async->init.data.cnd);
}
}
