static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q)
{
#ifdef USE_VM_PROBES
int len;
#endif
if (is_internal_port(a->port)) {
#if ERTS_USE_ASYNC_READY_Q
ErtsAsyncReadyQ *arq = async_ready_q(a->sched_id);
a->q.prep_enq = erts_thr_q_prepare_enqueue(&arq->thr_q);
#endif
/* make sure the driver will stay around */
if (a->hndl)
erts_ddll_reference_referenced_driver(a->hndl);
}
#if ERTS_ASYNC_PRINT_JOB
erts_fprintf(stderr, "-> %ld\n", a->async_id);
#endif
erts_thr_q_enqueue(&q->thr_q, a);
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(aio_pool_add)) {
DTRACE_CHARBUF(port_str, 16);
erts_snprintf(port_str, sizeof(port_str), "%T", a->port);
/* DTRACE TODO: Get the queue length from erts_thr_q_enqueue() ? */
len = -1;
DTRACE2(aio_pool_add, port_str, len);
}
gcc_optimizer_hack++;
#endif
}
/**
* Use mpmcqs to allow stealing directly from a victim, without waiting for a
* response.
*/
static pony_actor_t* steal(scheduler_t* sched, pony_actor_t* prev)
{
send_msg(0, SCHED_BLOCK, 0);
uint64_t tsc = ponyint_cpu_tick();
pony_actor_t* actor;
while(true)
{
scheduler_t* victim = choose_victim(sched);
if(victim == NULL)
actor = (pony_actor_t*)ponyint_mpmcq_pop(&inject);
else
actor = pop_global(victim);
if(actor != NULL)
{
DTRACE3(WORK_STEAL_SUCCESSFUL, (uintptr_t)sched, (uintptr_t)victim, (uintptr_t)actor);
break;
}
uint64_t tsc2 = ponyint_cpu_tick();
if(quiescent(sched, tsc, tsc2))
{
DTRACE2(WORK_STEAL_FAILURE, (uintptr_t)sched, (uintptr_t)victim);
return NULL;
}
// If we have been passed an actor (implicitly, the cycle detector), and
// enough time has elapsed without stealing or quiescing, return the actor
// we were passed (allowing the cycle detector to run).
if((prev != NULL) && ((tsc2 - tsc) > 10000000000))
{
actor = prev;
break;
}
}
send_msg(0, SCHED_UNBLOCK, 0);
return actor;
}
/*
* Copy object "obj" to process p.
*/
Eterm
copy_object(Eterm obj, Process* to)
{
Uint size = size_object(obj);
Eterm* hp = HAlloc(to, size);
Eterm res;
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(copy_object)) {
DTRACE_CHARBUF(proc_name, 64);
erts_snprintf(proc_name, sizeof(proc_name), "%T", to->common.id);
DTRACE2(copy_object, proc_name, size);
}
#endif
res = copy_struct(obj, size, &hp, &to->off_heap);
#ifdef DEBUG
if (eq(obj, res) == 0) {
erl_exit(ERTS_ABORT_EXIT, "copy not equal to source\n");
}
#endif
return res;
}
static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q,
erts_tse_t *tse,
ErtsThrQPrepEnQ_t **prep_enq)
{
#if ERTS_USE_ASYNC_READY_Q
int saved_fin_deq = 0;
ErtsThrQFinDeQ_t fin_deq;
#endif
#ifdef USE_VM_PROBES
int len;
#endif
while (1) {
ErtsAsync *a = (ErtsAsync *) erts_thr_q_dequeue(q);
if (a) {
#if ERTS_USE_ASYNC_READY_Q
*prep_enq = a->q.prep_enq;
erts_thr_q_get_finalize_dequeue_data(q, &a->q.fin_deq);
if (saved_fin_deq)
erts_thr_q_append_finalize_dequeue_data(&a->q.fin_deq, &fin_deq);
#endif
#ifdef USE_LTTNG_VM_TRACEPOINTS
if (LTTNG_ENABLED(aio_pool_get)) {
lttng_decl_portbuf(port_str);
int length = erts_thr_q_length_dirty(q);
lttng_portid_to_str(a->port, port_str);
LTTNG2(aio_pool_get, port_str, length);
}
#endif
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(aio_pool_get)) {
DTRACE_CHARBUF(port_str, 16);
erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)),
"%T", a->port);
/* DTRACE TODO: Get the length from erts_thr_q_dequeue() ? */
len = -1;
DTRACE2(aio_pool_get, port_str, len);
}
#endif
return a;
}
if (ERTS_THR_Q_DIRTY != erts_thr_q_clean(q)) {
ErtsThrQFinDeQ_t tmp_fin_deq;
erts_tse_reset(tse);
#if ERTS_USE_ASYNC_READY_Q
chk_fin_deq:
if (erts_thr_q_get_finalize_dequeue_data(q, &tmp_fin_deq)) {
if (!saved_fin_deq) {
erts_thr_q_finalize_dequeue_state_init(&fin_deq);
saved_fin_deq = 1;
}
erts_thr_q_append_finalize_dequeue_data(&fin_deq,
&tmp_fin_deq);
}
#endif
switch (erts_thr_q_inspect(q, 1)) {
case ERTS_THR_Q_DIRTY:
break;
case ERTS_THR_Q_NEED_THR_PRGR:
#ifdef ERTS_SMP
{
ErtsThrPrgrVal prgr = erts_thr_q_need_thr_progress(q);
erts_thr_progress_wakeup(NULL, prgr);
/*
* We do no dequeue finalizing in hope that a new async
* job will arrive before we are woken due to thread
* progress...
*/
erts_tse_wait(tse);
break;
}
#endif
case ERTS_THR_Q_CLEAN:
#if ERTS_USE_ASYNC_READY_Q
if (saved_fin_deq) {
if (erts_thr_q_finalize_dequeue(&fin_deq))
goto chk_fin_deq;
else
saved_fin_deq = 0;
}
#endif
erts_tse_wait(tse);
break;
default:
ASSERT(0);
break;
}
}
}
}
/**
* Run a scheduler thread until termination.
*/
static void run(scheduler_t* sched)
{
pony_actor_t* actor = pop_global(sched);
if (DTRACE_ENABLED(ACTOR_SCHEDULED) && actor != NULL) {
DTRACE2(ACTOR_SCHEDULED, (uintptr_t)sched, (uintptr_t)actor);
}
while(true)
{
if(actor == NULL)
{
// We had an empty queue and no rescheduled actor.
actor = steal(sched, NULL);
if(actor == NULL)
{
// Termination.
pony_assert(pop(sched) == NULL);
return;
}
DTRACE2(ACTOR_SCHEDULED, (uintptr_t)sched, (uintptr_t)actor);
}
// Run the current actor and get the next actor.
bool reschedule = ponyint_actor_run(&sched->ctx, actor, SCHED_BATCH);
pony_actor_t* next = pop_global(sched);
if(reschedule)
{
if(next != NULL)
{
// If we have a next actor, we go on the back of the queue. Otherwise,
// we continue to run this actor.
push(sched, actor);
DTRACE2(ACTOR_DESCHEDULED, (uintptr_t)sched, (uintptr_t)actor);
actor = next;
DTRACE2(ACTOR_SCHEDULED, (uintptr_t)sched, (uintptr_t)actor);
} else if(ponyint_is_cycle(actor)) {
// If all we have is the cycle detector, try to steal something else to
// run as well.
next = steal(sched, actor);
if(next == NULL)
{
// Termination.
DTRACE2(ACTOR_DESCHEDULED, (uintptr_t)sched, (uintptr_t)actor);
return;
}
// Push the cycle detector and run the actor we stole.
if(actor != next)
{
push(sched, actor);
DTRACE2(ACTOR_DESCHEDULED, (uintptr_t)sched, (uintptr_t)actor);
actor = next;
DTRACE2(ACTOR_SCHEDULED, (uintptr_t)sched, (uintptr_t)actor);
}
}
} else {
// We aren't rescheduling, so run the next actor. This may be NULL if our
// queue was empty.
DTRACE2(ACTOR_DESCHEDULED, (uintptr_t)sched, (uintptr_t)actor);
actor = next;
if (DTRACE_ENABLED(ACTOR_SCHEDULED) && actor != NULL) {
DTRACE2(ACTOR_SCHEDULED, (uintptr_t)sched, (uintptr_t)actor);
}
}
}
}