static void async_add(ErlAsync* a, AsyncQueue* q)
{
/* XXX:PaN Is this still necessary when ports lock drivers? */
if (is_internal_port(a->port)) {
ERTS_LC_ASSERT(erts_drvportid2port(a->port));
/* make sure the driver will stay around */
driver_lock_driver(internal_port_index(a->port));
}
erts_mtx_lock(&q->mtx);
if (q->len == 0) {
q->head = a;
q->tail = a;
q->len = 1;
erts_cnd_signal(&q->cv);
}
else { /* no need to signal (since the worker is working) */
a->next = q->head;
q->head->prev = a;
q->head = a;
q->len++;
}
erts_mtx_unlock(&q->mtx);
}
int
erts_port_task_schedule(Eterm id,
ErtsPortTaskHandle *pthp,
ErtsPortTaskType type,
ErlDrvEvent event,
ErlDrvEventData event_data)
{
ErtsRunQueue *runq;
Port *pp;
ErtsPortTask *ptp;
int enq_port = 0;
/*
* NOTE: We might not have the port lock here. We are only
* allowed to access the 'sched', 'tab_status',
* and 'id' fields of the port struct while
* tasks_lock is held.
*/
if (pthp && erts_port_task_is_scheduled(pthp)) {
ASSERT(0);
erts_port_task_abort(id, pthp);
}
ptp = port_task_alloc();
ASSERT(is_internal_port(id));
pp = &erts_port[internal_port_index(id)];
runq = erts_port_runq(pp);
if (!runq || ERTS_PORT_TASK_INVALID_PORT(pp, id)) {
if (runq)
erts_smp_runq_unlock(runq);
return -1;
}
ASSERT(!erts_port_task_is_scheduled(pthp));
ERTS_PT_CHK_PRES_PORTQ(runq, pp);
if (!pp->sched.taskq) {
pp->sched.taskq = port_taskq_init(port_taskq_alloc(), pp);
enq_port = !pp->sched.exe_taskq;
}
#ifdef ERTS_SMP
if (enq_port) {
ErtsRunQueue *xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL);
if (xrunq) {
/* Port emigrated ... */
erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq);
erts_smp_runq_unlock(runq);
runq = xrunq;
}
}
#endif
ASSERT(!enq_port || !(runq->flags & ERTS_RUNQ_FLG_SUSPENDED));
ASSERT(pp->sched.taskq);
ASSERT(ptp);
ptp->type = type;
ptp->event = event;
ptp->event_data = event_data;
set_handle(ptp, pthp);
switch (type) {
case ERTS_PORT_TASK_FREE:
erl_exit(ERTS_ABORT_EXIT,
"erts_port_task_schedule(): Cannot schedule free task\n");
break;
case ERTS_PORT_TASK_INPUT:
case ERTS_PORT_TASK_OUTPUT:
case ERTS_PORT_TASK_EVENT:
erts_smp_atomic_inc_relb(&erts_port_task_outstanding_io_tasks);
/* Fall through... */
default:
enqueue_task(pp->sched.taskq, ptp);
break;
}
#ifndef ERTS_SMP
/*
* When (!enq_port && !pp->sched.exe_taskq) is true in the smp case,
* the port might not be in the run queue. If this is the case, another
* thread is in the process of enqueueing the port. This very seldom
* occur, but do occur and is a valid scenario. Debug info showing this
* enqueue in progress must be introduced before we can enable (modified
* versions of these) assertions in the smp case again.
*/
#if defined(HARD_DEBUG)
if (pp->sched.exe_taskq || enq_port)
ERTS_PT_CHK_NOT_IN_PORTQ(runq, pp);
else
ERTS_PT_CHK_IN_PORTQ(runq, pp);
#elif defined(DEBUG)
if (!enq_port && !pp->sched.exe_taskq) {
/* We should be in port run q */
ASSERT(pp->sched.prev || runq->ports.start == pp);
}
#endif
#endif
if (!enq_port) {
ERTS_PT_CHK_PRES_PORTQ(runq, pp);
erts_smp_runq_unlock(runq);
}
else {
enqueue_port(runq, pp);
ERTS_PT_CHK_PRES_PORTQ(runq, pp);
if (erts_system_profile_flags.runnable_ports) {
profile_runnable_port(pp, am_active);
}
erts_smp_runq_unlock(runq);
erts_smp_notify_inc_runq(runq);
}
return 0;
}
int
erts_port_task_abort(Eterm id, ErtsPortTaskHandle *pthp)
{
ErtsRunQueue *runq;
ErtsPortTaskQueue *ptqp;
ErtsPortTask *ptp;
Port *pp;
int port_is_dequeued = 0;
pp = &erts_port[internal_port_index(id)];
runq = erts_port_runq(pp);
ptp = handle2task(pthp);
if (!ptp) {
erts_smp_runq_unlock(runq);
return 1;
}
ASSERT(ptp->handle == pthp);
ptqp = ptp->queue;
ASSERT(pp == ptqp->port);
ERTS_PT_CHK_PRES_PORTQ(runq, pp);
ASSERT(ptqp);
ASSERT(ptqp->first);
dequeue_task(ptp);
reset_handle(ptp);
switch (ptp->type) {
case ERTS_PORT_TASK_INPUT:
case ERTS_PORT_TASK_OUTPUT:
case ERTS_PORT_TASK_EVENT:
ASSERT(erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) > 0);
erts_smp_atomic_dec_relb(&erts_port_task_outstanding_io_tasks);
break;
default:
break;
}
ASSERT(ptqp == pp->sched.taskq || ptqp == pp->sched.exe_taskq);
if (ptqp->first || pp->sched.taskq != ptqp)
ptqp = NULL;
else {
pp->sched.taskq = NULL;
if (!pp->sched.exe_taskq) {
dequeue_port(runq, pp);
ERTS_PORT_NOT_IN_RUNQ(pp);
port_is_dequeued = 1;
}
}
ERTS_PT_CHK_PRES_PORTQ(runq, pp);
erts_smp_runq_unlock(runq);
if (erts_system_profile_flags.runnable_ports && port_is_dequeued) {
profile_runnable_port(pp, am_inactive);
}
port_task_free(ptp);
if (ptqp)
port_taskq_free(ptqp);
return 0;
}
int
erts_port_task_schedule(Eterm id,
ErtsPortTaskHandle *pthp,
ErtsPortTaskType type,
ErlDrvEvent event,
ErlDrvEventData event_data)
{
ErtsRunQueue *runq;
Port *pp;
ErtsPortTask *ptp;
int enq_port = 0;
/*
* NOTE: We might not have the port lock here. We are only
* allowed to access the 'sched', 'tab_status',
* and 'id' fields of the port struct while
* tasks_lock is held.
*/
if (pthp && erts_port_task_is_scheduled(pthp)) {
ASSERT(0);
erts_port_task_abort(id, pthp);
}
ptp = port_task_alloc();
ASSERT(is_internal_port(id));
pp = &erts_port[internal_port_index(id)];
runq = erts_port_runq(pp);
if (!runq || ERTS_PORT_TASK_INVALID_PORT(pp, id)) {
if (runq)
erts_smp_runq_unlock(runq);
return -1;
}
ASSERT(!erts_port_task_is_scheduled(pthp));
ERTS_PT_CHK_PRES_PORTQ(runq, pp);
#ifdef DEBUG
/* If we have a taskq and not executing, we should be in port run q */
if (pp->sched.taskq && !pp->sched.exe_taskq) {
ASSERT(pp->sched.prev || runq->ports.start == pp);
}
#endif
if (!pp->sched.taskq) {
pp->sched.taskq = port_taskq_init(port_taskq_alloc(), pp);
enq_port = !pp->sched.exe_taskq;
}
#ifdef ERTS_SMP
if (enq_port) {
ErtsRunQueue *xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL);
if (xrunq) {
/* Port emigrated ... */
erts_smp_atomic_set(&pp->run_queue, (long) xrunq);
erts_smp_runq_unlock(runq);
runq = xrunq;
}
}
#endif
ASSERT(!(runq->flags & ERTS_RUNQ_FLG_SUSPENDED));
ASSERT(pp->sched.taskq);
ASSERT(ptp);
ptp->type = type;
ptp->event = event;
ptp->event_data = event_data;
set_handle(ptp, pthp);
switch (type) {
case ERTS_PORT_TASK_FREE:
erl_exit(ERTS_ABORT_EXIT,
"erts_port_task_schedule(): Cannot schedule free task\n");
break;
case ERTS_PORT_TASK_INPUT:
case ERTS_PORT_TASK_OUTPUT:
case ERTS_PORT_TASK_EVENT:
erts_smp_atomic_inc(&erts_port_task_outstanding_io_tasks);
/* Fall through... */
default:
enqueue_task(pp->sched.taskq, ptp);
break;
}
#if defined(HARD_DEBUG)
if (pp->sched.exe_taskq || enq_port)
ERTS_PT_CHK_NOT_IN_PORTQ(runq, pp);
else
ERTS_PT_CHK_IN_PORTQ(runq, pp);
#elif defined(DEBUG)
if (!enq_port && !pp->sched.exe_taskq) {
/* We should be in port run q */
ASSERT(pp->sched.prev || runq->ports.start == pp);
}
#endif
if (!enq_port) {
ERTS_PT_CHK_PRES_PORTQ(runq, pp);
}
else {
enqueue_port(runq, pp);
ERTS_PT_CHK_PRES_PORTQ(runq, pp);
if (erts_system_profile_flags.runnable_ports) {
profile_runnable_port(pp, am_active);
}
erts_smp_notify_inc_runq(runq);
}
erts_smp_runq_unlock(runq);
return 0;
}
