void
erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp)
{
ErtsThrPrgrData *tpd = thr_prgr_data(esdp);
ErtsThrPrgrVal current, val;
#ifdef ERTS_ENABLE_LOCK_CHECK
erts_lc_check_exact(NULL, 0);
#endif
/*
* We aren't allowed to continue until our thread
* progress is past global current.
*/
val = current = read_acqb(&erts_thr_prgr__.current);
while (1) {
val++;
if (val == ERTS_THR_PRGR_VAL_WAITING)
val = 0;
tpd->confirmed = val;
set_mb(&intrnl->thr[tpd->id].data.current, val);
val = read_acqb(&erts_thr_prgr__.current);
if (current == val)
break;
current = val;
}
if (block_count_inc())
block_thread(tpd);
if (update(tpd))
leader_update(tpd);
}
void
erts_thr_progress_prepare_wait(ErtsSchedulerData *esdp)
{
erts_aint32_t lflgs;
ErtsThrPrgrData *tpd = thr_prgr_data(esdp);
#ifdef ERTS_ENABLE_LOCK_CHECK
erts_lc_check_exact(NULL, 0);
#endif
block_count_dec();
tpd->confirmed = ERTS_THR_PRGR_VAL_WAITING;
set_mb(&intrnl->thr[tpd->id].data.current, ERTS_THR_PRGR_VAL_WAITING);
lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs);
if ((lflgs & (ERTS_THR_PRGR_LFLG_NO_LEADER
| ERTS_THR_PRGR_LFLG_WAITING_UM
| ERTS_THR_PRGR_LFLG_ACTIVE_MASK))
== ERTS_THR_PRGR_LFLG_NO_LEADER
&& got_sched_wakeups()) {
/* Someone need to make progress */
wakeup_managed(0);
}
}
static int
update(ErtsThrPrgrData *tpd)
{
int res;
ErtsThrPrgrVal val;
if (tpd->leader)
res = 1;
else {
erts_aint32_t lflgs;
res = 0;
val = read_acqb(&erts_thr_prgr__.current);
if (tpd->confirmed == val) {
val++;
if (val == ERTS_THR_PRGR_VAL_WAITING)
val = 0;
tpd->confirmed = val;
set_mb(&intrnl->thr[tpd->id].data.current, val);
}
lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs);
if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK)
res = 1; /* Need to block in leader_update() */
if ((lflgs & ERTS_THR_PRGR_LFLG_NO_LEADER)
&& (tpd->active || ERTS_THR_PRGR_LFLGS_ACTIVE(lflgs) == 0)) {
/* Try to take over leadership... */
erts_aint32_t olflgs;
olflgs = erts_atomic32_read_band_acqb(
&intrnl->misc.data.lflgs,
~ERTS_THR_PRGR_LFLG_NO_LEADER);
if (olflgs & ERTS_THR_PRGR_LFLG_NO_LEADER) {
tpd->leader = 1;
#if ERTS_THR_PRGR_PRINT_LEADER
erts_fprintf(stderr, "L -> %d\n", tpd->id);
#endif
ERTS_THR_PROGRESS_STATE_DEBUG_SET_LEADER(tpd->id, 1);
}
}
res |= tpd->leader;
}
return res;
}
/*
* Interface for grader driver to add grader description
*/
int tegra_simon_add_grader(struct tegra_simon_grader_desc *desc)
{
int ret;
struct tegra_simon_grader *grader;
if (!desc || (desc->domain >= TEGRA_SIMON_DOMAIN_NUM)) {
pr_err("%s: Invalid grader data\n", __func__);
return -EINVAL;
}
grader = &simon_graders[desc->domain];
if (grader->desc) {
pr_err("%s: Duplicate grader for %s\n",
__func__, grader->domain_name);
return -EINVAL;
}
set_mb(grader->desc, desc);
switch (grader->domain) {
case TEGRA_SIMON_DOMAIN_CPU:
ret = tegra_simon_init_cpu();
break;
case TEGRA_SIMON_DOMAIN_GPU:
ret = tegra_simon_init_gpu();
break;
default:
pr_err("%s: Grader for %s is not supported\n",
__func__, grader->domain_name);
return -EINVAL;
}
if (ret) {
grader->desc = NULL;
pr_err("%s: Failed to initialize grader for %s\n",
__func__, grader->domain_name);
return ret;
}
return 0;
}
/*
* DEFINE_WAIT_FUNC(wait, woken_wake_func);
*
* add_wait_queue(&wq, &wait);
* for (;;) {
* if (condition)
* break;
*
* p->state = mode; condition = true;
* smp_mb(); // A smp_wmb(); // C
* if (!wait->flags & WQ_FLAG_WOKEN) wait->flags |= WQ_FLAG_WOKEN;
* schedule() try_to_wake_up();
* p->state = TASK_RUNNING; ~~~~~~~~~~~~~~~~~~
* wait->flags &= ~WQ_FLAG_WOKEN; condition = true;
* smp_mb() // B smp_wmb(); // C
* wait->flags |= WQ_FLAG_WOKEN;
* }
* remove_wait_queue(&wq, &wait);
*
*/
long wait_woken(wait_queue_t *wait, unsigned mode, long timeout)
{
set_current_state(mode); /* A */
/*
* The above implies an smp_mb(), which matches with the smp_wmb() from
* woken_wake_function() such that if we observe WQ_FLAG_WOKEN we must
* also observe all state before the wakeup.
*/
if (!(wait->flags & WQ_FLAG_WOKEN) && !is_kthread_should_stop())
timeout = schedule_timeout(timeout);
__set_current_state(TASK_RUNNING);
/*
* The below implies an smp_mb(), it too pairs with the smp_wmb() from
* woken_wake_function() such that we must either observe the wait
* condition being true _OR_ WQ_FLAG_WOKEN such that we will not miss
* an event.
*/
set_mb(wait->flags, wait->flags & ~WQ_FLAG_WOKEN); /* B */
return timeout;
}
static ERTS_INLINE int
leader_update(ErtsThrPrgrData *tpd)
{
#ifdef ERTS_ENABLE_LOCK_CHECK
erts_lc_check_exact(NULL, 0);
#endif
if (!tpd->leader) {
/* Probably need to block... */
block_thread(tpd);
}
else {
ErtsThrPrgrVal current;
int ix, chk_next_ix, umrefc_ix, my_ix, no_managed, waiting_unmanaged;
erts_aint32_t lflgs;
ErtsThrPrgrVal next;
erts_aint_t refc;
my_ix = tpd->id;
if (tpd->leader_state.current == ERTS_THR_PRGR_VAL_WAITING) {
/* Took over as leader from another thread */
tpd->leader_state.current = read_nob(&erts_thr_prgr__.current);
tpd->leader_state.next = tpd->leader_state.current;
tpd->leader_state.next++;
if (tpd->leader_state.next == ERTS_THR_PRGR_VAL_WAITING)
tpd->leader_state.next = 0;
tpd->leader_state.chk_next_ix = intrnl->misc.data.chk_next_ix;
tpd->leader_state.umrefc_ix.waiting = intrnl->misc.data.umrefc_ix.waiting;
tpd->leader_state.umrefc_ix.current =
(int) erts_atomic32_read_nob(&intrnl->misc.data.umrefc_ix.current);
if (tpd->confirmed == tpd->leader_state.current) {
ErtsThrPrgrVal val = tpd->leader_state.current + 1;
if (val == ERTS_THR_PRGR_VAL_WAITING)
val = 0;
tpd->confirmed = val;
set_mb(&intrnl->thr[my_ix].data.current, val);
}
}
next = tpd->leader_state.next;
waiting_unmanaged = 0;
umrefc_ix = -1; /* Shut up annoying warning */
chk_next_ix = tpd->leader_state.chk_next_ix;
no_managed = intrnl->managed.no;
ASSERT(0 <= chk_next_ix && chk_next_ix <= no_managed);
/* Check manged threads */
if (chk_next_ix < no_managed) {
for (ix = chk_next_ix; ix < no_managed; ix++) {
ErtsThrPrgrVal tmp;
if (ix == my_ix)
continue;
tmp = read_nob(&intrnl->thr[ix].data.current);
if (tmp != next && tmp != ERTS_THR_PRGR_VAL_WAITING) {
tpd->leader_state.chk_next_ix = ix;
ASSERT(erts_thr_progress_has_passed__(next, tmp));
goto done;
}
}
}
/* Check unmanged threads */
waiting_unmanaged = tpd->leader_state.umrefc_ix.waiting != -1;
umrefc_ix = (waiting_unmanaged
? tpd->leader_state.umrefc_ix.waiting
: tpd->leader_state.umrefc_ix.current);
refc = erts_atomic_read_nob(&intrnl->umrefc[umrefc_ix].refc);
ASSERT(refc >= 0);
if (refc != 0) {
int new_umrefc_ix;
if (waiting_unmanaged)
goto done;
new_umrefc_ix = (umrefc_ix + 1) & 0x1;
tpd->leader_state.umrefc_ix.waiting = umrefc_ix;
tpd->leader_state.chk_next_ix = no_managed;
erts_atomic32_set_nob(&intrnl->misc.data.umrefc_ix.current,
(erts_aint32_t) new_umrefc_ix);
ETHR_MEMBAR(ETHR_StoreLoad);
refc = erts_atomic_read_nob(&intrnl->umrefc[umrefc_ix].refc);
ASSERT(refc >= 0);
waiting_unmanaged = 1;
if (refc != 0)
goto done;
}
/* Make progress */
current = next;
next++;
if (next == ERTS_THR_PRGR_VAL_WAITING)
next = 0;
set_nob(&intrnl->thr[my_ix].data.current, next);
set_mb(&erts_thr_prgr__.current, current);
tpd->confirmed = next;
tpd->leader_state.next = next;
tpd->leader_state.current = current;
#if ERTS_THR_PRGR_PRINT_VAL
if (current % 1000 == 0)
erts_fprintf(stderr, "%b64u\n", current);
#endif
handle_wakeup_requests(current);
if (waiting_unmanaged) {
waiting_unmanaged = 0;
tpd->leader_state.umrefc_ix.waiting = -1;
erts_atomic32_read_band_nob(&intrnl->misc.data.lflgs,
~ERTS_THR_PRGR_LFLG_WAITING_UM);
}
tpd->leader_state.chk_next_ix = 0;
done:
if (tpd->active) {
lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs);
if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK)
(void) block_thread(tpd);
}
else {
int force_wakeup_check = 0;
erts_aint32_t set_flags = ERTS_THR_PRGR_LFLG_NO_LEADER;
tpd->leader = 0;
tpd->leader_state.current = ERTS_THR_PRGR_VAL_WAITING;
#if ERTS_THR_PRGR_PRINT_LEADER
erts_fprintf(stderr, "L <- %d\n", tpd->id);
#endif
ERTS_THR_PROGRESS_STATE_DEBUG_SET_LEADER(tpd->id, 0);
intrnl->misc.data.umrefc_ix.waiting
= tpd->leader_state.umrefc_ix.waiting;
if (waiting_unmanaged)
set_flags |= ERTS_THR_PRGR_LFLG_WAITING_UM;
lflgs = erts_atomic32_read_bor_relb(&intrnl->misc.data.lflgs,
set_flags);
lflgs |= set_flags;
if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK)
lflgs = block_thread(tpd);
if (waiting_unmanaged) {
/* Need to check umrefc again */
ETHR_MEMBAR(ETHR_StoreLoad);
refc = erts_atomic_read_nob(&intrnl->umrefc[umrefc_ix].refc);
if (refc == 0) {
/* Need to force wakeup check */
force_wakeup_check = 1;
}
}
if ((force_wakeup_check
|| ((lflgs & (ERTS_THR_PRGR_LFLG_NO_LEADER
| ERTS_THR_PRGR_LFLG_WAITING_UM
| ERTS_THR_PRGR_LFLG_ACTIVE_MASK))
== ERTS_THR_PRGR_LFLG_NO_LEADER))
&& got_sched_wakeups()) {
/* Someone need to make progress */
wakeup_managed(0);
}
}
}
return tpd->leader;
}
