void erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp) { ErtsThrPrgrData *tpd = thr_prgr_data(esdp); ErtsThrPrgrVal current, val; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_check_exact(NULL, 0); #endif /* * We aren't allowed to continue until our thread * progress is past global current. */ val = current = read_acqb(&erts_thr_prgr__.current); while (1) { val++; if (val == ERTS_THR_PRGR_VAL_WAITING) val = 0; tpd->confirmed = val; set_mb(&intrnl->thr[tpd->id].data.current, val); val = read_acqb(&erts_thr_prgr__.current); if (current == val) break; current = val; } if (block_count_inc()) block_thread(tpd); if (update(tpd)) leader_update(tpd); }
void erts_thr_progress_prepare_wait(ErtsSchedulerData *esdp) { erts_aint32_t lflgs; ErtsThrPrgrData *tpd = thr_prgr_data(esdp); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_check_exact(NULL, 0); #endif block_count_dec(); tpd->confirmed = ERTS_THR_PRGR_VAL_WAITING; set_mb(&intrnl->thr[tpd->id].data.current, ERTS_THR_PRGR_VAL_WAITING); lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs); if ((lflgs & (ERTS_THR_PRGR_LFLG_NO_LEADER | ERTS_THR_PRGR_LFLG_WAITING_UM | ERTS_THR_PRGR_LFLG_ACTIVE_MASK)) == ERTS_THR_PRGR_LFLG_NO_LEADER && got_sched_wakeups()) { /* Someone need to make progress */ wakeup_managed(0); } }
static int update(ErtsThrPrgrData *tpd) { int res; ErtsThrPrgrVal val; if (tpd->leader) res = 1; else { erts_aint32_t lflgs; res = 0; val = read_acqb(&erts_thr_prgr__.current); if (tpd->confirmed == val) { val++; if (val == ERTS_THR_PRGR_VAL_WAITING) val = 0; tpd->confirmed = val; set_mb(&intrnl->thr[tpd->id].data.current, val); } lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs); if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK) res = 1; /* Need to block in leader_update() */ if ((lflgs & ERTS_THR_PRGR_LFLG_NO_LEADER) && (tpd->active || ERTS_THR_PRGR_LFLGS_ACTIVE(lflgs) == 0)) { /* Try to take over leadership... */ erts_aint32_t olflgs; olflgs = erts_atomic32_read_band_acqb( &intrnl->misc.data.lflgs, ~ERTS_THR_PRGR_LFLG_NO_LEADER); if (olflgs & ERTS_THR_PRGR_LFLG_NO_LEADER) { tpd->leader = 1; #if ERTS_THR_PRGR_PRINT_LEADER erts_fprintf(stderr, "L -> %d\n", tpd->id); #endif ERTS_THR_PROGRESS_STATE_DEBUG_SET_LEADER(tpd->id, 1); } } res |= tpd->leader; } return res; }
/* * Interface for grader driver to add grader description */ int tegra_simon_add_grader(struct tegra_simon_grader_desc *desc) { int ret; struct tegra_simon_grader *grader; if (!desc || (desc->domain >= TEGRA_SIMON_DOMAIN_NUM)) { pr_err("%s: Invalid grader data\n", __func__); return -EINVAL; } grader = &simon_graders[desc->domain]; if (grader->desc) { pr_err("%s: Duplicate grader for %s\n", __func__, grader->domain_name); return -EINVAL; } set_mb(grader->desc, desc); switch (grader->domain) { case TEGRA_SIMON_DOMAIN_CPU: ret = tegra_simon_init_cpu(); break; case TEGRA_SIMON_DOMAIN_GPU: ret = tegra_simon_init_gpu(); break; default: pr_err("%s: Grader for %s is not supported\n", __func__, grader->domain_name); return -EINVAL; } if (ret) { grader->desc = NULL; pr_err("%s: Failed to initialize grader for %s\n", __func__, grader->domain_name); return ret; } return 0; }
/* * DEFINE_WAIT_FUNC(wait, woken_wake_func); * * add_wait_queue(&wq, &wait); * for (;;) { * if (condition) * break; * * p->state = mode; condition = true; * smp_mb(); // A smp_wmb(); // C * if (!wait->flags & WQ_FLAG_WOKEN) wait->flags |= WQ_FLAG_WOKEN; * schedule() try_to_wake_up(); * p->state = TASK_RUNNING; ~~~~~~~~~~~~~~~~~~ * wait->flags &= ~WQ_FLAG_WOKEN; condition = true; * smp_mb() // B smp_wmb(); // C * wait->flags |= WQ_FLAG_WOKEN; * } * remove_wait_queue(&wq, &wait); * */ long wait_woken(wait_queue_t *wait, unsigned mode, long timeout) { set_current_state(mode); /* A */ /* * The above implies an smp_mb(), which matches with the smp_wmb() from * woken_wake_function() such that if we observe WQ_FLAG_WOKEN we must * also observe all state before the wakeup. */ if (!(wait->flags & WQ_FLAG_WOKEN) && !is_kthread_should_stop()) timeout = schedule_timeout(timeout); __set_current_state(TASK_RUNNING); /* * The below implies an smp_mb(), it too pairs with the smp_wmb() from * woken_wake_function() such that we must either observe the wait * condition being true _OR_ WQ_FLAG_WOKEN such that we will not miss * an event. */ set_mb(wait->flags, wait->flags & ~WQ_FLAG_WOKEN); /* B */ return timeout; }
static ERTS_INLINE int leader_update(ErtsThrPrgrData *tpd) { #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_check_exact(NULL, 0); #endif if (!tpd->leader) { /* Probably need to block... */ block_thread(tpd); } else { ErtsThrPrgrVal current; int ix, chk_next_ix, umrefc_ix, my_ix, no_managed, waiting_unmanaged; erts_aint32_t lflgs; ErtsThrPrgrVal next; erts_aint_t refc; my_ix = tpd->id; if (tpd->leader_state.current == ERTS_THR_PRGR_VAL_WAITING) { /* Took over as leader from another thread */ tpd->leader_state.current = read_nob(&erts_thr_prgr__.current); tpd->leader_state.next = tpd->leader_state.current; tpd->leader_state.next++; if (tpd->leader_state.next == ERTS_THR_PRGR_VAL_WAITING) tpd->leader_state.next = 0; tpd->leader_state.chk_next_ix = intrnl->misc.data.chk_next_ix; tpd->leader_state.umrefc_ix.waiting = intrnl->misc.data.umrefc_ix.waiting; tpd->leader_state.umrefc_ix.current = (int) erts_atomic32_read_nob(&intrnl->misc.data.umrefc_ix.current); if (tpd->confirmed == tpd->leader_state.current) { ErtsThrPrgrVal val = tpd->leader_state.current + 1; if (val == ERTS_THR_PRGR_VAL_WAITING) val = 0; tpd->confirmed = val; set_mb(&intrnl->thr[my_ix].data.current, val); } } next = tpd->leader_state.next; waiting_unmanaged = 0; umrefc_ix = -1; /* Shut up annoying warning */ chk_next_ix = tpd->leader_state.chk_next_ix; no_managed = intrnl->managed.no; ASSERT(0 <= chk_next_ix && chk_next_ix <= no_managed); /* Check manged threads */ if (chk_next_ix < no_managed) { for (ix = chk_next_ix; ix < no_managed; ix++) { ErtsThrPrgrVal tmp; if (ix == my_ix) continue; tmp = read_nob(&intrnl->thr[ix].data.current); if (tmp != next && tmp != ERTS_THR_PRGR_VAL_WAITING) { tpd->leader_state.chk_next_ix = ix; ASSERT(erts_thr_progress_has_passed__(next, tmp)); goto done; } } } /* Check unmanged threads */ waiting_unmanaged = tpd->leader_state.umrefc_ix.waiting != -1; umrefc_ix = (waiting_unmanaged ? tpd->leader_state.umrefc_ix.waiting : tpd->leader_state.umrefc_ix.current); refc = erts_atomic_read_nob(&intrnl->umrefc[umrefc_ix].refc); ASSERT(refc >= 0); if (refc != 0) { int new_umrefc_ix; if (waiting_unmanaged) goto done; new_umrefc_ix = (umrefc_ix + 1) & 0x1; tpd->leader_state.umrefc_ix.waiting = umrefc_ix; tpd->leader_state.chk_next_ix = no_managed; erts_atomic32_set_nob(&intrnl->misc.data.umrefc_ix.current, (erts_aint32_t) new_umrefc_ix); ETHR_MEMBAR(ETHR_StoreLoad); refc = erts_atomic_read_nob(&intrnl->umrefc[umrefc_ix].refc); ASSERT(refc >= 0); waiting_unmanaged = 1; if (refc != 0) goto done; } /* Make progress */ current = next; next++; if (next == ERTS_THR_PRGR_VAL_WAITING) next = 0; set_nob(&intrnl->thr[my_ix].data.current, next); set_mb(&erts_thr_prgr__.current, current); tpd->confirmed = next; tpd->leader_state.next = next; tpd->leader_state.current = current; #if ERTS_THR_PRGR_PRINT_VAL if (current % 1000 == 0) erts_fprintf(stderr, "%b64u\n", current); #endif handle_wakeup_requests(current); if (waiting_unmanaged) { waiting_unmanaged = 0; tpd->leader_state.umrefc_ix.waiting = -1; erts_atomic32_read_band_nob(&intrnl->misc.data.lflgs, ~ERTS_THR_PRGR_LFLG_WAITING_UM); } tpd->leader_state.chk_next_ix = 0; done: if (tpd->active) { lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs); if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK) (void) block_thread(tpd); } else { int force_wakeup_check = 0; erts_aint32_t set_flags = ERTS_THR_PRGR_LFLG_NO_LEADER; tpd->leader = 0; tpd->leader_state.current = ERTS_THR_PRGR_VAL_WAITING; #if ERTS_THR_PRGR_PRINT_LEADER erts_fprintf(stderr, "L <- %d\n", tpd->id); #endif ERTS_THR_PROGRESS_STATE_DEBUG_SET_LEADER(tpd->id, 0); intrnl->misc.data.umrefc_ix.waiting = tpd->leader_state.umrefc_ix.waiting; if (waiting_unmanaged) set_flags |= ERTS_THR_PRGR_LFLG_WAITING_UM; lflgs = erts_atomic32_read_bor_relb(&intrnl->misc.data.lflgs, set_flags); lflgs |= set_flags; if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK) lflgs = block_thread(tpd); if (waiting_unmanaged) { /* Need to check umrefc again */ ETHR_MEMBAR(ETHR_StoreLoad); refc = erts_atomic_read_nob(&intrnl->umrefc[umrefc_ix].refc); if (refc == 0) { /* Need to force wakeup check */ force_wakeup_check = 1; } } if ((force_wakeup_check || ((lflgs & (ERTS_THR_PRGR_LFLG_NO_LEADER | ERTS_THR_PRGR_LFLG_WAITING_UM | ERTS_THR_PRGR_LFLG_ACTIVE_MASK)) == ERTS_THR_PRGR_LFLG_NO_LEADER)) && got_sched_wakeups()) { /* Someone need to make progress */ wakeup_managed(0); } } } return tpd->leader; }