void check_queue(erts_proc_lock_t *lck) { int lock_no; ErtsProcLocks lflgs = ERTS_PROC_LOCK_FLGS_READ_(lck); for (lock_no = 0; lock_no <= ERTS_PROC_LOCK_MAX_BIT; lock_no++) { ErtsProcLocks wtr; wtr = (((ErtsProcLocks) 1) << lock_no) << ERTS_PROC_LOCK_WAITER_SHIFT; if (lflgs & wtr) { int n; erts_tse_t *wtr; ERTS_LC_ASSERT(lck->queues && lck->queues->queue[lock_no]); wtr = lck->queues->queue[lock_no]; n = 0; do { wtr = wtr->next; n++; } while (wtr != lck->queues->queue[lock_no]); do { wtr = wtr->prev; n--; } while (wtr != lck->queues->queue[lock_no]); ERTS_LC_ASSERT(n == 0); } else { ERTS_LC_ASSERT(!lck->queues || !lck->queues->queue[lock_no]); } } }
/* * Tries to aquire as many locks as possible in lock order, * and sets the wait flag on the first lock not possible to * aquire. * * Note: We need the pix lock during this operation. Wait * flags are only allowed to be manipulated under pix * lock. */ static ERTS_INLINE void try_aquire(erts_proc_lock_t *lck, erts_tse_t *wtr) { ErtsProcLocks got_locks = (ErtsProcLocks) 0; ErtsProcLocks locks = wtr->uflgs; int lock_no; ERTS_LC_ASSERT(lck->queues); ERTS_LC_ASSERT(got_locks != locks); for (lock_no = 0; lock_no <= ERTS_PROC_LOCK_MAX_BIT; lock_no++) { ErtsProcLocks lock = ((ErtsProcLocks) 1) << lock_no; if (locks & lock) { ErtsProcLocks wflg, old_lflgs; if (lck->queues->queue[lock_no]) { /* Others already waiting */ enqueue: ERTS_LC_ASSERT(ERTS_PROC_LOCK_FLGS_READ_(lck) & (lock << ERTS_PROC_LOCK_WAITER_SHIFT)); enqueue_waiter(lck->queues, lock_no, wtr); break; } wflg = lock << ERTS_PROC_LOCK_WAITER_SHIFT; old_lflgs = ERTS_PROC_LOCK_FLGS_BOR_ACQB_(lck, wflg | lock); if (old_lflgs & lock) { /* Didn't get the lock */ goto enqueue; } else { /* Got the lock */ got_locks |= lock; ERTS_LC_ASSERT(!(old_lflgs & wflg)); /* No one else can be waiting for the lock; remove wait flag */ (void) ERTS_PROC_LOCK_FLGS_BAND_(lck, ~wflg); if (got_locks == locks) break; } } } wtr->uflgs &= ~got_locks; }
/* * erts_proc_lock_failed() is called when erts_smp_proc_lock() * wasn't able to lock all locks. We may need to transfer locks * to waiters and wait for our turn on locks. * * Iff !ERTS_PROC_LOCK_ATOMIC_IMPL, the pix lock is locked on entry. * * This always returns with the pix lock unlocked. */ void erts_proc_lock_failed(Process *p, erts_pix_lock_t *pixlck, ErtsProcLocks locks, ErtsProcLocks old_lflgs) { int until_yield = ERTS_PROC_LOCK_SPIN_UNTIL_YIELD; int thr_spin_count; int spin_count; ErtsProcLocks need_locks = locks; ErtsProcLocks olflgs = old_lflgs; if (erts_thr_get_main_status()) thr_spin_count = proc_lock_spin_count; else thr_spin_count = aux_thr_proc_lock_spin_count; spin_count = thr_spin_count; while (need_locks != 0) { ErtsProcLocks can_grab; can_grab = in_order_locks(olflgs, need_locks); if (can_grab == 0) { /* Someone already has the lowest-numbered lock we want. */ if (spin_count-- <= 0) { /* Too many retries, give up and sleep for the lock. */ wait_for_locks(p, pixlck, locks, need_locks, olflgs); return; } ERTS_SPIN_BODY; if (--until_yield == 0) { until_yield = ERTS_PROC_LOCK_SPIN_UNTIL_YIELD; erts_thr_yield(); } olflgs = ERTS_PROC_LOCK_FLGS_READ_(&p->lock); } else { /* Try to grab all of the grabbable locks at once with cmpxchg. */ ErtsProcLocks grabbed = olflgs | can_grab; ErtsProcLocks nflgs = ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(&p->lock, grabbed, olflgs); if (nflgs == olflgs) { /* Success! We grabbed the 'can_grab' locks. */ olflgs = grabbed; need_locks &= ~can_grab; /* Since we made progress, reset the spin count. */ spin_count = thr_spin_count; } else { /* Compare-and-exchange failed, try again. */ olflgs = nflgs; } } } /* Now we have all of the locks we wanted. */ #if !ERTS_PROC_LOCK_ATOMIC_IMPL erts_pix_unlock(pixlck); #endif }
/* * Try to grab locks one at a time in lock order and wait on the lowest * lock we fail to grab, if any. * * If successful, this returns 0 and all locks in 'need_locks' are held. * * On entry, the pix lock is held iff !ERTS_PROC_LOCK_ATOMIC_IMPL. * On exit it is not held. */ static void wait_for_locks(Process *p, erts_pix_lock_t *pixlck, ErtsProcLocks locks, ErtsProcLocks need_locks, ErtsProcLocks olflgs) { erts_pix_lock_t *pix_lock = pixlck ? pixlck : ERTS_PID2PIXLOCK(p->id); erts_tse_t *wtr; erts_proc_lock_queues_t *qs; /* Acquire a waiter object on which this thread can wait. */ wtr = tse_fetch(pix_lock); /* Record which locks this waiter needs. */ wtr->uflgs = need_locks; ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); #if ERTS_PROC_LOCK_ATOMIC_IMPL erts_pix_lock(pix_lock); #endif ERTS_LC_ASSERT(erts_lc_pix_lock_is_locked(pix_lock)); qs = wtr->udata; ASSERT(qs); /* Provide the process with waiter queues, if it doesn't have one. */ if (!p->lock.queues) { qs->next = NULL; p->lock.queues = qs; } else { qs->next = p->lock.queues->next; p->lock.queues->next = qs; } #ifdef ERTS_PROC_LOCK_HARD_DEBUG check_queue(&p->lock); #endif /* Try to aquire locks one at a time in lock order and set wait flag */ try_aquire(&p->lock, wtr); ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); #ifdef ERTS_PROC_LOCK_HARD_DEBUG check_queue(&p->lock); #endif if (wtr->uflgs) { /* We didn't get them all; need to wait... */ ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); erts_atomic32_set_nob(&wtr->uaflgs, 1); erts_pix_unlock(pix_lock); while (1) { int res; erts_tse_reset(wtr); if (erts_atomic32_read_nob(&wtr->uaflgs) == 0) break; /* * Wait for needed locks. When we are woken all needed locks have * have been acquired by other threads and transfered to us. * However, we need to be prepared for spurious wakeups. */ do { res = erts_tse_wait(wtr); /* might return EINTR */ } while (res != 0); } erts_pix_lock(pix_lock); ASSERT(wtr->uflgs == 0); } /* Recover some queues to store in the waiter. */ ERTS_LC_ASSERT(p->lock.queues); if (p->lock.queues->next) { qs = p->lock.queues->next; p->lock.queues->next = qs->next; } else { qs = p->lock.queues; p->lock.queues = NULL; } wtr->udata = qs; erts_pix_unlock(pix_lock); ERTS_LC_ASSERT(locks == (ERTS_PROC_LOCK_FLGS_READ_(&p->lock) & locks)); tse_return(wtr, 0); }
/* * Transfer 'trnsfr_lcks' held by this executing thread to other * threads waiting for the locks. When a lock has been transferred * we also have to try to aquire as many lock as possible for the * other thread. */ static int transfer_locks(Process *p, ErtsProcLocks trnsfr_lcks, erts_pix_lock_t *pix_lock, int unlock) { int transferred = 0; erts_tse_t *wake = NULL; erts_tse_t *wtr; ErtsProcLocks unset_waiter = 0; ErtsProcLocks tlocks = trnsfr_lcks; int lock_no; ERTS_LC_ASSERT(erts_lc_pix_lock_is_locked(pix_lock)); #ifdef ERTS_PROC_LOCK_HARD_DEBUG check_queue(&p->lock); #endif for (lock_no = 0; tlocks && lock_no <= ERTS_PROC_LOCK_MAX_BIT; lock_no++) { ErtsProcLocks lock = ((ErtsProcLocks) 1) << lock_no; if (tlocks & lock) { erts_proc_lock_queues_t *qs = p->lock.queues; /* Transfer lock */ #ifdef ERTS_ENABLE_LOCK_CHECK tlocks &= ~lock; #endif ERTS_LC_ASSERT(ERTS_PROC_LOCK_FLGS_READ_(&p->lock) & (lock << ERTS_PROC_LOCK_WAITER_SHIFT)); transferred++; wtr = dequeue_waiter(qs, lock_no); ERTS_LC_ASSERT(wtr); if (!qs->queue[lock_no]) unset_waiter |= lock; ERTS_LC_ASSERT(wtr->uflgs & lock); wtr->uflgs &= ~lock; if (wtr->uflgs) try_aquire(&p->lock, wtr); if (!wtr->uflgs) { /* * The other thread got all locks it needs; * need to wake it up. */ wtr->next = wake; wake = wtr; } } } if (unset_waiter) { unset_waiter <<= ERTS_PROC_LOCK_WAITER_SHIFT; (void) ERTS_PROC_LOCK_FLGS_BAND_(&p->lock, ~unset_waiter); } #ifdef ERTS_PROC_LOCK_HARD_DEBUG check_queue(&p->lock); #endif ERTS_LC_ASSERT(tlocks == 0); /* We should have transferred all of them */ if (!wake) { if (unlock) erts_pix_unlock(pix_lock); } else { erts_pix_unlock(pix_lock); do { erts_tse_t *tmp = wake; wake = wake->next; erts_atomic32_set_nob(&tmp->uaflgs, 0); erts_tse_set(tmp); } while (wake); if (!unlock) erts_pix_lock(pix_lock); } return transferred; }
Process * erts_pid2proc_opt(Process *c_p, ErtsProcLocks c_p_have_locks, Eterm pid, ErtsProcLocks pid_need_locks, int flags) { Process *dec_refc_proc = NULL; ErtsThrPrgrDelayHandle dhndl; ErtsProcLocks need_locks; Uint pix; Process *proc; #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) ErtsProcLocks lcnt_locks; #endif #ifdef ERTS_ENABLE_LOCK_CHECK if (c_p) { ErtsProcLocks might_unlock = c_p_have_locks & pid_need_locks; if (might_unlock) erts_proc_lc_might_unlock(c_p, might_unlock); } #endif if (is_not_internal_pid(pid)) return NULL; pix = internal_pid_index(pid); ERTS_LC_ASSERT((pid_need_locks & ERTS_PROC_LOCKS_ALL) == pid_need_locks); need_locks = pid_need_locks; if (c_p && c_p->common.id == pid) { ASSERT(c_p->common.id != ERTS_INVALID_PID); ASSERT(c_p == erts_pix2proc(pix)); if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) && ERTS_PROC_IS_EXITING(c_p)) return NULL; need_locks &= ~c_p_have_locks; if (!need_locks) { if (flags & ERTS_P2P_FLG_INC_REFC) erts_proc_inc_refc(c_p); return c_p; } } dhndl = erts_thr_progress_unmanaged_delay(); proc = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, pix); if (proc) { if (proc->common.id != pid) proc = NULL; else if (!need_locks) { if (flags & ERTS_P2P_FLG_INC_REFC) erts_proc_inc_refc(proc); } else { int busy; #if ERTS_PROC_LOCK_OWN_IMPL #ifdef ERTS_ENABLE_LOCK_COUNT lcnt_locks = need_locks; if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) { erts_lcnt_proc_lock(&proc->lock, need_locks); } #endif #ifdef ERTS_ENABLE_LOCK_CHECK /* Make sure erts_pid2proc_safelock() is enough to handle a potential lock order violation situation... */ busy = erts_proc_lc_trylock_force_busy(proc, need_locks); if (!busy) #endif #endif /* ERTS_PROC_LOCK_OWN_IMPL */ { /* Try a quick trylock to grab all the locks we need. */ busy = (int) erts_smp_proc_raw_trylock__(proc, need_locks); #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_CHECK) erts_proc_lc_trylock(proc, need_locks, !busy, __FILE__,__LINE__); #endif #ifdef ERTS_PROC_LOCK_DEBUG if (!busy) erts_proc_lock_op_debug(proc, need_locks, 1); #endif } #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) if (flags & ERTS_P2P_FLG_TRY_LOCK) erts_lcnt_proc_trylock(&proc->lock, need_locks, busy ? EBUSY : 0); #endif if (!busy) { if (flags & ERTS_P2P_FLG_INC_REFC) erts_proc_inc_refc(proc); #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) /* all is great */ if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) erts_lcnt_proc_lock_post_x(&proc->lock, lcnt_locks, __FILE__, __LINE__); #endif } else { if (flags & ERTS_P2P_FLG_TRY_LOCK) proc = ERTS_PROC_LOCK_BUSY; else { int managed; if (flags & ERTS_P2P_FLG_INC_REFC) erts_proc_inc_refc(proc); #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks); #endif managed = dhndl == ERTS_THR_PRGR_DHANDLE_MANAGED; if (!managed) { erts_proc_inc_refc(proc); erts_thr_progress_unmanaged_continue(dhndl); dec_refc_proc = proc; /* * We don't want to call * erts_thr_progress_unmanaged_continue() * again. */ dhndl = ERTS_THR_PRGR_DHANDLE_MANAGED; } proc_safelock(managed, c_p, c_p_have_locks, c_p_have_locks, proc, 0, need_locks); } } } } if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) erts_thr_progress_unmanaged_continue(dhndl); if (need_locks && proc && proc != ERTS_PROC_LOCK_BUSY && (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) ? ERTS_PROC_IS_EXITING(proc) : (proc != (Process *) erts_ptab_pix2intptr_nob(&erts_proc, pix)))) { erts_smp_proc_unlock(proc, need_locks); if (flags & ERTS_P2P_FLG_INC_REFC) dec_refc_proc = proc; proc = NULL; } if (dec_refc_proc) erts_proc_dec_refc(dec_refc_proc); #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_PROC_LOCK_DEBUG) ERTS_LC_ASSERT(!proc || proc == ERTS_PROC_LOCK_BUSY || (pid_need_locks == (ERTS_PROC_LOCK_FLGS_READ_(&proc->lock) & pid_need_locks))); #endif return proc; }
/* * Try to grab locks one at a time in lock order and wait on the lowest * lock we fail to grab, if any. * * If successful, this returns 0 and all locks in 'need_locks' are held. * * On entry, the pix lock is held iff !ERTS_PROC_LOCK_ATOMIC_IMPL. * On exit it is not held. */ static void wait_for_locks(Process *p, erts_pix_lock_t *pixlck, ErtsProcLocks locks, ErtsProcLocks need_locks, ErtsProcLocks olflgs) { erts_pix_lock_t *pix_lock = pixlck ? pixlck : ERTS_PID2PIXLOCK(p->common.id); erts_tse_t *wtr; /* Acquire a waiter object on which this thread can wait. */ wtr = tse_fetch(pix_lock); /* Record which locks this waiter needs. */ wtr->uflgs = need_locks; ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); #if ERTS_PROC_LOCK_ATOMIC_IMPL erts_pix_lock(pix_lock); #endif ERTS_LC_ASSERT(erts_lc_pix_lock_is_locked(pix_lock)); #ifdef ERTS_PROC_LOCK_HARD_DEBUG check_queue(&p->lock); #endif /* Try to aquire locks one at a time in lock order and set wait flag */ try_aquire(&p->lock, wtr); ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); #ifdef ERTS_PROC_LOCK_HARD_DEBUG check_queue(&p->lock); #endif if (wtr->uflgs == 0) erts_pix_unlock(pix_lock); else { /* We didn't get them all; need to wait... */ ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); erts_atomic32_set_nob(&wtr->uaflgs, 1); erts_pix_unlock(pix_lock); while (1) { int res; erts_tse_reset(wtr); if (erts_atomic32_read_nob(&wtr->uaflgs) == 0) break; /* * Wait for needed locks. When we are woken all needed locks have * have been acquired by other threads and transfered to us. * However, we need to be prepared for spurious wakeups. */ do { res = erts_tse_wait(wtr); /* might return EINTR */ } while (res != 0); } ASSERT(wtr->uflgs == 0); } ERTS_LC_ASSERT(locks == (ERTS_PROC_LOCK_FLGS_READ_(&p->lock) & locks)); tse_return(wtr); }
Process * erts_pid2proc_opt(Process *c_p, ErtsProcLocks c_p_have_locks, Eterm pid, ErtsProcLocks pid_need_locks, int flags) { Process *dec_refc_proc = NULL; int need_ptl; ErtsProcLocks need_locks; Uint pix; Process *proc; #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) ErtsProcLocks lcnt_locks; #endif #ifdef ERTS_ENABLE_LOCK_CHECK if (c_p) { ErtsProcLocks might_unlock = c_p_have_locks & pid_need_locks; if (might_unlock) erts_proc_lc_might_unlock(c_p, might_unlock); } #endif if (is_not_internal_pid(pid)) return NULL; pix = internal_pid_index(pid); ERTS_LC_ASSERT((pid_need_locks & ERTS_PROC_LOCKS_ALL) == pid_need_locks); need_locks = pid_need_locks; if (c_p && c_p->id == pid) { ASSERT(c_p->id != ERTS_INVALID_PID); ASSERT(c_p == erts_pix2proc(pix)); if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) && ERTS_PROC_IS_EXITING(c_p)) return NULL; need_locks &= ~c_p_have_locks; if (!need_locks) { if (flags & ERTS_P2P_FLG_SMP_INC_REFC) erts_smp_proc_inc_refc(c_p); return c_p; } } need_ptl = !erts_get_scheduler_id(); if (need_ptl) erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); proc = (Process *) erts_smp_atomic_read_ddrb(&erts_proc.tab[pix]); if (proc) { if (proc->id != pid) proc = NULL; else if (!need_locks) { if (flags & ERTS_P2P_FLG_SMP_INC_REFC) erts_smp_proc_inc_refc(proc); } else { int busy; #if ERTS_PROC_LOCK_OWN_IMPL #ifdef ERTS_ENABLE_LOCK_COUNT lcnt_locks = need_locks; if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) { erts_lcnt_proc_lock(&proc->lock, need_locks); } #endif #ifdef ERTS_ENABLE_LOCK_CHECK /* Make sure erts_pid2proc_safelock() is enough to handle a potential lock order violation situation... */ busy = erts_proc_lc_trylock_force_busy(proc, need_locks); if (!busy) #endif #endif /* ERTS_PROC_LOCK_OWN_IMPL */ { /* Try a quick trylock to grab all the locks we need. */ busy = (int) erts_smp_proc_raw_trylock__(proc, need_locks); #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_CHECK) erts_proc_lc_trylock(proc, need_locks, !busy); #endif #ifdef ERTS_PROC_LOCK_DEBUG if (!busy) erts_proc_lock_op_debug(proc, need_locks, 1); #endif } #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) if (flags & ERTS_P2P_FLG_TRY_LOCK) erts_lcnt_proc_trylock(&proc->lock, need_locks, busy ? EBUSY : 0); #endif if (!busy) { if (flags & ERTS_P2P_FLG_SMP_INC_REFC) erts_smp_proc_inc_refc(proc); #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) /* all is great */ if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) erts_lcnt_proc_lock_post_x(&proc->lock, lcnt_locks, __FILE__, __LINE__); #endif } else { if (flags & ERTS_P2P_FLG_TRY_LOCK) proc = ERTS_PROC_LOCK_BUSY; else { if (flags & ERTS_P2P_FLG_SMP_INC_REFC) erts_smp_proc_inc_refc(proc); #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks); #endif if (need_ptl) { erts_smp_proc_inc_refc(proc); dec_refc_proc = proc; erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); need_ptl = 0; } proc_safelock(!need_ptl, c_p, c_p_have_locks, c_p_have_locks, proc, 0, need_locks); } } } } if (need_ptl) erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); if (need_locks && proc && proc != ERTS_PROC_LOCK_BUSY && (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) ? ERTS_PROC_IS_EXITING(proc) : (proc != (Process *) erts_smp_atomic_read_nob(&erts_proc.tab[pix])))) { erts_smp_proc_unlock(proc, need_locks); if (flags & ERTS_P2P_FLG_SMP_INC_REFC) dec_refc_proc = proc; proc = NULL; } if (dec_refc_proc) erts_smp_proc_dec_refc(dec_refc_proc); #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_PROC_LOCK_DEBUG) ERTS_LC_ASSERT(!proc || proc == ERTS_PROC_LOCK_BUSY || (pid_need_locks == (ERTS_PROC_LOCK_FLGS_READ_(&proc->lock) & pid_need_locks))); #endif return proc; }