/*
* Transfer 'trnsfr_lcks' held by this executing thread to other
* threads waiting for the locks. When a lock has been transferred
* we also have to try to aquire as many lock as possible for the
* other thread.
*/
static int
transfer_locks(Process *p,
ErtsProcLocks trnsfr_lcks,
erts_pix_lock_t *pix_lock,
int unlock)
{
int transferred = 0;
erts_tse_t *wake = NULL;
erts_tse_t *wtr;
ErtsProcLocks unset_waiter = 0;
ErtsProcLocks tlocks = trnsfr_lcks;
int lock_no;
ERTS_LC_ASSERT(erts_lc_pix_lock_is_locked(pix_lock));
#ifdef ERTS_PROC_LOCK_HARD_DEBUG
check_queue(&p->lock);
#endif
for (lock_no = 0; tlocks && lock_no <= ERTS_PROC_LOCK_MAX_BIT; lock_no++) {
ErtsProcLocks lock = ((ErtsProcLocks) 1) << lock_no;
if (tlocks & lock) {
erts_proc_lock_queues_t *qs = p->lock.queues;
/* Transfer lock */
#ifdef ERTS_ENABLE_LOCK_CHECK
tlocks &= ~lock;
#endif
ERTS_LC_ASSERT(ERTS_PROC_LOCK_FLGS_READ_(&p->lock)
& (lock << ERTS_PROC_LOCK_WAITER_SHIFT));
transferred++;
wtr = dequeue_waiter(qs, lock_no);
ERTS_LC_ASSERT(wtr);
if (!qs->queue[lock_no])
unset_waiter |= lock;
ERTS_LC_ASSERT(wtr->uflgs & lock);
wtr->uflgs &= ~lock;
if (wtr->uflgs)
try_aquire(&p->lock, wtr);
if (!wtr->uflgs) {
/*
* The other thread got all locks it needs;
* need to wake it up.
*/
wtr->next = wake;
wake = wtr;
}
}
}
if (unset_waiter) {
unset_waiter <<= ERTS_PROC_LOCK_WAITER_SHIFT;
(void) ERTS_PROC_LOCK_FLGS_BAND_(&p->lock, ~unset_waiter);
}
#ifdef ERTS_PROC_LOCK_HARD_DEBUG
check_queue(&p->lock);
#endif
ERTS_LC_ASSERT(tlocks == 0); /* We should have transferred all of them */
if (!wake) {
if (unlock)
erts_pix_unlock(pix_lock);
}
else {
erts_pix_unlock(pix_lock);
do {
erts_tse_t *tmp = wake;
wake = wake->next;
erts_atomic32_set_nob(&tmp->uaflgs, 0);
erts_tse_set(tmp);
} while (wake);
if (!unlock)
erts_pix_lock(pix_lock);
}
return transferred;
}
/*
* Try to grab locks one at a time in lock order and wait on the lowest
* lock we fail to grab, if any.
*
* If successful, this returns 0 and all locks in 'need_locks' are held.
*
* On entry, the pix lock is held iff !ERTS_PROC_LOCK_ATOMIC_IMPL.
* On exit it is not held.
*/
static void
wait_for_locks(Process *p,
erts_pix_lock_t *pixlck,
ErtsProcLocks locks,
ErtsProcLocks need_locks,
ErtsProcLocks olflgs)
{
erts_pix_lock_t *pix_lock = pixlck ? pixlck : ERTS_PID2PIXLOCK(p->id);
erts_tse_t *wtr;
erts_proc_lock_queues_t *qs;
/* Acquire a waiter object on which this thread can wait. */
wtr = tse_fetch(pix_lock);
/* Record which locks this waiter needs. */
wtr->uflgs = need_locks;
ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
#if ERTS_PROC_LOCK_ATOMIC_IMPL
erts_pix_lock(pix_lock);
#endif
ERTS_LC_ASSERT(erts_lc_pix_lock_is_locked(pix_lock));
qs = wtr->udata;
ASSERT(qs);
/* Provide the process with waiter queues, if it doesn't have one. */
if (!p->lock.queues) {
qs->next = NULL;
p->lock.queues = qs;
}
else {
qs->next = p->lock.queues->next;
p->lock.queues->next = qs;
}
#ifdef ERTS_PROC_LOCK_HARD_DEBUG
check_queue(&p->lock);
#endif
/* Try to aquire locks one at a time in lock order and set wait flag */
try_aquire(&p->lock, wtr);
ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
#ifdef ERTS_PROC_LOCK_HARD_DEBUG
check_queue(&p->lock);
#endif
if (wtr->uflgs) {
/* We didn't get them all; need to wait... */
ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
erts_atomic32_set_nob(&wtr->uaflgs, 1);
erts_pix_unlock(pix_lock);
while (1) {
int res;
erts_tse_reset(wtr);
if (erts_atomic32_read_nob(&wtr->uaflgs) == 0)
break;
/*
* Wait for needed locks. When we are woken all needed locks have
* have been acquired by other threads and transfered to us.
* However, we need to be prepared for spurious wakeups.
*/
do {
res = erts_tse_wait(wtr); /* might return EINTR */
} while (res != 0);
}
erts_pix_lock(pix_lock);
ASSERT(wtr->uflgs == 0);
}
/* Recover some queues to store in the waiter. */
ERTS_LC_ASSERT(p->lock.queues);
if (p->lock.queues->next) {
qs = p->lock.queues->next;
p->lock.queues->next = qs->next;
}
else {
qs = p->lock.queues;
p->lock.queues = NULL;
}
wtr->udata = qs;
erts_pix_unlock(pix_lock);
ERTS_LC_ASSERT(locks == (ERTS_PROC_LOCK_FLGS_READ_(&p->lock) & locks));
tse_return(wtr, 0);
}
/*
* Try to grab locks one at a time in lock order and wait on the lowest
* lock we fail to grab, if any.
*
* If successful, this returns 0 and all locks in 'need_locks' are held.
*
* On entry, the pix lock is held iff !ERTS_PROC_LOCK_ATOMIC_IMPL.
* On exit it is not held.
*/
static void
wait_for_locks(Process *p,
erts_pix_lock_t *pixlck,
ErtsProcLocks locks,
ErtsProcLocks need_locks,
ErtsProcLocks olflgs)
{
erts_pix_lock_t *pix_lock = pixlck ? pixlck : ERTS_PID2PIXLOCK(p->common.id);
erts_tse_t *wtr;
/* Acquire a waiter object on which this thread can wait. */
wtr = tse_fetch(pix_lock);
/* Record which locks this waiter needs. */
wtr->uflgs = need_locks;
ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
#if ERTS_PROC_LOCK_ATOMIC_IMPL
erts_pix_lock(pix_lock);
#endif
ERTS_LC_ASSERT(erts_lc_pix_lock_is_locked(pix_lock));
#ifdef ERTS_PROC_LOCK_HARD_DEBUG
check_queue(&p->lock);
#endif
/* Try to aquire locks one at a time in lock order and set wait flag */
try_aquire(&p->lock, wtr);
ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
#ifdef ERTS_PROC_LOCK_HARD_DEBUG
check_queue(&p->lock);
#endif
if (wtr->uflgs == 0)
erts_pix_unlock(pix_lock);
else {
/* We didn't get them all; need to wait... */
ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
erts_atomic32_set_nob(&wtr->uaflgs, 1);
erts_pix_unlock(pix_lock);
while (1) {
int res;
erts_tse_reset(wtr);
if (erts_atomic32_read_nob(&wtr->uaflgs) == 0)
break;
/*
* Wait for needed locks. When we are woken all needed locks have
* have been acquired by other threads and transfered to us.
* However, we need to be prepared for spurious wakeups.
*/
do {
res = erts_tse_wait(wtr); /* might return EINTR */
} while (res != 0);
}
ASSERT(wtr->uflgs == 0);
}
ERTS_LC_ASSERT(locks == (ERTS_PROC_LOCK_FLGS_READ_(&p->lock) & locks));
tse_return(wtr);
}
