void
check_queue(erts_proc_lock_t *lck)
{
int lock_no;
ErtsProcLocks lflgs = ERTS_PROC_LOCK_FLGS_READ_(lck);
for (lock_no = 0; lock_no <= ERTS_PROC_LOCK_MAX_BIT; lock_no++) {
ErtsProcLocks wtr;
wtr = (((ErtsProcLocks) 1) << lock_no) << ERTS_PROC_LOCK_WAITER_SHIFT;
if (lflgs & wtr) {
int n;
erts_tse_t *wtr;
ERTS_LC_ASSERT(lck->queues && lck->queues->queue[lock_no]);
wtr = lck->queues->queue[lock_no];
n = 0;
do {
wtr = wtr->next;
n++;
} while (wtr != lck->queues->queue[lock_no]);
do {
wtr = wtr->prev;
n--;
} while (wtr != lck->queues->queue[lock_no]);
ERTS_LC_ASSERT(n == 0);
}
else {
ERTS_LC_ASSERT(!lck->queues || !lck->queues->queue[lock_no]);
}
}
}
static void
check_unused_tse(erts_tse_t *wtr)
{
int i;
erts_proc_lock_queues_t *queues = wtr->udata;
ERTS_LC_ASSERT(wtr->uflgs == 0);
for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++)
ERTS_LC_ASSERT(!queues->queue[i]);
}
Sint
erts_complete_off_heap_message_queue_change(Process *c_p)
{
int reds = 1;
ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
ASSERT(c_p->flags & F_OFF_HEAP_MSGQ_CHNG);
ASSERT(erts_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ);
/*
* This job was first initiated when the process changed to off heap
* message queue management. Since then ERTS_PSFLG_OFF_HEAP_MSGQ
* has been set. However, the management state might have been changed
* again (multiple times) since then. Check users last requested state
* (the flags F_OFF_HEAP_MSGQ, and F_ON_HEAP_MSGQ), and make the state
* consistent with that.
*/
if (!(c_p->flags & F_OFF_HEAP_MSGQ))
erts_atomic32_read_band_nob(&c_p->state,
~ERTS_PSFLG_OFF_HEAP_MSGQ);
else {
reds += 2;
erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ);
ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ);
reds += erts_move_messages_off_heap(c_p);
}
c_p->flags &= ~F_OFF_HEAP_MSGQ_CHNG;
return reds;
}
static void async_add(ErlAsync* a, AsyncQueue* q)
{
/* XXX:PaN Is this still necessary when ports lock drivers? */
if (is_internal_port(a->port)) {
ERTS_LC_ASSERT(erts_drvportid2port(a->port));
/* make sure the driver will stay around */
driver_lock_driver(internal_port_index(a->port));
}
erts_mtx_lock(&q->mtx);
if (q->len == 0) {
q->head = a;
q->tail = a;
q->len = 1;
erts_cnd_signal(&q->cv);
}
else { /* no need to signal (since the worker is working) */
a->next = q->head;
q->head->prev = a;
q->head = a;
q->len++;
}
erts_mtx_unlock(&q->mtx);
}
static Eterm notify_when_unloaded(Process *p, Eterm name_term, char *name, ErtsProcLocks plocks, Uint flag)
{
Eterm r = NIL;
Eterm immediate_tag = NIL;
Eterm immediate_type = NIL;
erts_driver_t *drv;
ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & plocks);
lock_drv_list();
if ((drv = lookup_driver(name)) == NULL) {
immediate_tag = am_unloaded;
immediate_type = am_DOWN;
goto immediate;
}
if (drv->handle == NULL || drv->handle->status == ERL_DE_PERMANENT) {
immediate_tag = am_permanent;
immediate_type = am_UP;
goto immediate;
}
p->flags |= F_USING_DDLL;
r = add_monitor(p, drv->handle, flag);
unlock_drv_list();
BIF_RET(r);
immediate:
r = erts_make_ref(p);
erts_proc_unlock(p, plocks);
notify_proc(p, r, name_term, immediate_type, immediate_tag, 0);
unlock_drv_list();
erts_proc_lock(p, plocks);
BIF_RET(r);
}
static void *
signal_dispatcher_thread_func(void *unused)
{
#ifdef ERTS_ENABLE_LOCK_CHECK
erts_lc_set_thread_name("signal_dispatcher");
#endif
while (1) {
union {int signum; char buf[4];} sb;
Eterm signal;
int res, i = 0;
/* Block on read() waiting for a signal notification to arrive... */
do {
res = read(sig_notify_fds[0], (void *) &sb.buf[i], sizeof(int) - i);
i += res > 0 ? res : 0;
} while ((i < sizeof(int) && res >= 0) || (res < 0 && errno == EINTR));
if (res < 0) {
erts_exit(ERTS_ABORT_EXIT,
"signal-dispatcher thread got unexpected error: %s (%d)\n",
erl_errno_id(errno),
errno);
}
/*
* NOTE 1: The signal dispatcher thread should not do work
* that takes a substantial amount of time (except
* perhaps in test and debug builds). It needs to
* be responsive, i.e, it should only dispatch work
* to other threads.
*
* NOTE 2: The signal dispatcher thread is not a blockable
* thread (i.e., not a thread managed by the
* erl_thr_progress module). This is intentional.
* We want to be able to interrupt writing of a crash
* dump by hitting C-c twice. Since it isn't a
* blockable thread it is important that it doesn't
* change the state of any data that a blocking thread
* expects to have exclusive access to (unless the
* signal dispatcher itself explicitly is blocking all
* blockable threads).
*/
switch (sb.signum) {
case 0: continue;
case SIGINT:
break_requested();
break;
default:
if ((signal = signum_to_signalterm(sb.signum)) == am_error) {
erts_exit(ERTS_ABORT_EXIT,
"signal-dispatcher thread received unknown "
"signal notification: '%d'\n",
sb.signum);
}
signal_notify_requested(signal);
}
ERTS_LC_ASSERT(!erts_thr_progress_is_blocking());
}
return NULL;
}
Module*
erts_put_module(Eterm mod)
{
ERTS_LC_ASSERT(erts_initialized == 0
|| erts_has_code_write_permission());
return put_module(mod, &module_tables[erts_staging_code_ix()]);
}
void
erts_queue_dist_message(Process *rcvr,
ErtsProcLocks rcvr_locks,
ErtsDistExternal *dist_ext,
Eterm token,
Eterm from)
{
ErtsMessage* mp;
erts_aint_t state;
ERTS_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
mp = erts_alloc_message(0, NULL);
mp->data.dist_ext = dist_ext;
ERL_MESSAGE_FROM(mp) = dist_ext->dep->sysname;
ERL_MESSAGE_TERM(mp) = THE_NON_VALUE;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = NIL;
if (token == am_have_dt_utag)
ERL_MESSAGE_TOKEN(mp) = NIL;
else
#endif
ERL_MESSAGE_TOKEN(mp) = token;
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) {
if (erts_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
ErtsProcLocks unlocks =
rcvr_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ);
if (unlocks) {
erts_proc_unlock(rcvr, unlocks);
need_locks |= unlocks;
}
erts_proc_lock(rcvr, need_locks);
}
}
state = erts_atomic32_read_acqb(&rcvr->state);
if (state & ERTS_PSFLG_EXITING) {
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
/* Drop message if receiver is exiting or has a pending exit ... */
erts_cleanup_messages(mp);
}
else {
LINK_MESSAGE(rcvr, mp);
if (rcvr_locks & ERTS_PROC_LOCK_MAIN)
erts_proc_sig_fetch(rcvr);
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
erts_proc_notify_new_message(rcvr, rcvr_locks);
}
}
/*
* erts_pid2proc_safelock() is called from erts_pid2proc_opt() when
* it wasn't possible to trylock all locks needed.
* c_p - current process
* c_p_have_locks - locks held on c_p
* pid - process id of process we are looking up
* proc - process struct of process we are looking
* up (both in and out argument)
* need_locks - all locks we need (including have_locks)
* pix_lock - pix lock for process we are looking up
* flags - option flags
*/
void
erts_pid2proc_safelock(Process *c_p,
ErtsProcLocks c_p_have_locks,
Process **proc,
ErtsProcLocks need_locks,
erts_pix_lock_t *pix_lock,
int flags)
{
Process *p = *proc;
ERTS_LC_ASSERT(p->lock.refc > 0);
ERTS_LC_ASSERT(process_tab[internal_pid_index(p->id)] == p);
p->lock.refc++;
erts_pix_unlock(pix_lock);
proc_safelock(c_p,
c_p ? ERTS_PID2PIXLOCK(c_p->id) : NULL,
c_p_have_locks,
c_p_have_locks,
p,
pix_lock,
0,
need_locks);
erts_pix_lock(pix_lock);
if (!p->is_exiting
|| ((flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
&& process_tab[internal_pid_index(p->id)] == p)) {
ERTS_LC_ASSERT(p->lock.refc > 1);
p->lock.refc--;
}
else {
/* No proc. Note, we need to keep refc until after process unlock */
erts_pix_unlock(pix_lock);
erts_smp_proc_unlock__(p, pix_lock, need_locks);
*proc = NULL;
erts_pix_lock(pix_lock);
ERTS_LC_ASSERT(p->lock.refc > 0);
if (--p->lock.refc == 0) {
erts_pix_unlock(pix_lock);
erts_free_proc(p);
erts_pix_lock(pix_lock);
}
}
}
static erts_tse_t *
dequeue_waiter(erts_proc_lock_queues_t *qs, int ix)
{
erts_tse_t *wtr = qs->queue[ix];
ERTS_LC_ASSERT(qs->queue[ix]);
if (wtr->next == wtr) {
ERTS_LC_ASSERT(qs->queue[ix]->prev == wtr);
qs->queue[ix] = NULL;
}
else {
ERTS_LC_ASSERT(wtr->next != wtr);
ERTS_LC_ASSERT(wtr->prev != wtr);
wtr->next->prev = wtr->prev;
wtr->prev->next = wtr->next;
qs->queue[ix] = wtr->next;
}
return wtr;
}
static Eterm build_load_error(Process *p, int code)
{
int need = load_error_need(code);
Eterm *hp = NULL;
ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));
if (need) {
hp = HAlloc(p,need);
}
return build_load_error_hp(hp,code);
}
/*
* Tries to aquire as many locks as possible in lock order,
* and sets the wait flag on the first lock not possible to
* aquire.
*
* Note: We need the pix lock during this operation. Wait
* flags are only allowed to be manipulated under pix
* lock.
*/
static ERTS_INLINE void
try_aquire(erts_proc_lock_t *lck, erts_tse_t *wtr)
{
ErtsProcLocks got_locks = (ErtsProcLocks) 0;
ErtsProcLocks locks = wtr->uflgs;
int lock_no;
ERTS_LC_ASSERT(lck->queues);
ERTS_LC_ASSERT(got_locks != locks);
for (lock_no = 0; lock_no <= ERTS_PROC_LOCK_MAX_BIT; lock_no++) {
ErtsProcLocks lock = ((ErtsProcLocks) 1) << lock_no;
if (locks & lock) {
ErtsProcLocks wflg, old_lflgs;
if (lck->queues->queue[lock_no]) {
/* Others already waiting */
enqueue:
ERTS_LC_ASSERT(ERTS_PROC_LOCK_FLGS_READ_(lck)
& (lock << ERTS_PROC_LOCK_WAITER_SHIFT));
enqueue_waiter(lck->queues, lock_no, wtr);
break;
}
wflg = lock << ERTS_PROC_LOCK_WAITER_SHIFT;
old_lflgs = ERTS_PROC_LOCK_FLGS_BOR_ACQB_(lck, wflg | lock);
if (old_lflgs & lock) {
/* Didn't get the lock */
goto enqueue;
}
else {
/* Got the lock */
got_locks |= lock;
ERTS_LC_ASSERT(!(old_lflgs & wflg));
/* No one else can be waiting for the lock; remove wait flag */
(void) ERTS_PROC_LOCK_FLGS_BAND_(lck, ~wflg);
if (got_locks == locks)
break;
}
}
}
wtr->uflgs &= ~got_locks;
}
char *getenv_string(GETENV_STATE *state)
{
ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&environ_rwmtx));
if (state->next_string[0] == L'\0') {
return NULL;
} else {
WCHAR *res = state->next_string;
state->next_string += wcslen(res) + 1;
return (char *) res;
}
}
void
erts_thr_progress_unmanaged_continue__(ErtsThrPrgrDelayHandle handle)
{
#ifdef ERTS_ENABLE_LOCK_CHECK
ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL);
ERTS_LC_ASSERT(tpd && tpd->is_delaying);
tpd->is_delaying = 0;
return_tmp_thr_prgr_data(tpd);
#endif
ASSERT(!erts_thr_progress_is_managed_thread());
unmanaged_continue(handle);
}
/*
* Utilities
*/
static Eterm notify_when_loaded(Process *p, Eterm name_term, char *name, ErtsProcLocks plocks)
{
Eterm r = NIL;
Eterm immediate_tag = NIL;
Eterm immediate_type = NIL;
erts_driver_t *drv;
ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & plocks);
lock_drv_list();
if ((drv = lookup_driver(name)) == NULL) {
immediate_tag = am_unloaded;
immediate_type = am_DOWN;
goto immediate;
}
if (drv->handle == NULL || drv->handle->status == ERL_DE_PERMANENT) {
immediate_tag = am_permanent;
immediate_type = am_UP;
goto immediate;
}
switch (drv->handle->status) {
case ERL_DE_OK:
immediate_tag = am_loaded;
immediate_type = am_UP;
goto immediate;
case ERL_DE_UNLOAD:
case ERL_DE_FORCE_UNLOAD:
immediate_tag = am_load_cancelled;
immediate_type = am_DOWN;
goto immediate;
case ERL_DE_RELOAD:
case ERL_DE_FORCE_RELOAD:
break;
default:
erts_exit(ERTS_ERROR_EXIT,"Internal error, unknown state %u in dynamic driver.", drv->handle->status);
}
p->flags |= F_USING_DDLL;
r = add_monitor(p, drv->handle, ERL_DE_PROC_AWAIT_LOAD);
unlock_drv_list();
BIF_RET(r);
immediate:
r = erts_make_ref(p);
erts_proc_unlock(p, plocks);
notify_proc(p, r, name_term, immediate_type, immediate_tag, 0);
unlock_drv_list();
erts_proc_lock(p, plocks);
BIF_RET(r);
}
static ERTS_INLINE void
enqueue_waiter(erts_proc_lock_t *lck, int ix, erts_tse_t *wtr)
{
if (!lck->queue[ix]) {
lck->queue[ix] = wtr;
wtr->next = wtr;
wtr->prev = wtr;
}
else {
ERTS_LC_ASSERT(lck->queue[ix]->next && lck->queue[ix]->prev);
wtr->next = lck->queue[ix];
wtr->prev = lck->queue[ix]->prev;
wtr->prev->next = wtr;
lck->queue[ix]->prev = wtr;
}
}
static ERTS_INLINE void
enqueue_waiter(erts_proc_lock_queues_t *qs,
int ix,
erts_tse_t *wtr)
{
if (!qs->queue[ix]) {
qs->queue[ix] = wtr;
wtr->next = wtr;
wtr->prev = wtr;
}
else {
ERTS_LC_ASSERT(qs->queue[ix]->next && qs->queue[ix]->prev);
wtr->next = qs->queue[ix];
wtr->prev = qs->queue[ix]->prev;
wtr->prev->next = wtr;
qs->queue[ix]->prev = wtr;
}
}
Process *
erts_pid2proc_opt(Process *c_p,
ErtsProcLocks c_p_have_locks,
Eterm pid,
ErtsProcLocks pid_need_locks,
int flags)
{
Process *dec_refc_proc = NULL;
int need_ptl;
ErtsProcLocks need_locks;
Uint pix;
Process *proc;
#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
ErtsProcLocks lcnt_locks;
#endif
#ifdef ERTS_ENABLE_LOCK_CHECK
if (c_p) {
ErtsProcLocks might_unlock = c_p_have_locks & pid_need_locks;
if (might_unlock)
erts_proc_lc_might_unlock(c_p, might_unlock);
}
#endif
if (is_not_internal_pid(pid))
return NULL;
pix = internal_pid_index(pid);
ERTS_LC_ASSERT((pid_need_locks & ERTS_PROC_LOCKS_ALL) == pid_need_locks);
need_locks = pid_need_locks;
if (c_p && c_p->id == pid) {
ASSERT(c_p->id != ERTS_INVALID_PID);
ASSERT(c_p == erts_pix2proc(pix));
if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
&& ERTS_PROC_IS_EXITING(c_p))
return NULL;
need_locks &= ~c_p_have_locks;
if (!need_locks) {
if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
erts_smp_proc_inc_refc(c_p);
return c_p;
}
}
need_ptl = !erts_get_scheduler_id();
if (need_ptl)
erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
proc = (Process *) erts_smp_atomic_read_ddrb(&erts_proc.tab[pix]);
if (proc) {
if (proc->id != pid)
proc = NULL;
else if (!need_locks) {
if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
erts_smp_proc_inc_refc(proc);
}
else {
int busy;
#if ERTS_PROC_LOCK_OWN_IMPL
#ifdef ERTS_ENABLE_LOCK_COUNT
lcnt_locks = need_locks;
if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) {
erts_lcnt_proc_lock(&proc->lock, need_locks);
}
#endif
#ifdef ERTS_ENABLE_LOCK_CHECK
/* Make sure erts_pid2proc_safelock() is enough to handle
a potential lock order violation situation... */
busy = erts_proc_lc_trylock_force_busy(proc, need_locks);
if (!busy)
#endif
#endif /* ERTS_PROC_LOCK_OWN_IMPL */
{
/* Try a quick trylock to grab all the locks we need. */
busy = (int) erts_smp_proc_raw_trylock__(proc, need_locks);
#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_CHECK)
erts_proc_lc_trylock(proc, need_locks, !busy);
#endif
#ifdef ERTS_PROC_LOCK_DEBUG
if (!busy)
erts_proc_lock_op_debug(proc, need_locks, 1);
#endif
}
#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
if (flags & ERTS_P2P_FLG_TRY_LOCK)
erts_lcnt_proc_trylock(&proc->lock, need_locks,
busy ? EBUSY : 0);
#endif
if (!busy) {
if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
erts_smp_proc_inc_refc(proc);
#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
/* all is great */
if (!(flags & ERTS_P2P_FLG_TRY_LOCK))
erts_lcnt_proc_lock_post_x(&proc->lock, lcnt_locks,
__FILE__, __LINE__);
#endif
}
else {
if (flags & ERTS_P2P_FLG_TRY_LOCK)
proc = ERTS_PROC_LOCK_BUSY;
else {
if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
erts_smp_proc_inc_refc(proc);
#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks);
#endif
if (need_ptl) {
erts_smp_proc_inc_refc(proc);
dec_refc_proc = proc;
erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
need_ptl = 0;
}
proc_safelock(!need_ptl,
c_p,
c_p_have_locks,
c_p_have_locks,
proc,
0,
need_locks);
}
}
}
}
if (need_ptl)
erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
if (need_locks
&& proc
&& proc != ERTS_PROC_LOCK_BUSY
&& (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
? ERTS_PROC_IS_EXITING(proc)
: (proc
!= (Process *) erts_smp_atomic_read_nob(&erts_proc.tab[pix])))) {
erts_smp_proc_unlock(proc, need_locks);
if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
dec_refc_proc = proc;
proc = NULL;
}
if (dec_refc_proc)
erts_smp_proc_dec_refc(dec_refc_proc);
#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_PROC_LOCK_DEBUG)
ERTS_LC_ASSERT(!proc
|| proc == ERTS_PROC_LOCK_BUSY
|| (pid_need_locks ==
(ERTS_PROC_LOCK_FLGS_READ_(&proc->lock)
& pid_need_locks)));
#endif
return proc;
}
void
erts_queue_dist_message(Process *rcvr,
ErtsProcLocks rcvr_locks,
ErtsDistExternal *dist_ext,
ErlHeapFragment *hfrag,
Eterm token,
Eterm from)
{
ErtsMessage* mp;
erts_aint_t state;
ERTS_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
if (hfrag) {
/* Fragmented message, allocate a message reference */
mp = erts_alloc_message(0, NULL);
mp->data.heap_frag = hfrag;
} else {
/* Un-fragmented message, allocate space for
token and dist_ext in message. */
Uint dist_ext_sz = erts_dist_ext_size(dist_ext) / sizeof(Eterm);
Uint token_sz = size_object(token);
Uint sz = token_sz + dist_ext_sz;
Eterm *hp;
mp = erts_alloc_message(sz, &hp);
mp->data.heap_frag = &mp->hfrag;
mp->hfrag.used_size = token_sz;
erts_make_dist_ext_copy(dist_ext, erts_get_dist_ext(mp->data.heap_frag));
token = copy_struct(token, token_sz, &hp, &mp->data.heap_frag->off_heap);
}
ERL_MESSAGE_FROM(mp) = dist_ext->dep->sysname;
ERL_MESSAGE_TERM(mp) = THE_NON_VALUE;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = NIL;
if (token == am_have_dt_utag)
ERL_MESSAGE_TOKEN(mp) = NIL;
else
#endif
ERL_MESSAGE_TOKEN(mp) = token;
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) {
if (erts_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
ErtsProcLocks unlocks =
rcvr_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ);
if (unlocks) {
erts_proc_unlock(rcvr, unlocks);
need_locks |= unlocks;
}
erts_proc_lock(rcvr, need_locks);
}
}
state = erts_atomic32_read_acqb(&rcvr->state);
if (state & ERTS_PSFLG_EXITING) {
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
/* Drop message if receiver is exiting or has a pending exit ... */
erts_cleanup_messages(mp);
}
else {
LINK_MESSAGE(rcvr, mp);
if (rcvr_locks & ERTS_PROC_LOCK_MAIN)
erts_proc_sig_fetch(rcvr);
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
erts_proc_notify_new_message(rcvr, rcvr_locks);
}
}
/*
* Try to load. If the driver is OK, add as LOADED. If the driver is
* UNLOAD, possibly change to reload and add as LOADED,
* there should be no other
* LOADED tagged pid's. If the driver is RELOAD then add/increment as
* LOADED (should be some LOADED pid). If the driver is not present,
* really load and add as LOADED {ok,loaded} {ok,pending_driver}
* {error, permanent} {error,load_error()}
*/
BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3)
{
Eterm path_term = BIF_ARG_1;
Eterm name_term = BIF_ARG_2;
Eterm options = BIF_ARG_3;
char *path = NULL;
Sint path_len;
char *name = NULL;
DE_Handle *dh;
erts_driver_t *drv;
int res;
Eterm soft_error_term = NIL;
Eterm ok_term = NIL;
Eterm *hp;
Eterm t;
int monitor = 0;
int reload = 0;
Eterm l;
Uint flags = 0;
int kill_ports = 0;
int do_build_load_error = 0;
int build_this_load_error = 0;
int encoding;
for(l = options; is_list(l); l = CDR(list_val(l))) {
Eterm opt = CAR(list_val(l));
Eterm *tp;
if (is_not_tuple(opt)) {
goto error;
}
tp = tuple_val(opt);
if (*tp != make_arityval(2) || is_not_atom(tp[1])) {
goto error;
}
switch (tp[1]) {
case am_driver_options:
{
Eterm ll;
for(ll = tp[2]; is_list(ll); ll = CDR(list_val(ll))) {
Eterm dopt = CAR(list_val(ll));
if (dopt == am_kill_ports) {
flags |= ERL_DE_FL_KILL_PORTS;
} else {
goto error;
}
}
if (is_not_nil(ll)) {
goto error;
}
}
break;
case am_monitor:
if (tp[2] == am_pending_driver) {
monitor = 1;
} else if (tp[2] == am_pending ) {
monitor = 2;
} else {
goto error;
}
break;
case am_reload:
if (tp[2] == am_pending_driver) {
reload = 1;
} else if (tp[2] == am_pending ) {
reload = 2;
} else {
goto error;
}
break;
default:
goto error;
}
}
if (is_not_nil(l)) {
goto error;
}
if ((name = pick_list_or_atom(name_term)) == NULL) {
goto error;
}
encoding = erts_get_native_filename_encoding();
if (encoding == ERL_FILENAME_WIN_WCHAR) {
/* Do not convert the lib name to utf-16le yet, do that in win32 specific code */
/* since lib_name is used in error messages */
encoding = ERL_FILENAME_UTF8;
}
path = erts_convert_filename_to_encoding(path_term, NULL, 0,
ERTS_ALC_T_DDLL_TMP_BUF, 1, 0,
encoding, &path_len,
sys_strlen(name) + 2); /* might need path separator */
if (!path) {
goto error;
}
ASSERT(path_len > 0 && path[path_len-1] == 0);
while (--path_len > 0 && (path[path_len-1] == '\\' || path[path_len-1] == '/'))
;
path[path_len++] = '/';
sys_strcpy(path+path_len,name);
erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
lock_drv_list();
if ((drv = lookup_driver(name)) != NULL) {
if (drv->handle == NULL) {
/* static_driver */
soft_error_term = am_linked_in_driver;
goto soft_error;
} else {
dh = drv->handle;
if (dh->status == ERL_DE_OK) {
int is_last = is_last_user(dh, BIF_P);
if (reload == 1 && !is_last) {
/*Want reload if no other users,
but there are others...*/
soft_error_term = am_pending_process;
goto soft_error;
}
if (reload != 0) {
DE_ProcEntry *old;
if ((dh->flags & ERL_FL_CONSISTENT_MASK) !=
(flags & ERL_FL_CONSISTENT_MASK)) {
soft_error_term = am_inconsistent;
goto soft_error;
}
if ((old = find_proc_entry(dh, BIF_P,
ERL_DE_PROC_LOADED)) ==
NULL) {
soft_error_term = am_not_loaded_by_this_process;
goto soft_error;
} else {
remove_proc_entry(dh, old);
erts_ddll_dereference_driver(dh);
erts_free(ERTS_ALC_T_DDLL_PROCESS, old);
}
/* Reload requested and granted */
dereference_all_processes(dh);
set_driver_reloading(dh, BIF_P, path, name, flags);
if (dh->flags & ERL_DE_FL_KILL_PORTS) {
kill_ports = 1;
}
ok_term = (reload == 1) ? am_pending_driver :
am_pending_process;
} else {
/* Already loaded and healthy (might be by me) */
if (sys_strcmp(dh->full_path, path) ||
(dh->flags & ERL_FL_CONSISTENT_MASK) !=
(flags & ERL_FL_CONSISTENT_MASK)) {
soft_error_term = am_inconsistent;
goto soft_error;
}
add_proc_loaded(dh, BIF_P);
erts_ddll_reference_driver(dh);
monitor = 0;
ok_term = mkatom("already_loaded");
}
} else if (dh->status == ERL_DE_UNLOAD ||
dh->status == ERL_DE_FORCE_UNLOAD) {
/* pending driver */
if (reload != 0) {
soft_error_term = am_not_loaded_by_this_process;
goto soft_error;
}
if (sys_strcmp(dh->full_path, path) ||
(dh->flags & ERL_FL_CONSISTENT_MASK) !=
(flags & ERL_FL_CONSISTENT_MASK)) {
soft_error_term = am_inconsistent;
goto soft_error;
}
dh->status = ERL_DE_OK;
notify_all(dh, drv->name,
ERL_DE_PROC_AWAIT_UNLOAD, am_UP,
am_unload_cancelled);
add_proc_loaded(dh, BIF_P);
erts_ddll_reference_driver(dh);
monitor = 0;
ok_term = mkatom("already_loaded");
} else if (dh->status == ERL_DE_RELOAD ||
dh->status == ERL_DE_FORCE_RELOAD) {
if (reload != 0) {
soft_error_term = am_pending_reload;
goto soft_error;
}
if (sys_strcmp(dh->reload_full_path, path) ||
(dh->reload_flags & ERL_FL_CONSISTENT_MASK) !=
(flags & ERL_FL_CONSISTENT_MASK)) {
soft_error_term = am_inconsistent;
goto soft_error;
}
/* Load of granted unload... */
/* Don't reference, will happen after reload */
add_proc_loaded_deref(dh, BIF_P);
++monitor;
ok_term = am_pending_driver;
} else { /* ERL_DE_PERMANENT */
soft_error_term = am_permanent;
goto soft_error;
}
}
} else { /* driver non-existing */
if (reload != 0) {
soft_error_term = am_not_loaded;
goto soft_error;
}
if ((res = load_driver_entry(&dh, path, name)) != ERL_DE_NO_ERROR) {
build_this_load_error = res;
do_build_load_error = 1;
soft_error_term = am_undefined;
goto soft_error;
} else {
dh->flags = flags;
add_proc_loaded(dh, BIF_P);
first_ddll_reference(dh);
monitor = 0;
ok_term = mkatom("loaded");
}
}
assert_drv_list_rwlocked();
if (kill_ports) {
/* Avoid closing the driver by referencing it */
erts_ddll_reference_driver(dh);
ASSERT(dh->status == ERL_DE_RELOAD);
dh->status = ERL_DE_FORCE_RELOAD;
unlock_drv_list();
kill_ports_driver_unloaded(dh);
/* Dereference, eventually causing driver destruction */
lock_drv_list();
erts_ddll_dereference_driver(dh);
}
erts_ddll_reference_driver(dh);
unlock_drv_list();
erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
lock_drv_list();
erts_ddll_dereference_driver(dh);
BIF_P->flags |= F_USING_DDLL;
if (monitor) {
Eterm mref = add_monitor(BIF_P, dh, ERL_DE_PROC_AWAIT_LOAD);
hp = HAlloc(BIF_P, 4);
t = TUPLE3(hp, am_ok, ok_term, mref);
} else {
hp = HAlloc(BIF_P, 3);
t = TUPLE2(hp, am_ok, ok_term);
}
unlock_drv_list();
erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) path);
erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name);
ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P));
BIF_RET(t);
soft_error:
unlock_drv_list();
erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
if (do_build_load_error) {
soft_error_term = build_load_error(BIF_P, build_this_load_error);
}
hp = HAlloc(BIF_P, 3);
t = TUPLE2(hp, am_error, soft_error_term);
erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) path);
erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name);
ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P));
BIF_RET(t);
error:
assert_drv_list_not_locked();
ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P));
if (path != NULL) {
erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) path);
}
if (name != NULL) {
erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name);
}
BIF_ERROR(BIF_P, BADARG);
}
/* Add messages last in message queue */
static void
queue_messages(Process* receiver,
ErtsProcLocks receiver_locks,
ErtsMessage* first,
ErtsMessage** last,
Uint len)
{
int locked_msgq = 0;
erts_aint32_t state;
#ifdef DEBUG
{
ErtsMessage* fmsg = ERTS_SIG_IS_MSG(first) ? first : first->next;
ASSERT(fmsg);
ASSERT(is_value(ERL_MESSAGE_TERM(fmsg)));
ASSERT(is_value(ERL_MESSAGE_FROM(fmsg)));
ASSERT(ERL_MESSAGE_TOKEN(fmsg) == am_undefined ||
ERL_MESSAGE_TOKEN(fmsg) == NIL ||
is_tuple(ERL_MESSAGE_TOKEN(fmsg)));
}
#endif
ERTS_LC_ASSERT((erts_proc_lc_my_proc_locks(receiver) & ERTS_PROC_LOCK_MSGQ)
== (receiver_locks & ERTS_PROC_LOCK_MSGQ));
if (!(receiver_locks & ERTS_PROC_LOCK_MSGQ)) {
erts_proc_lock(receiver, ERTS_PROC_LOCK_MSGQ);
locked_msgq = 1;
}
state = erts_atomic32_read_nob(&receiver->state);
if (state & ERTS_PSFLG_EXITING) {
/* Drop message if receiver is exiting or has a pending exit... */
if (locked_msgq)
erts_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
if (ERTS_SIG_IS_NON_MSG(first)) {
ErtsSchedulerData* esdp = erts_get_scheduler_data();
ASSERT(esdp);
ASSERT(!esdp->pending_signal.sig);
esdp->pending_signal.sig = (ErtsSignal*) first;
esdp->pending_signal.to = receiver->common.id;
first = first->next;
}
erts_cleanup_messages(first);
return;
}
if (last == &first->next) {
ASSERT(len == 1);
LINK_MESSAGE(receiver, first);
}
else {
erts_enqueue_signals(receiver, first, last, NULL, len, state);
}
if (receiver_locks & ERTS_PROC_LOCK_MAIN)
erts_proc_sig_fetch(receiver);
if (locked_msgq) {
erts_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
}
if (last == &first->next)
erts_proc_notify_new_message(receiver, receiver_locks);
else
erts_proc_notify_new_sig(receiver, state, ERTS_PSFLG_ACTIVE);
}
/* Add messages last in message queue */
static Sint
queue_messages(Process* receiver,
erts_aint32_t *receiver_state,
ErtsProcLocks receiver_locks,
ErtsMessage* first,
ErtsMessage** last,
Uint len,
Eterm from)
{
ErtsTracingEvent* te;
Sint res;
int locked_msgq = 0;
erts_aint32_t state;
ASSERT(is_value(ERL_MESSAGE_TERM(first)));
ASSERT(ERL_MESSAGE_TOKEN(first) == am_undefined ||
ERL_MESSAGE_TOKEN(first) == NIL ||
is_tuple(ERL_MESSAGE_TOKEN(first)));
#ifdef ERTS_ENABLE_LOCK_CHECK
ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(receiver) < ERTS_PROC_LOCK_MSGQ ||
receiver_locks == erts_proc_lc_my_proc_locks(receiver));
#endif
if (!(receiver_locks & ERTS_PROC_LOCK_MSGQ)) {
if (erts_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
ErtsProcLocks need_locks;
if (receiver_state)
state = *receiver_state;
else
state = erts_atomic32_read_nob(&receiver->state);
if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
goto exiting;
need_locks = receiver_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ);
if (need_locks) {
erts_proc_unlock(receiver, need_locks);
}
need_locks |= ERTS_PROC_LOCK_MSGQ;
erts_proc_lock(receiver, need_locks);
}
locked_msgq = 1;
}
state = erts_atomic32_read_nob(&receiver->state);
if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
exiting:
/* Drop message if receiver is exiting or has a pending exit... */
if (locked_msgq)
erts_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
erts_cleanup_messages(first);
return 0;
}
res = receiver->msg.len;
if (receiver_locks & ERTS_PROC_LOCK_MAIN) {
/*
* We move 'in queue' to 'private queue' and place
* message at the end of 'private queue' in order
* to ensure that the 'in queue' doesn't contain
* references into the heap. By ensuring this,
* we don't need to include the 'in queue' in
* the root set when garbage collecting.
*/
res += receiver->msg_inq.len;
ERTS_MSGQ_MV_INQ2PRIVQ(receiver);
LINK_MESSAGE_PRIVQ(receiver, first, last, len);
}
else
{
LINK_MESSAGE(receiver, first, last, len);
}
if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)
&& (te = &erts_receive_tracing[erts_active_bp_ix()],
te->on)) {
ErtsMessage *msg = first;
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
Sint tok_label = 0;
Sint tok_lastcnt = 0;
Sint tok_serial = 0;
Eterm seq_trace_token = ERL_MESSAGE_TOKEN(msg);
dtrace_proc_str(receiver, receiver_name);
if (seq_trace_token != NIL && is_tuple(seq_trace_token)) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token));
}
DTRACE6(message_queued,
receiver_name, size_object(ERL_MESSAGE_TERM(msg)),
receiver->msg.len,
tok_label, tok_lastcnt, tok_serial);
}
#endif
while (msg) {
trace_receive(receiver, from, ERL_MESSAGE_TERM(msg), te);
msg = msg->next;
}
}
if (locked_msgq) {
erts_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
}
erts_proc_notify_new_message(receiver, receiver_locks);
return res;
}
void
erts_queue_dist_message(Process *rcvr,
ErtsProcLocks rcvr_locks,
ErtsDistExternal *dist_ext,
Eterm token,
Eterm from)
{
ErtsMessage* mp;
#ifdef USE_VM_PROBES
Sint tok_label = 0;
Sint tok_lastcnt = 0;
Sint tok_serial = 0;
#endif
erts_aint_t state;
ERTS_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
mp = erts_alloc_message(0, NULL);
mp->data.dist_ext = dist_ext;
ERL_MESSAGE_TERM(mp) = THE_NON_VALUE;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = NIL;
if (token == am_have_dt_utag)
ERL_MESSAGE_TOKEN(mp) = NIL;
else
#endif
ERL_MESSAGE_TOKEN(mp) = token;
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) {
if (erts_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
ErtsProcLocks unlocks =
rcvr_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ);
if (unlocks) {
erts_proc_unlock(rcvr, unlocks);
need_locks |= unlocks;
}
erts_proc_lock(rcvr, need_locks);
}
}
state = erts_atomic32_read_acqb(&rcvr->state);
if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
/* Drop message if receiver is exiting or has a pending exit ... */
erts_cleanup_messages(mp);
}
else
if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) {
if (from == am_Empty)
from = dist_ext->dep->sysname;
/* Ahh... need to decode it in order to trace it... */
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
if (!erts_decode_dist_message(rcvr, rcvr_locks, mp, 0))
erts_free_message(mp);
else {
Eterm msg = ERL_MESSAGE_TERM(mp);
token = ERL_MESSAGE_TOKEN(mp);
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
dtrace_proc_str(rcvr, receiver_name);
if (have_seqtrace(token)) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
}
DTRACE6(message_queued,
receiver_name, size_object(msg), rcvr->msg.len,
tok_label, tok_lastcnt, tok_serial);
}
#endif
erts_queue_message(rcvr, rcvr_locks, mp, msg, from);
}
}
else {
/* Enqueue message on external format */
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
dtrace_proc_str(rcvr, receiver_name);
if (have_seqtrace(token)) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
}
/*
* TODO: We don't know the real size of the external message here.
* -1 will appear to a D script as 4294967295.
*/
DTRACE6(message_queued, receiver_name, -1, rcvr->msg.len + 1,
tok_label, tok_lastcnt, tok_serial);
}
#endif
LINK_MESSAGE(rcvr, mp, &mp->next, 1);
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
erts_proc_notify_new_message(rcvr,
rcvr_locks
);
}
}
/*
* Try to grab locks one at a time in lock order and wait on the lowest
* lock we fail to grab, if any.
*
* If successful, this returns 0 and all locks in 'need_locks' are held.
*
* On entry, the pix lock is held iff !ERTS_PROC_LOCK_ATOMIC_IMPL.
* On exit it is not held.
*/
static void
wait_for_locks(Process *p,
erts_pix_lock_t *pixlck,
ErtsProcLocks locks,
ErtsProcLocks need_locks,
ErtsProcLocks olflgs)
{
erts_pix_lock_t *pix_lock = pixlck ? pixlck : ERTS_PID2PIXLOCK(p->common.id);
erts_tse_t *wtr;
/* Acquire a waiter object on which this thread can wait. */
wtr = tse_fetch(pix_lock);
/* Record which locks this waiter needs. */
wtr->uflgs = need_locks;
ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
#if ERTS_PROC_LOCK_ATOMIC_IMPL
erts_pix_lock(pix_lock);
#endif
ERTS_LC_ASSERT(erts_lc_pix_lock_is_locked(pix_lock));
#ifdef ERTS_PROC_LOCK_HARD_DEBUG
check_queue(&p->lock);
#endif
/* Try to aquire locks one at a time in lock order and set wait flag */
try_aquire(&p->lock, wtr);
ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
#ifdef ERTS_PROC_LOCK_HARD_DEBUG
check_queue(&p->lock);
#endif
if (wtr->uflgs == 0)
erts_pix_unlock(pix_lock);
else {
/* We didn't get them all; need to wait... */
ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
erts_atomic32_set_nob(&wtr->uaflgs, 1);
erts_pix_unlock(pix_lock);
while (1) {
int res;
erts_tse_reset(wtr);
if (erts_atomic32_read_nob(&wtr->uaflgs) == 0)
break;
/*
* Wait for needed locks. When we are woken all needed locks have
* have been acquired by other threads and transfered to us.
* However, we need to be prepared for spurious wakeups.
*/
do {
res = erts_tse_wait(wtr); /* might return EINTR */
} while (res != 0);
}
ASSERT(wtr->uflgs == 0);
}
ERTS_LC_ASSERT(locks == (ERTS_PROC_LOCK_FLGS_READ_(&p->lock) & locks));
tse_return(wtr);
}
static void
proc_safelock(Process *a_proc,
erts_pix_lock_t *a_pix_lck,
ErtsProcLocks a_have_locks,
ErtsProcLocks a_need_locks,
Process *b_proc,
erts_pix_lock_t *b_pix_lck,
ErtsProcLocks b_have_locks,
ErtsProcLocks b_need_locks)
{
Process *p1, *p2;
#ifdef ERTS_ENABLE_LOCK_CHECK
Eterm pid1, pid2;
#endif
erts_pix_lock_t *pix_lck1, *pix_lck2;
ErtsProcLocks need_locks1, have_locks1, need_locks2, have_locks2;
ErtsProcLocks unlock_mask;
int lock_no, refc1 = 0, refc2 = 0;
ERTS_LC_ASSERT(b_proc);
/* Determine inter process lock order...
* Locks with the same lock order should be locked on p1 before p2.
*/
if (a_proc) {
if (a_proc->id < b_proc->id) {
p1 = a_proc;
#ifdef ERTS_ENABLE_LOCK_CHECK
pid1 = a_proc->id;
#endif
pix_lck1 = a_pix_lck;
need_locks1 = a_need_locks;
have_locks1 = a_have_locks;
p2 = b_proc;
#ifdef ERTS_ENABLE_LOCK_CHECK
pid2 = b_proc->id;
#endif
pix_lck2 = b_pix_lck;
need_locks2 = b_need_locks;
have_locks2 = b_have_locks;
}
else if (a_proc->id > b_proc->id) {
p1 = b_proc;
#ifdef ERTS_ENABLE_LOCK_CHECK
pid1 = b_proc->id;
#endif
pix_lck1 = b_pix_lck;
need_locks1 = b_need_locks;
have_locks1 = b_have_locks;
p2 = a_proc;
#ifdef ERTS_ENABLE_LOCK_CHECK
pid2 = a_proc->id;
#endif
pix_lck2 = a_pix_lck;
need_locks2 = a_need_locks;
have_locks2 = a_have_locks;
}
else {
ERTS_LC_ASSERT(a_proc == b_proc);
ERTS_LC_ASSERT(a_proc->id == b_proc->id);
p1 = a_proc;
#ifdef ERTS_ENABLE_LOCK_CHECK
pid1 = a_proc->id;
#endif
pix_lck1 = a_pix_lck;
need_locks1 = a_need_locks | b_need_locks;
have_locks1 = a_have_locks | b_have_locks;
p2 = NULL;
#ifdef ERTS_ENABLE_LOCK_CHECK
pid2 = 0;
#endif
pix_lck2 = NULL;
need_locks2 = 0;
have_locks2 = 0;
}
}
else {
p1 = b_proc;
#ifdef ERTS_ENABLE_LOCK_CHECK
pid1 = b_proc->id;
#endif
pix_lck1 = b_pix_lck;
need_locks1 = b_need_locks;
have_locks1 = b_have_locks;
p2 = NULL;
#ifdef ERTS_ENABLE_LOCK_CHECK
pid2 = 0;
#endif
pix_lck2 = NULL;
need_locks2 = 0;
have_locks2 = 0;
#ifdef ERTS_ENABLE_LOCK_CHECK
a_need_locks = 0;
a_have_locks = 0;
#endif
}
#ifdef ERTS_ENABLE_LOCK_CHECK
if (p1)
erts_proc_lc_chk_proc_locks(p1, have_locks1);
if (p2)
erts_proc_lc_chk_proc_locks(p2, have_locks2);
if ((need_locks1 & have_locks1) != have_locks1)
erts_lc_fail("Thread tries to release process lock(s) "
"on %T via erts_proc_safelock().", pid1);
if ((need_locks2 & have_locks2) != have_locks2)
erts_lc_fail("Thread tries to release process lock(s) "
"on %T via erts_proc_safelock().",
pid2);
#endif
need_locks1 &= ~have_locks1;
need_locks2 &= ~have_locks2;
/* Figure out the range of locks that needs to be unlocked... */
unlock_mask = ERTS_PROC_LOCKS_ALL;
for (lock_no = 0;
lock_no <= ERTS_PROC_LOCK_MAX_BIT;
lock_no++) {
ErtsProcLocks lock = (1 << lock_no);
if (lock & need_locks1)
break;
unlock_mask &= ~lock;
if (lock & need_locks2)
break;
}
/* ... and unlock locks in that range... */
if (have_locks1 || have_locks2) {
ErtsProcLocks unlock_locks;
unlock_locks = unlock_mask & have_locks1;
if (unlock_locks) {
have_locks1 &= ~unlock_locks;
need_locks1 |= unlock_locks;
if (!have_locks1) {
refc1 = 1;
erts_smp_proc_inc_refc(p1);
}
erts_smp_proc_unlock__(p1, pix_lck1, unlock_locks);
}
unlock_locks = unlock_mask & have_locks2;
if (unlock_locks) {
have_locks2 &= ~unlock_locks;
need_locks2 |= unlock_locks;
if (!have_locks2) {
refc2 = 1;
erts_smp_proc_inc_refc(p2);
}
erts_smp_proc_unlock__(p2, pix_lck2, unlock_locks);
}
}
/*
* lock_no equals the number of the first lock to lock on
* either p1 *or* p2.
*/
#ifdef ERTS_ENABLE_LOCK_CHECK
if (p1)
erts_proc_lc_chk_proc_locks(p1, have_locks1);
if (p2)
erts_proc_lc_chk_proc_locks(p2, have_locks2);
#endif
/* Lock locks in lock order... */
while (lock_no <= ERTS_PROC_LOCK_MAX_BIT) {
ErtsProcLocks locks;
ErtsProcLocks lock = (1 << lock_no);
ErtsProcLocks lock_mask = 0;
if (need_locks1 & lock) {
do {
lock = (1 << lock_no++);
lock_mask |= lock;
} while (lock_no <= ERTS_PROC_LOCK_MAX_BIT
&& !(need_locks2 & lock));
if (need_locks2 & lock)
lock_no--;
locks = need_locks1 & lock_mask;
erts_smp_proc_lock__(p1, pix_lck1, locks);
have_locks1 |= locks;
need_locks1 &= ~locks;
}
else if (need_locks2 & lock) {
while (lock_no <= ERTS_PROC_LOCK_MAX_BIT
&& !(need_locks1 & lock)) {
lock_mask |= lock;
lock = (1 << ++lock_no);
}
locks = need_locks2 & lock_mask;
erts_smp_proc_lock__(p2, pix_lck2, locks);
have_locks2 |= locks;
need_locks2 &= ~locks;
}
else
lock_no++;
}
#ifdef ERTS_ENABLE_LOCK_CHECK
if (p1)
erts_proc_lc_chk_proc_locks(p1, have_locks1);
if (p2)
erts_proc_lc_chk_proc_locks(p2, have_locks2);
if (p1 && p2) {
if (p1 == a_proc) {
ERTS_LC_ASSERT(a_need_locks == have_locks1);
ERTS_LC_ASSERT(b_need_locks == have_locks2);
}
else {
ERTS_LC_ASSERT(a_need_locks == have_locks2);
ERTS_LC_ASSERT(b_need_locks == have_locks1);
}
}
else {
ERTS_LC_ASSERT(p1);
if (a_proc) {
ERTS_LC_ASSERT(have_locks1 == (a_need_locks | b_need_locks));
}
else {
ERTS_LC_ASSERT(have_locks1 == b_need_locks);
}
}
#endif
if (refc1)
erts_smp_proc_dec_refc(p1);
if (refc2)
erts_smp_proc_dec_refc(p2);
}
/*
* Try to grab locks one at a time in lock order and wait on the lowest
* lock we fail to grab, if any.
*
* If successful, this returns 0 and all locks in 'need_locks' are held.
*
* On entry, the pix lock is held iff !ERTS_PROC_LOCK_ATOMIC_IMPL.
* On exit it is not held.
*/
static void
wait_for_locks(Process *p,
erts_pix_lock_t *pixlck,
ErtsProcLocks locks,
ErtsProcLocks need_locks,
ErtsProcLocks olflgs)
{
erts_pix_lock_t *pix_lock = pixlck ? pixlck : ERTS_PID2PIXLOCK(p->id);
erts_tse_t *wtr;
erts_proc_lock_queues_t *qs;
/* Acquire a waiter object on which this thread can wait. */
wtr = tse_fetch(pix_lock);
/* Record which locks this waiter needs. */
wtr->uflgs = need_locks;
ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
#if ERTS_PROC_LOCK_ATOMIC_IMPL
erts_pix_lock(pix_lock);
#endif
ERTS_LC_ASSERT(erts_lc_pix_lock_is_locked(pix_lock));
qs = wtr->udata;
ASSERT(qs);
/* Provide the process with waiter queues, if it doesn't have one. */
if (!p->lock.queues) {
qs->next = NULL;
p->lock.queues = qs;
}
else {
qs->next = p->lock.queues->next;
p->lock.queues->next = qs;
}
#ifdef ERTS_PROC_LOCK_HARD_DEBUG
check_queue(&p->lock);
#endif
/* Try to aquire locks one at a time in lock order and set wait flag */
try_aquire(&p->lock, wtr);
ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
#ifdef ERTS_PROC_LOCK_HARD_DEBUG
check_queue(&p->lock);
#endif
if (wtr->uflgs) {
/* We didn't get them all; need to wait... */
ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
erts_atomic32_set_nob(&wtr->uaflgs, 1);
erts_pix_unlock(pix_lock);
while (1) {
int res;
erts_tse_reset(wtr);
if (erts_atomic32_read_nob(&wtr->uaflgs) == 0)
break;
/*
* Wait for needed locks. When we are woken all needed locks have
* have been acquired by other threads and transfered to us.
* However, we need to be prepared for spurious wakeups.
*/
do {
res = erts_tse_wait(wtr); /* might return EINTR */
} while (res != 0);
}
erts_pix_lock(pix_lock);
ASSERT(wtr->uflgs == 0);
}
/* Recover some queues to store in the waiter. */
ERTS_LC_ASSERT(p->lock.queues);
if (p->lock.queues->next) {
qs = p->lock.queues->next;
p->lock.queues->next = qs->next;
}
else {
qs = p->lock.queues;
p->lock.queues = NULL;
}
wtr->udata = qs;
erts_pix_unlock(pix_lock);
ERTS_LC_ASSERT(locks == (ERTS_PROC_LOCK_FLGS_READ_(&p->lock) & locks));
tse_return(wtr, 0);
}
/*
* Transfer 'trnsfr_lcks' held by this executing thread to other
* threads waiting for the locks. When a lock has been transferred
* we also have to try to aquire as many lock as possible for the
* other thread.
*/
static int
transfer_locks(Process *p,
ErtsProcLocks trnsfr_lcks,
erts_pix_lock_t *pix_lock,
int unlock)
{
int transferred = 0;
erts_tse_t *wake = NULL;
erts_tse_t *wtr;
ErtsProcLocks unset_waiter = 0;
ErtsProcLocks tlocks = trnsfr_lcks;
int lock_no;
ERTS_LC_ASSERT(erts_lc_pix_lock_is_locked(pix_lock));
#ifdef ERTS_PROC_LOCK_HARD_DEBUG
check_queue(&p->lock);
#endif
for (lock_no = 0; tlocks && lock_no <= ERTS_PROC_LOCK_MAX_BIT; lock_no++) {
ErtsProcLocks lock = ((ErtsProcLocks) 1) << lock_no;
if (tlocks & lock) {
erts_proc_lock_queues_t *qs = p->lock.queues;
/* Transfer lock */
#ifdef ERTS_ENABLE_LOCK_CHECK
tlocks &= ~lock;
#endif
ERTS_LC_ASSERT(ERTS_PROC_LOCK_FLGS_READ_(&p->lock)
& (lock << ERTS_PROC_LOCK_WAITER_SHIFT));
transferred++;
wtr = dequeue_waiter(qs, lock_no);
ERTS_LC_ASSERT(wtr);
if (!qs->queue[lock_no])
unset_waiter |= lock;
ERTS_LC_ASSERT(wtr->uflgs & lock);
wtr->uflgs &= ~lock;
if (wtr->uflgs)
try_aquire(&p->lock, wtr);
if (!wtr->uflgs) {
/*
* The other thread got all locks it needs;
* need to wake it up.
*/
wtr->next = wake;
wake = wtr;
}
}
}
if (unset_waiter) {
unset_waiter <<= ERTS_PROC_LOCK_WAITER_SHIFT;
(void) ERTS_PROC_LOCK_FLGS_BAND_(&p->lock, ~unset_waiter);
}
#ifdef ERTS_PROC_LOCK_HARD_DEBUG
check_queue(&p->lock);
#endif
ERTS_LC_ASSERT(tlocks == 0); /* We should have transferred all of them */
if (!wake) {
if (unlock)
erts_pix_unlock(pix_lock);
}
else {
erts_pix_unlock(pix_lock);
do {
erts_tse_t *tmp = wake;
wake = wake->next;
erts_atomic32_set_nob(&tmp->uaflgs, 0);
erts_tse_set(tmp);
} while (wake);
if (!unlock)
erts_pix_lock(pix_lock);
}
return transferred;
}
Process *
erts_pid2proc_opt(Process *c_p,
ErtsProcLocks c_p_have_locks,
Eterm pid,
ErtsProcLocks pid_need_locks,
int flags)
{
Process *dec_refc_proc = NULL;
ErtsThrPrgrDelayHandle dhndl;
ErtsProcLocks need_locks;
Uint pix;
Process *proc;
#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
ErtsProcLocks lcnt_locks;
#endif
#ifdef ERTS_ENABLE_LOCK_CHECK
if (c_p) {
ErtsProcLocks might_unlock = c_p_have_locks & pid_need_locks;
if (might_unlock)
erts_proc_lc_might_unlock(c_p, might_unlock);
}
#endif
if (is_not_internal_pid(pid))
return NULL;
pix = internal_pid_index(pid);
ERTS_LC_ASSERT((pid_need_locks & ERTS_PROC_LOCKS_ALL) == pid_need_locks);
need_locks = pid_need_locks;
if (c_p && c_p->common.id == pid) {
ASSERT(c_p->common.id != ERTS_INVALID_PID);
ASSERT(c_p == erts_pix2proc(pix));
if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
&& ERTS_PROC_IS_EXITING(c_p))
return NULL;
need_locks &= ~c_p_have_locks;
if (!need_locks) {
if (flags & ERTS_P2P_FLG_INC_REFC)
erts_proc_inc_refc(c_p);
return c_p;
}
}
dhndl = erts_thr_progress_unmanaged_delay();
proc = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, pix);
if (proc) {
if (proc->common.id != pid)
proc = NULL;
else if (!need_locks) {
if (flags & ERTS_P2P_FLG_INC_REFC)
erts_proc_inc_refc(proc);
}
else {
int busy;
#if ERTS_PROC_LOCK_OWN_IMPL
#ifdef ERTS_ENABLE_LOCK_COUNT
lcnt_locks = need_locks;
if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) {
erts_lcnt_proc_lock(&proc->lock, need_locks);
}
#endif
#ifdef ERTS_ENABLE_LOCK_CHECK
/* Make sure erts_pid2proc_safelock() is enough to handle
a potential lock order violation situation... */
busy = erts_proc_lc_trylock_force_busy(proc, need_locks);
if (!busy)
#endif
#endif /* ERTS_PROC_LOCK_OWN_IMPL */
{
/* Try a quick trylock to grab all the locks we need. */
busy = (int) erts_smp_proc_raw_trylock__(proc, need_locks);
#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_CHECK)
erts_proc_lc_trylock(proc, need_locks, !busy, __FILE__,__LINE__);
#endif
#ifdef ERTS_PROC_LOCK_DEBUG
if (!busy)
erts_proc_lock_op_debug(proc, need_locks, 1);
#endif
}
#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
if (flags & ERTS_P2P_FLG_TRY_LOCK)
erts_lcnt_proc_trylock(&proc->lock, need_locks,
busy ? EBUSY : 0);
#endif
if (!busy) {
if (flags & ERTS_P2P_FLG_INC_REFC)
erts_proc_inc_refc(proc);
#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
/* all is great */
if (!(flags & ERTS_P2P_FLG_TRY_LOCK))
erts_lcnt_proc_lock_post_x(&proc->lock, lcnt_locks,
__FILE__, __LINE__);
#endif
}
else {
if (flags & ERTS_P2P_FLG_TRY_LOCK)
proc = ERTS_PROC_LOCK_BUSY;
else {
int managed;
if (flags & ERTS_P2P_FLG_INC_REFC)
erts_proc_inc_refc(proc);
#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks);
#endif
managed = dhndl == ERTS_THR_PRGR_DHANDLE_MANAGED;
if (!managed) {
erts_proc_inc_refc(proc);
erts_thr_progress_unmanaged_continue(dhndl);
dec_refc_proc = proc;
/*
* We don't want to call
* erts_thr_progress_unmanaged_continue()
* again.
*/
dhndl = ERTS_THR_PRGR_DHANDLE_MANAGED;
}
proc_safelock(managed,
c_p,
c_p_have_locks,
c_p_have_locks,
proc,
0,
need_locks);
}
}
}
}
if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
erts_thr_progress_unmanaged_continue(dhndl);
if (need_locks
&& proc
&& proc != ERTS_PROC_LOCK_BUSY
&& (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
? ERTS_PROC_IS_EXITING(proc)
: (proc
!= (Process *) erts_ptab_pix2intptr_nob(&erts_proc, pix)))) {
erts_smp_proc_unlock(proc, need_locks);
if (flags & ERTS_P2P_FLG_INC_REFC)
dec_refc_proc = proc;
proc = NULL;
}
if (dec_refc_proc)
erts_proc_dec_refc(dec_refc_proc);
#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_PROC_LOCK_DEBUG)
ERTS_LC_ASSERT(!proc
|| proc == ERTS_PROC_LOCK_BUSY
|| (pid_need_locks ==
(ERTS_PROC_LOCK_FLGS_READ_(&proc->lock)
& pid_need_locks)));
#endif
return proc;
}
