void
erts_deep_process_dump(fmtfn_t to, void *to_arg)
{
int i, max = erts_ptab_max(&erts_proc);
all_binaries = NULL;
init_literal_areas();
erts_init_persistent_dumping();
for (i = 0; i < max; i++) {
Process *p = erts_pix2proc(i);
if (p && p->i != ENULL) {
erts_aint32_t state = erts_atomic32_read_acqb(&p->state);
if (state & ERTS_PSFLG_EXITING)
continue;
if (state & ERTS_PSFLG_GC) {
ErtsSchedulerData *sdp = erts_get_scheduler_data();
if (!sdp || p != sdp->current_process)
continue;
/* We want to dump the garbing process that caused the dump */
}
dump_process_info(to, to_arg, p);
}
}
dump_persistent_terms(to, to_arg);
dump_literals(to, to_arg);
dump_binaries(to, to_arg, all_binaries);
}
void
erts_thr_progress_fatal_error_wait(SWord timeout) {
erts_aint32_t bc;
SWord time_left = timeout;
ErtsMonotonicTime timeout_time;
ErtsSchedulerData *esdp = erts_get_scheduler_data();
/*
* Counting poll intervals may give us a too long timeout
* if cpu is busy. We use timeout time to try to prevent
* this. In case we havn't got time correction this may
* however fail too...
*/
timeout_time = erts_get_monotonic_time(esdp);
timeout_time += ERTS_MSEC_TO_MONOTONIC((ErtsMonotonicTime) timeout);
while (1) {
if (erts_milli_sleep(ERTS_THR_PRGR_FTL_ERR_BLCK_POLL_INTERVAL) == 0)
time_left -= ERTS_THR_PRGR_FTL_ERR_BLCK_POLL_INTERVAL;
bc = erts_atomic32_read_acqb(&intrnl->misc.data.block_count);
if (bc == 0)
break; /* Succefully blocked all managed threads */
if (time_left <= 0)
break; /* Timeout */
if (timeout_time <= erts_get_monotonic_time(esdp))
break; /* Timeout */
}
}
void
erts_queue_dist_message(Process *rcvr,
ErtsProcLocks rcvr_locks,
ErtsDistExternal *dist_ext,
Eterm token,
Eterm from)
{
ErtsMessage* mp;
erts_aint_t state;
ERTS_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
mp = erts_alloc_message(0, NULL);
mp->data.dist_ext = dist_ext;
ERL_MESSAGE_FROM(mp) = dist_ext->dep->sysname;
ERL_MESSAGE_TERM(mp) = THE_NON_VALUE;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = NIL;
if (token == am_have_dt_utag)
ERL_MESSAGE_TOKEN(mp) = NIL;
else
#endif
ERL_MESSAGE_TOKEN(mp) = token;
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) {
if (erts_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
ErtsProcLocks unlocks =
rcvr_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ);
if (unlocks) {
erts_proc_unlock(rcvr, unlocks);
need_locks |= unlocks;
}
erts_proc_lock(rcvr, need_locks);
}
}
state = erts_atomic32_read_acqb(&rcvr->state);
if (state & ERTS_PSFLG_EXITING) {
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
/* Drop message if receiver is exiting or has a pending exit ... */
erts_cleanup_messages(mp);
}
else {
LINK_MESSAGE(rcvr, mp);
if (rcvr_locks & ERTS_PROC_LOCK_MAIN)
erts_proc_sig_fetch(rcvr);
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
erts_proc_notify_new_message(rcvr, rcvr_locks);
}
}
ErtsThrPrgrDelayHandle
erts_thr_progress_unmanaged_delay__(void)
{
int umrefc_ix;
ASSERT(!erts_thr_progress_is_managed_thread());
umrefc_ix = (int) erts_atomic32_read_acqb(&intrnl->misc.data.umrefc_ix.current);
while (1) {
int tmp_ix;
erts_atomic_inc_acqb(&intrnl->umrefc[umrefc_ix].refc);
tmp_ix = (int) erts_atomic32_read_acqb(&intrnl->misc.data.umrefc_ix.current);
if (tmp_ix == umrefc_ix)
break;
unmanaged_continue(umrefc_ix);
umrefc_ix = tmp_ix;
}
#ifdef ERTS_ENABLE_LOCK_CHECK
{
ErtsThrPrgrData *tpd = tmp_thr_prgr_data(NULL);
tpd->is_delaying = 1;
}
#endif
return (ErtsThrPrgrDelayHandle) umrefc_ix;
}
/* This is a special close operation used by the erts_prim_file process for
* cleaning up orphaned files. It differs from the ordinary close_nif in that
* it only works for files that have already entered the CLOSED state. */
static ERL_NIF_TERM delayed_close_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
posix_errno_t ignored;
efile_data_t *d;
ASSERT(argc == 1);
if(!get_file_data(env, argv[0], &d)) {
return enif_make_badarg(env);
}
ASSERT(erts_atomic32_read_acqb(&d->state) == EFILE_STATE_CLOSED);
efile_close(d, &ignored);
return am_ok;
}
BIF_RETTYPE erts_internal_port_control_3(BIF_ALIST_3)
{
Port* prt;
Eterm retval;
Uint uint_op;
unsigned int op;
erts_aint32_t state;
prt = sig_lookup_port(BIF_P, BIF_ARG_1);
if (!prt)
BIF_RET(am_badarg);
if (!term_to_Uint(BIF_ARG_2, &uint_op))
BIF_RET(am_badarg);
if (uint_op > (Uint) UINT_MAX)
BIF_RET(am_badarg);
op = (unsigned int) uint_op;
switch (erts_port_control(BIF_P, prt, op, BIF_ARG_3, &retval)) {
case ERTS_PORT_OP_CALLER_EXIT:
case ERTS_PORT_OP_BADARG:
case ERTS_PORT_OP_DROPPED:
retval = am_badarg;
break;
case ERTS_PORT_OP_SCHEDULED:
ASSERT(is_internal_ordinary_ref(retval));
break;
case ERTS_PORT_OP_DONE:
ASSERT(is_not_internal_ref(retval));
break;
default:
ERTS_INTERNAL_ERROR("Unexpected erts_port_control() result");
retval = am_internal_error;
break;
}
state = erts_atomic32_read_acqb(&BIF_P->state);
if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) {
if (state & ERTS_PSFLG_PENDING_EXIT)
erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN);
ERTS_BIF_EXITED(BIF_P);
}
BIF_RET(retval);
}
static erts_aint32_t
thr_progress_block(ErtsThrPrgrData *tpd, int wait)
{
erts_tse_t *event = NULL; /* Remove erroneous warning... sigh... */
erts_aint32_t lflgs, bc;
if (tpd->is_blocking++)
return (erts_aint32_t) 0;
while (1) {
lflgs = erts_atomic32_read_bor_nob(&intrnl->misc.data.lflgs,
ERTS_THR_PRGR_LFLG_BLOCK);
if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK)
block_thread(tpd);
else
break;
}
#if ERTS_THR_PRGR_PRINT_BLOCKERS
erts_fprintf(stderr, "block(%d)\n", tpd->id);
#endif
ASSERT(ERTS_AINT_NULL
== erts_atomic_read_nob(&intrnl->misc.data.blocker_event));
if (wait) {
event = erts_tse_fetch();
erts_tse_reset(event);
erts_atomic_set_nob(&intrnl->misc.data.blocker_event,
(erts_aint_t) event);
}
if (tpd->is_managed)
erts_atomic32_dec_nob(&intrnl->misc.data.block_count);
bc = erts_atomic32_read_band_mb(&intrnl->misc.data.block_count,
~ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING);
bc &= ~ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING;
if (wait) {
while (bc != 0) {
erts_tse_wait(event);
erts_tse_reset(event);
bc = erts_atomic32_read_acqb(&intrnl->misc.data.block_count);
}
}
return bc;
}
void
erts_deep_process_dump(fmtfn_t to, void *to_arg)
{
int i, max = erts_ptab_max(&erts_proc);
all_binaries = NULL;
init_literal_areas();
for (i = 0; i < max; i++) {
Process *p = erts_pix2proc(i);
if (p && p->i != ENULL) {
erts_aint32_t state = erts_atomic32_read_acqb(&p->state);
if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_GC)))
dump_process_info(to, to_arg, p);
}
}
dump_literals(to, to_arg);
dump_binaries(to, to_arg, all_binaries);
}
void
erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp,
ErtsThrPrgrCallbacks *callbacks,
int pref_wakeup)
{
ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL);
int is_blocking = 0, managed;
if (tpd) {
if (!tpd->is_temporary)
erl_exit(ERTS_ABORT_EXIT,
"%s:%d:%s(): Double register of thread\n",
__FILE__, __LINE__, __func__);
is_blocking = tpd->is_blocking;
return_tmp_thr_prgr_data(tpd);
}
if (esdp)
tpd = &esdp->thr_progress_data;
else
tpd = erts_alloc(ERTS_ALC_T_THR_PRGR_DATA, sizeof(ErtsThrPrgrData));
if (pref_wakeup
&& !erts_atomic32_xchg_nob(&intrnl->misc.data.pref_wakeup_used, 1))
tpd->id = 0;
else if (esdp)
tpd->id = (int) esdp->no;
else
tpd->id = erts_atomic32_inc_read_nob(&intrnl->misc.data.managed_id);
ASSERT(tpd->id >= 0);
if (tpd->id >= intrnl->managed.no)
erl_exit(ERTS_ABORT_EXIT,
"%s:%d:%s(): Too many managed registered threads\n",
__FILE__, __LINE__, __func__);
tpd->is_managed = 1;
tpd->is_blocking = is_blocking;
tpd->is_temporary = 0;
#ifdef ERTS_ENABLE_LOCK_CHECK
tpd->is_delaying = 1;
#endif
init_wakeup_request_array(&tpd->wakeup_request[0]);
ERTS_THR_PROGRESS_STATE_DEBUG_INIT(tpd->id);
tpd->leader = 0;
tpd->active = 1;
tpd->confirmed = 0;
tpd->leader_state.current = ERTS_THR_PRGR_VAL_WAITING;
erts_tsd_set(erts_thr_prgr_data_key__, (void *) tpd);
erts_atomic32_inc_nob(&intrnl->misc.data.lflgs);
ASSERT(callbacks->wakeup);
ASSERT(callbacks->prepare_wait);
ASSERT(callbacks->wait);
ASSERT(callbacks->finalize_wait);
intrnl->managed.callbacks[tpd->id] = *callbacks;
callbacks->prepare_wait(callbacks->arg);
managed = erts_atomic32_inc_read_relb(&intrnl->misc.data.managed_count);
if (managed != intrnl->managed.no) {
/* Wait until all managed threads have registered... */
do {
callbacks->wait(callbacks->arg);
callbacks->prepare_wait(callbacks->arg);
managed = erts_atomic32_read_acqb(&intrnl->misc.data.managed_count);
} while (managed != intrnl->managed.no);
}
else {
int id;
/* All managed threads have registered; lets go... */
for (id = 0; id < managed; id++)
if (id != tpd->id)
wakeup_managed(id);
}
callbacks->finalize_wait(callbacks->arg);
}
void
erts_queue_dist_message(Process *rcvr,
ErtsProcLocks rcvr_locks,
ErtsDistExternal *dist_ext,
Eterm token,
Eterm from)
{
ErtsMessage* mp;
#ifdef USE_VM_PROBES
Sint tok_label = 0;
Sint tok_lastcnt = 0;
Sint tok_serial = 0;
#endif
erts_aint_t state;
ERTS_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
mp = erts_alloc_message(0, NULL);
mp->data.dist_ext = dist_ext;
ERL_MESSAGE_TERM(mp) = THE_NON_VALUE;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = NIL;
if (token == am_have_dt_utag)
ERL_MESSAGE_TOKEN(mp) = NIL;
else
#endif
ERL_MESSAGE_TOKEN(mp) = token;
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) {
if (erts_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
ErtsProcLocks unlocks =
rcvr_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ);
if (unlocks) {
erts_proc_unlock(rcvr, unlocks);
need_locks |= unlocks;
}
erts_proc_lock(rcvr, need_locks);
}
}
state = erts_atomic32_read_acqb(&rcvr->state);
if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
/* Drop message if receiver is exiting or has a pending exit ... */
erts_cleanup_messages(mp);
}
else
if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) {
if (from == am_Empty)
from = dist_ext->dep->sysname;
/* Ahh... need to decode it in order to trace it... */
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
if (!erts_decode_dist_message(rcvr, rcvr_locks, mp, 0))
erts_free_message(mp);
else {
Eterm msg = ERL_MESSAGE_TERM(mp);
token = ERL_MESSAGE_TOKEN(mp);
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
dtrace_proc_str(rcvr, receiver_name);
if (have_seqtrace(token)) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
}
DTRACE6(message_queued,
receiver_name, size_object(msg), rcvr->msg.len,
tok_label, tok_lastcnt, tok_serial);
}
#endif
erts_queue_message(rcvr, rcvr_locks, mp, msg, from);
}
}
else {
/* Enqueue message on external format */
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
dtrace_proc_str(rcvr, receiver_name);
if (have_seqtrace(token)) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
}
/*
* TODO: We don't know the real size of the external message here.
* -1 will appear to a D script as 4294967295.
*/
DTRACE6(message_queued, receiver_name, -1, rcvr->msg.len + 1,
tok_label, tok_lastcnt, tok_serial);
}
#endif
LINK_MESSAGE(rcvr, mp, &mp->next, 1);
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
erts_proc_notify_new_message(rcvr,
rcvr_locks
);
}
}
Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
{
unsigned result;
Eterm reds_in = p->def_arg_reg[5];
/*
* Process is in the normal case scheduled out when reduction
* count reach zero. When "save calls" is enabled reduction
* count is subtracted with CONTEXT_REDS, i.e. initial reduction
* count will be zero or less and process is scheduled out
* when -CONTEXT_REDS is reached.
*
* HiPE does not support the "save calls" feature, so we switch
* to using a positive reduction counter when executing in
* hipe mode, but need to restore the "save calls" when
* returning to beam. We also need to hide the save calls buffer
* from BIFs. We do that by moving the saved calls buf to
* suspended saved calls buf.
*
* Beam has initial reduction count in stored in p->def_arg_reg[5].
*
* Beam expects -neg_o_reds to be found in p->def_arg_reg[4]
* on return to beam.
*/
{
struct saved_calls *scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, NULL);
if (scb) {
reds_in += CONTEXT_REDS;
p->fcalls += CONTEXT_REDS;
ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(p, scb);
}
}
p->flags |= F_HIPE_MODE; /* inform bifs where we are comming from... */
p->i = NULL;
/* Set current_function to undefined. stdlib hibernate tests rely on it. */
p->current = NULL;
DPRINTF("cmd == %#x (%s)", cmd, code_str(cmd));
HIPE_CHECK_PCB(p);
p->arity = 0;
switch (cmd & 0xFF) {
case HIPE_MODE_SWITCH_CMD_CALL: {
/* BEAM calls a native code function */
unsigned arity = cmd >> 8;
/* p->hipe.u.ncallee set in beam_emu */
if (p->cp == hipe_beam_pc_return) {
/* Native called BEAM, which now tailcalls native. */
hipe_pop_beam_trap_frame(p);
result = hipe_tailcall_to_native(p, arity, reg);
break;
}
DPRINTF("calling %#lx/%u", (long)p->hipe.u.ncallee, arity);
result = hipe_call_to_native(p, arity, reg);
break;
}
case HIPE_MODE_SWITCH_CMD_CALL_CLOSURE: {
/* BEAM calls a native code closure */
unsigned arity = cmd >> 8; /* #formals + #fvs (closure not counted) */
Eterm fun;
ErlFunThing *funp;
/* drop the fvs, move the closure, correct arity */
fun = reg[arity];
HIPE_ASSERT(is_fun(fun));
funp = (ErlFunThing*)fun_val(fun);
HIPE_ASSERT(funp->num_free <= arity);
arity -= funp->num_free; /* arity == #formals */
reg[arity] = fun;
++arity; /* correct for having added the closure */
/* HIPE_ASSERT(p->hipe.u.ncallee == (void(*)(void))funp->native_address); */
/* just like a normal call from now on */
/* p->hipe.u.ncallee set in beam_emu */
if (p->cp == hipe_beam_pc_return) {
/* Native called BEAM, which now tailcalls native. */
hipe_pop_beam_trap_frame(p);
result = hipe_tailcall_to_native(p, arity, reg);
break;
}
DPRINTF("calling %#lx/%u", (long)p->hipe.u.ncallee, arity);
result = hipe_call_to_native(p, arity, reg);
break;
}
case HIPE_MODE_SWITCH_CMD_THROW: {
/* BEAM just executed hipe_beam_pc_throw[] */
/* Native called BEAM, which now throws an exception back to native. */
DPRINTF("beam throws freason %#lx fvalue %#lx", p->freason, p->fvalue);
hipe_pop_beam_trap_frame(p);
do_throw_to_native:
p->def_arg_reg[0] = exception_tag[GET_EXC_CLASS(p->freason)];
hipe_find_handler(p);
result = hipe_throw_to_native(p);
break;
}
case HIPE_MODE_SWITCH_CMD_RETURN: {
/* BEAM just executed hipe_beam_pc_return[] */
/* Native called BEAM, which now returns back to native. */
/* pop trap frame off estack */
hipe_pop_beam_trap_frame(p);
p->def_arg_reg[0] = reg[0];
result = hipe_return_to_native(p);
break;
}
do_resume:
case HIPE_MODE_SWITCH_CMD_RESUME: {
/* BEAM just executed hipe_beam_pc_resume[] */
/* BEAM called native, which suspended. */
if (p->flags & F_TIMO) {
/* XXX: The process will immediately execute 'clear_timeout',
repeating these two statements. Remove them? */
p->flags &= ~F_TIMO;
JOIN_MESSAGE(p);
p->def_arg_reg[0] = 0; /* make_small(0)? */
} else
p->def_arg_reg[0] = 1; /* make_small(1)? */
result = hipe_return_to_native(p);
break;
}
default:
erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: cmd %#x\r\n", cmd);
}
do_return_from_native:
DPRINTF("result == %#x (%s)", result, code_str(result));
HIPE_CHECK_PCB(p);
switch (result) {
case HIPE_MODE_SWITCH_RES_RETURN: {
hipe_return_from_native(p);
reg[0] = p->def_arg_reg[0];
DPRINTF("returning with r(0) == %#lx", reg[0]);
break;
}
case HIPE_MODE_SWITCH_RES_THROW: {
DPRINTF("native throws freason %#lx fvalue %#lx", p->freason, p->fvalue);
hipe_throw_from_native(p);
break;
}
case HIPE_MODE_SWITCH_RES_TRAP: {
/*
* Native code called a BIF, which "failed" with a TRAP to BEAM.
* Prior to returning, the BIF stored (see BIF_TRAP<N>):
* the callee's address in p->i
* the callee's parameters in reg[0..2]
* the callee's arity in p->arity (for BEAM gc purposes)
*
* We need to remove the BIF's parameters from the native
* stack: to this end hipe_${ARCH}_glue.S stores the BIF's
* arity in p->hipe.narity.
*
* If the BIF emptied the stack (typically hibernate), p->hipe.nstack
* is NULL and there is no need to get rid of stacked parameters.
*/
unsigned int i, is_recursive = 0;
if (p->hipe.nstack != NULL) {
ASSERT(p->hipe.nsp != NULL);
is_recursive = hipe_trap_from_native_is_recursive(p);
}
else {
/* Some architectures (risc) need this re-reset of nsp as the
* BIF wrapper do not detect stack change and causes an obsolete
* stack pointer to be saved in p->hipe.nsp before return to us.
*/
p->hipe.nsp = NULL;
}
/* Schedule next process if current process was hibernated or is waiting
for messages */
if (p->flags & F_HIBERNATE_SCHED) {
p->flags &= ~F_HIBERNATE_SCHED;
goto do_schedule;
}
if (!(erts_atomic32_read_acqb(&p->state) & ERTS_PSFLG_ACTIVE)) {
for (i = 0; i < p->arity; ++i)
p->arg_reg[i] = reg[i];
goto do_schedule;
}
if (is_recursive)
hipe_push_beam_trap_frame(p, reg, p->arity);
result = HIPE_MODE_SWITCH_RES_CALL_BEAM;
break;
}
case HIPE_MODE_SWITCH_RES_CALL_EXPORTED: {
/* Native code calls or tailcalls BEAM.
*
* p->hipe.u.callee_exp is the callee's export entry
* p->arity is the callee's arity
* p->def_arg_reg[] contains the register parameters
* p->hipe.nsp[] contains the stacked parameters
*/
if (hipe_call_from_native_is_recursive(p, reg)) {
/* BEAM called native, which now calls BEAM */
hipe_push_beam_trap_frame(p, reg, p->arity);
}
break;
}
case HIPE_MODE_SWITCH_RES_CALL_CLOSURE: {
/* Native code calls or tailcalls a closure in BEAM
*
* In native code a call to a closure of arity n looks like
* F(A1, ..., AN, Closure),
* The BEAM code for a closure expects to get:
* F(A1, ..., AN, FV1, ..., FVM, Closure)
* (Where Ai is argument i and FVj is free variable j)
*
* p->hipe.u.closure contains the closure
* p->def_arg_reg[] contains the register parameters
* p->hipe.nsp[] contains the stacked parameters
*/
ErlFunThing *closure;
unsigned num_free, arity, i, is_recursive;
HIPE_ASSERT(is_fun(p->hipe.u.closure));
closure = (ErlFunThing*)fun_val(p->hipe.u.closure);
num_free = closure->num_free;
arity = closure->fe->arity;
/* Store the arity in p->arity for the stack popping. */
/* Note: we already have the closure so only need to move arity
values to reg[]. However, there are arity+1 parameters in the
native code state that need to be removed. */
p->arity = arity+1; /* +1 for the closure */
/* Get parameters, don't do GC just yet. */
is_recursive = hipe_call_from_native_is_recursive(p, reg);
if ((Sint)closure->fe->address[-1] < 0) {
/* Unloaded. Let beam_emu.c:call_fun() deal with it. */
result = HIPE_MODE_SWITCH_RES_CALL_CLOSURE;
} else {
/* The BEAM code is present. Prepare to call it. */
/* Append the free vars after the actual parameters. */
for (i = 0; i < num_free; ++i)
reg[arity+i] = closure->env[i];
/* Update arity to reflect the new parameters. */
arity += i;
/* Make a call to the closure's BEAM code. */
p->i = closure->fe->address;
/* Change result code to the faster plain CALL type. */
result = HIPE_MODE_SWITCH_RES_CALL_BEAM;
}
/* Append the closure as the last parameter. Don't increment arity. */
reg[arity] = p->hipe.u.closure;
if (is_recursive) {
/* BEAM called native, which now calls BEAM.
Need to put a trap-frame on the beam stack.
This may cause GC, which is safe now that
the arguments, free vars, and most
importantly the closure, all are in reg[]. */
hipe_push_beam_trap_frame(p, reg, arity+1);
}
break;
}
case HIPE_MODE_SWITCH_RES_SUSPEND: {
p->i = hipe_beam_pc_resume;
p->arity = 0;
goto do_schedule;
}
case HIPE_MODE_SWITCH_RES_WAIT:
case HIPE_MODE_SWITCH_RES_WAIT_TIMEOUT: {
/* same semantics, different debug trace messages */
/* XXX: BEAM has different entries for the locked and unlocked
cases. HiPE doesn't, so we must check dynamically. */
if (p->flags & F_HIPE_RECV_LOCKED)
p->flags &= ~F_HIPE_RECV_LOCKED;
else
erts_proc_lock(p, ERTS_PROC_LOCKS_MSG_RECEIVE);
p->i = hipe_beam_pc_resume;
p->arity = 0;
if (erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_EXITING)
ASSERT(erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_ACTIVE);
else if (!(p->flags & F_HIPE_RECV_YIELD))
erts_atomic32_read_band_relb(&p->state, ~ERTS_PSFLG_ACTIVE);
else {
/* Yielded from receive */
ERTS_VBUMP_ALL_REDS(p);
p->flags &= ~F_HIPE_RECV_YIELD;
}
erts_proc_unlock(p, ERTS_PROC_LOCKS_MSG_RECEIVE);
do_schedule:
{
struct saved_calls *scb;
scb = ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(p, NULL);
if (scb)
ERTS_PROC_SET_SAVED_CALLS_BUF(p, scb);
/* The process may have died while it was executing,
if so we return out from native code to the interpreter */
if (erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_EXITING)
p->i = beam_exit;
#ifdef DEBUG
ASSERT(p->debug_reds_in == reds_in);
#endif
p->flags &= ~F_HIPE_MODE;
ERTS_UNREQ_PROC_MAIN_LOCK(p);
p = erts_schedule(NULL, p, reds_in - p->fcalls);
ERTS_REQ_PROC_MAIN_LOCK(p);
ASSERT(!(p->flags & F_HIPE_MODE));
p->flags &= ~F_HIPE_RECV_LOCKED;
reg = p->scheduler_data->x_reg_array;
}
{
Eterm *argp;
int i;
argp = p->arg_reg;
for (i = p->arity; --i >= 0;)
reg[i] = argp[i];
}
{
struct saved_calls *scb;
reds_in = p->fcalls;
p->def_arg_reg[5] = reds_in;
#ifdef DEBUG
p->debug_reds_in = reds_in;
#endif
if (p->i == hipe_beam_pc_resume) {
p->flags |= F_HIPE_MODE; /* inform bifs where we are comming from... */
scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, NULL);
if (scb)
ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(p, scb);
p->i = NULL;
p->arity = 0;
goto do_resume;
}
scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p);
if (!scb)
p->def_arg_reg[4] = 0;
else {
p->def_arg_reg[4] = CONTEXT_REDS;
p->fcalls = -CONTEXT_REDS + reds_in;
}
}
HIPE_CHECK_PCB(p);
result = HIPE_MODE_SWITCH_RES_CALL_BEAM;
p->def_arg_reg[3] = result;
return p;
}
case HIPE_MODE_SWITCH_RES_APPLY: {
Eterm mfa[3], args;
unsigned int arity;
void *address;
hipe_pop_params(p, 3, &mfa[0]);
/* Unroll the arglist onto reg[]. */
args = mfa[2];
arity = 0;
while (is_list(args)) {
if (arity < 255) {
reg[arity++] = CAR(list_val(args));
args = CDR(list_val(args));
} else
goto do_apply_fail;
}
if (is_not_nil(args))
goto do_apply_fail;
/* find a native code entry point for {M,F,A} for a remote call */
address = hipe_get_remote_na(mfa[0], mfa[1], arity);
if (!address)
goto do_apply_fail;
p->hipe.u.ncallee = (void(*)(void)) address;
result = hipe_tailcall_to_native(p, arity, reg);
goto do_return_from_native;
do_apply_fail:
p->freason = BADARG;
goto do_throw_to_native;
}
default:
erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: result %#x\r\n", result);
}
{
struct saved_calls *scb = ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(p, NULL);
if (!scb)
p->def_arg_reg[4] = 0;
else {
p->def_arg_reg[4] = CONTEXT_REDS;
p->fcalls -= CONTEXT_REDS;
ERTS_PROC_SET_SAVED_CALLS_BUF(p, scb);
}
}
HIPE_CHECK_PCB(p);
p->def_arg_reg[3] = result;
#if NR_ARG_REGS > 5
/*
* When NR_ARG_REGS > 5, we need to protect the process' input
* reduction count (which BEAM stores in def_arg_reg[5]) from
* being clobbered by the arch glue code.
*/
p->def_arg_reg[5] = reds_in;
#endif
p->flags &= ~F_HIPE_MODE;
return p;
}
void
erts_queue_dist_message(Process *rcvr,
ErtsProcLocks rcvr_locks,
ErtsDistExternal *dist_ext,
ErlHeapFragment *hfrag,
Eterm token,
Eterm from)
{
ErtsMessage* mp;
erts_aint_t state;
ERTS_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
if (hfrag) {
/* Fragmented message, allocate a message reference */
mp = erts_alloc_message(0, NULL);
mp->data.heap_frag = hfrag;
} else {
/* Un-fragmented message, allocate space for
token and dist_ext in message. */
Uint dist_ext_sz = erts_dist_ext_size(dist_ext) / sizeof(Eterm);
Uint token_sz = size_object(token);
Uint sz = token_sz + dist_ext_sz;
Eterm *hp;
mp = erts_alloc_message(sz, &hp);
mp->data.heap_frag = &mp->hfrag;
mp->hfrag.used_size = token_sz;
erts_make_dist_ext_copy(dist_ext, erts_get_dist_ext(mp->data.heap_frag));
token = copy_struct(token, token_sz, &hp, &mp->data.heap_frag->off_heap);
}
ERL_MESSAGE_FROM(mp) = dist_ext->dep->sysname;
ERL_MESSAGE_TERM(mp) = THE_NON_VALUE;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = NIL;
if (token == am_have_dt_utag)
ERL_MESSAGE_TOKEN(mp) = NIL;
else
#endif
ERL_MESSAGE_TOKEN(mp) = token;
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) {
if (erts_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
ErtsProcLocks unlocks =
rcvr_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ);
if (unlocks) {
erts_proc_unlock(rcvr, unlocks);
need_locks |= unlocks;
}
erts_proc_lock(rcvr, need_locks);
}
}
state = erts_atomic32_read_acqb(&rcvr->state);
if (state & ERTS_PSFLG_EXITING) {
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
/* Drop message if receiver is exiting or has a pending exit ... */
erts_cleanup_messages(mp);
}
else {
LINK_MESSAGE(rcvr, mp);
if (rcvr_locks & ERTS_PROC_LOCK_MAIN)
erts_proc_sig_fetch(rcvr);
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
erts_proc_notify_new_message(rcvr, rcvr_locks);
}
}
