static ERTS_INLINE void
notify_new_message(Process *receiver)
{
ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS
& erts_proc_lc_my_proc_locks(receiver));
switch (receiver->status) {
case P_GARBING:
switch (receiver->gcstatus) {
case P_SUSPENDED:
goto suspended;
case P_WAITING:
goto waiting;
default:
break;
}
break;
case P_SUSPENDED:
suspended:
receiver->rstatus = P_RUNABLE;
break;
case P_WAITING:
waiting:
erts_add_to_runq(receiver);
break;
default:
break;
}
}
Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
{
unsigned result;
#if NR_ARG_REGS > 5
/* When NR_ARG_REGS > 5, we need to protect the process' input
reduction count (which BEAM stores in def_arg_reg[5]) from
being clobbered by the arch glue code. */
Eterm reds_in = p->def_arg_reg[5];
#endif
#if NR_ARG_REGS > 4
Eterm o_reds = p->def_arg_reg[4];
#endif
p->i = NULL;
DPRINTF("cmd == %#x (%s)", cmd, code_str(cmd));
HIPE_CHECK_PCB(p);
p->arity = 0;
switch (cmd & 0xFF) {
case HIPE_MODE_SWITCH_CMD_CALL: {
/* BEAM calls a native code function */
unsigned arity = cmd >> 8;
/* p->hipe.ncallee set in beam_emu */
if (p->cp == hipe_beam_pc_return) {
/* Native called BEAM, which now tailcalls native. */
hipe_pop_beam_trap_frame(p);
result = hipe_tailcall_to_native(p, arity, reg);
break;
}
DPRINTF("calling %#lx/%u", (long)p->hipe.ncallee, arity);
result = hipe_call_to_native(p, arity, reg);
break;
}
case HIPE_MODE_SWITCH_CMD_CALL_CLOSURE: {
/* BEAM calls a native code closure */
unsigned arity = cmd >> 8; /* #formals + #fvs (closure not counted) */
Eterm fun;
ErlFunThing *funp;
/* drop the fvs, move the closure, correct arity */
fun = reg[arity];
HIPE_ASSERT(is_fun(fun));
funp = (ErlFunThing*)fun_val(fun);
HIPE_ASSERT(funp->num_free <= arity);
arity -= funp->num_free; /* arity == #formals */
reg[arity] = fun;
++arity; /* correct for having added the closure */
/* HIPE_ASSERT(p->hipe.ncallee == (void(*)(void))funp->native_address); */
/* just like a normal call from now on */
/* p->hipe.ncallee set in beam_emu */
if (p->cp == hipe_beam_pc_return) {
/* Native called BEAM, which now tailcalls native. */
hipe_pop_beam_trap_frame(p);
result = hipe_tailcall_to_native(p, arity, reg);
break;
}
DPRINTF("calling %#lx/%u", (long)p->hipe.ncallee, arity);
result = hipe_call_to_native(p, arity, reg);
break;
}
case HIPE_MODE_SWITCH_CMD_THROW: {
/* BEAM just executed hipe_beam_pc_throw[] */
/* Native called BEAM, which now throws an exception back to native. */
DPRINTF("beam throws freason %#lx fvalue %#lx", p->freason, p->fvalue);
hipe_pop_beam_trap_frame(p);
do_throw_to_native:
p->def_arg_reg[0] = exception_tag[GET_EXC_CLASS(p->freason)];
hipe_find_handler(p);
result = hipe_throw_to_native(p);
break;
}
case HIPE_MODE_SWITCH_CMD_RETURN: {
/* BEAM just executed hipe_beam_pc_return[] */
/* Native called BEAM, which now returns back to native. */
/* pop trap frame off estack */
hipe_pop_beam_trap_frame(p);
p->def_arg_reg[0] = reg[0];
result = hipe_return_to_native(p);
break;
}
do_resume:
case HIPE_MODE_SWITCH_CMD_RESUME: {
/* BEAM just executed hipe_beam_pc_resume[] */
/* BEAM called native, which suspended. */
if (p->flags & F_TIMO) {
/* XXX: The process will immediately execute 'clear_timeout',
repeating these two statements. Remove them? */
p->flags &= ~F_TIMO;
JOIN_MESSAGE(p);
p->def_arg_reg[0] = 0; /* make_small(0)? */
} else
p->def_arg_reg[0] = 1; /* make_small(1)? */
result = hipe_return_to_native(p);
break;
}
default:
erl_exit(1, "hipe_mode_switch: cmd %#x\r\n", cmd);
}
do_return_from_native:
DPRINTF("result == %#x (%s)", result, code_str(result));
HIPE_CHECK_PCB(p);
switch (result) {
case HIPE_MODE_SWITCH_RES_RETURN: {
hipe_return_from_native(p);
reg[0] = p->def_arg_reg[0];
DPRINTF("returning with r(0) == %#lx", reg[0]);
break;
}
case HIPE_MODE_SWITCH_RES_THROW: {
DPRINTF("native throws freason %#lx fvalue %#lx", p->freason, p->fvalue);
hipe_throw_from_native(p);
break;
}
case HIPE_MODE_SWITCH_RES_TRAP: {
/*
* Native code called a BIF, which "failed" with a TRAP to BEAM.
* Prior to returning, the BIF stored (see BIF_TRAP<N>):
* the callee's address in p->def_arg_reg[3]
* the callee's parameters in p->def_arg_reg[0..2]
* the callee's arity in p->arity (for BEAM gc purposes)
*
* We need to remove the BIF's parameters from the native
* stack: to this end hipe_${ARCH}_glue.S stores the BIF's
* arity in p->hipe.narity.
*/
unsigned int i, is_recursive, callee_arity;
/* Save p->arity, then update it with the original BIF's arity.
Get rid of any stacked parameters in that call. */
/* XXX: hipe_call_from_native_is_recursive() copies data to
reg[], which is useless in the TRAP case. Maybe write a
specialised hipe_trap_from_native_is_recursive() later. */
callee_arity = p->arity;
p->arity = p->hipe.narity; /* caller's arity */
is_recursive = hipe_call_from_native_is_recursive(p, reg);
p->i = (Eterm *)(p->def_arg_reg[3]);
p->arity = callee_arity;
for (i = 0; i < p->arity; ++i)
reg[i] = p->def_arg_reg[i];
if (is_recursive)
hipe_push_beam_trap_frame(p, reg, p->arity);
result = HIPE_MODE_SWITCH_RES_CALL;
break;
}
case HIPE_MODE_SWITCH_RES_CALL: {
/* Native code calls or tailcalls BEAM.
*
* p->i is the callee's BEAM code
* p->arity is the callee's arity
* p->def_arg_reg[] contains the register parameters
* p->hipe.nsp[] contains the stacked parameters
*/
if (hipe_call_from_native_is_recursive(p, reg)) {
/* BEAM called native, which now calls BEAM */
hipe_push_beam_trap_frame(p, reg, p->arity);
}
break;
}
case HIPE_MODE_SWITCH_RES_CALL_CLOSURE: {
/* Native code calls or tailcalls a closure in BEAM
*
* In native code a call to a closure of arity n looks like
* F(A1, ..., AN, Closure),
* The BEAM code for a closure expects to get:
* F(A1, ..., AN, FV1, ..., FVM, Closure)
* (Where Ai is argument i and FVj is free variable j)
*
* p->hipe.closure contains the closure
* p->def_arg_reg[] contains the register parameters
* p->hipe.nsp[] contains the stacked parameters
*/
ErlFunThing *closure;
unsigned num_free, arity, i, is_recursive;
HIPE_ASSERT(is_fun(p->hipe.closure));
closure = (ErlFunThing*)fun_val(p->hipe.closure);
num_free = closure->num_free;
arity = closure->fe->arity;
/* Store the arity in p->arity for the stack popping. */
/* Note: we already have the closure so only need to move arity
values to reg[]. However, there are arity+1 parameters in the
native code state that need to be removed. */
p->arity = arity+1; /* +1 for the closure */
/* Get parameters, don't do GC just yet. */
is_recursive = hipe_call_from_native_is_recursive(p, reg);
if ((Sint)closure->fe->address[-1] < 0) {
/* Unloaded. Let beam_emu.c:call_fun() deal with it. */
result = HIPE_MODE_SWITCH_RES_CALL_CLOSURE;
} else {
/* The BEAM code is present. Prepare to call it. */
/* Append the free vars after the actual parameters. */
for (i = 0; i < num_free; ++i)
reg[arity+i] = closure->env[i];
/* Update arity to reflect the new parameters. */
arity += i;
/* Make a call to the closure's BEAM code. */
p->i = closure->fe->address;
/* Change result code to the faster plain CALL type. */
result = HIPE_MODE_SWITCH_RES_CALL;
}
/* Append the closure as the last parameter. Don't increment arity. */
reg[arity] = p->hipe.closure;
if (is_recursive) {
/* BEAM called native, which now calls BEAM.
Need to put a trap-frame on the beam stack.
This may cause GC, which is safe now that
the arguments, free vars, and most
importantly the closure, all are in reg[]. */
hipe_push_beam_trap_frame(p, reg, arity+1);
}
break;
}
case HIPE_MODE_SWITCH_RES_SUSPEND: {
p->i = hipe_beam_pc_resume;
p->arity = 0;
erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
if (p->status != P_SUSPENDED)
erts_add_to_runq(p);
erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
goto do_schedule;
}
case HIPE_MODE_SWITCH_RES_WAIT:
case HIPE_MODE_SWITCH_RES_WAIT_TIMEOUT: {
/* same semantics, different debug trace messages */
#ifdef ERTS_SMP
/* XXX: BEAM has different entries for the locked and unlocked
cases. HiPE doesn't, so we must check dynamically. */
if (p->hipe_smp.have_receive_locks)
p->hipe_smp.have_receive_locks = 0;
else
erts_smp_proc_lock(p, ERTS_PROC_LOCKS_MSG_RECEIVE);
#endif
p->i = hipe_beam_pc_resume;
p->arity = 0;
p->status = P_WAITING;
erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_MSG_RECEIVE);
do_schedule:
{
#if !(NR_ARG_REGS > 5)
int reds_in = p->def_arg_reg[5];
#endif
p = schedule(p, reds_in - p->fcalls);
#ifdef ERTS_SMP
p->hipe_smp.have_receive_locks = 0;
reg = p->scheduler_data->save_reg;
#endif
}
{
Eterm *argp;
int i;
argp = p->arg_reg;
for (i = p->arity; --i >= 0;)
reg[i] = argp[i];
}
{
#if !(NR_ARG_REGS > 5)
Eterm reds_in;
#endif
#if !(NR_ARG_REGS > 4)
Eterm o_reds;
#endif
reds_in = p->fcalls;
o_reds = 0;
if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) {
o_reds = reds_in;
reds_in = 0;
p->fcalls = 0;
}
p->def_arg_reg[4] = o_reds;
p->def_arg_reg[5] = reds_in;
if (p->i == hipe_beam_pc_resume) {
p->i = NULL;
p->arity = 0;
goto do_resume;
}
}
HIPE_CHECK_PCB(p);
result = HIPE_MODE_SWITCH_RES_CALL;
p->def_arg_reg[3] = result;
return p;
}
case HIPE_MODE_SWITCH_RES_APPLY: {
Eterm mfa[3], args;
unsigned int arity;
void *address;
hipe_pop_params(p, 3, &mfa[0]);
/* Unroll the arglist onto reg[]. */
args = mfa[2];
arity = 0;
while (is_list(args)) {
if (arity < 255) {
reg[arity++] = CAR(list_val(args));
args = CDR(list_val(args));
} else
goto do_apply_fail;
}
if (is_not_nil(args))
goto do_apply_fail;
/* find a native code entry point for {M,F,A} for a remote call */
address = hipe_get_remote_na(mfa[0], mfa[1], arity);
if (!address)
goto do_apply_fail;
p->hipe.ncallee = (void(*)(void)) address;
result = hipe_tailcall_to_native(p, arity, reg);
goto do_return_from_native;
do_apply_fail:
p->freason = BADARG;
goto do_throw_to_native;
}
default:
erl_exit(1, "hipe_mode_switch: result %#x\r\n", result);
}
HIPE_CHECK_PCB(p);
p->def_arg_reg[3] = result;
#if NR_ARG_REGS > 4
p->def_arg_reg[4] = o_reds;
#endif
#if NR_ARG_REGS > 5
p->def_arg_reg[5] = reds_in;
#endif
return p;
}
void
erts_send_message(Process* sender,
Process* receiver,
ErtsProcLocks *receiver_locks,
Eterm message,
unsigned flags)
{
Uint msize;
ErlHeapFragment* bp = NULL;
Eterm token = NIL;
#ifdef USE_VM_PROBES
DTRACE_CHARBUF(sender_name, 64);
DTRACE_CHARBUF(receiver_name, 64);
Sint tok_label = 0;
Sint tok_lastcnt = 0;
Sint tok_serial = 0;
#endif
BM_STOP_TIMER(system);
BM_MESSAGE(message,sender,receiver);
BM_START_TIMER(send);
#ifdef USE_VM_PROBES
*sender_name = *receiver_name = '\0';
if (DTRACE_ENABLED(message_send)) {
erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), "%T", sender->id);
erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "%T", receiver->id);
}
#endif
if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) {
Eterm* hp;
Eterm stoken = SEQ_TRACE_TOKEN(sender);
Uint seq_trace_size = 0;
#ifdef USE_VM_PROBES
Uint dt_utag_size = 0;
Eterm utag = NIL;
#endif
BM_SWAP_TIMER(send,size);
msize = size_object(message);
BM_SWAP_TIMER(size,send);
#ifdef USE_VM_PROBES
if (stoken != am_have_dt_utag) {
#endif
seq_trace_update_send(sender);
seq_trace_output(stoken, message, SEQ_TRACE_SEND,
receiver->id, sender);
seq_trace_size = 6; /* TUPLE5 */
#ifdef USE_VM_PROBES
}
if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) {
dt_utag_size = size_object(DT_UTAG(sender));
} else if (stoken == am_have_dt_utag ) {
stoken = NIL;
}
#endif
bp = new_message_buffer(msize + seq_trace_size
#ifdef USE_VM_PROBES
+ dt_utag_size
#endif
);
hp = bp->mem;
BM_SWAP_TIMER(send,copy);
token = copy_struct(stoken,
seq_trace_size,
&hp,
&bp->off_heap);
message = copy_struct(message, msize, &hp, &bp->off_heap);
#ifdef USE_VM_PROBES
if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) {
utag = copy_struct(DT_UTAG(sender), dt_utag_size, &hp, &bp->off_heap);
#ifdef DTRACE_TAG_HARDDEBUG
erts_fprintf(stderr,
"Dtrace -> (%T) Spreading tag (%T) with "
"message %T!\r\n",sender->id, utag, message);
#endif
}
#endif
BM_MESSAGE_COPIED(msize);
BM_SWAP_TIMER(copy,send);
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_send)) {
if (stoken != NIL && stoken != am_have_dt_utag) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(stoken));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(stoken));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(stoken));
}
DTRACE6(message_send, sender_name, receiver_name,
msize, tok_label, tok_lastcnt, tok_serial);
}
#endif
erts_queue_message(receiver,
receiver_locks,
bp,
message,
token
#ifdef USE_VM_PROBES
, utag
#endif
);
BM_SWAP_TIMER(send,system);
} else if (sender == receiver) {
/* Drop message if receiver has a pending exit ... */
#ifdef ERTS_SMP
ErtsProcLocks need_locks = (~(*receiver_locks)
& (ERTS_PROC_LOCK_MSGQ
| ERTS_PROC_LOCK_STATUS));
if (need_locks) {
*receiver_locks |= need_locks;
if (erts_smp_proc_trylock(receiver, need_locks) == EBUSY) {
if (need_locks == ERTS_PROC_LOCK_MSGQ) {
erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS);
need_locks = ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS;
}
erts_smp_proc_lock(receiver, need_locks);
}
}
if (!ERTS_PROC_PENDING_EXIT(receiver))
#endif
{
ErlMessage* mp = message_alloc();
DTRACE6(message_send, sender_name, receiver_name,
size_object(message), tok_label, tok_lastcnt, tok_serial);
mp->data.attached = NULL;
ERL_MESSAGE_TERM(mp) = message;
ERL_MESSAGE_TOKEN(mp) = NIL;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = NIL;
#endif
mp->next = NULL;
/*
* We move 'in queue' to 'private queue' and place
* message at the end of 'private queue' in order
* to ensure that the 'in queue' doesn't contain
* references into the heap. By ensuring this,
* we don't need to include the 'in queue' in
* the root set when garbage collecting.
*/
ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver);
LINK_MESSAGE_PRIVQ(receiver, mp);
if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) {
trace_receive(receiver, message);
}
}
BM_SWAP_TIMER(send,system);
return;
} else {
#ifdef ERTS_SMP
ErlOffHeap *ohp;
Eterm *hp;
BM_SWAP_TIMER(send,size);
msize = size_object(message);
BM_SWAP_TIMER(size,send);
hp = erts_alloc_message_heap(msize,&bp,&ohp,receiver,receiver_locks);
BM_SWAP_TIMER(send,copy);
message = copy_struct(message, msize, &hp, ohp);
BM_MESSAGE_COPIED(msz);
BM_SWAP_TIMER(copy,send);
DTRACE6(message_send, sender_name, receiver_name,
msize, tok_label, tok_lastcnt, tok_serial);
erts_queue_message(receiver, receiver_locks, bp, message, token
#ifdef USE_VM_PROBES
, NIL
#endif
);
BM_SWAP_TIMER(send,system);
#else
ErlMessage* mp = message_alloc();
Eterm *hp;
BM_SWAP_TIMER(send,size);
msize = size_object(message);
BM_SWAP_TIMER(size,send);
if (receiver->stop - receiver->htop <= msize) {
BM_SWAP_TIMER(send,system);
erts_garbage_collect(receiver, msize, receiver->arg_reg, receiver->arity);
BM_SWAP_TIMER(system,send);
}
hp = receiver->htop;
receiver->htop = hp + msize;
BM_SWAP_TIMER(send,copy);
message = copy_struct(message, msize, &hp, &receiver->off_heap);
BM_MESSAGE_COPIED(msize);
BM_SWAP_TIMER(copy,send);
DTRACE6(message_send, sender_name, receiver_name,
(uint32_t)msize, tok_label, tok_lastcnt, tok_serial);
ERL_MESSAGE_TERM(mp) = message;
ERL_MESSAGE_TOKEN(mp) = NIL;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = NIL;
#endif
mp->next = NULL;
mp->data.attached = NULL;
LINK_MESSAGE(receiver, mp);
if (receiver->status == P_WAITING) {
erts_add_to_runq(receiver);
} else if (receiver->status == P_SUSPENDED) {
receiver->rstatus = P_RUNABLE;
}
if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) {
trace_receive(receiver, message);
}
BM_SWAP_TIMER(send,system);
#endif /* #ifndef ERTS_SMP */
return;
}
}
void
erts_send_message(Process* sender,
Process* receiver,
ErtsProcLocks *receiver_locks,
Eterm message,
unsigned flags)
{
Uint msize;
ErlHeapFragment* bp = NULL;
Eterm token = NIL;
BM_STOP_TIMER(system);
BM_MESSAGE(message,sender,receiver);
BM_START_TIMER(send);
if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) {
Eterm* hp;
BM_SWAP_TIMER(send,size);
msize = size_object(message);
BM_SWAP_TIMER(size,send);
seq_trace_update_send(sender);
seq_trace_output(SEQ_TRACE_TOKEN(sender), message, SEQ_TRACE_SEND,
receiver->id, sender);
bp = new_message_buffer(msize + 6 /* TUPLE5 */);
hp = bp->mem;
BM_SWAP_TIMER(send,copy);
token = copy_struct(SEQ_TRACE_TOKEN(sender),
6 /* TUPLE5 */,
&hp,
&bp->off_heap);
message = copy_struct(message, msize, &hp, &bp->off_heap);
BM_MESSAGE_COPIED(msize);
BM_SWAP_TIMER(copy,send);
erts_queue_message(receiver,
receiver_locks,
bp,
message,
token);
BM_SWAP_TIMER(send,system);
#ifdef HYBRID
} else {
ErlMessage* mp = message_alloc();
BM_SWAP_TIMER(send,copy);
#ifdef INCREMENTAL
/* TODO: During GC activate processes if the message relies in
* the fromspace and the sender is active. During major
* collections add the message to the gray stack if it relies
* in the old generation and the sender is active and the
* receiver is inactive.
if (!IS_CONST(message) && (ma_gc_flags & GC_CYCLE) &&
(ptr_val(message) >= inc_fromspc &&
ptr_val(message) < inc_fromend) && INC_IS_ACTIVE(sender))
INC_ACTIVATE(receiver);
else if (!IS_CONST(message) && (ma_gc_flags & GC_CYCLE) &&
(ptr_val(message) >= global_old_heap &&
ptr_val(message) < global_old_hend) &&
INC_IS_ACTIVE(sender) && !INC_IS_ACTIVE(receiver))
Mark message in blackmap and add it to the gray stack
*/
if (!IS_CONST(message))
INC_ACTIVATE(receiver);
#endif
LAZY_COPY(sender,message);
BM_SWAP_TIMER(copy,send);
ERL_MESSAGE_TERM(mp) = message;
ERL_MESSAGE_TOKEN(mp) = NIL;
mp->next = NULL;
LINK_MESSAGE(receiver, mp);
ACTIVATE(receiver);
if (receiver->status == P_WAITING) {
erts_add_to_runq(receiver);
} else if (receiver->status == P_SUSPENDED) {
receiver->rstatus = P_RUNABLE;
}
if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) {
trace_receive(receiver, message);
}
BM_SWAP_TIMER(send,system);
return;
#else
} else if (sender == receiver) {
