void
erts_install_breakpoints(BpFunctions* f)
{
Uint i;
Uint n = f->matched;
BeamInstr br = (BeamInstr) BeamOp(op_i_generic_breakpoint);
for (i = 0; i < n; i++) {
BeamInstr* pc = f->matching[i].pc;
GenericBp* g = (GenericBp *) pc[-4];
if (*pc != br && g) {
Module* modp = f->matching[i].mod;
/*
* The breakpoint must be disabled in the active data
* (it will enabled later by switching bp indices),
* and enabled in the staging data.
*/
ASSERT(g->data[erts_active_bp_ix()].flags == 0);
ASSERT(g->data[erts_staging_bp_ix()].flags != 0);
/*
* The following write is not protected by any lock. We
* assume that the hardware guarantees that a write of an
* aligned word-size (or half-word) writes is atomic
* (i.e. that other processes executing this code will not
* see a half pointer).
*/
*pc = br;
modp->curr.num_breakpoints++;
}
}
}
void
erts_commit_staged_bp(void)
{
ErtsBpIndex staging = erts_staging_bp_ix();
ErtsBpIndex active = erts_active_bp_ix();
erts_smp_atomic32_set_nob(&erts_active_bp_index, staging);
erts_smp_atomic32_set_nob(&erts_staging_bp_index, active);
}
static void
uninstall_breakpoint(BeamInstr* pc)
{
if (*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
GenericBp* g = (GenericBp *) pc[-4];
if (g->data[erts_active_bp_ix()].flags == 0) {
/*
* The following write is not protected by any lock. We
* assume that the hardware guarantees that a write of an
* aligned word-size (or half-word) writes is atomic
* (i.e. that other processes executing this code will not
* see a half pointer).
*/
*pc = g->orig_instr;
}
}
}
/* Add messages last in message queue */
static Sint
queue_messages(Process* receiver,
erts_aint32_t *receiver_state,
ErtsProcLocks receiver_locks,
ErtsMessage* first,
ErtsMessage** last,
Uint len,
Eterm from)
{
ErtsTracingEvent* te;
Sint res;
int locked_msgq = 0;
erts_aint32_t state;
ASSERT(is_value(ERL_MESSAGE_TERM(first)));
ASSERT(ERL_MESSAGE_TOKEN(first) == am_undefined ||
ERL_MESSAGE_TOKEN(first) == NIL ||
is_tuple(ERL_MESSAGE_TOKEN(first)));
#ifdef ERTS_ENABLE_LOCK_CHECK
ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(receiver) < ERTS_PROC_LOCK_MSGQ ||
receiver_locks == erts_proc_lc_my_proc_locks(receiver));
#endif
if (!(receiver_locks & ERTS_PROC_LOCK_MSGQ)) {
if (erts_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
ErtsProcLocks need_locks;
if (receiver_state)
state = *receiver_state;
else
state = erts_atomic32_read_nob(&receiver->state);
if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
goto exiting;
need_locks = receiver_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ);
if (need_locks) {
erts_proc_unlock(receiver, need_locks);
}
need_locks |= ERTS_PROC_LOCK_MSGQ;
erts_proc_lock(receiver, need_locks);
}
locked_msgq = 1;
}
state = erts_atomic32_read_nob(&receiver->state);
if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
exiting:
/* Drop message if receiver is exiting or has a pending exit... */
if (locked_msgq)
erts_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
erts_cleanup_messages(first);
return 0;
}
res = receiver->msg.len;
if (receiver_locks & ERTS_PROC_LOCK_MAIN) {
/*
* We move 'in queue' to 'private queue' and place
* message at the end of 'private queue' in order
* to ensure that the 'in queue' doesn't contain
* references into the heap. By ensuring this,
* we don't need to include the 'in queue' in
* the root set when garbage collecting.
*/
res += receiver->msg_inq.len;
ERTS_MSGQ_MV_INQ2PRIVQ(receiver);
LINK_MESSAGE_PRIVQ(receiver, first, last, len);
}
else
{
LINK_MESSAGE(receiver, first, last, len);
}
if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)
&& (te = &erts_receive_tracing[erts_active_bp_ix()],
te->on)) {
ErtsMessage *msg = first;
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
Sint tok_label = 0;
Sint tok_lastcnt = 0;
Sint tok_serial = 0;
Eterm seq_trace_token = ERL_MESSAGE_TOKEN(msg);
dtrace_proc_str(receiver, receiver_name);
if (seq_trace_token != NIL && is_tuple(seq_trace_token)) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token));
}
DTRACE6(message_queued,
receiver_name, size_object(ERL_MESSAGE_TERM(msg)),
receiver->msg.len,
tok_label, tok_lastcnt, tok_serial);
}
#endif
while (msg) {
trace_receive(receiver, from, ERL_MESSAGE_TERM(msg), te);
msg = msg->next;
}
}
if (locked_msgq) {
erts_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
}
erts_proc_notify_new_message(receiver, receiver_locks);
return res;
}
/*
* Entry point called by the trace wrap functions in erl_bif_wrap.c
*
* The trace wrap functions are themselves called through the export
* entries instead of the original BIF functions.
*/
Eterm
erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
{
Eterm result;
Eterm (*func)(Process*, Eterm*, BeamInstr*);
Export* ep = bif_export[bif_index];
Uint32 flags = 0, flags_meta = 0;
ErtsTracer meta_tracer = erts_tracer_nil;
int applying = (I == &(ep->code[3])); /* Yup, the apply code for a bif
* is actually in the
* export entry */
BeamInstr *cp = p->cp;
GenericBp* g;
GenericBpData* bp = NULL;
Uint bp_flags = 0;
ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
g = (GenericBp *) ep->fake_op_func_info_for_hipe[1];
if (g) {
bp = &g->data[erts_active_bp_ix()];
bp_flags = bp->flags;
}
/*
* Make continuation pointer OK, it is not during direct BIF calls,
* but it is correct during apply of bif.
*/
if (!applying) {
p->cp = I;
}
if (bp_flags & (ERTS_BPF_LOCAL_TRACE|ERTS_BPF_GLOBAL_TRACE) &&
IS_TRACED_FL(p, F_TRACE_CALLS)) {
int local = !!(bp_flags & ERTS_BPF_LOCAL_TRACE);
flags = erts_call_trace(p, ep->code, bp->local_ms, args,
local, &ERTS_TRACER(p));
}
if (bp_flags & ERTS_BPF_META_TRACE) {
ErtsTracer old_tracer;
meta_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer);
old_tracer = meta_tracer;
flags_meta = erts_call_trace(p, ep->code, bp->meta_ms, args,
0, &meta_tracer);
if (!ERTS_TRACER_COMPARE(old_tracer, meta_tracer)) {
ErtsTracer new_tracer = erts_tracer_nil;
erts_tracer_update(&new_tracer, meta_tracer);
if (old_tracer == erts_smp_atomic_cmpxchg_acqb(
&bp->meta_tracer->tracer,
(erts_aint_t)new_tracer,
(erts_aint_t)old_tracer)) {
ERTS_TRACER_CLEAR(&old_tracer);
} else {
ERTS_TRACER_CLEAR(&new_tracer);
}
}
}
if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE &&
IS_TRACED_FL(p, F_TRACE_CALLS)) {
BeamInstr *pc = (BeamInstr *)ep->code+3;
erts_trace_time_call(p, pc, bp->time);
}
/* Restore original continuation pointer (if changed). */
p->cp = cp;
func = bif_table[bif_index].f;
result = func(p, args, I);
if (applying && (flags & MATCH_SET_RETURN_TO_TRACE)) {
BeamInstr i_return_trace = beam_return_trace[0];
BeamInstr i_return_to_trace = beam_return_to_trace[0];
BeamInstr i_return_time_trace = beam_return_time_trace[0];
Eterm *cpp;
/* Maybe advance cp to skip trace stack frames */
for (cpp = p->stop; ; cp = cp_val(*cpp++)) {
if (*cp == i_return_trace) {
/* Skip stack frame variables */
while (is_not_CP(*cpp)) cpp++;
cpp += 2; /* Skip return_trace parameters */
} else if (*cp == i_return_time_trace) {
/* Skip stack frame variables */
while (is_not_CP(*cpp)) cpp++;
cpp += 1; /* Skip return_time_trace parameters */
} else if (*cp == i_return_to_trace) {
/* A return_to trace message is going to be generated
* by normal means, so we do not have to.
*/
cp = NULL;
break;
} else break;
}
}
/* Try to get these in the order
* they usually appear in normal code... */
if (is_non_value(result)) {
Uint reason = p->freason;
if (reason != TRAP) {
Eterm class;
Eterm value = p->fvalue;
/* Expand error value like in handle_error() */
if (reason & EXF_ARGLIST) {
Eterm *tp;
ASSERT(is_tuple(value));
tp = tuple_val(value);
value = tp[1];
}
if ((reason & EXF_THROWN) && (p->catches <= 0)) {
Eterm *hp = HAlloc(p, 3);
value = TUPLE2(hp, am_nocatch, value);
reason = EXC_ERROR;
}
/* Note: expand_error_value() could theoretically
* allocate on the heap, but not for any error
* returned by a BIF, and it would do no harm,
* just be annoying.
*/
value = expand_error_value(p, reason, value);
class = exception_tag[GET_EXC_CLASS(reason)];
if (flags_meta & MATCH_SET_EXCEPTION_TRACE) {
erts_trace_exception(p, ep->code, class, value,
&meta_tracer);
}
if (flags & MATCH_SET_EXCEPTION_TRACE) {
erts_trace_exception(p, ep->code, class, value,
&ERTS_TRACER(p));
}
BeamInstr
erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg)
{
GenericBp* g;
GenericBpData* bp;
Uint bp_flags;
ErtsBpIndex ix = erts_active_bp_ix();
g = (GenericBp *) I[-4];
bp = &g->data[ix];
bp_flags = bp->flags;
ASSERT((bp_flags & ~ERTS_BPF_ALL) == 0);
if (bp_flags & (ERTS_BPF_LOCAL_TRACE|
ERTS_BPF_GLOBAL_TRACE|
ERTS_BPF_TIME_TRACE_ACTIVE) &&
!IS_TRACED_FL(c_p, F_TRACE_CALLS)) {
bp_flags &= ~(ERTS_BPF_LOCAL_TRACE|
ERTS_BPF_GLOBAL_TRACE|
ERTS_BPF_TIME_TRACE|
ERTS_BPF_TIME_TRACE_ACTIVE);
if (bp_flags == 0) { /* Quick exit */
return g->orig_instr;
}
}
if (bp_flags & ERTS_BPF_LOCAL_TRACE) {
ASSERT((bp_flags & ERTS_BPF_GLOBAL_TRACE) == 0);
(void) do_call_trace(c_p, I, reg, 1, bp->local_ms, erts_tracer_true);
} else if (bp_flags & ERTS_BPF_GLOBAL_TRACE) {
(void) do_call_trace(c_p, I, reg, 0, bp->local_ms, erts_tracer_true);
}
if (bp_flags & ERTS_BPF_META_TRACE) {
ErtsTracer old_tracer, new_tracer;
old_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer);
new_tracer = do_call_trace(c_p, I, reg, 1, bp->meta_ms, old_tracer);
if (!ERTS_TRACER_COMPARE(new_tracer, old_tracer)) {
if (old_tracer == erts_smp_atomic_cmpxchg_acqb(
&bp->meta_tracer->tracer,
(erts_aint_t)new_tracer,
(erts_aint_t)old_tracer)) {
ERTS_TRACER_CLEAR(&old_tracer);
} else {
ERTS_TRACER_CLEAR(&new_tracer);
}
}
}
if (bp_flags & ERTS_BPF_COUNT_ACTIVE) {
erts_smp_atomic_inc_nob(&bp->count->acount);
}
if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE) {
Eterm w;
erts_trace_time_call(c_p, I, bp->time);
w = (BeamInstr) *c_p->cp;
if (! (w == (BeamInstr) BeamOp(op_i_return_time_trace) ||
w == (BeamInstr) BeamOp(op_return_trace) ||
w == (BeamInstr) BeamOp(op_i_return_to_trace)) ) {
Eterm* E = c_p->stop;
ASSERT(c_p->htop <= E && E <= c_p->hend);
if (E - 2 < c_p->htop) {
(void) erts_garbage_collect(c_p, 2, reg, I[-1]);
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
}
E = c_p->stop;
ASSERT(c_p->htop <= E && E <= c_p->hend);
E -= 2;
E[0] = make_cp(I);
E[1] = make_cp(c_p->cp); /* original return address */
c_p->cp = beam_return_time_trace;
c_p->stop = E;
}
}
if (bp_flags & ERTS_BPF_DEBUG) {
return (BeamInstr) BeamOp(op_i_debug_breakpoint);
} else {
return g->orig_instr;
}
}
static void
consolidate_bp_data(Module* modp, BeamInstr* pc, int local)
{
GenericBp* g = (GenericBp *) pc[-4];
GenericBpData* src;
GenericBpData* dst;
Uint flags;
if (g == 0) {
return;
}
src = &g->data[erts_active_bp_ix()];
dst = &g->data[erts_staging_bp_ix()];
/*
* The contents of the staging area may be out of date.
* Decrement all reference pointers.
*/
flags = dst->flags;
if (flags & (ERTS_BPF_LOCAL_TRACE|ERTS_BPF_GLOBAL_TRACE)) {
MatchSetUnref(dst->local_ms);
}
if (flags & ERTS_BPF_META_TRACE) {
bp_meta_unref(dst->meta_tracer);
MatchSetUnref(dst->meta_ms);
}
if (flags & ERTS_BPF_COUNT) {
bp_count_unref(dst->count);
}
if (flags & ERTS_BPF_TIME_TRACE) {
bp_time_unref(dst->time);
}
/*
* If all flags are zero, deallocate all breakpoint data.
*/
flags = dst->flags = src->flags;
if (flags == 0) {
if (modp) {
if (local) {
modp->curr.num_breakpoints--;
} else {
modp->curr.num_traced_exports--;
}
ASSERT(modp->curr.num_breakpoints >= 0);
ASSERT(modp->curr.num_traced_exports >= 0);
ASSERT(*pc != (BeamInstr) BeamOp(op_i_generic_breakpoint));
}
pc[-4] = 0;
Free(g);
return;
}
/*
* Copy the active data to the staging area (making it ready
* for the next time it will be used).
*/
if (flags & (ERTS_BPF_LOCAL_TRACE|ERTS_BPF_GLOBAL_TRACE)) {
dst->local_ms = src->local_ms;
MatchSetRef(dst->local_ms);
}
if (flags & ERTS_BPF_META_TRACE) {
dst->meta_tracer = src->meta_tracer;
erts_refc_inc(&dst->meta_tracer->refc, 1);
dst->meta_ms = src->meta_ms;
MatchSetRef(dst->meta_ms);
}
if (flags & ERTS_BPF_COUNT) {
dst->count = src->count;
erts_refc_inc(&dst->count->refc, 1);
}
if (flags & ERTS_BPF_TIME_TRACE) {
dst->time = src->time;
erts_refc_inc(&dst->time->refc, 1);
ASSERT(dst->time->hash);
}
}
/*
* Entry point called by the trace wrap functions in erl_bif_wrap.c
*
* The trace wrap functions are themselves called through the export
* entries instead of the original BIF functions.
*/
Eterm
erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
{
Eterm result;
Eterm (*func)(Process*, Eterm*, BeamInstr*);
Export* ep = bif_export[bif_index];
Uint32 flags = 0, flags_meta = 0;
ErtsTracer meta_tracer = erts_tracer_nil;
int applying = (I == ep->beam); /* Yup, the apply code for a bif
* is actually in the
* export entry */
BeamInstr *cp = p->cp;
GenericBp* g;
GenericBpData* bp = NULL;
Uint bp_flags = 0;
ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
g = ep->info.u.gen_bp;
if (g) {
bp = &g->data[erts_active_bp_ix()];
bp_flags = bp->flags;
}
/*
* Make continuation pointer OK, it is not during direct BIF calls,
* but it is correct during apply of bif.
*/
if (!applying) {
p->cp = I;
}
if (bp_flags & (ERTS_BPF_LOCAL_TRACE|ERTS_BPF_GLOBAL_TRACE) &&
IS_TRACED_FL(p, F_TRACE_CALLS)) {
int local = !!(bp_flags & ERTS_BPF_LOCAL_TRACE);
flags = erts_call_trace(p, &ep->info, bp->local_ms, args,
local, &ERTS_TRACER(p));
}
if (bp_flags & ERTS_BPF_META_TRACE) {
ErtsTracer old_tracer;
meta_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer);
old_tracer = meta_tracer;
flags_meta = erts_call_trace(p, &ep->info, bp->meta_ms, args,
0, &meta_tracer);
if (!ERTS_TRACER_COMPARE(old_tracer, meta_tracer)) {
ErtsTracer new_tracer = erts_tracer_nil;
erts_tracer_update(&new_tracer, meta_tracer);
if (old_tracer == erts_smp_atomic_cmpxchg_acqb(
&bp->meta_tracer->tracer,
(erts_aint_t)new_tracer,
(erts_aint_t)old_tracer)) {
ERTS_TRACER_CLEAR(&old_tracer);
} else {
ERTS_TRACER_CLEAR(&new_tracer);
}
}
}
if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE &&
IS_TRACED_FL(p, F_TRACE_CALLS)) {
erts_trace_time_call(p, &ep->info, bp->time);
}
/* Restore original continuation pointer (if changed). */
p->cp = cp;
func = bif_table[bif_index].f;
result = func(p, args, I);
if (erts_nif_export_check_save_trace(p, result,
applying, ep,
cp, flags,
flags_meta, I,
meta_tracer)) {
/*
* erts_bif_trace_epilogue() will be called
* later when appropriate via the NIF export
* scheduling functionality...
*/
return result;
}
return erts_bif_trace_epilogue(p, result, applying, ep, cp,
flags, flags_meta, I,
meta_tracer);
}