void
erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
Eterm reason, Eterm token)
{
Eterm mess;
Eterm save;
Eterm from_copy;
Uint sz_reason;
Uint sz_token;
Uint sz_from;
Eterm* hp;
Eterm temptoken;
ErlHeapFragment* bp = NULL;
if (token != NIL
#ifdef USE_VM_PROBES
&& token != am_have_dt_utag
#endif
) {
ASSERT(is_tuple(token));
sz_reason = size_object(reason);
sz_token = size_object(token);
sz_from = size_object(from);
bp = new_message_buffer(sz_reason + sz_from + sz_token + 4);
hp = bp->mem;
mess = copy_struct(reason, sz_reason, &hp, &bp->off_heap);
from_copy = copy_struct(from, sz_from, &hp, &bp->off_heap);
save = TUPLE3(hp, am_EXIT, from_copy, mess);
hp += 4;
/* the trace token must in this case be updated by the caller */
seq_trace_output(token, save, SEQ_TRACE_SEND, to->common.id, NULL);
temptoken = copy_struct(token, sz_token, &hp, &bp->off_heap);
erts_queue_message(to, to_locksp, bp, save, temptoken);
} else {
ErlOffHeap *ohp;
sz_reason = size_object(reason);
sz_from = IS_CONST(from) ? 0 : size_object(from);
hp = erts_alloc_message_heap(sz_reason+sz_from+4,
&bp,
&ohp,
to,
to_locksp);
mess = copy_struct(reason, sz_reason, &hp, ohp);
from_copy = (IS_CONST(from)
? from
: copy_struct(from, sz_from, &hp, ohp));
save = TUPLE3(hp, am_EXIT, from_copy, mess);
erts_queue_message(to, to_locksp, bp, save, NIL);
}
}
static void send_reply(ErtsMsAcc *msacc, ErtsMSAccReq *msaccrp) {
ErtsSchedulerData *esdp = erts_get_scheduler_data();
Process *rp = msaccrp->proc;
ErtsMessage *msgp = NULL;
Eterm **hpp, *hp;
Eterm ref_copy = NIL, msg;
Uint sz, *szp;
ErlOffHeap *ohp = NULL;
ErtsProcLocks rp_locks = (esdp && msaccrp->req_sched == esdp->no
? ERTS_PROC_LOCK_MAIN : 0);
sz = 0;
hpp = NULL;
szp = &sz;
if (msacc->unmanaged) erts_mtx_lock(&msacc->mtx);
while (1) {
if (hpp)
ref_copy = STORE_NC(hpp, ohp, msaccrp->ref);
else
*szp += REF_THING_SIZE;
if (msaccrp->action != ERTS_MSACC_GATHER)
msg = ref_copy;
else {
msg = erts_msacc_gather_stats(msacc, hpp, szp);
msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg);
}
if (hpp)
break;
msgp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
hpp = &hp;
szp = NULL;
}
if (msacc->unmanaged) erts_mtx_unlock(&msacc->mtx);
erts_queue_message(rp, rp_locks, msgp, msg, am_system);
if (esdp && msaccrp->req_sched == esdp->no)
rp_locks &= ~ERTS_PROC_LOCK_MAIN;
if (rp_locks)
erts_smp_proc_unlock(rp, rp_locks);
}
static void signal_notify_requested(Eterm type) {
Process* p = NULL;
Eterm msg, *hp;
ErtsProcLocks locks = 0;
ErlOffHeap *ohp;
Eterm id = erts_whereis_name_to_id(NULL, am_erl_signal_server);
if ((p = (erts_pid2proc_opt(NULL, 0, id, 0, ERTS_P2P_FLG_INC_REFC))) != NULL) {
ErtsMessage *msgp = erts_alloc_message_heap(p, &locks, 3, &hp, &ohp);
/* erl_signal_server ! {notify, sighup} */
msg = TUPLE2(hp, am_notify, type);
erts_queue_message(p, locks, msgp, msg, am_system);
if (locks)
erts_smp_proc_unlock(p, locks);
erts_proc_dec_refc(p);
}
}
static int
dirty_send_message(Process *c_p, Eterm to, Eterm tag)
{
ErtsProcLocks c_p_locks, rp_locks;
Process *rp, *real_c_p;
Eterm msg, *hp;
ErlOffHeap *ohp;
ErtsMessage *mp;
ASSERT(is_immed(tag));
real_c_p = erts_proc_shadow2real(c_p);
if (real_c_p != c_p)
c_p_locks = 0;
else
c_p_locks = ERTS_PROC_LOCK_MAIN;
ASSERT(real_c_p->common.id == c_p->common.id);
rp = erts_pid2proc_opt(real_c_p, c_p_locks,
to, 0,
ERTS_P2P_FLG_INC_REFC);
if (!rp)
return 0;
rp_locks = 0;
mp = erts_alloc_message_heap(rp, &rp_locks, 3, &hp, &ohp);
msg = TUPLE2(hp, tag, c_p->common.id);
erts_queue_message(rp, rp_locks, mp, msg, c_p->common.id);
if (rp == real_c_p)
rp_locks &= ~c_p_locks;
if (rp_locks)
erts_proc_unlock(rp, rp_locks);
erts_proc_dec_refc(rp);
return 1;
}
static void send_reply(ErtsMsAcc *msacc, ErtsMSAccReq *msaccrp) {
ErtsSchedulerData *esdp = erts_get_scheduler_data();
Process *rp = msaccrp->proc;
ErtsMessage *msgp = NULL;
Eterm *hp;
Eterm ref_copy = NIL, msg;
ErtsProcLocks rp_locks = (esdp && msaccrp->req_sched == esdp->no
? ERTS_PROC_LOCK_MAIN : 0);
ErtsHeapFactory factory;
if (msaccrp->action == ERTS_MSACC_GATHER) {
msgp = erts_factory_message_create(&factory, rp, &rp_locks, DEFAULT_MSACC_MSG_SIZE);
if (msacc->unmanaged) erts_mtx_lock(&msacc->mtx);
hp = erts_produce_heap(&factory, REF_THING_SIZE + 3 /* tuple */, 0);
ref_copy = STORE_NC(&hp, &msgp->hfrag.off_heap, msaccrp->ref);
msg = erts_msacc_gather_stats(msacc, &factory);
msg = TUPLE2(hp, ref_copy, msg);
if (msacc->unmanaged) erts_mtx_unlock(&msacc->mtx);
erts_factory_close(&factory);
} else {
ErlOffHeap *ohp = NULL;
msgp = erts_alloc_message_heap(rp, &rp_locks, REF_THING_SIZE, &hp, &ohp);
msg = STORE_NC(&hp, &msgp->hfrag.off_heap, msaccrp->ref);
}
erts_queue_message(rp, rp_locks, msgp, msg, am_system);
if (esdp && msaccrp->req_sched == esdp->no)
rp_locks &= ~ERTS_PROC_LOCK_MAIN;
if (rp_locks)
erts_smp_proc_unlock(rp, rp_locks);
}
static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type,
Eterm tag, int errcode)
{
Eterm mess;
Eterm r;
Eterm *hp;
ErtsMessage *mp;
ErtsProcLocks rp_locks = 0;
ErlOffHeap *ohp;
ERTS_CHK_NO_PROC_LOCKS;
assert_drv_list_rwlocked();
if (errcode != 0) {
int need = load_error_need(errcode);
Eterm e;
mp = erts_alloc_message_heap(proc, &rp_locks,
(6 /* tuple */ + 3 /* Error tuple */ +
ERTS_REF_THING_SIZE + need),
&hp, &ohp);
r = copy_ref(ref,hp);
hp += ERTS_REF_THING_SIZE;
e = build_load_error_hp(hp, errcode);
hp += need;
mess = TUPLE2(hp,tag,e);
hp += 3;
mess = TUPLE5(hp,type,r,am_driver,driver_name,mess);
} else {
mp = erts_alloc_message_heap(proc, &rp_locks,
6 /* tuple */ + ERTS_REF_THING_SIZE,
&hp, &ohp);
r = copy_ref(ref,hp);
hp += ERTS_REF_THING_SIZE;
mess = TUPLE5(hp,type,r,am_driver,driver_name,tag);
}
erts_queue_message(proc, rp_locks, mp, mess, am_system);
erts_proc_unlock(proc, rp_locks);
ERTS_CHK_NO_PROC_LOCKS;
}
void
erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
Eterm reason, Eterm token)
{
Eterm mess;
Eterm save;
Eterm from_copy;
Uint sz_reason;
Uint sz_token;
Uint sz_from;
Eterm* hp;
Eterm temptoken;
ErtsMessage* mp;
ErlOffHeap *ohp;
#ifdef SHCOPY_SEND
erts_shcopy_t info;
#endif
if (have_seqtrace(token)) {
ASSERT(is_tuple(token));
sz_token = size_object(token);
sz_from = size_object(from);
#ifdef SHCOPY_SEND
INITIALIZE_SHCOPY(info);
sz_reason = copy_shared_calculate(reason, &info);
#else
sz_reason = size_object(reason);
#endif
mp = erts_alloc_message_heap(to, to_locksp,
sz_reason + sz_from + sz_token + 4,
&hp, &ohp);
#ifdef SHCOPY_SEND
mess = copy_shared_perform(reason, sz_reason, &info, &hp, ohp);
DESTROY_SHCOPY(info);
#else
mess = copy_struct(reason, sz_reason, &hp, ohp);
#endif
from_copy = copy_struct(from, sz_from, &hp, ohp);
save = TUPLE3(hp, am_EXIT, from_copy, mess);
hp += 4;
/* the trace token must in this case be updated by the caller */
seq_trace_output(token, save, SEQ_TRACE_SEND, to->common.id, NULL);
temptoken = copy_struct(token, sz_token, &hp, ohp);
ERL_MESSAGE_TOKEN(mp) = temptoken;
erts_queue_message(to, *to_locksp, mp, save, am_system);
} else {
sz_from = IS_CONST(from) ? 0 : size_object(from);
#ifdef SHCOPY_SEND
INITIALIZE_SHCOPY(info);
sz_reason = copy_shared_calculate(reason, &info);
#else
sz_reason = size_object(reason);
#endif
mp = erts_alloc_message_heap(to, to_locksp,
sz_reason+sz_from+4, &hp, &ohp);
#ifdef SHCOPY_SEND
mess = copy_shared_perform(reason, sz_reason, &info, &hp, ohp);
DESTROY_SHCOPY(info);
#else
mess = copy_struct(reason, sz_reason, &hp, ohp);
#endif
from_copy = (IS_CONST(from)
? from
: copy_struct(from, sz_from, &hp, ohp));
save = TUPLE3(hp, am_EXIT, from_copy, mess);
erts_queue_message(to, *to_locksp, mp, save, am_system);
}
}
void
erts_queue_dist_message(Process *rcvr,
ErtsProcLocks rcvr_locks,
ErtsDistExternal *dist_ext,
Eterm token,
Eterm from)
{
ErtsMessage* mp;
#ifdef USE_VM_PROBES
Sint tok_label = 0;
Sint tok_lastcnt = 0;
Sint tok_serial = 0;
#endif
erts_aint_t state;
ERTS_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
mp = erts_alloc_message(0, NULL);
mp->data.dist_ext = dist_ext;
ERL_MESSAGE_TERM(mp) = THE_NON_VALUE;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = NIL;
if (token == am_have_dt_utag)
ERL_MESSAGE_TOKEN(mp) = NIL;
else
#endif
ERL_MESSAGE_TOKEN(mp) = token;
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) {
if (erts_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
ErtsProcLocks unlocks =
rcvr_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ);
if (unlocks) {
erts_proc_unlock(rcvr, unlocks);
need_locks |= unlocks;
}
erts_proc_lock(rcvr, need_locks);
}
}
state = erts_atomic32_read_acqb(&rcvr->state);
if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
/* Drop message if receiver is exiting or has a pending exit ... */
erts_cleanup_messages(mp);
}
else
if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) {
if (from == am_Empty)
from = dist_ext->dep->sysname;
/* Ahh... need to decode it in order to trace it... */
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
if (!erts_decode_dist_message(rcvr, rcvr_locks, mp, 0))
erts_free_message(mp);
else {
Eterm msg = ERL_MESSAGE_TERM(mp);
token = ERL_MESSAGE_TOKEN(mp);
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
dtrace_proc_str(rcvr, receiver_name);
if (have_seqtrace(token)) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
}
DTRACE6(message_queued,
receiver_name, size_object(msg), rcvr->msg.len,
tok_label, tok_lastcnt, tok_serial);
}
#endif
erts_queue_message(rcvr, rcvr_locks, mp, msg, from);
}
}
else {
/* Enqueue message on external format */
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
dtrace_proc_str(rcvr, receiver_name);
if (have_seqtrace(token)) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
}
/*
* TODO: We don't know the real size of the external message here.
* -1 will appear to a D script as 4294967295.
*/
DTRACE6(message_queued, receiver_name, -1, rcvr->msg.len + 1,
tok_label, tok_lastcnt, tok_serial);
}
#endif
LINK_MESSAGE(rcvr, mp, &mp->next, 1);
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
erts_proc_notify_new_message(rcvr,
rcvr_locks
);
}
}
int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
ErlNifEnv* msg_env, ERL_NIF_TERM msg)
{
struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)msg_env;
ErtsProcLocks rp_locks = 0;
Process* rp;
Process* c_p;
ErlHeapFragment* frags;
#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
ErtsProcLocks rp_had_locks;
#endif
Eterm receiver = to_pid->pid;
int flush_me = 0;
if (env != NULL) {
c_p = env->proc;
if (receiver == c_p->id) {
rp_locks = ERTS_PROC_LOCK_MAIN;
flush_me = 1;
}
}
else {
#ifdef ERTS_SMP
c_p = NULL;
#else
erl_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM");
#endif
}
#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
rp_had_locks = rp_locks;
#endif
rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN,
receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC);
if (rp == NULL) {
ASSERT(env == NULL || receiver != c_p->id);
return 0;
}
flush_env(msg_env);
frags = menv->env.heap_frag;
ASSERT(frags == MBUF(&menv->phony_proc));
if (frags != NULL) {
/* Move all offheap's from phony proc to the first fragment.
Quick and dirty, but erts_move_msg_mbuf_to_heap doesn't care. */
ASSERT(!is_offheap(&frags->off_heap));
frags->off_heap = MSO(&menv->phony_proc);
clear_offheap(&MSO(&menv->phony_proc));
menv->env.heap_frag = NULL;
MBUF(&menv->phony_proc) = NULL;
}
ASSERT(!is_offheap(&MSO(&menv->phony_proc)));
if (flush_me) {
flush_env(env); /* Needed for ERTS_HOLE_CHECK */
}
erts_queue_message(rp, &rp_locks, frags, msg, am_undefined);
if (rp_locks) {
ERTS_SMP_LC_ASSERT(rp_locks == (rp_had_locks | (ERTS_PROC_LOCK_MSGQ |
ERTS_PROC_LOCK_STATUS)));
erts_smp_proc_unlock(rp, (ERTS_PROC_LOCK_MSGQ | ERTS_PROC_LOCK_STATUS));
}
erts_smp_proc_dec_refc(rp);
if (flush_me) {
cache_env(env);
}
return 1;
}
BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
{
if (BIF_P != erts_code_purger)
BIF_ERROR(BIF_P, EXC_NOTSUP);
if (is_not_atom(BIF_ARG_1))
BIF_ERROR(BIF_P, BADARG);
switch (BIF_ARG_2) {
case am_prepare:
case am_prepare_on_load: {
/*
* Prepare for purge by marking all fun
* entries referring to the code to purge
* with "pending purge" markers.
*/
ErtsCodeIndex code_ix;
Module* modp;
Eterm res;
if (is_value(purge_state.module))
BIF_ERROR(BIF_P, BADARG);
code_ix = erts_active_code_ix();
/*
* Correct module?
*/
modp = erts_get_module(BIF_ARG_1, code_ix);
if (!modp)
res = am_false;
else {
/*
* Any code to purge?
*/
if (BIF_ARG_2 == am_prepare_on_load) {
erts_rwlock_old_code(code_ix);
} else {
erts_rlock_old_code(code_ix);
}
if (BIF_ARG_2 == am_prepare_on_load) {
ASSERT(modp->on_load);
ASSERT(modp->on_load->code_hdr);
purge_state.saved_old = modp->old;
modp->old = *modp->on_load;
erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) modp->on_load);
modp->on_load = 0;
}
if (!modp->old.code_hdr)
res = am_false;
else {
BeamInstr* code;
BeamInstr* end;
erts_smp_mtx_lock(&purge_state.mtx);
purge_state.module = BIF_ARG_1;
erts_smp_mtx_unlock(&purge_state.mtx);
res = am_true;
code = (BeamInstr*) modp->old.code_hdr;
end = (BeamInstr *)((char *)code + modp->old.code_length);
erts_fun_purge_prepare(code, end);
}
if (BIF_ARG_2 == am_prepare_on_load) {
erts_rwunlock_old_code(code_ix);
} else {
erts_runlock_old_code(code_ix);
}
}
#ifndef ERTS_SMP
BIF_RET(res);
#else
if (res != am_true)
BIF_RET(res);
else {
/*
* We'll be resumed when all schedulers are guaranteed
* to see the "pending purge" markers that we've made on
* all fun entries of the code that we are about to purge.
* Processes trying to call these funs will be suspended
* before calling the funs. That is we are guaranteed not
* to get any more direct references into the code while
* checking for such references...
*/
erts_schedule_thr_prgr_later_op(resume_purger,
NULL,
&purger_lop_data);
erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL);
ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
}
#endif
}
case am_abort: {
/*
* Soft purge that detected direct references into the code
* we set out to purge. Abort the purge.
*/
if (purge_state.module != BIF_ARG_1)
BIF_ERROR(BIF_P, BADARG);
erts_fun_purge_abort_prepare(purge_state.funs, purge_state.fe_ix);
#ifndef ERTS_SMP
erts_fun_purge_abort_finalize(purge_state.funs, purge_state.fe_ix);
finalize_purge_operation(BIF_P, 0);
BIF_RET(am_false);
#else
/*
* We need to restore the code addresses of the funs in
* two stages in order to ensure that we do not get any
* stale suspended processes due to the purge abort.
* Restore address pointer (erts_fun_purge_abort_prepare);
* wait for thread progress; clear pending purge address
* pointer (erts_fun_purge_abort_finalize), and then
* resume processes that got suspended
* (finalize_purge_operation).
*/
erts_schedule_thr_prgr_later_op(finalize_purge_abort,
NULL,
&purger_lop_data);
erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL);
ERTS_BIF_YIELD_RETURN(BIF_P, am_false);
#endif
}
case am_complete: {
ErtsCodeIndex code_ix;
BeamInstr* code;
Module* modp;
int is_blocking = 0;
Eterm ret;
ErtsLiteralArea *literals = NULL;
/*
* We have no direct references into the code.
* Complete to purge.
*/
if (purge_state.module != BIF_ARG_1)
BIF_ERROR(BIF_P, BADARG);
if (!erts_try_seize_code_write_permission(BIF_P)) {
ERTS_BIF_YIELD2(bif_export[BIF_erts_internal_purge_module_2],
BIF_P, BIF_ARG_1, BIF_ARG_2);
}
code_ix = erts_active_code_ix();
/*
* Correct module?
*/
if ((modp = erts_get_module(BIF_ARG_1, code_ix)) == NULL) {
ERTS_BIF_PREP_RET(ret, am_false);
}
else {
erts_rwlock_old_code(code_ix);
/*
* Any code to purge?
*/
if (!modp->old.code_hdr) {
ERTS_BIF_PREP_RET(ret, am_false);
}
else {
/*
* Unload any NIF library
*/
if (modp->old.nif != NULL
|| IF_HIPE(hipe_purge_need_blocking(modp))) {
/* ToDo: Do unload nif without blocking */
erts_rwunlock_old_code(code_ix);
erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
erts_smp_thr_progress_block();
is_blocking = 1;
erts_rwlock_old_code(code_ix);
if (modp->old.nif) {
erts_unload_nif(modp->old.nif);
modp->old.nif = NULL;
}
}
/*
* Remove the old code.
*/
ASSERT(erts_total_code_size >= modp->old.code_length);
erts_total_code_size -= modp->old.code_length;
code = (BeamInstr*) modp->old.code_hdr;
erts_fun_purge_complete(purge_state.funs, purge_state.fe_ix);
beam_catches_delmod(modp->old.catches, code, modp->old.code_length,
code_ix);
literals = modp->old.code_hdr->literal_area;
modp->old.code_hdr->literal_area = NULL;
erts_free(ERTS_ALC_T_CODE, (void *) code);
modp->old.code_hdr = NULL;
modp->old.code_length = 0;
modp->old.catches = BEAM_CATCHES_NIL;
erts_remove_from_ranges(code);
#ifdef HIPE
hipe_purge_module(modp, is_blocking);
#endif
ERTS_BIF_PREP_RET(ret, am_true);
}
if (purge_state.saved_old.code_hdr) {
modp->old = purge_state.saved_old;
purge_state.saved_old.code_hdr = 0;
}
erts_rwunlock_old_code(code_ix);
}
if (is_blocking) {
erts_smp_thr_progress_unblock();
erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
}
erts_release_code_write_permission();
finalize_purge_operation(BIF_P, ret == am_true);
if (literals) {
ErtsLiteralAreaRef *ref;
ref = erts_alloc(ERTS_ALC_T_LITERAL_REF,
sizeof(ErtsLiteralAreaRef));
ref->literal_area = literals;
ref->next = NULL;
erts_smp_mtx_lock(&release_literal_areas.mtx);
if (release_literal_areas.last) {
release_literal_areas.last->next = ref;
release_literal_areas.last = ref;
}
else {
release_literal_areas.first = ref;
release_literal_areas.last = ref;
}
erts_smp_mtx_unlock(&release_literal_areas.mtx);
erts_queue_message(erts_literal_area_collector,
0,
erts_alloc_message(0, NULL),
am_copy_literals,
BIF_P->common.id);
}
return ret;
}
default:
BIF_ERROR(BIF_P, BADARG);
}
}
void
erts_queue_dist_message(Process *rcvr,
ErtsProcLocks *rcvr_locks,
ErtsDistExternal *dist_ext,
Eterm token)
{
ErlMessage* mp;
#ifdef USE_VM_PROBES
Sint tok_label = 0;
Sint tok_lastcnt = 0;
Sint tok_serial = 0;
#endif
#ifdef ERTS_SMP
erts_aint_t state;
#endif
ERTS_SMP_LC_ASSERT(*rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
mp = message_alloc();
#ifdef ERTS_SMP
if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) {
if (erts_smp_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
if (*rcvr_locks & ERTS_PROC_LOCK_STATUS) {
erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_STATUS);
need_locks |= ERTS_PROC_LOCK_STATUS;
}
erts_smp_proc_lock(rcvr, need_locks);
}
}
state = erts_smp_atomic32_read_acqb(&rcvr->state);
if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
/* Drop message if receiver is exiting or has a pending exit ... */
if (is_not_nil(token)) {
ErlHeapFragment *heap_frag;
heap_frag = erts_dist_ext_trailer(mp->data.dist_ext);
erts_cleanup_offheap(&heap_frag->off_heap);
}
erts_free_dist_ext_copy(dist_ext);
message_free(mp);
}
else
#endif
if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) {
/* Ahh... need to decode it in order to trace it... */
ErlHeapFragment *mbuf;
Eterm msg;
if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
message_free(mp);
msg = erts_msg_distext2heap(rcvr, rcvr_locks, &mbuf, &token, dist_ext);
if (is_value(msg))
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
dtrace_proc_str(rcvr, receiver_name);
if (token != NIL && token != am_have_dt_utag) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
}
DTRACE6(message_queued,
receiver_name, size_object(msg), rcvr->msg.len,
tok_label, tok_lastcnt, tok_serial);
}
#endif
erts_queue_message(rcvr, rcvr_locks, mbuf, msg, token);
}
else {
/* Enqueue message on external format */
ERL_MESSAGE_TERM(mp) = THE_NON_VALUE;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = NIL;
if (token == am_have_dt_utag) {
ERL_MESSAGE_TOKEN(mp) = NIL;
} else {
#endif
ERL_MESSAGE_TOKEN(mp) = token;
#ifdef USE_VM_PROBES
}
#endif
mp->next = NULL;
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
dtrace_proc_str(rcvr, receiver_name);
if (token != NIL && token != am_have_dt_utag) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
}
/*
* TODO: We don't know the real size of the external message here.
* -1 will appear to a D script as 4294967295.
*/
DTRACE6(message_queued, receiver_name, -1, rcvr->msg.len + 1,
tok_label, tok_lastcnt, tok_serial);
}
#endif
mp->data.dist_ext = dist_ext;
LINK_MESSAGE(rcvr, mp);
if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
erts_proc_notify_new_message(rcvr,
#ifdef ERTS_SMP
*rcvr_locks
#else
0
#endif
);
}
}
void
erts_send_message(Process* sender,
Process* receiver,
ErtsProcLocks *receiver_locks,
Eterm message,
unsigned flags)
{
Uint msize;
ErlHeapFragment* bp = NULL;
Eterm token = NIL;
#ifdef USE_VM_PROBES
DTRACE_CHARBUF(sender_name, 64);
DTRACE_CHARBUF(receiver_name, 64);
Sint tok_label = 0;
Sint tok_lastcnt = 0;
Sint tok_serial = 0;
#endif
BM_STOP_TIMER(system);
BM_MESSAGE(message,sender,receiver);
BM_START_TIMER(send);
#ifdef USE_VM_PROBES
*sender_name = *receiver_name = '\0';
if (DTRACE_ENABLED(message_send)) {
erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), "%T", sender->id);
erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "%T", receiver->id);
}
#endif
if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) {
Eterm* hp;
Eterm stoken = SEQ_TRACE_TOKEN(sender);
Uint seq_trace_size = 0;
#ifdef USE_VM_PROBES
Uint dt_utag_size = 0;
Eterm utag = NIL;
#endif
BM_SWAP_TIMER(send,size);
msize = size_object(message);
BM_SWAP_TIMER(size,send);
#ifdef USE_VM_PROBES
if (stoken != am_have_dt_utag) {
#endif
seq_trace_update_send(sender);
seq_trace_output(stoken, message, SEQ_TRACE_SEND,
receiver->id, sender);
seq_trace_size = 6; /* TUPLE5 */
#ifdef USE_VM_PROBES
}
if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) {
dt_utag_size = size_object(DT_UTAG(sender));
} else if (stoken == am_have_dt_utag ) {
stoken = NIL;
}
#endif
bp = new_message_buffer(msize + seq_trace_size
#ifdef USE_VM_PROBES
+ dt_utag_size
#endif
);
hp = bp->mem;
BM_SWAP_TIMER(send,copy);
token = copy_struct(stoken,
seq_trace_size,
&hp,
&bp->off_heap);
message = copy_struct(message, msize, &hp, &bp->off_heap);
#ifdef USE_VM_PROBES
if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) {
utag = copy_struct(DT_UTAG(sender), dt_utag_size, &hp, &bp->off_heap);
#ifdef DTRACE_TAG_HARDDEBUG
erts_fprintf(stderr,
"Dtrace -> (%T) Spreading tag (%T) with "
"message %T!\r\n",sender->id, utag, message);
#endif
}
#endif
BM_MESSAGE_COPIED(msize);
BM_SWAP_TIMER(copy,send);
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_send)) {
if (stoken != NIL && stoken != am_have_dt_utag) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(stoken));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(stoken));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(stoken));
}
DTRACE6(message_send, sender_name, receiver_name,
msize, tok_label, tok_lastcnt, tok_serial);
}
#endif
erts_queue_message(receiver,
receiver_locks,
bp,
message,
token
#ifdef USE_VM_PROBES
, utag
#endif
);
BM_SWAP_TIMER(send,system);
} else if (sender == receiver) {
/* Drop message if receiver has a pending exit ... */
#ifdef ERTS_SMP
ErtsProcLocks need_locks = (~(*receiver_locks)
& (ERTS_PROC_LOCK_MSGQ
| ERTS_PROC_LOCK_STATUS));
if (need_locks) {
*receiver_locks |= need_locks;
if (erts_smp_proc_trylock(receiver, need_locks) == EBUSY) {
if (need_locks == ERTS_PROC_LOCK_MSGQ) {
erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS);
need_locks = ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS;
}
erts_smp_proc_lock(receiver, need_locks);
}
}
if (!ERTS_PROC_PENDING_EXIT(receiver))
#endif
{
ErlMessage* mp = message_alloc();
DTRACE6(message_send, sender_name, receiver_name,
size_object(message), tok_label, tok_lastcnt, tok_serial);
mp->data.attached = NULL;
ERL_MESSAGE_TERM(mp) = message;
ERL_MESSAGE_TOKEN(mp) = NIL;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = NIL;
#endif
mp->next = NULL;
/*
* We move 'in queue' to 'private queue' and place
* message at the end of 'private queue' in order
* to ensure that the 'in queue' doesn't contain
* references into the heap. By ensuring this,
* we don't need to include the 'in queue' in
* the root set when garbage collecting.
*/
ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver);
LINK_MESSAGE_PRIVQ(receiver, mp);
if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) {
trace_receive(receiver, message);
}
}
BM_SWAP_TIMER(send,system);
return;
} else {
#ifdef ERTS_SMP
ErlOffHeap *ohp;
Eterm *hp;
BM_SWAP_TIMER(send,size);
msize = size_object(message);
BM_SWAP_TIMER(size,send);
hp = erts_alloc_message_heap(msize,&bp,&ohp,receiver,receiver_locks);
BM_SWAP_TIMER(send,copy);
message = copy_struct(message, msize, &hp, ohp);
BM_MESSAGE_COPIED(msz);
BM_SWAP_TIMER(copy,send);
DTRACE6(message_send, sender_name, receiver_name,
msize, tok_label, tok_lastcnt, tok_serial);
erts_queue_message(receiver, receiver_locks, bp, message, token
#ifdef USE_VM_PROBES
, NIL
#endif
);
BM_SWAP_TIMER(send,system);
#else
ErlMessage* mp = message_alloc();
Eterm *hp;
BM_SWAP_TIMER(send,size);
msize = size_object(message);
BM_SWAP_TIMER(size,send);
if (receiver->stop - receiver->htop <= msize) {
BM_SWAP_TIMER(send,system);
erts_garbage_collect(receiver, msize, receiver->arg_reg, receiver->arity);
BM_SWAP_TIMER(system,send);
}
hp = receiver->htop;
receiver->htop = hp + msize;
BM_SWAP_TIMER(send,copy);
message = copy_struct(message, msize, &hp, &receiver->off_heap);
BM_MESSAGE_COPIED(msize);
BM_SWAP_TIMER(copy,send);
DTRACE6(message_send, sender_name, receiver_name,
(uint32_t)msize, tok_label, tok_lastcnt, tok_serial);
ERL_MESSAGE_TERM(mp) = message;
ERL_MESSAGE_TOKEN(mp) = NIL;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = NIL;
#endif
mp->next = NULL;
mp->data.attached = NULL;
LINK_MESSAGE(receiver, mp);
if (receiver->status == P_WAITING) {
erts_add_to_runq(receiver);
} else if (receiver->status == P_SUSPENDED) {
receiver->rstatus = P_RUNABLE;
}
if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) {
trace_receive(receiver, message);
}
BM_SWAP_TIMER(send,system);
#endif /* #ifndef ERTS_SMP */
return;
}
}
void
erts_send_message(Process* sender,
Process* receiver,
ErtsProcLocks *receiver_locks,
Eterm message,
unsigned flags)
{
Uint msize;
ErlHeapFragment* bp = NULL;
Eterm token = NIL;
BM_STOP_TIMER(system);
BM_MESSAGE(message,sender,receiver);
BM_START_TIMER(send);
if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) {
Eterm* hp;
BM_SWAP_TIMER(send,size);
msize = size_object(message);
BM_SWAP_TIMER(size,send);
seq_trace_update_send(sender);
seq_trace_output(SEQ_TRACE_TOKEN(sender), message, SEQ_TRACE_SEND,
receiver->id, sender);
bp = new_message_buffer(msize + 6 /* TUPLE5 */);
hp = bp->mem;
BM_SWAP_TIMER(send,copy);
token = copy_struct(SEQ_TRACE_TOKEN(sender),
6 /* TUPLE5 */,
&hp,
&bp->off_heap);
message = copy_struct(message, msize, &hp, &bp->off_heap);
BM_MESSAGE_COPIED(msize);
BM_SWAP_TIMER(copy,send);
erts_queue_message(receiver,
receiver_locks,
bp,
message,
token);
BM_SWAP_TIMER(send,system);
#ifdef HYBRID
} else {
ErlMessage* mp = message_alloc();
BM_SWAP_TIMER(send,copy);
#ifdef INCREMENTAL
/* TODO: During GC activate processes if the message relies in
* the fromspace and the sender is active. During major
* collections add the message to the gray stack if it relies
* in the old generation and the sender is active and the
* receiver is inactive.
if (!IS_CONST(message) && (ma_gc_flags & GC_CYCLE) &&
(ptr_val(message) >= inc_fromspc &&
ptr_val(message) < inc_fromend) && INC_IS_ACTIVE(sender))
INC_ACTIVATE(receiver);
else if (!IS_CONST(message) && (ma_gc_flags & GC_CYCLE) &&
(ptr_val(message) >= global_old_heap &&
ptr_val(message) < global_old_hend) &&
INC_IS_ACTIVE(sender) && !INC_IS_ACTIVE(receiver))
Mark message in blackmap and add it to the gray stack
*/
if (!IS_CONST(message))
INC_ACTIVATE(receiver);
#endif
LAZY_COPY(sender,message);
BM_SWAP_TIMER(copy,send);
ERL_MESSAGE_TERM(mp) = message;
ERL_MESSAGE_TOKEN(mp) = NIL;
mp->next = NULL;
LINK_MESSAGE(receiver, mp);
ACTIVATE(receiver);
if (receiver->status == P_WAITING) {
erts_add_to_runq(receiver);
} else if (receiver->status == P_SUSPENDED) {
receiver->rstatus = P_RUNABLE;
}
if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) {
trace_receive(receiver, message);
}
BM_SWAP_TIMER(send,system);
return;
#else
} else if (sender == receiver) {
void
erts_queue_dist_message(Process *rcvr,
ErtsProcLocks *rcvr_locks,
ErtsDistExternal *dist_ext,
Eterm token)
{
ErlMessage* mp;
#ifdef ERTS_SMP
ErtsProcLocks need_locks;
#endif
ERTS_SMP_LC_ASSERT(*rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
mp = message_alloc();
#ifdef ERTS_SMP
need_locks = ~(*rcvr_locks) & (ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
if (need_locks) {
*rcvr_locks |= need_locks;
if (erts_smp_proc_trylock(rcvr, need_locks) == EBUSY) {
if (need_locks == ERTS_PROC_LOCK_MSGQ) {
erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_STATUS);
need_locks = (ERTS_PROC_LOCK_MSGQ
| ERTS_PROC_LOCK_STATUS);
}
erts_smp_proc_lock(rcvr, need_locks);
}
}
if (rcvr->is_exiting || ERTS_PROC_PENDING_EXIT(rcvr)) {
/* Drop message if receiver is exiting or has a pending exit ... */
if (is_not_nil(token)) {
ErlHeapFragment *heap_frag;
heap_frag = erts_dist_ext_trailer(mp->data.dist_ext);
erts_cleanup_offheap(&heap_frag->off_heap);
}
erts_free_dist_ext_copy(dist_ext);
message_free(mp);
}
else
#endif
if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) {
/* Ahh... need to decode it in order to trace it... */
ErlHeapFragment *mbuf;
Eterm msg;
message_free(mp);
msg = erts_msg_distext2heap(rcvr, rcvr_locks, &mbuf, &token, dist_ext);
if (is_value(msg))
erts_queue_message(rcvr, rcvr_locks, mbuf, msg, token);
}
else {
/* Enqueue message on external format */
ERL_MESSAGE_TERM(mp) = THE_NON_VALUE;
ERL_MESSAGE_TOKEN(mp) = token;
mp->next = NULL;
mp->data.dist_ext = dist_ext;
LINK_MESSAGE(rcvr, mp);
notify_new_message(rcvr);
}
}
