void
erts_queue_dist_message(Process *rcvr,
ErtsProcLocks rcvr_locks,
ErtsDistExternal *dist_ext,
Eterm token,
Eterm from)
{
ErtsMessage* mp;
erts_aint_t state;
ERTS_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
mp = erts_alloc_message(0, NULL);
mp->data.dist_ext = dist_ext;
ERL_MESSAGE_FROM(mp) = dist_ext->dep->sysname;
ERL_MESSAGE_TERM(mp) = THE_NON_VALUE;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = NIL;
if (token == am_have_dt_utag)
ERL_MESSAGE_TOKEN(mp) = NIL;
else
#endif
ERL_MESSAGE_TOKEN(mp) = token;
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) {
if (erts_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
ErtsProcLocks unlocks =
rcvr_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ);
if (unlocks) {
erts_proc_unlock(rcvr, unlocks);
need_locks |= unlocks;
}
erts_proc_lock(rcvr, need_locks);
}
}
state = erts_atomic32_read_acqb(&rcvr->state);
if (state & ERTS_PSFLG_EXITING) {
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
/* Drop message if receiver is exiting or has a pending exit ... */
erts_cleanup_messages(mp);
}
else {
LINK_MESSAGE(rcvr, mp);
if (rcvr_locks & ERTS_PROC_LOCK_MAIN)
erts_proc_sig_fetch(rcvr);
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
erts_proc_notify_new_message(rcvr, rcvr_locks);
}
}
Sint
erts_move_messages_off_heap(Process *c_p)
{
int reds = 1;
/*
* Move all messages off heap. This *only* occurs when the
* process had off heap message disabled and just enabled
* it...
*/
ErtsMessage *mp;
reds += c_p->msg.len / 10;
ASSERT(erts_atomic32_read_nob(&c_p->state)
& ERTS_PSFLG_OFF_HEAP_MSGQ);
ASSERT(c_p->flags & F_OFF_HEAP_MSGQ_CHNG);
for (mp = c_p->msg.first; mp; mp = mp->next) {
Uint msg_sz, token_sz;
#ifdef USE_VM_PROBES
Uint utag_sz;
#endif
Eterm *hp;
ErlHeapFragment *hfrag;
if (mp->data.attached)
continue;
if (is_immed(ERL_MESSAGE_TERM(mp))
#ifdef USE_VM_PROBES
&& is_immed(ERL_MESSAGE_DT_UTAG(mp))
#endif
&& is_not_immed(ERL_MESSAGE_TOKEN(mp)))
continue;
/*
* The message refers into the heap. Copy the message
* from the heap into a heap fragment and attach
* it to the message...
*/
msg_sz = size_object(ERL_MESSAGE_TERM(mp));
#ifdef USE_VM_PROBES
utag_sz = size_object(ERL_MESSAGE_DT_UTAG(mp));
#endif
token_sz = size_object(ERL_MESSAGE_TOKEN(mp));
hfrag = new_message_buffer(msg_sz
#ifdef USE_VM_PROBES
+ utag_sz
#endif
+ token_sz);
hp = hfrag->mem;
if (is_not_immed(ERL_MESSAGE_TERM(mp)))
ERL_MESSAGE_TERM(mp) = copy_struct(ERL_MESSAGE_TERM(mp),
msg_sz, &hp,
&hfrag->off_heap);
if (is_not_immed(ERL_MESSAGE_TOKEN(mp)))
ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp),
token_sz, &hp,
&hfrag->off_heap);
#ifdef USE_VM_PROBES
if (is_not_immed(ERL_MESSAGE_DT_UTAG(mp)))
ERL_MESSAGE_DT_UTAG(mp) = copy_struct(ERL_MESSAGE_DT_UTAG(mp),
utag_sz, &hp,
&hfrag->off_heap);
#endif
mp->data.heap_frag = hfrag;
reds += 1;
}
return reds;
}
Sint
erts_send_message(Process* sender,
Process* receiver,
ErtsProcLocks *receiver_locks,
Eterm message,
unsigned flags)
{
Uint msize;
ErtsMessage* mp;
ErlOffHeap *ohp;
Eterm token = NIL;
Sint res = 0;
#ifdef USE_VM_PROBES
DTRACE_CHARBUF(sender_name, 64);
DTRACE_CHARBUF(receiver_name, 64);
Sint tok_label = 0;
Sint tok_lastcnt = 0;
Sint tok_serial = 0;
Eterm utag = NIL;
#endif
erts_aint32_t receiver_state;
#ifdef SHCOPY_SEND
erts_shcopy_t info;
#else
erts_literal_area_t litarea;
INITIALIZE_LITERAL_PURGE_AREA(litarea);
#endif
#ifdef USE_VM_PROBES
*sender_name = *receiver_name = '\0';
if (DTRACE_ENABLED(message_send)) {
erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)),
"%T", sender->common.id);
erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)),
"%T", receiver->common.id);
}
#endif
receiver_state = erts_atomic32_read_nob(&receiver->state);
if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) {
Eterm* hp;
Eterm stoken = SEQ_TRACE_TOKEN(sender);
Uint seq_trace_size = 0;
#ifdef USE_VM_PROBES
Uint dt_utag_size = 0;
#endif
/* SHCOPY corrupts the heap between
* copy_shared_calculate, and
* copy_shared_perform. (it inserts move_markers like the gc).
* Make sure we don't use the heap between those instances.
*/
if (have_seqtrace(stoken)) {
seq_trace_update_send(sender);
seq_trace_output(stoken, message, SEQ_TRACE_SEND,
receiver->common.id, sender);
seq_trace_size = 6; /* TUPLE5 */
}
#ifdef USE_VM_PROBES
if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) {
dt_utag_size = size_object(DT_UTAG(sender));
} else if (stoken == am_have_dt_utag ) {
stoken = NIL;
}
#endif
#ifdef SHCOPY_SEND
INITIALIZE_SHCOPY(info);
msize = copy_shared_calculate(message, &info);
#else
msize = size_object_litopt(message, &litarea);
#endif
mp = erts_alloc_message_heap_state(receiver,
&receiver_state,
receiver_locks,
(msize
#ifdef USE_VM_PROBES
+ dt_utag_size
#endif
+ seq_trace_size),
&hp,
&ohp);
#ifdef SHCOPY_SEND
if (is_not_immed(message))
message = copy_shared_perform(message, msize, &info, &hp, ohp);
DESTROY_SHCOPY(info);
#else
if (is_not_immed(message))
message = copy_struct_litopt(message, msize, &hp, ohp, &litarea);
#endif
if (is_immed(stoken))
token = stoken;
else
token = copy_struct(stoken, seq_trace_size, &hp, ohp);
#ifdef USE_VM_PROBES
if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) {
if (is_immed(DT_UTAG(sender)))
utag = DT_UTAG(sender);
else
utag = copy_struct(DT_UTAG(sender), dt_utag_size, &hp, ohp);
}
if (DTRACE_ENABLED(message_send)) {
if (have_seqtrace(stoken)) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(stoken));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(stoken));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(stoken));
}
DTRACE6(message_send, sender_name, receiver_name,
msize, tok_label, tok_lastcnt, tok_serial);
}
#endif
} else {
Eterm *hp;
if (receiver == sender && !(receiver_state & ERTS_PSFLG_OFF_HEAP_MSGQ)) {
mp = erts_alloc_message(0, NULL);
msize = 0;
}
else {
#ifdef SHCOPY_SEND
INITIALIZE_SHCOPY(info);
msize = copy_shared_calculate(message, &info);
#else
msize = size_object_litopt(message, &litarea);
#endif
mp = erts_alloc_message_heap_state(receiver,
&receiver_state,
receiver_locks,
msize,
&hp,
&ohp);
#ifdef SHCOPY_SEND
if (is_not_immed(message))
message = copy_shared_perform(message, msize, &info, &hp, ohp);
DESTROY_SHCOPY(info);
#else
if (is_not_immed(message))
message = copy_struct_litopt(message, msize, &hp, ohp, &litarea);
#endif
}
#ifdef USE_VM_PROBES
DTRACE6(message_send, sender_name, receiver_name,
msize, tok_label, tok_lastcnt, tok_serial);
#endif
}
ERL_MESSAGE_TOKEN(mp) = token;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = utag;
#endif
res = queue_message(receiver,
&receiver_state,
*receiver_locks,
mp, message,
sender->common.id);
return res;
}
void
erts_queue_dist_message(Process *rcvr,
ErtsProcLocks rcvr_locks,
ErtsDistExternal *dist_ext,
Eterm token,
Eterm from)
{
ErtsMessage* mp;
#ifdef USE_VM_PROBES
Sint tok_label = 0;
Sint tok_lastcnt = 0;
Sint tok_serial = 0;
#endif
erts_aint_t state;
ERTS_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
mp = erts_alloc_message(0, NULL);
mp->data.dist_ext = dist_ext;
ERL_MESSAGE_TERM(mp) = THE_NON_VALUE;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = NIL;
if (token == am_have_dt_utag)
ERL_MESSAGE_TOKEN(mp) = NIL;
else
#endif
ERL_MESSAGE_TOKEN(mp) = token;
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) {
if (erts_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
ErtsProcLocks unlocks =
rcvr_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ);
if (unlocks) {
erts_proc_unlock(rcvr, unlocks);
need_locks |= unlocks;
}
erts_proc_lock(rcvr, need_locks);
}
}
state = erts_atomic32_read_acqb(&rcvr->state);
if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
/* Drop message if receiver is exiting or has a pending exit ... */
erts_cleanup_messages(mp);
}
else
if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) {
if (from == am_Empty)
from = dist_ext->dep->sysname;
/* Ahh... need to decode it in order to trace it... */
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
if (!erts_decode_dist_message(rcvr, rcvr_locks, mp, 0))
erts_free_message(mp);
else {
Eterm msg = ERL_MESSAGE_TERM(mp);
token = ERL_MESSAGE_TOKEN(mp);
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
dtrace_proc_str(rcvr, receiver_name);
if (have_seqtrace(token)) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
}
DTRACE6(message_queued,
receiver_name, size_object(msg), rcvr->msg.len,
tok_label, tok_lastcnt, tok_serial);
}
#endif
erts_queue_message(rcvr, rcvr_locks, mp, msg, from);
}
}
else {
/* Enqueue message on external format */
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
dtrace_proc_str(rcvr, receiver_name);
if (have_seqtrace(token)) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
}
/*
* TODO: We don't know the real size of the external message here.
* -1 will appear to a D script as 4294967295.
*/
DTRACE6(message_queued, receiver_name, -1, rcvr->msg.len + 1,
tok_label, tok_lastcnt, tok_serial);
}
#endif
LINK_MESSAGE(rcvr, mp, &mp->next, 1);
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
erts_proc_notify_new_message(rcvr,
rcvr_locks
);
}
}
/* Add a message last in message queue */
static Sint
queue_message(Process *c_p,
Process* receiver,
erts_aint32_t *receiver_state,
ErtsProcLocks *receiver_locks,
ErtsMessage* mp,
Eterm message,
Eterm seq_trace_token
#ifdef USE_VM_PROBES
, Eterm dt_utag
#endif
)
{
Sint res;
int locked_msgq = 0;
erts_aint32_t state;
ERTS_SMP_LC_ASSERT(*receiver_locks == erts_proc_lc_my_proc_locks(receiver));
#ifdef ERTS_SMP
if (!(*receiver_locks & ERTS_PROC_LOCK_MSGQ)) {
if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
if (receiver_state)
state = *receiver_state;
else
state = erts_smp_atomic32_read_nob(&receiver->state);
if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
goto exiting;
if (*receiver_locks & ERTS_PROC_LOCK_STATUS) {
erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS);
need_locks |= ERTS_PROC_LOCK_STATUS;
}
erts_smp_proc_lock(receiver, need_locks);
}
locked_msgq = 1;
}
#endif
state = erts_smp_atomic32_read_nob(&receiver->state);
if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
#ifdef ERTS_SMP
exiting:
#endif
/* Drop message if receiver is exiting or has a pending exit... */
if (locked_msgq)
erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
erts_cleanup_messages(mp);
return 0;
}
ERL_MESSAGE_TERM(mp) = message;
ERL_MESSAGE_TOKEN(mp) = seq_trace_token;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = dt_utag;
#endif
res = receiver->msg.len;
#ifdef ERTS_SMP
if (*receiver_locks & ERTS_PROC_LOCK_MAIN) {
/*
* We move 'in queue' to 'private queue' and place
* message at the end of 'private queue' in order
* to ensure that the 'in queue' doesn't contain
* references into the heap. By ensuring this,
* we don't need to include the 'in queue' in
* the root set when garbage collecting.
*/
res += receiver->msg_inq.len;
ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver);
LINK_MESSAGE_PRIVQ(receiver, mp);
}
else
#endif
{
LINK_MESSAGE(receiver, mp);
}
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
Sint tok_label = 0;
Sint tok_lastcnt = 0;
Sint tok_serial = 0;
dtrace_proc_str(receiver, receiver_name);
if (seq_trace_token != NIL && is_tuple(seq_trace_token)) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token));
}
DTRACE6(message_queued,
receiver_name, size_object(message), receiver->msg.len,
tok_label, tok_lastcnt, tok_serial);
}
#endif
if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE))
trace_receive(receiver, message);
if (locked_msgq)
erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
erts_proc_notify_new_message(receiver,
#ifdef ERTS_SMP
*receiver_locks
#else
0
#endif
);
#ifndef ERTS_SMP
ERTS_HOLE_CHECK(receiver);
#endif
return res;
}
Sint
erts_send_message(Process* sender,
Process* receiver,
ErtsProcLocks *receiver_locks,
Eterm message,
unsigned flags)
{
Uint msize;
ErlHeapFragment* bp = NULL;
Eterm token = NIL;
Sint res = 0;
#ifdef USE_VM_PROBES
DTRACE_CHARBUF(sender_name, 64);
DTRACE_CHARBUF(receiver_name, 64);
Sint tok_label = 0;
Sint tok_lastcnt = 0;
Sint tok_serial = 0;
#endif
BM_STOP_TIMER(system);
BM_MESSAGE(message,sender,receiver);
BM_START_TIMER(send);
#ifdef USE_VM_PROBES
*sender_name = *receiver_name = '\0';
if (DTRACE_ENABLED(message_send)) {
erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)),
"%T", sender->common.id);
erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)),
"%T", receiver->common.id);
}
#endif
if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) {
Eterm* hp;
Eterm stoken = SEQ_TRACE_TOKEN(sender);
Uint seq_trace_size = 0;
#ifdef USE_VM_PROBES
Uint dt_utag_size = 0;
Eterm utag = NIL;
#endif
BM_SWAP_TIMER(send,size);
msize = size_object(message);
BM_SWAP_TIMER(size,send);
#ifdef USE_VM_PROBES
if (stoken != am_have_dt_utag) {
#endif
seq_trace_update_send(sender);
seq_trace_output(stoken, message, SEQ_TRACE_SEND,
receiver->common.id, sender);
seq_trace_size = 6; /* TUPLE5 */
#ifdef USE_VM_PROBES
}
if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) {
dt_utag_size = size_object(DT_UTAG(sender));
} else if (stoken == am_have_dt_utag ) {
stoken = NIL;
}
#endif
bp = new_message_buffer(msize + seq_trace_size
#ifdef USE_VM_PROBES
+ dt_utag_size
#endif
);
hp = bp->mem;
BM_SWAP_TIMER(send,copy);
token = copy_struct(stoken,
seq_trace_size,
&hp,
&bp->off_heap);
message = copy_struct(message, msize, &hp, &bp->off_heap);
#ifdef USE_VM_PROBES
if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) {
utag = copy_struct(DT_UTAG(sender), dt_utag_size, &hp, &bp->off_heap);
#ifdef DTRACE_TAG_HARDDEBUG
erts_fprintf(stderr,
"Dtrace -> (%T) Spreading tag (%T) with "
"message %T!\r\n",sender->common.id, utag, message);
#endif
}
#endif
BM_MESSAGE_COPIED(msize);
BM_SWAP_TIMER(copy,send);
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_send)) {
if (stoken != NIL && stoken != am_have_dt_utag) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(stoken));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(stoken));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(stoken));
}
DTRACE6(message_send, sender_name, receiver_name,
msize, tok_label, tok_lastcnt, tok_serial);
}
#endif
res = queue_message(NULL,
receiver,
receiver_locks,
NULL,
bp,
message,
token
#ifdef USE_VM_PROBES
, utag
#endif
);
BM_SWAP_TIMER(send,system);
} else if (sender == receiver) {
/* Drop message if receiver has a pending exit ... */
#ifdef ERTS_SMP
ErtsProcLocks need_locks = (~(*receiver_locks)
& (ERTS_PROC_LOCK_MSGQ
| ERTS_PROC_LOCK_STATUS));
if (need_locks) {
*receiver_locks |= need_locks;
if (erts_smp_proc_trylock(receiver, need_locks) == EBUSY) {
if (need_locks == ERTS_PROC_LOCK_MSGQ) {
erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS);
need_locks = ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS;
}
erts_smp_proc_lock(receiver, need_locks);
}
}
if (!ERTS_PROC_PENDING_EXIT(receiver))
#endif
{
ErlMessage* mp = message_alloc();
DTRACE6(message_send, sender_name, receiver_name,
size_object(message), tok_label, tok_lastcnt, tok_serial);
mp->data.attached = NULL;
ERL_MESSAGE_TERM(mp) = message;
ERL_MESSAGE_TOKEN(mp) = NIL;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = NIL;
#endif
mp->next = NULL;
/*
* We move 'in queue' to 'private queue' and place
* message at the end of 'private queue' in order
* to ensure that the 'in queue' doesn't contain
* references into the heap. By ensuring this,
* we don't need to include the 'in queue' in
* the root set when garbage collecting.
*/
ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver);
LINK_MESSAGE_PRIVQ(receiver, mp);
res = receiver->msg.len;
if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) {
trace_receive(receiver, message);
}
}
BM_SWAP_TIMER(send,system);
} else {
ErlOffHeap *ohp;
Eterm *hp;
erts_aint32_t state;
BM_SWAP_TIMER(send,size);
msize = size_object(message);
BM_SWAP_TIMER(size,send);
hp = erts_alloc_message_heap_state(msize,
&bp,
&ohp,
receiver,
receiver_locks,
&state);
BM_SWAP_TIMER(send,copy);
message = copy_struct(message, msize, &hp, ohp);
BM_MESSAGE_COPIED(msz);
BM_SWAP_TIMER(copy,send);
DTRACE6(message_send, sender_name, receiver_name,
msize, tok_label, tok_lastcnt, tok_serial);
res = queue_message(sender,
receiver,
receiver_locks,
&state,
bp,
message,
token
#ifdef USE_VM_PROBES
, NIL
#endif
);
BM_SWAP_TIMER(send,system);
}
return res;
}
/*
* Moves content of message buffer attached to a message into a heap.
* The message buffer is deallocated.
*/
void
erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
{
struct erl_off_heap_header* oh;
Eterm term, token, *fhp, *hp;
Sint offs;
Uint sz;
ErlHeapFragment *bp;
#ifdef USE_VM_PROBES
Eterm utag;
#endif
#ifdef HARD_DEBUG
struct erl_off_heap_header* dbg_oh_start = off_heap->first;
Eterm dbg_term, dbg_token;
ErlHeapFragment *dbg_bp;
Uint *dbg_hp, *dbg_thp_start;
Uint dbg_term_sz, dbg_token_sz;
#ifdef USE_VM_PROBES
Eterm dbg_utag;
Uint dbg_utag_sz;
#endif
#endif
bp = msg->data.heap_frag;
term = ERL_MESSAGE_TERM(msg);
token = ERL_MESSAGE_TOKEN(msg);
#ifdef USE_VM_PROBES
utag = ERL_MESSAGE_DT_UTAG(msg);
#endif
if (!bp) {
#ifdef USE_VM_PROBES
ASSERT(is_immed(term) && is_immed(token) && is_immed(utag));
#else
ASSERT(is_immed(term) && is_immed(token));
#endif
return;
}
#ifdef HARD_DEBUG
dbg_term_sz = size_object(term);
dbg_token_sz = size_object(token);
dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz);
#ifdef USE_VM_PROBES
dbg_utag_sz = size_object(utag);
dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz + dbg_utag_sz );
#endif
/*ASSERT(dbg_term_sz + dbg_token_sz == erts_msg_used_frag_sz(msg));
Copied size may be smaller due to removed SubBins's or garbage.
Copied size may be larger due to duplicated shared terms.
*/
dbg_hp = dbg_bp->mem;
dbg_term = copy_struct(term, dbg_term_sz, &dbg_hp, &dbg_bp->off_heap);
dbg_token = copy_struct(token, dbg_token_sz, &dbg_hp, &dbg_bp->off_heap);
#ifdef USE_VM_PROBES
dbg_utag = copy_struct(utag, dbg_utag_sz, &dbg_hp, &dbg_bp->off_heap);
#endif
dbg_thp_start = *hpp;
#endif
if (bp->next != NULL) {
move_multi_frags(hpp, off_heap, bp, msg->m,
#ifdef USE_VM_PROBES
3
#else
2
#endif
);
goto copy_done;
}
OH_OVERHEAD(off_heap, bp->off_heap.overhead);
sz = bp->used_size;
ASSERT(is_immed(term) || in_heapfrag(ptr_val(term),bp));
ASSERT(is_immed(token) || in_heapfrag(ptr_val(token),bp));
fhp = bp->mem;
hp = *hpp;
offs = hp - fhp;
oh = NULL;
while (sz--) {
Uint cpy_sz;
Eterm val = *fhp++;
switch (primary_tag(val)) {
case TAG_PRIMARY_IMMED1:
*hp++ = val;
break;
case TAG_PRIMARY_LIST:
case TAG_PRIMARY_BOXED:
ASSERT(in_heapfrag(ptr_val(val), bp));
*hp++ = offset_ptr(val, offs);
break;
case TAG_PRIMARY_HEADER:
*hp++ = val;
switch (val & _HEADER_SUBTAG_MASK) {
case ARITYVAL_SUBTAG:
break;
case REFC_BINARY_SUBTAG:
case FUN_SUBTAG:
case EXTERNAL_PID_SUBTAG:
case EXTERNAL_PORT_SUBTAG:
case EXTERNAL_REF_SUBTAG:
oh = (struct erl_off_heap_header*) (hp-1);
cpy_sz = thing_arityval(val);
goto cpy_words;
default:
cpy_sz = header_arity(val);
cpy_words:
ASSERT(sz >= cpy_sz);
sz -= cpy_sz;
while (cpy_sz >= 8) {
cpy_sz -= 8;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
}
switch (cpy_sz) {
case 7: *hp++ = *fhp++;
case 6: *hp++ = *fhp++;
case 5: *hp++ = *fhp++;
case 4: *hp++ = *fhp++;
case 3: *hp++ = *fhp++;
case 2: *hp++ = *fhp++;
case 1: *hp++ = *fhp++;
default: break;
}
if (oh) {
/* Add to offheap list */
oh->next = off_heap->first;
off_heap->first = oh;
ASSERT(*hpp <= (Eterm*)oh);
ASSERT(hp > (Eterm*)oh);
oh = NULL;
}
break;
}
break;
}
}
ASSERT(bp->used_size == hp - *hpp);
*hpp = hp;
if (is_not_immed(token)) {
ASSERT(in_heapfrag(ptr_val(token), bp));
ERL_MESSAGE_TOKEN(msg) = offset_ptr(token, offs);
#ifdef HARD_DEBUG
ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TOKEN(msg)));
ASSERT(hp > ptr_val(ERL_MESSAGE_TOKEN(msg)));
#endif
}
if (is_not_immed(term)) {
ASSERT(in_heapfrag(ptr_val(term),bp));
ERL_MESSAGE_TERM(msg) = offset_ptr(term, offs);
#ifdef HARD_DEBUG
ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TERM(msg)));
ASSERT(hp > ptr_val(ERL_MESSAGE_TERM(msg)));
#endif
}
#ifdef USE_VM_PROBES
if (is_not_immed(utag)) {
ASSERT(in_heapfrag(ptr_val(utag), bp));
ERL_MESSAGE_DT_UTAG(msg) = offset_ptr(utag, offs);
#ifdef HARD_DEBUG
ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_DT_UTAG(msg)));
ASSERT(hp > ptr_val(ERL_MESSAGE_DT_UTAG(msg)));
#endif
}
#endif
copy_done:
#ifdef HARD_DEBUG
{
int i, j;
ErlHeapFragment* frag;
{
struct erl_off_heap_header* dbg_oh = off_heap->first;
i = j = 0;
while (dbg_oh != dbg_oh_start) {
dbg_oh = dbg_oh->next;
i++;
}
for (frag=bp; frag; frag=frag->next) {
dbg_oh = frag->off_heap.first;
while (dbg_oh) {
dbg_oh = dbg_oh->next;
j++;
}
}
ASSERT(i == j);
}
}
#endif
bp->off_heap.first = NULL;
free_message_buffer(bp);
msg->data.heap_frag = NULL;
#ifdef HARD_DEBUG
ASSERT(eq(ERL_MESSAGE_TERM(msg), dbg_term));
ASSERT(eq(ERL_MESSAGE_TOKEN(msg), dbg_token));
#ifdef USE_VM_PROBES
ASSERT(eq(ERL_MESSAGE_DT_UTAG(msg), dbg_utag));
#endif
free_message_buffer(dbg_bp);
#endif
}
void
erts_queue_dist_message(Process *rcvr,
ErtsProcLocks *rcvr_locks,
ErtsDistExternal *dist_ext,
Eterm token)
{
ErlMessage* mp;
#ifdef USE_VM_PROBES
Sint tok_label = 0;
Sint tok_lastcnt = 0;
Sint tok_serial = 0;
#endif
#ifdef ERTS_SMP
erts_aint_t state;
#endif
ERTS_SMP_LC_ASSERT(*rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
mp = message_alloc();
#ifdef ERTS_SMP
if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) {
if (erts_smp_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
if (*rcvr_locks & ERTS_PROC_LOCK_STATUS) {
erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_STATUS);
need_locks |= ERTS_PROC_LOCK_STATUS;
}
erts_smp_proc_lock(rcvr, need_locks);
}
}
state = erts_smp_atomic32_read_acqb(&rcvr->state);
if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
/* Drop message if receiver is exiting or has a pending exit ... */
if (is_not_nil(token)) {
ErlHeapFragment *heap_frag;
heap_frag = erts_dist_ext_trailer(mp->data.dist_ext);
erts_cleanup_offheap(&heap_frag->off_heap);
}
erts_free_dist_ext_copy(dist_ext);
message_free(mp);
}
else
#endif
if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) {
/* Ahh... need to decode it in order to trace it... */
ErlHeapFragment *mbuf;
Eterm msg;
if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
message_free(mp);
msg = erts_msg_distext2heap(rcvr, rcvr_locks, &mbuf, &token, dist_ext);
if (is_value(msg))
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
dtrace_proc_str(rcvr, receiver_name);
if (token != NIL && token != am_have_dt_utag) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
}
DTRACE6(message_queued,
receiver_name, size_object(msg), rcvr->msg.len,
tok_label, tok_lastcnt, tok_serial);
}
#endif
erts_queue_message(rcvr, rcvr_locks, mbuf, msg, token);
}
else {
/* Enqueue message on external format */
ERL_MESSAGE_TERM(mp) = THE_NON_VALUE;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = NIL;
if (token == am_have_dt_utag) {
ERL_MESSAGE_TOKEN(mp) = NIL;
} else {
#endif
ERL_MESSAGE_TOKEN(mp) = token;
#ifdef USE_VM_PROBES
}
#endif
mp->next = NULL;
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
dtrace_proc_str(rcvr, receiver_name);
if (token != NIL && token != am_have_dt_utag) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
}
/*
* TODO: We don't know the real size of the external message here.
* -1 will appear to a D script as 4294967295.
*/
DTRACE6(message_queued, receiver_name, -1, rcvr->msg.len + 1,
tok_label, tok_lastcnt, tok_serial);
}
#endif
mp->data.dist_ext = dist_ext;
LINK_MESSAGE(rcvr, mp);
if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
erts_proc_notify_new_message(rcvr,
#ifdef ERTS_SMP
*rcvr_locks
#else
0
#endif
);
}
}
void
erts_queue_dist_message(Process *rcvr,
ErtsProcLocks rcvr_locks,
ErtsDistExternal *dist_ext,
ErlHeapFragment *hfrag,
Eterm token,
Eterm from)
{
ErtsMessage* mp;
erts_aint_t state;
ERTS_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
if (hfrag) {
/* Fragmented message, allocate a message reference */
mp = erts_alloc_message(0, NULL);
mp->data.heap_frag = hfrag;
} else {
/* Un-fragmented message, allocate space for
token and dist_ext in message. */
Uint dist_ext_sz = erts_dist_ext_size(dist_ext) / sizeof(Eterm);
Uint token_sz = size_object(token);
Uint sz = token_sz + dist_ext_sz;
Eterm *hp;
mp = erts_alloc_message(sz, &hp);
mp->data.heap_frag = &mp->hfrag;
mp->hfrag.used_size = token_sz;
erts_make_dist_ext_copy(dist_ext, erts_get_dist_ext(mp->data.heap_frag));
token = copy_struct(token, token_sz, &hp, &mp->data.heap_frag->off_heap);
}
ERL_MESSAGE_FROM(mp) = dist_ext->dep->sysname;
ERL_MESSAGE_TERM(mp) = THE_NON_VALUE;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = NIL;
if (token == am_have_dt_utag)
ERL_MESSAGE_TOKEN(mp) = NIL;
else
#endif
ERL_MESSAGE_TOKEN(mp) = token;
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) {
if (erts_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
ErtsProcLocks unlocks =
rcvr_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ);
if (unlocks) {
erts_proc_unlock(rcvr, unlocks);
need_locks |= unlocks;
}
erts_proc_lock(rcvr, need_locks);
}
}
state = erts_atomic32_read_acqb(&rcvr->state);
if (state & ERTS_PSFLG_EXITING) {
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
/* Drop message if receiver is exiting or has a pending exit ... */
erts_cleanup_messages(mp);
}
else {
LINK_MESSAGE(rcvr, mp);
if (rcvr_locks & ERTS_PROC_LOCK_MAIN)
erts_proc_sig_fetch(rcvr);
if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
erts_proc_notify_new_message(rcvr, rcvr_locks);
}
}
/* Add a message last in message queue */
void
erts_queue_message(Process* receiver,
ErtsProcLocks *receiver_locks,
ErlHeapFragment* bp,
Eterm message,
Eterm seq_trace_token
#ifdef USE_VM_PROBES
, Eterm dt_utag
#endif
)
{
ErlMessage* mp;
#ifdef ERTS_SMP
ErtsProcLocks need_locks;
#else
ASSERT(bp != NULL || receiver->mbuf == NULL);
#endif
ERTS_SMP_LC_ASSERT(*receiver_locks == erts_proc_lc_my_proc_locks(receiver));
mp = message_alloc();
#ifdef ERTS_SMP
need_locks = ~(*receiver_locks) & (ERTS_PROC_LOCK_MSGQ
| ERTS_PROC_LOCK_STATUS);
if (need_locks) {
*receiver_locks |= need_locks;
if (erts_smp_proc_trylock(receiver, need_locks) == EBUSY) {
if (need_locks == ERTS_PROC_LOCK_MSGQ) {
erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS);
need_locks = (ERTS_PROC_LOCK_MSGQ
| ERTS_PROC_LOCK_STATUS);
}
erts_smp_proc_lock(receiver, need_locks);
}
}
if (receiver->is_exiting || ERTS_PROC_PENDING_EXIT(receiver)) {
/* Drop message if receiver is exiting or has a pending
* exit ...
*/
if (bp)
free_message_buffer(bp);
message_free(mp);
return;
}
#endif
ERL_MESSAGE_TERM(mp) = message;
ERL_MESSAGE_TOKEN(mp) = seq_trace_token;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = dt_utag;
#endif
mp->next = NULL;
mp->data.heap_frag = bp;
#ifdef ERTS_SMP
if (*receiver_locks & ERTS_PROC_LOCK_MAIN) {
/*
* We move 'in queue' to 'private queue' and place
* message at the end of 'private queue' in order
* to ensure that the 'in queue' doesn't contain
* references into the heap. By ensuring this,
* we don't need to include the 'in queue' in
* the root set when garbage collecting.
*/
ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver);
LINK_MESSAGE_PRIVQ(receiver, mp);
}
else {
LINK_MESSAGE(receiver, mp);
}
#else
LINK_MESSAGE(receiver, mp);
#endif
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
Sint tok_label = 0;
Sint tok_lastcnt = 0;
Sint tok_serial = 0;
dtrace_proc_str(receiver, receiver_name);
if (seq_trace_token != NIL && is_tuple(seq_trace_token)) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token));
}
DTRACE6(message_queued,
receiver_name, size_object(message), receiver->msg.len,
tok_label, tok_lastcnt, tok_serial);
}
#endif
notify_new_message(receiver);
if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) {
trace_receive(receiver, message);
}
#ifndef ERTS_SMP
ERTS_HOLE_CHECK(receiver);
#endif
}
//将消息放入目标进程的消息队列中
static Sint
queue_message(Process *c_p,
Process* receiver,
ErtsProcLocks *receiver_locks,
erts_aint32_t *receiver_state,
ErlHeapFragment* bp,
Eterm message,
Eterm seq_trace_token
#ifdef USE_VM_PROBES
, Eterm dt_utag
#endif
)
{
Sint res;
ErlMessage* mp;
int locked_msgq = 0;
erts_aint_t state;
#ifndef ERTS_SMP
ASSERT(bp != NULL || receiver->mbuf == NULL);
#endif
ERTS_SMP_LC_ASSERT(*receiver_locks == erts_proc_lc_my_proc_locks(receiver));
mp = message_alloc();
if (receiver_state){
state = *receiver_state;
}else{
state = erts_smp_atomic32_read_acqb(&receiver->state);
}
#ifdef ERTS_SMP
//如果目标进程处在退出的状态，直接进入退出清理
if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)){
goto exiting;
}
//如果当前没有目标Erlang进程的消息队列锁
//尝试获取消息队列的锁
if (!(*receiver_locks & ERTS_PROC_LOCK_MSGQ)) {
if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
if (*receiver_locks & ERTS_PROC_LOCK_STATUS) {
erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS);
need_locks |= ERTS_PROC_LOCK_STATUS;
}
erts_smp_proc_lock(receiver, need_locks);
}
locked_msgq = 1;
state = erts_smp_atomic32_read_nob(&receiver->state);
//获取目标Erlang进程的状态
if (receiver_state){
*receiver_state = state;
}
}
#endif
//Erlang进程处于退出的状态
//直接释放目标进程的锁，和当前消息
if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
#ifdef ERTS_SMP
exiting:
#endif
/* Drop message if receiver is exiting or has a pending exit... */
if (locked_msgq){
erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
}
if (bp){
free_message_buffer(bp);
}
message_free(mp);
return 0;
}
//从此处可以看出，放入目标进程的消息，只是对当前进程的消息的一个指针应用
//那我们需要找出，message是如何被处理的，何时增加的引用计数
ERL_MESSAGE_TERM(mp) = message;
ERL_MESSAGE_TOKEN(mp) = seq_trace_token;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = dt_utag;
#endif
mp->next = NULL;
mp->data.heap_frag = bp;
#ifndef ERTS_SMP
res = receiver->msg.len;
#else
res = receiver->msg_inq.len;
//把消息attach的目标Erlang进程的消息队列或者private
//的消息队列
if (*receiver_locks & ERTS_PROC_LOCK_MAIN) {
/*
* We move 'in queue' to 'private queue' and place
* message at the end of 'private queue' in order
* to ensure that the 'in queue' doesn't contain
* references into the heap. By ensuring this,
* we don't need to include the 'in queue' in
* the root set when garbage collecting.
*/
//这个时候可以操作msg_inq和msg
res += receiver->msg.len;
ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver);
LINK_MESSAGE_PRIVQ(receiver, mp);
}
else
#endif
{
//这个时候只操作msg_inq
LINK_MESSAGE(receiver, mp);
}
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
Sint tok_label = 0;
Sint tok_lastcnt = 0;
Sint tok_serial = 0;
dtrace_proc_str(receiver, receiver_name);
if (seq_trace_token != NIL && is_tuple(seq_trace_token)) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token));
}
DTRACE6(message_queued,
receiver_name, size_object(message), receiver->msg.len,
tok_label, tok_lastcnt, tok_serial);
}
#endif
if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)){
trace_receive(receiver, message);
}
if (locked_msgq){
erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
}
erts_proc_notify_new_message(receiver,
#ifdef ERTS_SMP
*receiver_locks
#else
0
#endif
);
#ifndef ERTS_SMP
ERTS_HOLE_CHECK(receiver);
#endif
return res;
}
