void
erts_cleanup_messages(ErtsMessage *msgp)
{
ErtsMessage *mp = msgp;
while (mp) {
ErtsMessage *fmp;
ErlHeapFragment *bp;
if (is_non_value(ERL_MESSAGE_TERM(mp))) {
if (is_not_immed(ERL_MESSAGE_TOKEN(mp))) {
bp = (ErlHeapFragment *) mp->data.dist_ext->ext_endp;
erts_cleanup_offheap(&bp->off_heap);
}
if (mp->data.dist_ext)
erts_free_dist_ext_copy(mp->data.dist_ext);
}
else {
if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG)
bp = mp->data.heap_frag;
else {
bp = mp->hfrag.next;
erts_cleanup_offheap(&mp->hfrag.off_heap);
}
if (bp)
free_message_buffer(bp);
}
fmp = mp;
mp = mp->next;
erts_free_message(fmp);
}
}
Eterm
erts_msg_distext2heap(Process *pp,
ErtsProcLocks *plcksp,
ErlHeapFragment **bpp,
Eterm *tokenp,
ErtsDistExternal *dist_extp)
{
Eterm msg;
Uint tok_sz = 0;
Eterm *hp = NULL;
ErtsHeapFactory factory;
Sint sz;
*bpp = NULL;
sz = erts_decode_dist_ext_size(dist_extp);
if (sz < 0)
goto decode_error;
if (is_not_nil(*tokenp)) {
ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp);
tok_sz = heap_frag->used_size;
sz += tok_sz;
}
if (pp) {
ErlOffHeap *ohp;
hp = erts_alloc_message_heap(sz, bpp, &ohp, pp, plcksp);
}
else {
*bpp = new_message_buffer(sz);
hp = (*bpp)->mem;
}
erts_factory_message_init(&factory, pp, hp, *bpp);
msg = erts_decode_dist_ext(&factory, dist_extp);
if (is_non_value(msg))
goto decode_error;
if (is_not_nil(*tokenp)) {
ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp);
hp = erts_produce_heap(&factory, tok_sz, 0);
*tokenp = copy_struct(*tokenp, tok_sz, &hp, factory.off_heap);
erts_cleanup_offheap(&heap_frag->off_heap);
}
erts_free_dist_ext_copy(dist_extp);
erts_factory_close(&factory);
return msg;
decode_error:
if (is_not_nil(*tokenp)) {
ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp);
erts_cleanup_offheap(&heap_frag->off_heap);
}
erts_free_dist_ext_copy(dist_extp);
*bpp = NULL;
return THE_NON_VALUE;
}
static void
erts_cleanup_message(ErtsMessage *mp)
{
ErlHeapFragment *bp;
if (ERTS_SIG_IS_EXTERNAL_MSG(mp) || ERTS_SIG_IS_NON_MSG(mp)) {
ErtsDistExternal *edep = erts_proc_sig_get_external(mp);
if (edep) {
erts_free_dist_ext_copy(edep);
if (mp->data.heap_frag == &mp->hfrag) {
ASSERT(ERTS_SIG_IS_EXTERNAL_MSG(mp));
mp->data.heap_frag = ERTS_MSG_COMBINED_HFRAG;
}
}
}
if (ERTS_SIG_IS_MSG(mp) && mp->data.attached != ERTS_MSG_COMBINED_HFRAG) {
bp = mp->data.heap_frag;
} else {
/* All non msg signals are combined HFRAG messages,
but we overwrite the mp->data field with the
nm_signal queue ptr so have to fix that here
before freeing it. */
mp->data.attached = ERTS_MSG_COMBINED_HFRAG;
bp = mp->hfrag.next;
erts_cleanup_offheap(&mp->hfrag.off_heap);
}
if (bp)
free_message_buffer(bp);
}
Uint
erts_msg_attached_data_size_aux(ErtsMessage *msg)
{
Sint sz;
ASSERT(is_non_value(ERL_MESSAGE_TERM(msg)));
ASSERT(msg->data.dist_ext);
ASSERT(msg->data.dist_ext->heap_size < 0);
sz = erts_decode_dist_ext_size(msg->data.dist_ext);
if (sz < 0) {
/* Bad external; remove it */
if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) {
ErlHeapFragment *heap_frag;
heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
erts_cleanup_offheap(&heap_frag->off_heap);
}
erts_free_dist_ext_copy(msg->data.dist_ext);
msg->data.dist_ext = NULL;
return 0;
}
msg->data.dist_ext->heap_size = sz;
if (is_not_nil(msg->m[1])) {
ErlHeapFragment *heap_frag;
heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
sz += heap_frag->used_size;
}
return sz;
}
void
free_message_buffer(ErlHeapFragment* bp)
{
erts_cleanup_offheap(&bp->off_heap);
ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG,
(void *) bp,
(sizeof(ErlHeapFragment)
- sizeof(Eterm)
+ bp->size*sizeof(Eterm)));
}
void
free_message_buffer(ErlHeapFragment* bp)
{
ASSERT(bp != NULL);
do {
ErlHeapFragment* next_bp = bp->next;
erts_cleanup_offheap(&bp->off_heap);
ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp,
ERTS_HEAP_FRAG_SIZE(bp->alloc_size));
bp = next_bp;
}while (bp != NULL);
}
void enif_clear_env(ErlNifEnv* env)
{
struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)env;
Process* p = &menv->phony_proc;
ASSERT(p == menv->env.proc);
ASSERT(p->id == ERTS_INVALID_PID);
ASSERT(MBUF(p) == menv->env.heap_frag);
if (MBUF(p) != NULL) {
erts_cleanup_offheap(&MSO(p));
clear_offheap(&MSO(p));
free_message_buffer(MBUF(p));
MBUF(p) = NULL;
menv->env.heap_frag = NULL;
}
ASSERT(HEAP_TOP(p) == HEAP_END(p));
menv->env.hp = menv->env.hp_end = HEAP_TOP(p);
ASSERT(!is_offheap(&MSO(p)));
free_tmp_objs(env);
}
void
erts_move_msg_attached_data_to_heap(ErtsHeapFactory* factory,
ErlMessage *msg)
{
if (is_value(ERL_MESSAGE_TERM(msg)))
erts_move_msg_mbuf_to_heap(&factory->hp, factory->off_heap, msg);
else if (msg->data.dist_ext) {
ASSERT(msg->data.dist_ext->heap_size >= 0);
if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) {
ErlHeapFragment *heap_frag;
heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
ERL_MESSAGE_TOKEN(msg) = copy_struct(ERL_MESSAGE_TOKEN(msg),
heap_frag->used_size,
&factory->hp,
factory->off_heap);
erts_cleanup_offheap(&heap_frag->off_heap);
}
ERL_MESSAGE_TERM(msg) = erts_decode_dist_ext(factory,
msg->data.dist_ext);
erts_free_dist_ext_copy(msg->data.dist_ext);
msg->data.dist_ext = NULL;
}
/* else: bad external detected when calculating size */
}
static void
system_cleanup(int exit_code)
{
/* No cleanup wanted if ...
* 1. we are about to do an abnormal exit
* 2. we haven't finished initializing, or
* 3. another thread than the main thread is performing the exit
* (in threaded non smp case).
*/
if (exit_code != 0
|| !erts_initialized
#if defined(USE_THREADS) && !defined(ERTS_SMP)
|| !erts_equal_tids(main_thread, erts_thr_self())
#endif
)
return;
#ifdef ERTS_SMP
#ifdef ERTS_ENABLE_LOCK_CHECK
erts_lc_check_exact(NULL, 0);
#endif
erts_smp_block_system(ERTS_BS_FLG_ALLOW_GC); /* We never release it... */
#endif
#ifdef HYBRID
if (ma_src_stack) erts_free(ERTS_ALC_T_OBJECT_STACK,
(void *)ma_src_stack);
if (ma_dst_stack) erts_free(ERTS_ALC_T_OBJECT_STACK,
(void *)ma_dst_stack);
if (ma_offset_stack) erts_free(ERTS_ALC_T_OBJECT_STACK,
(void *)ma_offset_stack);
ma_src_stack = NULL;
ma_dst_stack = NULL;
ma_offset_stack = NULL;
erts_cleanup_offheap(&erts_global_offheap);
#endif
#if defined(HYBRID) && !defined(INCREMENTAL)
if (global_heap) {
ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
(void*) global_heap,
sizeof(Eterm) * global_heap_sz);
}
global_heap = NULL;
#endif
#ifdef INCREMENTAL
erts_cleanup_incgc();
#endif
#if defined(USE_THREADS)
exit_async();
#endif
#if HAVE_ERTS_MSEG
erts_mseg_exit();
#endif
/*
* A lot more cleaning could/should have been done...
*/
}
static void
system_cleanup(int flush_async)
{
/*
* Make sure only one thread exits the runtime system.
*/
if (erts_atomic_inc_read_nob(&exiting) != 1) {
/*
* Another thread is currently exiting the system;
* wait for it to do its job.
*/
#ifdef ERTS_SMP
if (erts_thr_progress_is_managed_thread()) {
/*
* The exiting thread might be waiting for
* us to block; need to update status...
*/
erts_thr_progress_active(NULL, 0);
erts_thr_progress_prepare_wait(NULL);
}
#endif
/* Wait forever... */
while (1)
erts_milli_sleep(10000000);
}
/* No cleanup wanted if ...
* 1. we are about to do an abnormal exit
* 2. we haven't finished initializing, or
* 3. another thread than the main thread is performing the exit
* (in threaded non smp case).
*/
if (!flush_async
|| !erts_initialized
#if defined(USE_THREADS) && !defined(ERTS_SMP)
|| !erts_equal_tids(main_thread, erts_thr_self())
#endif
)
return;
#ifdef ERTS_SMP
#ifdef ERTS_ENABLE_LOCK_CHECK
erts_lc_check_exact(NULL, 0);
#endif
#endif
#ifdef HYBRID
if (ma_src_stack) erts_free(ERTS_ALC_T_OBJECT_STACK,
(void *)ma_src_stack);
if (ma_dst_stack) erts_free(ERTS_ALC_T_OBJECT_STACK,
(void *)ma_dst_stack);
if (ma_offset_stack) erts_free(ERTS_ALC_T_OBJECT_STACK,
(void *)ma_offset_stack);
ma_src_stack = NULL;
ma_dst_stack = NULL;
ma_offset_stack = NULL;
erts_cleanup_offheap(&erts_global_offheap);
#endif
#if defined(HYBRID) && !defined(INCREMENTAL)
if (global_heap) {
ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
(void*) global_heap,
sizeof(Eterm) * global_heap_sz);
}
global_heap = NULL;
#endif
#ifdef INCREMENTAL
erts_cleanup_incgc();
#endif
erts_exit_flush_async();
}
void erts_factory_undo(ErtsHeapFactory* factory)
{
ErlHeapFragment* bp;
struct erl_off_heap_header *hdr, **hdr_nextp;
switch (factory->mode) {
case FACTORY_HALLOC:
case FACTORY_STATIC:
/* Cleanup off-heap
*/
hdr_nextp = NULL;
for (hdr = factory->off_heap->first;
hdr != factory->off_heap_saved.first;
hdr = hdr->next) {
hdr_nextp = &hdr->next;
}
if (hdr_nextp != NULL) {
*hdr_nextp = NULL;
erts_cleanup_offheap(factory->off_heap);
factory->off_heap->first = factory->off_heap_saved.first;
factory->off_heap->overhead = factory->off_heap_saved.overhead;
}
if (factory->mode == FACTORY_HALLOC) {
/* Free heap frags
*/
bp = factory->p->mbuf;
if (bp != factory->heap_frags_saved) {
do {
ErlHeapFragment *next_bp = bp->next;
ASSERT(bp->off_heap.first == NULL);
ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp,
ERTS_HEAP_FRAG_SIZE(bp->alloc_size));
bp = next_bp;
} while (bp != factory->heap_frags_saved);
factory->p->mbuf = bp;
}
/* Rollback heap top
*/
if (HEAP_START(factory->p) <= factory->hp_start
&& factory->hp_start <= HEAP_LIMIT(factory->p)) {
HEAP_TOP(factory->p) = factory->hp_start;
}
/* Fix last heap frag */
if (factory->heap_frags_saved) {
ASSERT(factory->heap_frags_saved == factory->p->mbuf);
if (factory->hp_start != factory->heap_frags_saved->mem)
factory->heap_frags_saved->used_size = factory->heap_frags_saved_used;
else {
factory->p->mbuf = factory->p->mbuf->next;
ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, factory->heap_frags_saved,
ERTS_HEAP_FRAG_SIZE(factory->heap_frags_saved->alloc_size));
}
}
}
break;
case FACTORY_MESSAGE:
if (factory->message->data.attached == ERTS_MSG_COMBINED_HFRAG)
factory->message->hfrag.next = factory->heap_frags;
else
factory->message->data.heap_frag = factory->heap_frags;
erts_cleanup_messages(factory->message);
break;
case FACTORY_TMP:
case FACTORY_HEAP_FRAGS:
erts_cleanup_offheap(factory->off_heap);
factory->off_heap->first = NULL;
bp = factory->heap_frags;
while (bp != NULL) {
ErlHeapFragment* next_bp = bp->next;
ASSERT(bp->off_heap.first == NULL);
ERTS_HEAP_FREE(factory->alloc_type, (void *) bp,
ERTS_HEAP_FRAG_SIZE(bp->alloc_size));
bp = next_bp;
}
break;
case FACTORY_CLOSED: break;
default:
ASSERT(!"Invalid factory mode");
}
factory->mode = FACTORY_CLOSED;
#ifdef DEBUG
factory->p = NULL;
factory->hp = NULL;
factory->heap_frags = NULL;
#endif
}
Eterm
erts_msg_distext2heap(Process *pp,
ErtsProcLocks *plcksp,
ErlHeapFragment **bpp,
Eterm *tokenp,
ErtsDistExternal *dist_extp)
{
Eterm msg;
Uint tok_sz = 0;
Eterm *hp = NULL;
Eterm *hp_end = NULL;
ErlOffHeap *ohp;
Sint sz;
*bpp = NULL;
sz = erts_decode_dist_ext_size(dist_extp);
if (sz < 0)
goto decode_error;
if (is_not_nil(*tokenp)) {
ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp);
tok_sz = heap_frag->used_size;
sz += tok_sz;
}
if (pp)
hp = erts_alloc_message_heap(sz, bpp, &ohp, pp, plcksp);
else {
*bpp = new_message_buffer(sz);
hp = (*bpp)->mem;
ohp = &(*bpp)->off_heap;
}
hp_end = hp + sz;
msg = erts_decode_dist_ext(&hp, ohp, dist_extp);
if (is_non_value(msg))
goto decode_error;
if (is_not_nil(*tokenp)) {
ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp);
*tokenp = copy_struct(*tokenp, tok_sz, &hp, ohp);
erts_cleanup_offheap(&heap_frag->off_heap);
}
erts_free_dist_ext_copy(dist_extp);
if (hp_end != hp) {
if (!(*bpp)) {
HRelease(pp, hp_end, hp);
}
else {
Uint final_size = hp - &(*bpp)->mem[0];
Eterm brefs[2] = {msg, *tokenp};
ASSERT(sz - (hp_end - hp) == final_size);
*bpp = erts_resize_message_buffer(*bpp, final_size, &brefs[0], 2);
msg = brefs[0];
*tokenp = brefs[1];
}
}
return msg;
decode_error:
if (is_not_nil(*tokenp)) {
ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp);
erts_cleanup_offheap(&heap_frag->off_heap);
}
erts_free_dist_ext_copy(dist_extp);
if (*bpp) {
free_message_buffer(*bpp);
*bpp = NULL;
}
else if (hp) {
HRelease(pp, hp_end, hp);
}
return THE_NON_VALUE;
}
int
erts_decode_dist_message(Process *proc, ErtsProcLocks proc_locks,
ErtsMessage *msgp, int force_off_heap)
{
ErtsHeapFactory factory;
Eterm msg;
ErlHeapFragment *bp;
Sint need;
int decode_in_heap_frag;
decode_in_heap_frag = (force_off_heap
|| !(proc_locks & ERTS_PROC_LOCK_MAIN)
|| (proc->flags & F_OFF_HEAP_MSGQ));
if (msgp->data.dist_ext->heap_size >= 0)
need = msgp->data.dist_ext->heap_size;
else {
need = erts_decode_dist_ext_size(msgp->data.dist_ext);
if (need < 0) {
/* bad msg; remove it... */
if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) {
bp = erts_dist_ext_trailer(msgp->data.dist_ext);
erts_cleanup_offheap(&bp->off_heap);
}
erts_free_dist_ext_copy(msgp->data.dist_ext);
msgp->data.dist_ext = NULL;
return 0;
}
msgp->data.dist_ext->heap_size = need;
}
if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) {
bp = erts_dist_ext_trailer(msgp->data.dist_ext);
need += bp->used_size;
}
if (decode_in_heap_frag)
erts_factory_heap_frag_init(&factory, new_message_buffer(need));
else
erts_factory_proc_prealloc_init(&factory, proc, need);
ASSERT(msgp->data.dist_ext->heap_size >= 0);
if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) {
ErlHeapFragment *heap_frag;
heap_frag = erts_dist_ext_trailer(msgp->data.dist_ext);
ERL_MESSAGE_TOKEN(msgp) = copy_struct(ERL_MESSAGE_TOKEN(msgp),
heap_frag->used_size,
&factory.hp,
factory.off_heap);
erts_cleanup_offheap(&heap_frag->off_heap);
}
msg = erts_decode_dist_ext(&factory, msgp->data.dist_ext);
ERL_MESSAGE_TERM(msgp) = msg;
erts_free_dist_ext_copy(msgp->data.dist_ext);
msgp->data.attached = NULL;
if (is_non_value(msg)) {
erts_factory_undo(&factory);
return 0;
}
erts_factory_trim_and_close(&factory, msgp->m,
ERL_MESSAGE_REF_ARRAY_SZ);
ASSERT(!msgp->data.heap_frag);
if (decode_in_heap_frag)
msgp->data.heap_frag = factory.heap_frags;
return 1;
}
void erts_factory_undo(ErtsHeapFactory* factory)
{
ErlHeapFragment* bp;
struct erl_off_heap_header *hdr, **hdr_nextp;
switch (factory->mode) {
case FACTORY_HALLOC:
case FACTORY_STATIC:
/* Cleanup off-heap
*/
hdr_nextp = NULL;
for (hdr = factory->off_heap->first;
hdr != factory->off_heap_saved.first;
hdr = hdr->next) {
hdr_nextp = &hdr->next;
}
if (hdr_nextp != NULL) {
*hdr_nextp = NULL;
erts_cleanup_offheap(factory->off_heap);
factory->off_heap->first = factory->off_heap_saved.first;
factory->off_heap->overhead = factory->off_heap_saved.overhead;
}
if (factory->mode == FACTORY_HALLOC) {
/* Free heap frags
*/
bp = factory->p->mbuf;
if (bp != factory->heap_frags_saved) {
do {
ErlHeapFragment *next_bp = bp->next;
ASSERT(bp->off_heap.first == NULL);
ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp,
ERTS_HEAP_FRAG_SIZE(bp->alloc_size));
bp = next_bp;
} while (bp != factory->heap_frags_saved);
factory->p->mbuf = bp;
}
/* Rollback heap top
*/
if (factory->heap_frags_saved == NULL) { /* No heap frags when we started */
ASSERT(factory->hp_start >= HEAP_START(factory->p));
ASSERT(factory->hp_start <= HEAP_LIMIT(factory->p));
HEAP_TOP(factory->p) = factory->hp_start;
}
else {
ASSERT(factory->heap_frags_saved == factory->p->mbuf);
if (factory->hp_start == factory->heap_frags_saved->mem) {
factory->p->mbuf = factory->p->mbuf->next;
ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, factory->heap_frags_saved,
ERTS_HEAP_FRAG_SIZE(factory->heap_frags_saved->alloc_size));
}
else if (factory->hp_start != factory->hp_end) {
unsigned remains = factory->hp_start - factory->heap_frags_saved->mem;
ASSERT(remains > 0 && remains < factory->heap_frags_saved->used_size);
factory->heap_frags_saved->used_size = remains;
}
}
}
break;
case FACTORY_TMP:
case FACTORY_HEAP_FRAGS:
erts_cleanup_offheap(factory->off_heap);
factory->off_heap->first = NULL;
bp = factory->heap_frags;
while (bp != NULL) {
ErlHeapFragment* next_bp = bp->next;
ASSERT(bp->off_heap.first == NULL);
ERTS_HEAP_FREE(factory->alloc_type, (void *) bp,
ERTS_HEAP_FRAG_SIZE(bp->alloc_size));
bp = next_bp;
}
break;
case FACTORY_CLOSED: break;
default:
ASSERT(!"Invalid factory mode");
}
factory->mode = FACTORY_CLOSED;
#ifdef DEBUG
factory->p = NULL;
factory->hp = NULL;
factory->heap_frags = NULL;
#endif
}
static void
free_port_data_heap(void *vpdhp)
{
erts_cleanup_offheap(&((ErtsPortDataHeap *) vpdhp)->off_heap);
erts_free(ERTS_ALC_T_PORT_DATA_HEAP, vpdhp);
}
void
erts_queue_dist_message(Process *rcvr,
ErtsProcLocks *rcvr_locks,
ErtsDistExternal *dist_ext,
Eterm token)
{
ErlMessage* mp;
#ifdef USE_VM_PROBES
Sint tok_label = 0;
Sint tok_lastcnt = 0;
Sint tok_serial = 0;
#endif
#ifdef ERTS_SMP
erts_aint_t state;
#endif
ERTS_SMP_LC_ASSERT(*rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
mp = message_alloc();
#ifdef ERTS_SMP
if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) {
if (erts_smp_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
if (*rcvr_locks & ERTS_PROC_LOCK_STATUS) {
erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_STATUS);
need_locks |= ERTS_PROC_LOCK_STATUS;
}
erts_smp_proc_lock(rcvr, need_locks);
}
}
state = erts_smp_atomic32_read_acqb(&rcvr->state);
if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
/* Drop message if receiver is exiting or has a pending exit ... */
if (is_not_nil(token)) {
ErlHeapFragment *heap_frag;
heap_frag = erts_dist_ext_trailer(mp->data.dist_ext);
erts_cleanup_offheap(&heap_frag->off_heap);
}
erts_free_dist_ext_copy(dist_ext);
message_free(mp);
}
else
#endif
if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) {
/* Ahh... need to decode it in order to trace it... */
ErlHeapFragment *mbuf;
Eterm msg;
if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
message_free(mp);
msg = erts_msg_distext2heap(rcvr, rcvr_locks, &mbuf, &token, dist_ext);
if (is_value(msg))
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
dtrace_proc_str(rcvr, receiver_name);
if (token != NIL && token != am_have_dt_utag) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
}
DTRACE6(message_queued,
receiver_name, size_object(msg), rcvr->msg.len,
tok_label, tok_lastcnt, tok_serial);
}
#endif
erts_queue_message(rcvr, rcvr_locks, mbuf, msg, token);
}
else {
/* Enqueue message on external format */
ERL_MESSAGE_TERM(mp) = THE_NON_VALUE;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = NIL;
if (token == am_have_dt_utag) {
ERL_MESSAGE_TOKEN(mp) = NIL;
} else {
#endif
ERL_MESSAGE_TOKEN(mp) = token;
#ifdef USE_VM_PROBES
}
#endif
mp->next = NULL;
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
dtrace_proc_str(rcvr, receiver_name);
if (token != NIL && token != am_have_dt_utag) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
}
/*
* TODO: We don't know the real size of the external message here.
* -1 will appear to a D script as 4294967295.
*/
DTRACE6(message_queued, receiver_name, -1, rcvr->msg.len + 1,
tok_label, tok_lastcnt, tok_serial);
}
#endif
mp->data.dist_ext = dist_ext;
LINK_MESSAGE(rcvr, mp);
if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ))
erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
erts_proc_notify_new_message(rcvr,
#ifdef ERTS_SMP
*rcvr_locks
#else
0
#endif
);
}
}
void
erts_queue_dist_message(Process *rcvr,
ErtsProcLocks *rcvr_locks,
ErtsDistExternal *dist_ext,
Eterm token)
{
ErlMessage* mp;
#ifdef ERTS_SMP
ErtsProcLocks need_locks;
#endif
ERTS_SMP_LC_ASSERT(*rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
mp = message_alloc();
#ifdef ERTS_SMP
need_locks = ~(*rcvr_locks) & (ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
if (need_locks) {
*rcvr_locks |= need_locks;
if (erts_smp_proc_trylock(rcvr, need_locks) == EBUSY) {
if (need_locks == ERTS_PROC_LOCK_MSGQ) {
erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_STATUS);
need_locks = (ERTS_PROC_LOCK_MSGQ
| ERTS_PROC_LOCK_STATUS);
}
erts_smp_proc_lock(rcvr, need_locks);
}
}
if (rcvr->is_exiting || ERTS_PROC_PENDING_EXIT(rcvr)) {
/* Drop message if receiver is exiting or has a pending exit ... */
if (is_not_nil(token)) {
ErlHeapFragment *heap_frag;
heap_frag = erts_dist_ext_trailer(mp->data.dist_ext);
erts_cleanup_offheap(&heap_frag->off_heap);
}
erts_free_dist_ext_copy(dist_ext);
message_free(mp);
}
else
#endif
if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) {
/* Ahh... need to decode it in order to trace it... */
ErlHeapFragment *mbuf;
Eterm msg;
message_free(mp);
msg = erts_msg_distext2heap(rcvr, rcvr_locks, &mbuf, &token, dist_ext);
if (is_value(msg))
erts_queue_message(rcvr, rcvr_locks, mbuf, msg, token);
}
else {
/* Enqueue message on external format */
ERL_MESSAGE_TERM(mp) = THE_NON_VALUE;
ERL_MESSAGE_TOKEN(mp) = token;
mp->next = NULL;
mp->data.dist_ext = dist_ext;
LINK_MESSAGE(rcvr, mp);
notify_new_message(rcvr);
}
}
