Uint erts_msg_attached_data_size_aux(ErtsMessage *msg) { Sint sz; ASSERT(is_non_value(ERL_MESSAGE_TERM(msg))); ASSERT(msg->data.dist_ext); ASSERT(msg->data.dist_ext->heap_size < 0); sz = erts_decode_dist_ext_size(msg->data.dist_ext); if (sz < 0) { /* Bad external; remove it */ if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) { ErlHeapFragment *heap_frag; heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); erts_cleanup_offheap(&heap_frag->off_heap); } erts_free_dist_ext_copy(msg->data.dist_ext); msg->data.dist_ext = NULL; return 0; } msg->data.dist_ext->heap_size = sz; if (is_not_nil(msg->m[1])) { ErlHeapFragment *heap_frag; heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); sz += heap_frag->used_size; } return sz; }
Eterm erts_msg_distext2heap(Process *pp, ErtsProcLocks *plcksp, ErlHeapFragment **bpp, Eterm *tokenp, ErtsDistExternal *dist_extp) { Eterm msg; Uint tok_sz = 0; Eterm *hp = NULL; ErtsHeapFactory factory; Sint sz; *bpp = NULL; sz = erts_decode_dist_ext_size(dist_extp); if (sz < 0) goto decode_error; if (is_not_nil(*tokenp)) { ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); tok_sz = heap_frag->used_size; sz += tok_sz; } if (pp) { ErlOffHeap *ohp; hp = erts_alloc_message_heap(sz, bpp, &ohp, pp, plcksp); } else { *bpp = new_message_buffer(sz); hp = (*bpp)->mem; } erts_factory_message_init(&factory, pp, hp, *bpp); msg = erts_decode_dist_ext(&factory, dist_extp); if (is_non_value(msg)) goto decode_error; if (is_not_nil(*tokenp)) { ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); hp = erts_produce_heap(&factory, tok_sz, 0); *tokenp = copy_struct(*tokenp, tok_sz, &hp, factory.off_heap); erts_cleanup_offheap(&heap_frag->off_heap); } erts_free_dist_ext_copy(dist_extp); erts_factory_close(&factory); return msg; decode_error: if (is_not_nil(*tokenp)) { ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); erts_cleanup_offheap(&heap_frag->off_heap); } erts_free_dist_ext_copy(dist_extp); *bpp = NULL; return THE_NON_VALUE; }
Uint erts_msg_attached_data_size_aux(ErtsMessage *msg) { Sint sz; ASSERT(is_non_value(ERL_MESSAGE_TERM(msg))); ASSERT(msg->data.dist_ext); ASSERT(msg->data.dist_ext->heap_size < 0); sz = erts_decode_dist_ext_size(msg->data.dist_ext); if (sz < 0) { /* Bad external * We leave the message intact in this case as it's not worth the trouble * to make all callers remove it from queue. It will be detected again * and removed from message queue later anyway. */ return 0; } msg->data.dist_ext->heap_size = sz; if (is_not_nil(msg->m[1])) { ErlHeapFragment *heap_frag; heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); sz += heap_frag->used_size; } return sz; }
static int within2(Eterm *ptr, Process *p, Eterm *real_htop) { ErlHeapFragment* bp = MBUF(p); ErlMessage* mp = p->msg.first; Eterm *htop = real_htop ? real_htop : HEAP_TOP(p); if (OLD_HEAP(p) && (OLD_HEAP(p) <= ptr && ptr < OLD_HEND(p))) { return 1; } if (HEAP_START(p) <= ptr && ptr < htop) { return 1; } while (bp != NULL) { if (bp->mem <= ptr && ptr < bp->mem + bp->used_size) { return 1; } bp = bp->next; } while (mp) { if (mp->data.attached) { ErlHeapFragment *hfp; if (is_value(ERL_MESSAGE_TERM(mp))) hfp = mp->data.heap_frag; else if (is_not_nil(ERL_MESSAGE_TOKEN(mp))) hfp = erts_dist_ext_trailer(mp->data.dist_ext); else hfp = NULL; if (hfp && hfp->mem <= ptr && ptr < hfp->mem + hfp->used_size) return 1; } mp = mp->next; } return 0; }
void erts_move_msg_attached_data_to_heap(ErtsHeapFactory* factory, ErlMessage *msg) { if (is_value(ERL_MESSAGE_TERM(msg))) erts_move_msg_mbuf_to_heap(&factory->hp, factory->off_heap, msg); else if (msg->data.dist_ext) { ASSERT(msg->data.dist_ext->heap_size >= 0); if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) { ErlHeapFragment *heap_frag; heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); ERL_MESSAGE_TOKEN(msg) = copy_struct(ERL_MESSAGE_TOKEN(msg), heap_frag->used_size, &factory->hp, factory->off_heap); erts_cleanup_offheap(&heap_frag->off_heap); } ERL_MESSAGE_TERM(msg) = erts_decode_dist_ext(factory, msg->data.dist_ext); erts_free_dist_ext_copy(msg->data.dist_ext); msg->data.dist_ext = NULL; } /* else: bad external detected when calculating size */ }
int erts_decode_dist_message(Process *proc, ErtsProcLocks proc_locks, ErtsMessage *msgp, int force_off_heap) { ErtsHeapFactory factory; Eterm msg; ErlHeapFragment *bp; Sint need; int decode_in_heap_frag; decode_in_heap_frag = (force_off_heap || !(proc_locks & ERTS_PROC_LOCK_MAIN) || (proc->flags & F_OFF_HEAP_MSGQ)); if (msgp->data.dist_ext->heap_size >= 0) need = msgp->data.dist_ext->heap_size; else { need = erts_decode_dist_ext_size(msgp->data.dist_ext); if (need < 0) { /* bad msg; remove it... */ if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) { bp = erts_dist_ext_trailer(msgp->data.dist_ext); erts_cleanup_offheap(&bp->off_heap); } erts_free_dist_ext_copy(msgp->data.dist_ext); msgp->data.dist_ext = NULL; return 0; } msgp->data.dist_ext->heap_size = need; } if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) { bp = erts_dist_ext_trailer(msgp->data.dist_ext); need += bp->used_size; } if (decode_in_heap_frag) erts_factory_heap_frag_init(&factory, new_message_buffer(need)); else erts_factory_proc_prealloc_init(&factory, proc, need); ASSERT(msgp->data.dist_ext->heap_size >= 0); if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) { ErlHeapFragment *heap_frag; heap_frag = erts_dist_ext_trailer(msgp->data.dist_ext); ERL_MESSAGE_TOKEN(msgp) = copy_struct(ERL_MESSAGE_TOKEN(msgp), heap_frag->used_size, &factory.hp, factory.off_heap); erts_cleanup_offheap(&heap_frag->off_heap); } msg = erts_decode_dist_ext(&factory, msgp->data.dist_ext); ERL_MESSAGE_TERM(msgp) = msg; erts_free_dist_ext_copy(msgp->data.dist_ext); msgp->data.attached = NULL; if (is_non_value(msg)) { erts_factory_undo(&factory); return 0; } erts_factory_trim_and_close(&factory, msgp->m, ERL_MESSAGE_REF_ARRAY_SZ); ASSERT(!msgp->data.heap_frag); if (decode_in_heap_frag) msgp->data.heap_frag = factory.heap_frags; return 1; }
static void setup_reference_table(void) { ErlHeapFragment *hfp; DistEntry *dep; HashInfo hi; int i, max; DeclareTmpHeapNoproc(heap,3); inserted_bins = NULL; hash_get_info(&hi, &erts_node_table); referred_nodes = erts_alloc(ERTS_ALC_T_NC_TMP, hi.objs*sizeof(ReferredNode)); no_referred_nodes = 0; hash_foreach(&erts_node_table, init_referred_node, NULL); ASSERT(no_referred_nodes == hi.objs); hash_get_info(&hi, &erts_dist_table); referred_dists = erts_alloc(ERTS_ALC_T_NC_TMP, hi.objs*sizeof(ReferredDist)); no_referred_dists = 0; hash_foreach(&erts_dist_table, init_referred_dist, NULL); ASSERT(no_referred_dists == hi.objs); /* Go through the hole system, and build a table of all references to ErlNode and DistEntry structures */ erts_debug_callback_timer_foreach(try_delete_node, insert_delayed_delete_node, NULL); erts_debug_callback_timer_foreach(try_delete_dist_entry, insert_delayed_delete_dist_entry, NULL); UseTmpHeapNoproc(3); insert_node(erts_this_node, SYSTEM_REF, TUPLE2(&heap[0], AM_system, am_undefined)); UnUseTmpHeapNoproc(3); max = erts_ptab_max(&erts_proc); /* Insert all processes */ for (i = 0; i < max; i++) { Process *proc = erts_pix2proc(i); if (proc) { int mli; ErtsMessage *msg_list[] = { proc->msg.first, #ifdef ERTS_SMP proc->msg_inq.first, #endif proc->msg_frag}; /* Insert Heap */ insert_offheap(&(proc->off_heap), HEAP_REF, proc->common.id); /* Insert heap fragments buffers */ for(hfp = proc->mbuf; hfp; hfp = hfp->next) insert_offheap(&(hfp->off_heap), HEAP_REF, proc->common.id); /* Insert msg buffers */ for (mli = 0; mli < sizeof(msg_list)/sizeof(msg_list[0]); mli++) { ErtsMessage *msg; for (msg = msg_list[mli]; msg; msg = msg->next) { ErlHeapFragment *heap_frag = NULL; if (msg->data.attached) { if (msg->data.attached == ERTS_MSG_COMBINED_HFRAG) heap_frag = &msg->hfrag; else if (is_value(ERL_MESSAGE_TERM(msg))) heap_frag = msg->data.heap_frag; else { if (msg->data.dist_ext->dep) insert_dist_entry(msg->data.dist_ext->dep, HEAP_REF, proc->common.id, 0); if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); } } while (heap_frag) { insert_offheap(&(heap_frag->off_heap), HEAP_REF, proc->common.id); heap_frag = heap_frag->next; } } } /* Insert links */ if (ERTS_P_LINKS(proc)) insert_links(ERTS_P_LINKS(proc), proc->common.id); if (ERTS_P_MONITORS(proc)) insert_monitors(ERTS_P_MONITORS(proc), proc->common.id); /* Insert controller */ { DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(proc); if (dep) insert_dist_entry(dep, CTRL_REF, proc->common.id, 0); } } } #ifdef ERTS_SMP erts_foreach_sys_msg_in_q(insert_sys_msg); #endif /* Insert all ports */ max = erts_ptab_max(&erts_port); for (i = 0; i < max; i++) { ErlOffHeap *ohp; erts_aint32_t state; Port *prt; prt = erts_pix2port(i); if (!prt) continue; state = erts_atomic32_read_nob(&prt->state); if (state & ERTS_PORT_SFLGS_DEAD) continue; /* Insert links */ if (ERTS_P_LINKS(prt)) insert_links(ERTS_P_LINKS(prt), prt->common.id); /* Insert monitors */ if (ERTS_P_MONITORS(prt)) insert_monitors(ERTS_P_MONITORS(prt), prt->common.id); /* Insert port data */ ohp = erts_port_data_offheap(prt); if (ohp) insert_offheap(ohp, HEAP_REF, prt->common.id); /* Insert controller */ if (prt->dist_entry) insert_dist_entry(prt->dist_entry, CTRL_REF, prt->common.id, 0); } { /* Add binaries stored elsewhere ... */ ErlOffHeap oh; ProcBin pb[2]; int i = 0; Binary *default_match_spec; Binary *default_meta_match_spec; oh.first = NULL; /* Only the ProcBin members thing_word, val and next will be inspected (by insert_offheap()) */ #undef ADD_BINARY #define ADD_BINARY(Bin) \ if ((Bin)) { \ pb[i].thing_word = REFC_BINARY_SUBTAG; \ pb[i].val = (Bin); \ pb[i].next = oh.first; \ oh.first = (struct erl_off_heap_header*) &pb[i]; \ i++; \ } erts_get_default_trace_pattern(NULL, &default_match_spec, &default_meta_match_spec, NULL, NULL); ADD_BINARY(default_match_spec); ADD_BINARY(default_meta_match_spec); insert_offheap(&oh, BIN_REF, AM_match_spec); #undef ADD_BINARY } /* Insert all dist links */ for(dep = erts_visible_dist_entries; dep; dep = dep->next) { if(dep->nlinks) insert_links2(dep->nlinks, dep->sysname); if(dep->node_links) insert_links(dep->node_links, dep->sysname); if(dep->monitors) insert_monitors(dep->monitors, dep->sysname); } for(dep = erts_hidden_dist_entries; dep; dep = dep->next) { if(dep->nlinks) insert_links2(dep->nlinks, dep->sysname); if(dep->node_links) insert_links(dep->node_links, dep->sysname); if(dep->monitors) insert_monitors(dep->monitors, dep->sysname); } /* Not connected dist entries should not have any links, but inspect them anyway */ for(dep = erts_not_connected_dist_entries; dep; dep = dep->next) { if(dep->nlinks) insert_links2(dep->nlinks, dep->sysname); if(dep->node_links) insert_links(dep->node_links, dep->sysname); if(dep->monitors) insert_monitors(dep->monitors, dep->sysname); } /* Insert all ets tables */ erts_db_foreach_table(insert_ets_table, NULL); /* Insert all bif timers */ erts_debug_bif_timer_foreach(insert_bif_timer, NULL); /* Insert node table (references to dist) */ hash_foreach(&erts_node_table, insert_erl_node, NULL); }
void erts_queue_dist_message(Process *rcvr, ErtsProcLocks *rcvr_locks, ErtsDistExternal *dist_ext, Eterm token) { ErlMessage* mp; #ifdef USE_VM_PROBES Sint tok_label = 0; Sint tok_lastcnt = 0; Sint tok_serial = 0; #endif #ifdef ERTS_SMP erts_aint_t state; #endif ERTS_SMP_LC_ASSERT(*rcvr_locks == erts_proc_lc_my_proc_locks(rcvr)); mp = message_alloc(); #ifdef ERTS_SMP if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) { if (erts_smp_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) { ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ; if (*rcvr_locks & ERTS_PROC_LOCK_STATUS) { erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_STATUS); need_locks |= ERTS_PROC_LOCK_STATUS; } erts_smp_proc_lock(rcvr, need_locks); } } state = erts_smp_atomic32_read_acqb(&rcvr->state); if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) { if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); /* Drop message if receiver is exiting or has a pending exit ... */ if (is_not_nil(token)) { ErlHeapFragment *heap_frag; heap_frag = erts_dist_ext_trailer(mp->data.dist_ext); erts_cleanup_offheap(&heap_frag->off_heap); } erts_free_dist_ext_copy(dist_ext); message_free(mp); } else #endif if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) { /* Ahh... need to decode it in order to trace it... */ ErlHeapFragment *mbuf; Eterm msg; if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); message_free(mp); msg = erts_msg_distext2heap(rcvr, rcvr_locks, &mbuf, &token, dist_ext); if (is_value(msg)) #ifdef USE_VM_PROBES if (DTRACE_ENABLED(message_queued)) { DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); dtrace_proc_str(rcvr, receiver_name); if (token != NIL && token != am_have_dt_utag) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); } DTRACE6(message_queued, receiver_name, size_object(msg), rcvr->msg.len, tok_label, tok_lastcnt, tok_serial); } #endif erts_queue_message(rcvr, rcvr_locks, mbuf, msg, token); } else { /* Enqueue message on external format */ ERL_MESSAGE_TERM(mp) = THE_NON_VALUE; #ifdef USE_VM_PROBES ERL_MESSAGE_DT_UTAG(mp) = NIL; if (token == am_have_dt_utag) { ERL_MESSAGE_TOKEN(mp) = NIL; } else { #endif ERL_MESSAGE_TOKEN(mp) = token; #ifdef USE_VM_PROBES } #endif mp->next = NULL; #ifdef USE_VM_PROBES if (DTRACE_ENABLED(message_queued)) { DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); dtrace_proc_str(rcvr, receiver_name); if (token != NIL && token != am_have_dt_utag) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); } /* * TODO: We don't know the real size of the external message here. * -1 will appear to a D script as 4294967295. */ DTRACE6(message_queued, receiver_name, -1, rcvr->msg.len + 1, tok_label, tok_lastcnt, tok_serial); } #endif mp->data.dist_ext = dist_ext; LINK_MESSAGE(rcvr, mp); if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); erts_proc_notify_new_message(rcvr, #ifdef ERTS_SMP *rcvr_locks #else 0 #endif ); } }
Eterm erts_msg_distext2heap(Process *pp, ErtsProcLocks *plcksp, ErlHeapFragment **bpp, Eterm *tokenp, ErtsDistExternal *dist_extp) { Eterm msg; Uint tok_sz = 0; Eterm *hp = NULL; Eterm *hp_end = NULL; ErlOffHeap *ohp; Sint sz; *bpp = NULL; sz = erts_decode_dist_ext_size(dist_extp); if (sz < 0) goto decode_error; if (is_not_nil(*tokenp)) { ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); tok_sz = heap_frag->used_size; sz += tok_sz; } if (pp) hp = erts_alloc_message_heap(sz, bpp, &ohp, pp, plcksp); else { *bpp = new_message_buffer(sz); hp = (*bpp)->mem; ohp = &(*bpp)->off_heap; } hp_end = hp + sz; msg = erts_decode_dist_ext(&hp, ohp, dist_extp); if (is_non_value(msg)) goto decode_error; if (is_not_nil(*tokenp)) { ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); *tokenp = copy_struct(*tokenp, tok_sz, &hp, ohp); erts_cleanup_offheap(&heap_frag->off_heap); } erts_free_dist_ext_copy(dist_extp); if (hp_end != hp) { if (!(*bpp)) { HRelease(pp, hp_end, hp); } else { Uint final_size = hp - &(*bpp)->mem[0]; Eterm brefs[2] = {msg, *tokenp}; ASSERT(sz - (hp_end - hp) == final_size); *bpp = erts_resize_message_buffer(*bpp, final_size, &brefs[0], 2); msg = brefs[0]; *tokenp = brefs[1]; } } return msg; decode_error: if (is_not_nil(*tokenp)) { ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); erts_cleanup_offheap(&heap_frag->off_heap); } erts_free_dist_ext_copy(dist_extp); if (*bpp) { free_message_buffer(*bpp); *bpp = NULL; } else if (hp) { HRelease(pp, hp_end, hp); } return THE_NON_VALUE; }
void erts_queue_dist_message(Process *rcvr, ErtsProcLocks *rcvr_locks, ErtsDistExternal *dist_ext, Eterm token) { ErlMessage* mp; #ifdef ERTS_SMP ErtsProcLocks need_locks; #endif ERTS_SMP_LC_ASSERT(*rcvr_locks == erts_proc_lc_my_proc_locks(rcvr)); mp = message_alloc(); #ifdef ERTS_SMP need_locks = ~(*rcvr_locks) & (ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); if (need_locks) { *rcvr_locks |= need_locks; if (erts_smp_proc_trylock(rcvr, need_locks) == EBUSY) { if (need_locks == ERTS_PROC_LOCK_MSGQ) { erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_STATUS); need_locks = (ERTS_PROC_LOCK_MSGQ | ERTS_PROC_LOCK_STATUS); } erts_smp_proc_lock(rcvr, need_locks); } } if (rcvr->is_exiting || ERTS_PROC_PENDING_EXIT(rcvr)) { /* Drop message if receiver is exiting or has a pending exit ... */ if (is_not_nil(token)) { ErlHeapFragment *heap_frag; heap_frag = erts_dist_ext_trailer(mp->data.dist_ext); erts_cleanup_offheap(&heap_frag->off_heap); } erts_free_dist_ext_copy(dist_ext); message_free(mp); } else #endif if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) { /* Ahh... need to decode it in order to trace it... */ ErlHeapFragment *mbuf; Eterm msg; message_free(mp); msg = erts_msg_distext2heap(rcvr, rcvr_locks, &mbuf, &token, dist_ext); if (is_value(msg)) erts_queue_message(rcvr, rcvr_locks, mbuf, msg, token); } else { /* Enqueue message on external format */ ERL_MESSAGE_TERM(mp) = THE_NON_VALUE; ERL_MESSAGE_TOKEN(mp) = token; mp->next = NULL; mp->data.dist_ext = dist_ext; LINK_MESSAGE(rcvr, mp); notify_new_message(rcvr); } }