void erts_destroy_monitor(ErtsMonitor *mon)
{
Uint mon_size = ERTS_MONITOR_SIZE;
ErlNode *node;
ASSERT(is_not_immed(mon->ref));
mon_size += NC_HEAP_SIZE(mon->ref);
if (is_external(mon->ref)) {
node = external_thing_ptr(mon->ref)->node;
erts_deref_node_entry(node);
}
if (mon->type != MON_NIF_TARGET && is_not_immed(mon->u.pid)) {
mon_size += NC_HEAP_SIZE(mon->u.pid);
if (is_external(mon->u.pid)) {
node = external_thing_ptr(mon->u.pid)->node;
erts_deref_node_entry(node);
}
}
if (mon_size <= ERTS_MONITOR_SH_SIZE) {
erts_free(ERTS_ALC_T_MONITOR_SH, (void *) mon);
} else {
erts_free(ERTS_ALC_T_MONITOR_LH, (void *) mon);
erts_smp_atomic_add_nob(&tot_link_lh_size, -1*mon_size*sizeof(Uint));
}
}
static void
print_lock2(char *prefix, Sint16 id, Wterm extra, Uint16 flags, char *suffix)
{
char *lname = (0 <= id && id < ERTS_LOCK_ORDER_SIZE
? erts_lock_order[id].name
: "unknown");
if (is_not_immed(extra))
erts_fprintf(stderr,
"%s'%s:%p%s'%s%s",
prefix,
lname,
_unchecked_boxed_val(extra),
lock_type(flags),
rw_op_str(flags),
suffix);
else
erts_fprintf(stderr,
"%s'%s:%T%s'%s%s",
prefix,
lname,
extra,
lock_type(flags),
rw_op_str(flags),
suffix);
}
void
erts_cleanup_messages(ErtsMessage *msgp)
{
ErtsMessage *mp = msgp;
while (mp) {
ErtsMessage *fmp;
ErlHeapFragment *bp;
if (is_non_value(ERL_MESSAGE_TERM(mp))) {
if (is_not_immed(ERL_MESSAGE_TOKEN(mp))) {
bp = (ErlHeapFragment *) mp->data.dist_ext->ext_endp;
erts_cleanup_offheap(&bp->off_heap);
}
if (mp->data.dist_ext)
erts_free_dist_ext_copy(mp->data.dist_ext);
}
else {
if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG)
bp = mp->data.heap_frag;
else {
bp = mp->hfrag.next;
erts_cleanup_offheap(&mp->hfrag.off_heap);
}
if (bp)
free_message_buffer(bp);
}
fmp = mp;
mp = mp->next;
erts_free_message(fmp);
}
}
BIF_RETTYPE lists_member_2(BIF_ALIST_2)
{
Eterm term;
Eterm list;
Eterm item;
int non_immed_key;
int max_iter = 10 * CONTEXT_REDS;
if (is_nil(BIF_ARG_2)) {
BIF_RET(am_false);
} else if (is_not_list(BIF_ARG_2)) {
BIF_ERROR(BIF_P, BADARG);
}
term = BIF_ARG_1;
non_immed_key = is_not_immed(term);
list = BIF_ARG_2;
while (is_list(list)) {
if (--max_iter < 0) {
BUMP_ALL_REDS(BIF_P);
BIF_TRAP2(bif_export[BIF_lists_member_2], BIF_P, term, list);
}
item = CAR(list_val(list));
if ((item == term) || (non_immed_key && eq(item, term))) {
BIF_RET2(am_true, CONTEXT_REDS - max_iter/10);
}
list = CDR(list_val(list));
}
if (is_not_nil(list)) {
BIF_ERROR(BIF_P, BADARG);
}
BIF_RET2(am_false, CONTEXT_REDS - max_iter/10);
}
static ErtsLink *create_link(Uint type, Eterm pid)
{
Uint lnk_size = ERTS_LINK_SIZE;
ErtsLink *n;
Eterm *hp;
if (is_not_immed(pid)) {
lnk_size += NC_HEAP_SIZE(pid);
}
if (lnk_size <= ERTS_LINK_SH_SIZE) {
n = (ErtsLink *) erts_alloc(ERTS_ALC_T_NLINK_SH,
lnk_size*sizeof(Uint));
} else {
n = (ErtsLink *) erts_alloc(ERTS_ALC_T_NLINK_LH,
lnk_size*sizeof(Uint));
erts_smp_atomic_add_nob(&tot_link_lh_size, lnk_size*sizeof(Uint));
}
hp = n->heap;
n->left = n->right = NULL; /* Always the same initial value*/
n->type = (Uint16) type;
n->balance = 0; /* Always the same initial value */
if (n->type == LINK_NODE) {
ERTS_LINK_REFC(n) = 0;
} else {
ERTS_LINK_ROOT(n) = NULL;
}
CP_LINK_VAL(n->pid, hp, pid);
return n;
}
static ErtsMonitor *create_monitor(Uint type, Eterm ref, UWord entity, Eterm name)
{
Uint mon_size = ERTS_MONITOR_SIZE;
ErtsMonitor *n;
Eterm *hp;
mon_size += NC_HEAP_SIZE(ref);
if (type != MON_NIF_TARGET && is_not_immed(entity)) {
mon_size += NC_HEAP_SIZE(entity);
}
if (mon_size <= ERTS_MONITOR_SH_SIZE) {
n = (ErtsMonitor *) erts_alloc(ERTS_ALC_T_MONITOR_SH,
mon_size*sizeof(Uint));
} else {
n = (ErtsMonitor *) erts_alloc(ERTS_ALC_T_MONITOR_LH,
mon_size*sizeof(Uint));
erts_smp_atomic_add_nob(&tot_link_lh_size, mon_size*sizeof(Uint));
}
hp = n->heap;
n->left = n->right = NULL; /* Always the same initial value*/
n->type = (Uint16) type;
n->balance = 0; /* Always the same initial value */
n->name = name; /* atom() or [] */
CP_LINK_VAL(n->ref, hp, ref); /*XXX Unnecessary check, never immediate*/
if (type == MON_NIF_TARGET)
n->u.resource = (ErtsResource*)entity;
else
CP_LINK_VAL(n->u.pid, hp, (Eterm)entity);
return n;
}
void
erts_lc_init_lock(erts_lc_lock_t *lck, char *name, Uint16 flags)
{
lck->id = erts_lc_get_lock_order_id(name);
lck->extra = (UWord) &lck->extra;
ASSERT(is_not_immed(lck->extra));
lck->flags = flags;
lck->inited = ERTS_LC_INITITALIZED;
}
void
erts_lc_init_lock(erts_lc_lock_t *lck, char *name, erts_lock_flags_t flags)
{
lck->id = erts_lc_get_lock_order_id(name);
lck->check_order = erts_lc_is_check_order(name);
lck->extra = (UWord) &lck->extra;
ASSERT(is_not_immed(lck->extra));
lck->flags = flags;
lck->taken_options = 0;
lck->inited = ERTS_LC_INITITALIZED;
}
static void
raw_print_lock(char *prefix, Sint16 id, Wterm extra, Uint16 flags,
char* file, unsigned int line, char *suffix)
{
char *lname = (0 <= id && id < ERTS_LOCK_ORDER_SIZE
? erts_lock_order[id].name
: "unknown");
erts_fprintf(stderr,"%s'%s:",prefix,lname);
if (is_not_immed(extra))
erts_fprintf(stderr,"%p",_unchecked_boxed_val(extra));
else
erts_fprintf(stderr,"%T",extra);
erts_fprintf(stderr,"%s",lock_type(flags));
if (file)
erts_fprintf(stderr,"(%s:%d)",file,line);
erts_fprintf(stderr,"'%s%s",rw_op_str(flags),suffix);
}
void erts_destroy_link(ErtsLink *lnk)
{
Uint lnk_size = ERTS_LINK_SIZE;
ErlNode *node;
ASSERT(lnk->type == LINK_NODE || ERTS_LINK_ROOT(lnk) == NULL);
if (is_not_immed(lnk->pid)) {
lnk_size += NC_HEAP_SIZE(lnk->pid);
if (is_external(lnk->pid)) {
node = external_thing_ptr(lnk->pid)->node;
erts_deref_node_entry(node);
}
}
if (lnk_size <= ERTS_LINK_SH_SIZE) {
erts_free(ERTS_ALC_T_NLINK_SH, (void *) lnk);
} else {
erts_free(ERTS_ALC_T_NLINK_LH, (void *) lnk);
erts_smp_atomic_add_nob(&tot_link_lh_size, -1*lnk_size*sizeof(Uint));
}
}
Sint
erts_move_messages_off_heap(Process *c_p)
{
int reds = 1;
/*
* Move all messages off heap. This *only* occurs when the
* process had off heap message disabled and just enabled
* it...
*/
ErtsMessage *mp;
reds += c_p->msg.len / 10;
ASSERT(erts_atomic32_read_nob(&c_p->state)
& ERTS_PSFLG_OFF_HEAP_MSGQ);
ASSERT(c_p->flags & F_OFF_HEAP_MSGQ_CHNG);
for (mp = c_p->msg.first; mp; mp = mp->next) {
Uint msg_sz, token_sz;
#ifdef USE_VM_PROBES
Uint utag_sz;
#endif
Eterm *hp;
ErlHeapFragment *hfrag;
if (mp->data.attached)
continue;
if (is_immed(ERL_MESSAGE_TERM(mp))
#ifdef USE_VM_PROBES
&& is_immed(ERL_MESSAGE_DT_UTAG(mp))
#endif
&& is_not_immed(ERL_MESSAGE_TOKEN(mp)))
continue;
/*
* The message refers into the heap. Copy the message
* from the heap into a heap fragment and attach
* it to the message...
*/
msg_sz = size_object(ERL_MESSAGE_TERM(mp));
#ifdef USE_VM_PROBES
utag_sz = size_object(ERL_MESSAGE_DT_UTAG(mp));
#endif
token_sz = size_object(ERL_MESSAGE_TOKEN(mp));
hfrag = new_message_buffer(msg_sz
#ifdef USE_VM_PROBES
+ utag_sz
#endif
+ token_sz);
hp = hfrag->mem;
if (is_not_immed(ERL_MESSAGE_TERM(mp)))
ERL_MESSAGE_TERM(mp) = copy_struct(ERL_MESSAGE_TERM(mp),
msg_sz, &hp,
&hfrag->off_heap);
if (is_not_immed(ERL_MESSAGE_TOKEN(mp)))
ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp),
token_sz, &hp,
&hfrag->off_heap);
#ifdef USE_VM_PROBES
if (is_not_immed(ERL_MESSAGE_DT_UTAG(mp)))
ERL_MESSAGE_DT_UTAG(mp) = copy_struct(ERL_MESSAGE_DT_UTAG(mp),
utag_sz, &hp,
&hfrag->off_heap);
#endif
mp->data.heap_frag = hfrag;
reds += 1;
}
return reds;
}
Sint
erts_send_message(Process* sender,
Process* receiver,
ErtsProcLocks *receiver_locks,
Eterm message,
unsigned flags)
{
Uint msize;
ErtsMessage* mp;
ErlOffHeap *ohp;
Eterm token = NIL;
Sint res = 0;
#ifdef USE_VM_PROBES
DTRACE_CHARBUF(sender_name, 64);
DTRACE_CHARBUF(receiver_name, 64);
Sint tok_label = 0;
Sint tok_lastcnt = 0;
Sint tok_serial = 0;
Eterm utag = NIL;
#endif
erts_aint32_t receiver_state;
#ifdef SHCOPY_SEND
erts_shcopy_t info;
#else
erts_literal_area_t litarea;
INITIALIZE_LITERAL_PURGE_AREA(litarea);
#endif
#ifdef USE_VM_PROBES
*sender_name = *receiver_name = '\0';
if (DTRACE_ENABLED(message_send)) {
erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)),
"%T", sender->common.id);
erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)),
"%T", receiver->common.id);
}
#endif
receiver_state = erts_atomic32_read_nob(&receiver->state);
if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) {
Eterm* hp;
Eterm stoken = SEQ_TRACE_TOKEN(sender);
Uint seq_trace_size = 0;
#ifdef USE_VM_PROBES
Uint dt_utag_size = 0;
#endif
/* SHCOPY corrupts the heap between
* copy_shared_calculate, and
* copy_shared_perform. (it inserts move_markers like the gc).
* Make sure we don't use the heap between those instances.
*/
if (have_seqtrace(stoken)) {
seq_trace_update_send(sender);
seq_trace_output(stoken, message, SEQ_TRACE_SEND,
receiver->common.id, sender);
seq_trace_size = 6; /* TUPLE5 */
}
#ifdef USE_VM_PROBES
if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) {
dt_utag_size = size_object(DT_UTAG(sender));
} else if (stoken == am_have_dt_utag ) {
stoken = NIL;
}
#endif
#ifdef SHCOPY_SEND
INITIALIZE_SHCOPY(info);
msize = copy_shared_calculate(message, &info);
#else
msize = size_object_litopt(message, &litarea);
#endif
mp = erts_alloc_message_heap_state(receiver,
&receiver_state,
receiver_locks,
(msize
#ifdef USE_VM_PROBES
+ dt_utag_size
#endif
+ seq_trace_size),
&hp,
&ohp);
#ifdef SHCOPY_SEND
if (is_not_immed(message))
message = copy_shared_perform(message, msize, &info, &hp, ohp);
DESTROY_SHCOPY(info);
#else
if (is_not_immed(message))
message = copy_struct_litopt(message, msize, &hp, ohp, &litarea);
#endif
if (is_immed(stoken))
token = stoken;
else
token = copy_struct(stoken, seq_trace_size, &hp, ohp);
#ifdef USE_VM_PROBES
if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) {
if (is_immed(DT_UTAG(sender)))
utag = DT_UTAG(sender);
else
utag = copy_struct(DT_UTAG(sender), dt_utag_size, &hp, ohp);
}
if (DTRACE_ENABLED(message_send)) {
if (have_seqtrace(stoken)) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(stoken));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(stoken));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(stoken));
}
DTRACE6(message_send, sender_name, receiver_name,
msize, tok_label, tok_lastcnt, tok_serial);
}
#endif
} else {
Eterm *hp;
if (receiver == sender && !(receiver_state & ERTS_PSFLG_OFF_HEAP_MSGQ)) {
mp = erts_alloc_message(0, NULL);
msize = 0;
}
else {
#ifdef SHCOPY_SEND
INITIALIZE_SHCOPY(info);
msize = copy_shared_calculate(message, &info);
#else
msize = size_object_litopt(message, &litarea);
#endif
mp = erts_alloc_message_heap_state(receiver,
&receiver_state,
receiver_locks,
msize,
&hp,
&ohp);
#ifdef SHCOPY_SEND
if (is_not_immed(message))
message = copy_shared_perform(message, msize, &info, &hp, ohp);
DESTROY_SHCOPY(info);
#else
if (is_not_immed(message))
message = copy_struct_litopt(message, msize, &hp, ohp, &litarea);
#endif
}
#ifdef USE_VM_PROBES
DTRACE6(message_send, sender_name, receiver_name,
msize, tok_label, tok_lastcnt, tok_serial);
#endif
}
ERL_MESSAGE_TOKEN(mp) = token;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = utag;
#endif
res = queue_message(receiver,
&receiver_state,
*receiver_locks,
mp, message,
sender->common.id);
return res;
}
Uint
erts_prep_msgq_for_inspection(Process *c_p, Process *rp,
ErtsProcLocks rp_locks, ErtsMessageInfo *mip)
{
Uint tot_heap_size;
ErtsMessage* mp;
Sint i;
int self_on_heap;
/*
* Prepare the message queue for inspection
* by process_info().
*
*
* - Decode all messages on external format
* - Remove all corrupt dist messages from queue
* - Save pointer to, and heap size need of each
* message in the mip array.
* - Return total heap size need for all messages
* that needs to be copied.
*
* If ERTS_INSPECT_MSGQ_KEEP_OH_MSGS == 0:
* - In case off heap messages is disabled and
* we are inspecting our own queue, move all
* off heap data into the heap.
*/
self_on_heap = c_p == rp && !(c_p->flags & F_OFF_HEAP_MSGQ);
tot_heap_size = 0;
i = 0;
mp = rp->msg.first;
while (mp) {
Eterm msg = ERL_MESSAGE_TERM(mp);
mip[i].size = 0;
if (is_non_value(msg)) {
/* Dist message on external format; decode it... */
if (mp->data.attached)
erts_decode_dist_message(rp, rp_locks, mp,
ERTS_INSPECT_MSGQ_KEEP_OH_MSGS);
msg = ERL_MESSAGE_TERM(mp);
if (is_non_value(msg)) {
ErtsMessage **mpp;
ErtsMessage *bad_mp = mp;
/*
* Bad distribution message; remove
* it from the queue...
*/
ASSERT(!mp->data.attached);
mpp = i == 0 ? &rp->msg.first : &mip[i-1].msgp->next;
ASSERT(*mpp == bad_mp);
erts_msgq_update_internal_pointers(&rp->msg, mpp, &bad_mp->next);
mp = mp->next;
*mpp = mp;
rp->msg.len--;
bad_mp->next = NULL;
erts_cleanup_messages(bad_mp);
continue;
}
}
ASSERT(is_value(msg));
#if ERTS_INSPECT_MSGQ_KEEP_OH_MSGS
if (is_not_immed(msg) && (!self_on_heap || mp->data.attached)) {
Uint sz = size_object(msg);
mip[i].size = sz;
tot_heap_size += sz;
}
#else
if (self_on_heap) {
if (mp->data.attached) {
ErtsMessage *tmp = NULL;
if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) {
erts_link_mbuf_to_proc(rp, mp->data.heap_frag);
mp->data.attached = NULL;
}
else {
/*
* Need to replace the message reference since
* we will get references to the message data
* from the heap...
*/
ErtsMessage **mpp;
tmp = erts_alloc_message(0, NULL);
sys_memcpy((void *) tmp->m, (void *) mp->m,
sizeof(Eterm)*ERL_MESSAGE_REF_ARRAY_SZ);
mpp = i == 0 ? &rp->msg.first : &mip[i-1].msgp->next;
erts_msgq_replace_msg_ref(&rp->msg, tmp, mpp);
erts_save_message_in_proc(rp, mp);
mp = tmp;
}
}
}
else if (is_not_immed(msg)) {
Uint sz = size_object(msg);
mip[i].size = sz;
tot_heap_size += sz;
}
#endif
mip[i].msgp = mp;
i++;
mp = mp->next;
}
return tot_heap_size;
}
int
erts_decode_dist_message(Process *proc, ErtsProcLocks proc_locks,
ErtsMessage *msgp, int force_off_heap)
{
ErtsHeapFactory factory;
Eterm msg;
ErlHeapFragment *bp;
Sint need;
int decode_in_heap_frag;
decode_in_heap_frag = (force_off_heap
|| !(proc_locks & ERTS_PROC_LOCK_MAIN)
|| (proc->flags & F_OFF_HEAP_MSGQ));
if (msgp->data.dist_ext->heap_size >= 0)
need = msgp->data.dist_ext->heap_size;
else {
need = erts_decode_dist_ext_size(msgp->data.dist_ext);
if (need < 0) {
/* bad msg; remove it... */
if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) {
bp = erts_dist_ext_trailer(msgp->data.dist_ext);
erts_cleanup_offheap(&bp->off_heap);
}
erts_free_dist_ext_copy(msgp->data.dist_ext);
msgp->data.dist_ext = NULL;
return 0;
}
msgp->data.dist_ext->heap_size = need;
}
if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) {
bp = erts_dist_ext_trailer(msgp->data.dist_ext);
need += bp->used_size;
}
if (decode_in_heap_frag)
erts_factory_heap_frag_init(&factory, new_message_buffer(need));
else
erts_factory_proc_prealloc_init(&factory, proc, need);
ASSERT(msgp->data.dist_ext->heap_size >= 0);
if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) {
ErlHeapFragment *heap_frag;
heap_frag = erts_dist_ext_trailer(msgp->data.dist_ext);
ERL_MESSAGE_TOKEN(msgp) = copy_struct(ERL_MESSAGE_TOKEN(msgp),
heap_frag->used_size,
&factory.hp,
factory.off_heap);
erts_cleanup_offheap(&heap_frag->off_heap);
}
msg = erts_decode_dist_ext(&factory, msgp->data.dist_ext);
ERL_MESSAGE_TERM(msgp) = msg;
erts_free_dist_ext_copy(msgp->data.dist_ext);
msgp->data.attached = NULL;
if (is_non_value(msg)) {
erts_factory_undo(&factory);
return 0;
}
erts_factory_trim_and_close(&factory, msgp->m,
ERL_MESSAGE_REF_ARRAY_SZ);
ASSERT(!msgp->data.heap_frag);
if (decode_in_heap_frag)
msgp->data.heap_frag = factory.heap_frags;
return 1;
}
/*
* Moves content of message buffer attached to a message into a heap.
* The message buffer is deallocated.
*/
void
erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
{
struct erl_off_heap_header* oh;
Eterm term, token, *fhp, *hp;
Sint offs;
Uint sz;
ErlHeapFragment *bp;
#ifdef USE_VM_PROBES
Eterm utag;
#endif
#ifdef HARD_DEBUG
struct erl_off_heap_header* dbg_oh_start = off_heap->first;
Eterm dbg_term, dbg_token;
ErlHeapFragment *dbg_bp;
Uint *dbg_hp, *dbg_thp_start;
Uint dbg_term_sz, dbg_token_sz;
#ifdef USE_VM_PROBES
Eterm dbg_utag;
Uint dbg_utag_sz;
#endif
#endif
bp = msg->data.heap_frag;
term = ERL_MESSAGE_TERM(msg);
token = ERL_MESSAGE_TOKEN(msg);
#ifdef USE_VM_PROBES
utag = ERL_MESSAGE_DT_UTAG(msg);
#endif
if (!bp) {
#ifdef USE_VM_PROBES
ASSERT(is_immed(term) && is_immed(token) && is_immed(utag));
#else
ASSERT(is_immed(term) && is_immed(token));
#endif
return;
}
#ifdef HARD_DEBUG
dbg_term_sz = size_object(term);
dbg_token_sz = size_object(token);
dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz);
#ifdef USE_VM_PROBES
dbg_utag_sz = size_object(utag);
dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz + dbg_utag_sz );
#endif
/*ASSERT(dbg_term_sz + dbg_token_sz == erts_msg_used_frag_sz(msg));
Copied size may be smaller due to removed SubBins's or garbage.
Copied size may be larger due to duplicated shared terms.
*/
dbg_hp = dbg_bp->mem;
dbg_term = copy_struct(term, dbg_term_sz, &dbg_hp, &dbg_bp->off_heap);
dbg_token = copy_struct(token, dbg_token_sz, &dbg_hp, &dbg_bp->off_heap);
#ifdef USE_VM_PROBES
dbg_utag = copy_struct(utag, dbg_utag_sz, &dbg_hp, &dbg_bp->off_heap);
#endif
dbg_thp_start = *hpp;
#endif
if (bp->next != NULL) {
move_multi_frags(hpp, off_heap, bp, msg->m,
#ifdef USE_VM_PROBES
3
#else
2
#endif
);
goto copy_done;
}
OH_OVERHEAD(off_heap, bp->off_heap.overhead);
sz = bp->used_size;
ASSERT(is_immed(term) || in_heapfrag(ptr_val(term),bp));
ASSERT(is_immed(token) || in_heapfrag(ptr_val(token),bp));
fhp = bp->mem;
hp = *hpp;
offs = hp - fhp;
oh = NULL;
while (sz--) {
Uint cpy_sz;
Eterm val = *fhp++;
switch (primary_tag(val)) {
case TAG_PRIMARY_IMMED1:
*hp++ = val;
break;
case TAG_PRIMARY_LIST:
case TAG_PRIMARY_BOXED:
ASSERT(in_heapfrag(ptr_val(val), bp));
*hp++ = offset_ptr(val, offs);
break;
case TAG_PRIMARY_HEADER:
*hp++ = val;
switch (val & _HEADER_SUBTAG_MASK) {
case ARITYVAL_SUBTAG:
break;
case REFC_BINARY_SUBTAG:
case FUN_SUBTAG:
case EXTERNAL_PID_SUBTAG:
case EXTERNAL_PORT_SUBTAG:
case EXTERNAL_REF_SUBTAG:
oh = (struct erl_off_heap_header*) (hp-1);
cpy_sz = thing_arityval(val);
goto cpy_words;
default:
cpy_sz = header_arity(val);
cpy_words:
ASSERT(sz >= cpy_sz);
sz -= cpy_sz;
while (cpy_sz >= 8) {
cpy_sz -= 8;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
}
switch (cpy_sz) {
case 7: *hp++ = *fhp++;
case 6: *hp++ = *fhp++;
case 5: *hp++ = *fhp++;
case 4: *hp++ = *fhp++;
case 3: *hp++ = *fhp++;
case 2: *hp++ = *fhp++;
case 1: *hp++ = *fhp++;
default: break;
}
if (oh) {
/* Add to offheap list */
oh->next = off_heap->first;
off_heap->first = oh;
ASSERT(*hpp <= (Eterm*)oh);
ASSERT(hp > (Eterm*)oh);
oh = NULL;
}
break;
}
break;
}
}
ASSERT(bp->used_size == hp - *hpp);
*hpp = hp;
if (is_not_immed(token)) {
ASSERT(in_heapfrag(ptr_val(token), bp));
ERL_MESSAGE_TOKEN(msg) = offset_ptr(token, offs);
#ifdef HARD_DEBUG
ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TOKEN(msg)));
ASSERT(hp > ptr_val(ERL_MESSAGE_TOKEN(msg)));
#endif
}
if (is_not_immed(term)) {
ASSERT(in_heapfrag(ptr_val(term),bp));
ERL_MESSAGE_TERM(msg) = offset_ptr(term, offs);
#ifdef HARD_DEBUG
ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TERM(msg)));
ASSERT(hp > ptr_val(ERL_MESSAGE_TERM(msg)));
#endif
}
#ifdef USE_VM_PROBES
if (is_not_immed(utag)) {
ASSERT(in_heapfrag(ptr_val(utag), bp));
ERL_MESSAGE_DT_UTAG(msg) = offset_ptr(utag, offs);
#ifdef HARD_DEBUG
ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_DT_UTAG(msg)));
ASSERT(hp > ptr_val(ERL_MESSAGE_DT_UTAG(msg)));
#endif
}
#endif
copy_done:
#ifdef HARD_DEBUG
{
int i, j;
ErlHeapFragment* frag;
{
struct erl_off_heap_header* dbg_oh = off_heap->first;
i = j = 0;
while (dbg_oh != dbg_oh_start) {
dbg_oh = dbg_oh->next;
i++;
}
for (frag=bp; frag; frag=frag->next) {
dbg_oh = frag->off_heap.first;
while (dbg_oh) {
dbg_oh = dbg_oh->next;
j++;
}
}
ASSERT(i == j);
}
}
#endif
bp->off_heap.first = NULL;
free_message_buffer(bp);
msg->data.heap_frag = NULL;
#ifdef HARD_DEBUG
ASSERT(eq(ERL_MESSAGE_TERM(msg), dbg_term));
ASSERT(eq(ERL_MESSAGE_TOKEN(msg), dbg_token));
#ifdef USE_VM_PROBES
ASSERT(eq(ERL_MESSAGE_DT_UTAG(msg), dbg_utag));
#endif
free_message_buffer(dbg_bp);
#endif
}
/*
* Moves content of message buffer attached to a message into a heap.
* The message buffer is deallocated.
*/
void
erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
{
/* Unions for typecasts avoids warnings about type-punned pointers and aliasing */
union {
Uint** upp;
ProcBin **pbpp;
ErlFunThing **efpp;
ExternalThing **etpp;
} oh_list_pp, oh_el_next_pp;
union {
Uint *up;
ProcBin *pbp;
ErlFunThing *efp;
ExternalThing *etp;
} oh_el_p;
Eterm term, token, *fhp, *hp;
Sint offs;
Uint sz;
ErlHeapFragment *bp;
#ifdef HARD_DEBUG
ProcBin *dbg_mso_start = off_heap->mso;
ErlFunThing *dbg_fun_start = off_heap->funs;
ExternalThing *dbg_external_start = off_heap->externals;
Eterm dbg_term, dbg_token;
ErlHeapFragment *dbg_bp;
Uint *dbg_hp, *dbg_thp_start;
Uint dbg_term_sz, dbg_token_sz;
#endif
bp = msg->data.heap_frag;
term = ERL_MESSAGE_TERM(msg);
token = ERL_MESSAGE_TOKEN(msg);
if (!bp) {
ASSERT(is_immed(term) && is_immed(token));
return;
}
#ifdef HARD_DEBUG
dbg_term_sz = size_object(term);
dbg_token_sz = size_object(token);
ASSERT(bp->size == dbg_term_sz + dbg_token_sz);
dbg_bp = new_message_buffer(bp->size);
dbg_hp = dbg_bp->mem;
dbg_term = copy_struct(term, dbg_term_sz, &dbg_hp, &dbg_bp->off_heap);
dbg_token = copy_struct(token, dbg_token_sz, &dbg_hp, &dbg_bp->off_heap);
dbg_thp_start = *hpp;
#endif
ASSERT(bp);
msg->data.attached = NULL;
off_heap->overhead += bp->off_heap.overhead;
sz = bp->size;
#ifdef DEBUG
if (is_not_immed(term)) {
ASSERT(bp->mem <= ptr_val(term));
ASSERT(bp->mem + bp->size > ptr_val(term));
}
if (is_not_immed(token)) {
ASSERT(bp->mem <= ptr_val(token));
ASSERT(bp->mem + bp->size > ptr_val(token));
}
#endif
fhp = bp->mem;
hp = *hpp;
offs = hp - fhp;
oh_list_pp.upp = NULL;
oh_el_next_pp.upp = NULL; /* Shut up compiler warning */
oh_el_p.up = NULL; /* Shut up compiler warning */
while (sz--) {
Uint cpy_sz;
Eterm val = *fhp++;
switch (primary_tag(val)) {
case TAG_PRIMARY_IMMED1:
*hp++ = val;
break;
case TAG_PRIMARY_LIST:
case TAG_PRIMARY_BOXED:
ASSERT(bp->mem <= ptr_val(val));
ASSERT(bp->mem + bp->size > ptr_val(val));
*hp++ = offset_ptr(val, offs);
break;
case TAG_PRIMARY_HEADER:
*hp++ = val;
switch (val & _HEADER_SUBTAG_MASK) {
case ARITYVAL_SUBTAG:
break;
case REFC_BINARY_SUBTAG:
oh_list_pp.pbpp = &off_heap->mso;
oh_el_p.up = (hp-1);
oh_el_next_pp.pbpp = &(oh_el_p.pbp)->next;
cpy_sz = thing_arityval(val);
goto cpy_words;
case FUN_SUBTAG:
#ifndef HYBRID
oh_list_pp.efpp = &off_heap->funs;
oh_el_p.up = (hp-1);
oh_el_next_pp.efpp = &(oh_el_p.efp)->next;
#endif
cpy_sz = thing_arityval(val);
goto cpy_words;
case EXTERNAL_PID_SUBTAG:
case EXTERNAL_PORT_SUBTAG:
case EXTERNAL_REF_SUBTAG:
oh_list_pp.etpp = &off_heap->externals;
oh_el_p.up = (hp-1);
oh_el_next_pp.etpp = &(oh_el_p.etp)->next;
cpy_sz = thing_arityval(val);
goto cpy_words;
default:
cpy_sz = header_arity(val);
cpy_words:
sz -= cpy_sz;
while (cpy_sz >= 8) {
cpy_sz -= 8;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
}
switch (cpy_sz) {
case 7: *hp++ = *fhp++;
case 6: *hp++ = *fhp++;
case 5: *hp++ = *fhp++;
case 4: *hp++ = *fhp++;
case 3: *hp++ = *fhp++;
case 2: *hp++ = *fhp++;
case 1: *hp++ = *fhp++;
default: break;
}
if (oh_list_pp.upp) {
#ifdef HARD_DEBUG
Uint *dbg_old_oh_list_p = *oh_list_pp.upp;
#endif
/* Add to offheap list */
*oh_el_next_pp.upp = *oh_list_pp.upp;
*oh_list_pp.upp = oh_el_p.up;
ASSERT(*hpp <= oh_el_p.up);
ASSERT(hp > oh_el_p.up);
#ifdef HARD_DEBUG
switch (val & _HEADER_SUBTAG_MASK) {
case REFC_BINARY_SUBTAG:
ASSERT(off_heap->mso == *oh_list_pp.pbpp);
ASSERT(off_heap->mso->next
== (ProcBin *) dbg_old_oh_list_p);
break;
#ifndef HYBRID
case FUN_SUBTAG:
ASSERT(off_heap->funs == *oh_list_pp.efpp);
ASSERT(off_heap->funs->next
== (ErlFunThing *) dbg_old_oh_list_p);
break;
#endif
case EXTERNAL_PID_SUBTAG:
case EXTERNAL_PORT_SUBTAG:
case EXTERNAL_REF_SUBTAG:
ASSERT(off_heap->externals
== *oh_list_pp.etpp);
ASSERT(off_heap->externals->next
== (ExternalThing *) dbg_old_oh_list_p);
break;
default:
ASSERT(0);
}
#endif
oh_list_pp.upp = NULL;
}
break;
}
break;
}
}
ASSERT(bp->size == hp - *hpp);
*hpp = hp;
if (is_not_immed(token)) {
ASSERT(bp->mem <= ptr_val(token));
ASSERT(bp->mem + bp->size > ptr_val(token));
ERL_MESSAGE_TOKEN(msg) = offset_ptr(token, offs);
#ifdef HARD_DEBUG
ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TOKEN(msg)));
ASSERT(hp > ptr_val(ERL_MESSAGE_TOKEN(msg)));
#endif
}
if (is_not_immed(term)) {
ASSERT(bp->mem <= ptr_val(term));
ASSERT(bp->mem + bp->size > ptr_val(term));
ERL_MESSAGE_TERM(msg) = offset_ptr(term, offs);
#ifdef HARD_DEBUG
ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TERM(msg)));
ASSERT(hp > ptr_val(ERL_MESSAGE_TERM(msg)));
#endif
}
#ifdef HARD_DEBUG
{
int i, j;
{
ProcBin *mso = off_heap->mso;
i = j = 0;
while (mso != dbg_mso_start) {
mso = mso->next;
i++;
}
mso = bp->off_heap.mso;
while (mso) {
mso = mso->next;
j++;
}
ASSERT(i == j);
}
{
ErlFunThing *fun = off_heap->funs;
i = j = 0;
while (fun != dbg_fun_start) {
fun = fun->next;
i++;
}
fun = bp->off_heap.funs;
while (fun) {
fun = fun->next;
j++;
}
ASSERT(i == j);
}
{
ExternalThing *external = off_heap->externals;
i = j = 0;
while (external != dbg_external_start) {
external = external->next;
i++;
}
external = bp->off_heap.externals;
while (external) {
external = external->next;
j++;
}
ASSERT(i == j);
}
}
#endif
bp->off_heap.mso = NULL;
#ifndef HYBRID
bp->off_heap.funs = NULL;
#endif
bp->off_heap.externals = NULL;
free_message_buffer(bp);
#ifdef HARD_DEBUG
ASSERT(eq(ERL_MESSAGE_TERM(msg), dbg_term));
ASSERT(eq(ERL_MESSAGE_TOKEN(msg), dbg_token));
free_message_buffer(dbg_bp);
#endif
}