void erts_cleanup_messages(ErtsMessage *msgp) { ErtsMessage *mp = msgp; while (mp) { ErtsMessage *fmp; ErlHeapFragment *bp; if (is_non_value(ERL_MESSAGE_TERM(mp))) { if (is_not_immed(ERL_MESSAGE_TOKEN(mp))) { bp = (ErlHeapFragment *) mp->data.dist_ext->ext_endp; erts_cleanup_offheap(&bp->off_heap); } if (mp->data.dist_ext) erts_free_dist_ext_copy(mp->data.dist_ext); } else { if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) bp = mp->data.heap_frag; else { bp = mp->hfrag.next; erts_cleanup_offheap(&mp->hfrag.off_heap); } if (bp) free_message_buffer(bp); } fmp = mp; mp = mp->next; erts_free_message(fmp); } }
static void erts_cleanup_message(ErtsMessage *mp) { ErlHeapFragment *bp; if (ERTS_SIG_IS_EXTERNAL_MSG(mp) || ERTS_SIG_IS_NON_MSG(mp)) { ErtsDistExternal *edep = erts_proc_sig_get_external(mp); if (edep) { erts_free_dist_ext_copy(edep); if (mp->data.heap_frag == &mp->hfrag) { ASSERT(ERTS_SIG_IS_EXTERNAL_MSG(mp)); mp->data.heap_frag = ERTS_MSG_COMBINED_HFRAG; } } } if (ERTS_SIG_IS_MSG(mp) && mp->data.attached != ERTS_MSG_COMBINED_HFRAG) { bp = mp->data.heap_frag; } else { /* All non msg signals are combined HFRAG messages, but we overwrite the mp->data field with the nm_signal queue ptr so have to fix that here before freeing it. */ mp->data.attached = ERTS_MSG_COMBINED_HFRAG; bp = mp->hfrag.next; erts_cleanup_offheap(&mp->hfrag.off_heap); } if (bp) free_message_buffer(bp); }
BIF_RETTYPE port_set_data_2(BIF_ALIST_2) { Port* prt; Eterm portid = BIF_ARG_1; Eterm data = BIF_ARG_2; prt = id_or_name2port(BIF_P, portid); if (!prt) { BIF_ERROR(BIF_P, BADARG); } if (prt->bp != NULL) { free_message_buffer(prt->bp); prt->bp = NULL; } if (IS_CONST(data)) { prt->data = data; } else { Uint size; ErlHeapFragment* bp; Eterm* hp; size = size_object(data); prt->bp = bp = new_message_buffer(size); hp = bp->mem; prt->data = copy_struct(data, size, &hp, &bp->off_heap); } erts_smp_port_unlock(prt); BIF_RET(am_true); }
void erts_factory_trim_and_close(ErtsHeapFactory* factory, Eterm *brefs, Uint brefs_size) { ErlHeapFragment *bp; switch (factory->mode) { case FACTORY_MESSAGE: { ErtsMessage *mp = factory->message; if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG) { if (!factory->heap_frags) { Uint sz = factory->hp - factory->hp_start; mp = erts_shrink_message(mp, sz, brefs, brefs_size); factory->message = mp; factory->mode = FACTORY_CLOSED; return; } /*else we don't trim multi fragmented messages for now (off_heap...) */ break; } /* Fall through... */ } case FACTORY_HEAP_FRAGS: bp = factory->heap_frags; if (!bp) break; if (bp->next == NULL) { Uint used_sz = factory->hp - bp->mem; ASSERT(used_sz <= bp->alloc_size); if (used_sz > 0) { if (used_sz != bp->alloc_size) bp = erts_resize_message_buffer(bp, used_sz, brefs, brefs_size); } else { free_message_buffer(bp); bp = NULL; } factory->heap_frags = bp; if (factory->mode == FACTORY_MESSAGE) factory->message->data.heap_frag = bp; factory->mode = FACTORY_CLOSED; return; } /*else we don't trim multi fragmented messages for now (off_heap...) */ default: break; } erts_factory_close(factory); }
void enif_clear_env(ErlNifEnv* env) { struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)env; Process* p = &menv->phony_proc; ASSERT(p == menv->env.proc); ASSERT(p->id == ERTS_INVALID_PID); ASSERT(MBUF(p) == menv->env.heap_frag); if (MBUF(p) != NULL) { erts_cleanup_offheap(&MSO(p)); clear_offheap(&MSO(p)); free_message_buffer(MBUF(p)); MBUF(p) = NULL; menv->env.heap_frag = NULL; } ASSERT(HEAP_TOP(p) == HEAP_END(p)); menv->env.hp = menv->env.hp_end = HEAP_TOP(p); ASSERT(!is_offheap(&MSO(p))); free_tmp_objs(env); }
ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, Eterm *brefs, Uint brefs_size) { #ifdef DEBUG int i; #endif #ifdef HARD_DEBUG ErlHeapFragment *dbg_bp; Eterm *dbg_brefs; Uint dbg_size; Uint dbg_tot_size; Eterm *dbg_hp; #endif ErlHeapFragment* nbp; #ifdef DEBUG { Uint off_sz = size < bp->used_size ? size : bp->used_size; for (i = 0; i < brefs_size; i++) { Eterm *ptr; if (is_immed(brefs[i])) continue; ptr = ptr_val(brefs[i]); ASSERT(&bp->mem[0] <= ptr && ptr < &bp->mem[0] + off_sz); } } #endif if (size >= (bp->used_size - bp->used_size / 16)) { bp->used_size = size; return bp; } #ifdef HARD_DEBUG dbg_brefs = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(Eterm *)*brefs_size); dbg_bp = new_message_buffer(bp->used_size); dbg_hp = dbg_bp->mem; dbg_tot_size = 0; for (i = 0; i < brefs_size; i++) { dbg_size = size_object(brefs[i]); dbg_tot_size += dbg_size; dbg_brefs[i] = copy_struct(brefs[i], dbg_size, &dbg_hp, &dbg_bp->off_heap); } ASSERT(dbg_tot_size == (size < bp->used_size ? size : bp->used_size)); #endif nbp = (ErlHeapFragment*) ERTS_HEAP_REALLOC(ERTS_ALC_T_HEAP_FRAG, (void *) bp, ERTS_HEAP_FRAG_SIZE(bp->alloc_size), ERTS_HEAP_FRAG_SIZE(size)); if (bp != nbp) { Uint off_sz = size < nbp->used_size ? size : nbp->used_size; Eterm *sp = &bp->mem[0]; Eterm *ep = sp + off_sz; Sint offs = &nbp->mem[0] - sp; erts_offset_off_heap(&nbp->off_heap, offs, sp, ep); erts_offset_heap(&nbp->mem[0], off_sz, offs, sp, ep); if (brefs && brefs_size) erts_offset_heap_ptr(brefs, brefs_size, offs, sp, ep); #ifdef DEBUG for (i = 0; i < brefs_size; i++) { Eterm *ptr; if (is_immed(brefs[i])) continue; ptr = ptr_val(brefs[i]); ASSERT(&nbp->mem[0] <= ptr && ptr < &nbp->mem[0] + off_sz); } #endif } nbp->alloc_size = size; nbp->used_size = size; #ifdef HARD_DEBUG for (i = 0; i < brefs_size; i++) ASSERT(eq(dbg_brefs[i], brefs[i])); free_message_buffer(dbg_bp); erts_free(ERTS_ALC_T_UNDEF, dbg_brefs); #endif return nbp; }
/* * Garbage collect a process. * * p: Pointer to the process structure. * need: Number of Eterm words needed on the heap. * objv: Array of terms to add to rootset; that is to preserve. * nobj: Number of objects in objv. */ int erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) { int ret; struct slave_syscall_gc *cmd = erts_alloc(ERTS_ALC_T_TMP, sizeof(*cmd)); Eterm *dram_objv; int copy_objv = nobj != 0 && !epiphany_in_dram(objv); if (copy_objv) { dram_objv = erts_alloc(ERTS_ALC_T_TMP, nobj*sizeof(Eterm)); memcpy(dram_objv, objv, nobj*sizeof(Eterm)); } else { dram_objv = objv; } #ifdef DEBUG { int i; for (i = 0; i < nobj; i++) ASSERT(is_immed(objv[i]) || epiphany_in_dram(ptr_val(objv[i]))); } #endif cmd->need = need; cmd->objv = dram_objv; cmd->nobj = nobj; slave_state_swapout(p, &cmd->state); erts_master_syscall(SLAVE_SYSCALL_GC, cmd); if (copy_objv) { memcpy(objv, dram_objv, nobj*sizeof(Eterm)); erts_free(ERTS_ALC_T_TMP, dram_objv); } /* * The garbage collector will have set mbuf to NULL without freeing it. We * do so here. See remove_message_buffers in the master. */ if (MBUF(p) != NULL) { free_message_buffer(MBUF(p)); ASSERT(cmd->state.mbuf == NULL); } slave_state_swapin(p, &cmd->state); #ifdef CHECK_FOR_HOLES /* * We intentionally do not rescan the areas copied by the GC. * We trust the GC not to leave any holes. */ p->last_htop = p->htop; p->last_mbuf = 0; #endif #ifdef DEBUG /* * The scanning for pointers from the old_heap into the new_heap or * heap fragments turned out to be costly, so we remember how far we * have scanned this time and will start scanning there next time. * (We will not detect wild writes into the old heap, or modifications * of the old heap in-between garbage collections.) */ p->last_old_htop = p->old_htop; #endif ret = cmd->ret; erts_free(ERTS_ALC_T_TMP, cmd); return ret; }
/* send object-communication packet */ void SendMessage(MessageBuffer mb, long long destination) { ExecTable et; SendPort sp,spe,free_sp; MessageBuffer mbp; int i,flag,rval; unsigned int least_access; #if 0 printf("SendMessage enter\n"); #endif /* find send port already assigned */ for(i=0,flag=0,free_sp=(SendPort)0,sp=apgw_send_tbl;i<MAX_APGW_SEND;i++,sp++) { if(sp->exid == destination) { flag=1; break;} if(sp->exid==0LL) { free_sp =sp; } } if(flag) { /* Send Port exists */ if(sp->fd != 0) { /* found connected port */ if((i = write(sp->fd, (char *)(&(mb->buffer_size)),sizeof(int))) <sizeof(int)) { /* error */ perror("SendMessage(write):"); SendError(mb); free_send_port(sp); return; } if((i = write(sp->fd, mb->buffer, mb->buffer_size)) < mb->buffer_size) { /* error */ perror("SendMessage(write):"); SendError(mb); free_send_port(sp); return; } set_last_access_send(sp); free_message_buffer(mb); #if 0 printf("Message sent using connected send port\n"); #endif return; } else if(sp->waiting != (MessageBuffer)0) { /* send port is waiting for address resolution */ for(mbp=sp->waiting; mbp->next != (MessageBuffer)0;mbp=mbp->next) ; mbp->next=mb; #if 0 printf("message is appended on waiting port\n"); #endif return; } else { /* if reach here, it's bug */ printf("Invalid send port ?? sp(%x) fd(%x) waiting(%x) exid(%08x%08x)\n", sp,sp->fd,sp->waiting, (int)(((sp->exid)>>32)&INTEGER_MASK), (int)((sp->exid)&INTEGER_MASK)); return; } } else { if(!free_sp)
/* * Moves content of message buffer attached to a message into a heap. * The message buffer is deallocated. */ void erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) { struct erl_off_heap_header* oh; Eterm term, token, *fhp, *hp; Sint offs; Uint sz; ErlHeapFragment *bp; #ifdef USE_VM_PROBES Eterm utag; #endif #ifdef HARD_DEBUG struct erl_off_heap_header* dbg_oh_start = off_heap->first; Eterm dbg_term, dbg_token; ErlHeapFragment *dbg_bp; Uint *dbg_hp, *dbg_thp_start; Uint dbg_term_sz, dbg_token_sz; #ifdef USE_VM_PROBES Eterm dbg_utag; Uint dbg_utag_sz; #endif #endif bp = msg->data.heap_frag; term = ERL_MESSAGE_TERM(msg); token = ERL_MESSAGE_TOKEN(msg); #ifdef USE_VM_PROBES utag = ERL_MESSAGE_DT_UTAG(msg); #endif if (!bp) { #ifdef USE_VM_PROBES ASSERT(is_immed(term) && is_immed(token) && is_immed(utag)); #else ASSERT(is_immed(term) && is_immed(token)); #endif return; } #ifdef HARD_DEBUG dbg_term_sz = size_object(term); dbg_token_sz = size_object(token); dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz); #ifdef USE_VM_PROBES dbg_utag_sz = size_object(utag); dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz + dbg_utag_sz ); #endif /*ASSERT(dbg_term_sz + dbg_token_sz == erts_msg_used_frag_sz(msg)); Copied size may be smaller due to removed SubBins's or garbage. Copied size may be larger due to duplicated shared terms. */ dbg_hp = dbg_bp->mem; dbg_term = copy_struct(term, dbg_term_sz, &dbg_hp, &dbg_bp->off_heap); dbg_token = copy_struct(token, dbg_token_sz, &dbg_hp, &dbg_bp->off_heap); #ifdef USE_VM_PROBES dbg_utag = copy_struct(utag, dbg_utag_sz, &dbg_hp, &dbg_bp->off_heap); #endif dbg_thp_start = *hpp; #endif if (bp->next != NULL) { move_multi_frags(hpp, off_heap, bp, msg->m, #ifdef USE_VM_PROBES 3 #else 2 #endif ); goto copy_done; } OH_OVERHEAD(off_heap, bp->off_heap.overhead); sz = bp->used_size; ASSERT(is_immed(term) || in_heapfrag(ptr_val(term),bp)); ASSERT(is_immed(token) || in_heapfrag(ptr_val(token),bp)); fhp = bp->mem; hp = *hpp; offs = hp - fhp; oh = NULL; while (sz--) { Uint cpy_sz; Eterm val = *fhp++; switch (primary_tag(val)) { case TAG_PRIMARY_IMMED1: *hp++ = val; break; case TAG_PRIMARY_LIST: case TAG_PRIMARY_BOXED: ASSERT(in_heapfrag(ptr_val(val), bp)); *hp++ = offset_ptr(val, offs); break; case TAG_PRIMARY_HEADER: *hp++ = val; switch (val & _HEADER_SUBTAG_MASK) { case ARITYVAL_SUBTAG: break; case REFC_BINARY_SUBTAG: case FUN_SUBTAG: case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: oh = (struct erl_off_heap_header*) (hp-1); cpy_sz = thing_arityval(val); goto cpy_words; default: cpy_sz = header_arity(val); cpy_words: ASSERT(sz >= cpy_sz); sz -= cpy_sz; while (cpy_sz >= 8) { cpy_sz -= 8; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; } switch (cpy_sz) { case 7: *hp++ = *fhp++; case 6: *hp++ = *fhp++; case 5: *hp++ = *fhp++; case 4: *hp++ = *fhp++; case 3: *hp++ = *fhp++; case 2: *hp++ = *fhp++; case 1: *hp++ = *fhp++; default: break; } if (oh) { /* Add to offheap list */ oh->next = off_heap->first; off_heap->first = oh; ASSERT(*hpp <= (Eterm*)oh); ASSERT(hp > (Eterm*)oh); oh = NULL; } break; } break; } } ASSERT(bp->used_size == hp - *hpp); *hpp = hp; if (is_not_immed(token)) { ASSERT(in_heapfrag(ptr_val(token), bp)); ERL_MESSAGE_TOKEN(msg) = offset_ptr(token, offs); #ifdef HARD_DEBUG ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TOKEN(msg))); ASSERT(hp > ptr_val(ERL_MESSAGE_TOKEN(msg))); #endif } if (is_not_immed(term)) { ASSERT(in_heapfrag(ptr_val(term),bp)); ERL_MESSAGE_TERM(msg) = offset_ptr(term, offs); #ifdef HARD_DEBUG ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TERM(msg))); ASSERT(hp > ptr_val(ERL_MESSAGE_TERM(msg))); #endif } #ifdef USE_VM_PROBES if (is_not_immed(utag)) { ASSERT(in_heapfrag(ptr_val(utag), bp)); ERL_MESSAGE_DT_UTAG(msg) = offset_ptr(utag, offs); #ifdef HARD_DEBUG ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_DT_UTAG(msg))); ASSERT(hp > ptr_val(ERL_MESSAGE_DT_UTAG(msg))); #endif } #endif copy_done: #ifdef HARD_DEBUG { int i, j; ErlHeapFragment* frag; { struct erl_off_heap_header* dbg_oh = off_heap->first; i = j = 0; while (dbg_oh != dbg_oh_start) { dbg_oh = dbg_oh->next; i++; } for (frag=bp; frag; frag=frag->next) { dbg_oh = frag->off_heap.first; while (dbg_oh) { dbg_oh = dbg_oh->next; j++; } } ASSERT(i == j); } } #endif bp->off_heap.first = NULL; free_message_buffer(bp); msg->data.heap_frag = NULL; #ifdef HARD_DEBUG ASSERT(eq(ERL_MESSAGE_TERM(msg), dbg_term)); ASSERT(eq(ERL_MESSAGE_TOKEN(msg), dbg_token)); #ifdef USE_VM_PROBES ASSERT(eq(ERL_MESSAGE_DT_UTAG(msg), dbg_utag)); #endif free_message_buffer(dbg_bp); #endif }
/* Add a message last in message queue */ static Sint queue_message(Process *c_p, Process* receiver, ErtsProcLocks *receiver_locks, erts_aint32_t *receiver_state, ErlHeapFragment* bp, Eterm message, Eterm seq_trace_token #ifdef USE_VM_PROBES , Eterm dt_utag #endif ) { Sint res; ErlMessage* mp; int locked_msgq = 0; erts_aint_t state; #ifndef ERTS_SMP ASSERT(bp != NULL || receiver->mbuf == NULL); #endif ERTS_SMP_LC_ASSERT(*receiver_locks == erts_proc_lc_my_proc_locks(receiver)); mp = message_alloc(); if (receiver_state) state = *receiver_state; else state = erts_smp_atomic32_read_acqb(&receiver->state); #ifdef ERTS_SMP if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) goto exiting; if (!(*receiver_locks & ERTS_PROC_LOCK_MSGQ)) { if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) { ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ; if (*receiver_locks & ERTS_PROC_LOCK_STATUS) { erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS); need_locks |= ERTS_PROC_LOCK_STATUS; } erts_smp_proc_lock(receiver, need_locks); } locked_msgq = 1; state = erts_smp_atomic32_read_nob(&receiver->state); if (receiver_state) *receiver_state = state; } #endif if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) { #ifdef ERTS_SMP exiting: #endif /* Drop message if receiver is exiting or has a pending exit... */ if (locked_msgq) erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); if (bp) free_message_buffer(bp); message_free(mp); return 0; } ERL_MESSAGE_TERM(mp) = message; ERL_MESSAGE_TOKEN(mp) = seq_trace_token; #ifdef USE_VM_PROBES ERL_MESSAGE_DT_UTAG(mp) = dt_utag; #endif mp->next = NULL; mp->data.heap_frag = bp; #ifndef ERTS_SMP res = receiver->msg.len; #else res = receiver->msg_inq.len; if (*receiver_locks & ERTS_PROC_LOCK_MAIN) { /* * We move 'in queue' to 'private queue' and place * message at the end of 'private queue' in order * to ensure that the 'in queue' doesn't contain * references into the heap. By ensuring this, * we don't need to include the 'in queue' in * the root set when garbage collecting. */ res += receiver->msg.len; ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver); LINK_MESSAGE_PRIVQ(receiver, mp); } else #endif { LINK_MESSAGE(receiver, mp); } #ifdef USE_VM_PROBES if (DTRACE_ENABLED(message_queued)) { DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); Sint tok_label = 0; Sint tok_lastcnt = 0; Sint tok_serial = 0; dtrace_proc_str(receiver, receiver_name); if (seq_trace_token != NIL && is_tuple(seq_trace_token)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token)); } DTRACE6(message_queued, receiver_name, size_object(message), receiver->msg.len, tok_label, tok_lastcnt, tok_serial); } #endif if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) trace_receive(receiver, message); if (locked_msgq) erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); erts_proc_notify_new_message(receiver, #ifdef ERTS_SMP *receiver_locks #else 0 #endif ); #ifndef ERTS_SMP ERTS_HOLE_CHECK(receiver); #endif return res; }
void erts_reserve_heap__(ErtsHeapFactory* factory, Uint need, Uint xtra) { /* internal... */ ErlHeapFragment* bp; switch (factory->mode) { case FACTORY_HALLOC: HRelease(factory->p, factory->hp_end, factory->hp); factory->hp = HAllocX(factory->p, need, xtra); factory->hp_end = factory->hp + need; return; case FACTORY_MESSAGE: { int replace_oh; int replace_msg_hfrag; if (!factory->heap_frags) { ASSERT(factory->message->data.attached == ERTS_MSG_COMBINED_HFRAG); bp = &factory->message->hfrag; } else { /* Fall through */ case FACTORY_HEAP_FRAGS: case FACTORY_TMP: bp = factory->heap_frags; } replace_oh = 0; replace_msg_hfrag = 0; if (bp) { ASSERT(factory->hp >= bp->mem); ASSERT(factory->hp <= factory->hp_end); ASSERT(factory->hp_end == bp->mem + bp->alloc_size); bp->used_size = factory->hp - bp->mem; if (!bp->used_size && factory->heap_frags) { factory->heap_frags = bp->next; bp->next = NULL; ASSERT(!bp->off_heap.first); if (factory->off_heap == &bp->off_heap) replace_oh = !0; if (factory->message && factory->message->data.heap_frag == bp) replace_msg_hfrag = !0; free_message_buffer(bp); } } bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(factory->alloc_type, ERTS_HEAP_FRAG_SIZE(need+xtra)); bp->next = factory->heap_frags; factory->heap_frags = bp; bp->alloc_size = need + xtra; bp->used_size = need + xtra; bp->off_heap.first = NULL; bp->off_heap.overhead = 0; if (replace_oh) { factory->off_heap = &bp->off_heap; factory->off_heap_saved.first = factory->off_heap->first; factory->off_heap_saved.overhead = factory->off_heap->overhead; } if (replace_msg_hfrag) factory->message->data.heap_frag = bp; factory->hp = bp->mem; factory->hp_end = bp->mem + bp->alloc_size; return; } case FACTORY_STATIC: case FACTORY_CLOSED: default: ASSERT(!"Invalid factory mode"); } }
/* Add a message last in message queue */ void erts_queue_message(Process* receiver, ErtsProcLocks *receiver_locks, ErlHeapFragment* bp, Eterm message, Eterm seq_trace_token #ifdef USE_VM_PROBES , Eterm dt_utag #endif ) { ErlMessage* mp; #ifdef ERTS_SMP ErtsProcLocks need_locks; #else ASSERT(bp != NULL || receiver->mbuf == NULL); #endif ERTS_SMP_LC_ASSERT(*receiver_locks == erts_proc_lc_my_proc_locks(receiver)); mp = message_alloc(); #ifdef ERTS_SMP need_locks = ~(*receiver_locks) & (ERTS_PROC_LOCK_MSGQ | ERTS_PROC_LOCK_STATUS); if (need_locks) { *receiver_locks |= need_locks; if (erts_smp_proc_trylock(receiver, need_locks) == EBUSY) { if (need_locks == ERTS_PROC_LOCK_MSGQ) { erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS); need_locks = (ERTS_PROC_LOCK_MSGQ | ERTS_PROC_LOCK_STATUS); } erts_smp_proc_lock(receiver, need_locks); } } if (receiver->is_exiting || ERTS_PROC_PENDING_EXIT(receiver)) { /* Drop message if receiver is exiting or has a pending * exit ... */ if (bp) free_message_buffer(bp); message_free(mp); return; } #endif ERL_MESSAGE_TERM(mp) = message; ERL_MESSAGE_TOKEN(mp) = seq_trace_token; #ifdef USE_VM_PROBES ERL_MESSAGE_DT_UTAG(mp) = dt_utag; #endif mp->next = NULL; mp->data.heap_frag = bp; #ifdef ERTS_SMP if (*receiver_locks & ERTS_PROC_LOCK_MAIN) { /* * We move 'in queue' to 'private queue' and place * message at the end of 'private queue' in order * to ensure that the 'in queue' doesn't contain * references into the heap. By ensuring this, * we don't need to include the 'in queue' in * the root set when garbage collecting. */ ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver); LINK_MESSAGE_PRIVQ(receiver, mp); } else { LINK_MESSAGE(receiver, mp); } #else LINK_MESSAGE(receiver, mp); #endif #ifdef USE_VM_PROBES if (DTRACE_ENABLED(message_queued)) { DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); Sint tok_label = 0; Sint tok_lastcnt = 0; Sint tok_serial = 0; dtrace_proc_str(receiver, receiver_name); if (seq_trace_token != NIL && is_tuple(seq_trace_token)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token)); } DTRACE6(message_queued, receiver_name, size_object(message), receiver->msg.len, tok_label, tok_lastcnt, tok_serial); } #endif notify_new_message(receiver); if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) { trace_receive(receiver, message); } #ifndef ERTS_SMP ERTS_HOLE_CHECK(receiver); #endif }
Eterm erts_msg_distext2heap(Process *pp, ErtsProcLocks *plcksp, ErlHeapFragment **bpp, Eterm *tokenp, ErtsDistExternal *dist_extp) { Eterm msg; Uint tok_sz = 0; Eterm *hp = NULL; Eterm *hp_end = NULL; ErlOffHeap *ohp; Sint sz; *bpp = NULL; sz = erts_decode_dist_ext_size(dist_extp); if (sz < 0) goto decode_error; if (is_not_nil(*tokenp)) { ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); tok_sz = heap_frag->used_size; sz += tok_sz; } if (pp) hp = erts_alloc_message_heap(sz, bpp, &ohp, pp, plcksp); else { *bpp = new_message_buffer(sz); hp = (*bpp)->mem; ohp = &(*bpp)->off_heap; } hp_end = hp + sz; msg = erts_decode_dist_ext(&hp, ohp, dist_extp); if (is_non_value(msg)) goto decode_error; if (is_not_nil(*tokenp)) { ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); *tokenp = copy_struct(*tokenp, tok_sz, &hp, ohp); erts_cleanup_offheap(&heap_frag->off_heap); } erts_free_dist_ext_copy(dist_extp); if (hp_end != hp) { if (!(*bpp)) { HRelease(pp, hp_end, hp); } else { Uint final_size = hp - &(*bpp)->mem[0]; Eterm brefs[2] = {msg, *tokenp}; ASSERT(sz - (hp_end - hp) == final_size); *bpp = erts_resize_message_buffer(*bpp, final_size, &brefs[0], 2); msg = brefs[0]; *tokenp = brefs[1]; } } return msg; decode_error: if (is_not_nil(*tokenp)) { ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); erts_cleanup_offheap(&heap_frag->off_heap); } erts_free_dist_ext_copy(dist_extp); if (*bpp) { free_message_buffer(*bpp); *bpp = NULL; } else if (hp) { HRelease(pp, hp_end, hp); } return THE_NON_VALUE; }
/* * Moves content of message buffer attached to a message into a heap. * The message buffer is deallocated. */ void erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) { /* Unions for typecasts avoids warnings about type-punned pointers and aliasing */ union { Uint** upp; ProcBin **pbpp; ErlFunThing **efpp; ExternalThing **etpp; } oh_list_pp, oh_el_next_pp; union { Uint *up; ProcBin *pbp; ErlFunThing *efp; ExternalThing *etp; } oh_el_p; Eterm term, token, *fhp, *hp; Sint offs; Uint sz; ErlHeapFragment *bp; #ifdef HARD_DEBUG ProcBin *dbg_mso_start = off_heap->mso; ErlFunThing *dbg_fun_start = off_heap->funs; ExternalThing *dbg_external_start = off_heap->externals; Eterm dbg_term, dbg_token; ErlHeapFragment *dbg_bp; Uint *dbg_hp, *dbg_thp_start; Uint dbg_term_sz, dbg_token_sz; #endif bp = msg->data.heap_frag; term = ERL_MESSAGE_TERM(msg); token = ERL_MESSAGE_TOKEN(msg); if (!bp) { ASSERT(is_immed(term) && is_immed(token)); return; } #ifdef HARD_DEBUG dbg_term_sz = size_object(term); dbg_token_sz = size_object(token); ASSERT(bp->size == dbg_term_sz + dbg_token_sz); dbg_bp = new_message_buffer(bp->size); dbg_hp = dbg_bp->mem; dbg_term = copy_struct(term, dbg_term_sz, &dbg_hp, &dbg_bp->off_heap); dbg_token = copy_struct(token, dbg_token_sz, &dbg_hp, &dbg_bp->off_heap); dbg_thp_start = *hpp; #endif ASSERT(bp); msg->data.attached = NULL; off_heap->overhead += bp->off_heap.overhead; sz = bp->size; #ifdef DEBUG if (is_not_immed(term)) { ASSERT(bp->mem <= ptr_val(term)); ASSERT(bp->mem + bp->size > ptr_val(term)); } if (is_not_immed(token)) { ASSERT(bp->mem <= ptr_val(token)); ASSERT(bp->mem + bp->size > ptr_val(token)); } #endif fhp = bp->mem; hp = *hpp; offs = hp - fhp; oh_list_pp.upp = NULL; oh_el_next_pp.upp = NULL; /* Shut up compiler warning */ oh_el_p.up = NULL; /* Shut up compiler warning */ while (sz--) { Uint cpy_sz; Eterm val = *fhp++; switch (primary_tag(val)) { case TAG_PRIMARY_IMMED1: *hp++ = val; break; case TAG_PRIMARY_LIST: case TAG_PRIMARY_BOXED: ASSERT(bp->mem <= ptr_val(val)); ASSERT(bp->mem + bp->size > ptr_val(val)); *hp++ = offset_ptr(val, offs); break; case TAG_PRIMARY_HEADER: *hp++ = val; switch (val & _HEADER_SUBTAG_MASK) { case ARITYVAL_SUBTAG: break; case REFC_BINARY_SUBTAG: oh_list_pp.pbpp = &off_heap->mso; oh_el_p.up = (hp-1); oh_el_next_pp.pbpp = &(oh_el_p.pbp)->next; cpy_sz = thing_arityval(val); goto cpy_words; case FUN_SUBTAG: #ifndef HYBRID oh_list_pp.efpp = &off_heap->funs; oh_el_p.up = (hp-1); oh_el_next_pp.efpp = &(oh_el_p.efp)->next; #endif cpy_sz = thing_arityval(val); goto cpy_words; case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: oh_list_pp.etpp = &off_heap->externals; oh_el_p.up = (hp-1); oh_el_next_pp.etpp = &(oh_el_p.etp)->next; cpy_sz = thing_arityval(val); goto cpy_words; default: cpy_sz = header_arity(val); cpy_words: sz -= cpy_sz; while (cpy_sz >= 8) { cpy_sz -= 8; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; } switch (cpy_sz) { case 7: *hp++ = *fhp++; case 6: *hp++ = *fhp++; case 5: *hp++ = *fhp++; case 4: *hp++ = *fhp++; case 3: *hp++ = *fhp++; case 2: *hp++ = *fhp++; case 1: *hp++ = *fhp++; default: break; } if (oh_list_pp.upp) { #ifdef HARD_DEBUG Uint *dbg_old_oh_list_p = *oh_list_pp.upp; #endif /* Add to offheap list */ *oh_el_next_pp.upp = *oh_list_pp.upp; *oh_list_pp.upp = oh_el_p.up; ASSERT(*hpp <= oh_el_p.up); ASSERT(hp > oh_el_p.up); #ifdef HARD_DEBUG switch (val & _HEADER_SUBTAG_MASK) { case REFC_BINARY_SUBTAG: ASSERT(off_heap->mso == *oh_list_pp.pbpp); ASSERT(off_heap->mso->next == (ProcBin *) dbg_old_oh_list_p); break; #ifndef HYBRID case FUN_SUBTAG: ASSERT(off_heap->funs == *oh_list_pp.efpp); ASSERT(off_heap->funs->next == (ErlFunThing *) dbg_old_oh_list_p); break; #endif case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: ASSERT(off_heap->externals == *oh_list_pp.etpp); ASSERT(off_heap->externals->next == (ExternalThing *) dbg_old_oh_list_p); break; default: ASSERT(0); } #endif oh_list_pp.upp = NULL; } break; } break; } } ASSERT(bp->size == hp - *hpp); *hpp = hp; if (is_not_immed(token)) { ASSERT(bp->mem <= ptr_val(token)); ASSERT(bp->mem + bp->size > ptr_val(token)); ERL_MESSAGE_TOKEN(msg) = offset_ptr(token, offs); #ifdef HARD_DEBUG ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TOKEN(msg))); ASSERT(hp > ptr_val(ERL_MESSAGE_TOKEN(msg))); #endif } if (is_not_immed(term)) { ASSERT(bp->mem <= ptr_val(term)); ASSERT(bp->mem + bp->size > ptr_val(term)); ERL_MESSAGE_TERM(msg) = offset_ptr(term, offs); #ifdef HARD_DEBUG ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TERM(msg))); ASSERT(hp > ptr_val(ERL_MESSAGE_TERM(msg))); #endif } #ifdef HARD_DEBUG { int i, j; { ProcBin *mso = off_heap->mso; i = j = 0; while (mso != dbg_mso_start) { mso = mso->next; i++; } mso = bp->off_heap.mso; while (mso) { mso = mso->next; j++; } ASSERT(i == j); } { ErlFunThing *fun = off_heap->funs; i = j = 0; while (fun != dbg_fun_start) { fun = fun->next; i++; } fun = bp->off_heap.funs; while (fun) { fun = fun->next; j++; } ASSERT(i == j); } { ExternalThing *external = off_heap->externals; i = j = 0; while (external != dbg_external_start) { external = external->next; i++; } external = bp->off_heap.externals; while (external) { external = external->next; j++; } ASSERT(i == j); } } #endif bp->off_heap.mso = NULL; #ifndef HYBRID bp->off_heap.funs = NULL; #endif bp->off_heap.externals = NULL; free_message_buffer(bp); #ifdef HARD_DEBUG ASSERT(eq(ERL_MESSAGE_TERM(msg), dbg_term)); ASSERT(eq(ERL_MESSAGE_TOKEN(msg), dbg_token)); free_message_buffer(dbg_bp); #endif }
/* Add a message last in message queue */ void erts_queue_message(Process* receiver, ErtsProcLocks *receiver_locks, ErlHeapFragment* bp, Eterm message, Eterm seq_trace_token) { ErlMessage* mp; #ifdef ERTS_SMP ErtsProcLocks need_locks; #else ASSERT(bp != NULL || receiver->mbuf == NULL); #endif ERTS_SMP_LC_ASSERT(*receiver_locks == erts_proc_lc_my_proc_locks(receiver)); mp = message_alloc(); #ifdef ERTS_SMP need_locks = ~(*receiver_locks) & (ERTS_PROC_LOCK_MSGQ | ERTS_PROC_LOCK_STATUS); if (need_locks) { *receiver_locks |= need_locks; if (erts_smp_proc_trylock(receiver, need_locks) == EBUSY) { if (need_locks == ERTS_PROC_LOCK_MSGQ) { erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS); need_locks = (ERTS_PROC_LOCK_MSGQ | ERTS_PROC_LOCK_STATUS); } erts_smp_proc_lock(receiver, need_locks); } } if (receiver->is_exiting || ERTS_PROC_PENDING_EXIT(receiver)) { /* Drop message if receiver is exiting or has a pending * exit ... */ if (bp) free_message_buffer(bp); message_free(mp); return; } #endif ERL_MESSAGE_TERM(mp) = message; ERL_MESSAGE_TOKEN(mp) = seq_trace_token; mp->next = NULL; #ifdef ERTS_SMP if (*receiver_locks & ERTS_PROC_LOCK_MAIN) { mp->data.heap_frag = bp; /* * We move 'in queue' to 'private queue' and place * message at the end of 'private queue' in order * to ensure that the 'in queue' doesn't contain * references into the heap. By ensuring this, * we don't need to include the 'in queue' in * the root set when garbage collecting. */ ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver); LINK_MESSAGE_PRIVQ(receiver, mp); } else { mp->data.heap_frag = bp; LINK_MESSAGE(receiver, mp); } #else mp->data.heap_frag = bp; LINK_MESSAGE(receiver, mp); #endif notify_new_message(receiver); if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) { trace_receive(receiver, message); } #ifndef ERTS_SMP ERTS_HOLE_CHECK(receiver); #endif }