void
erts_cleanup_messages(ErtsMessage *msgp)
{
ErtsMessage *mp = msgp;
while (mp) {
ErtsMessage *fmp;
ErlHeapFragment *bp;
if (is_non_value(ERL_MESSAGE_TERM(mp))) {
if (is_not_immed(ERL_MESSAGE_TOKEN(mp))) {
bp = (ErlHeapFragment *) mp->data.dist_ext->ext_endp;
erts_cleanup_offheap(&bp->off_heap);
}
if (mp->data.dist_ext)
erts_free_dist_ext_copy(mp->data.dist_ext);
}
else {
if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG)
bp = mp->data.heap_frag;
else {
bp = mp->hfrag.next;
erts_cleanup_offheap(&mp->hfrag.off_heap);
}
if (bp)
free_message_buffer(bp);
}
fmp = mp;
mp = mp->next;
erts_free_message(fmp);
}
}
static void
erts_cleanup_message(ErtsMessage *mp)
{
ErlHeapFragment *bp;
if (ERTS_SIG_IS_EXTERNAL_MSG(mp) || ERTS_SIG_IS_NON_MSG(mp)) {
ErtsDistExternal *edep = erts_proc_sig_get_external(mp);
if (edep) {
erts_free_dist_ext_copy(edep);
if (mp->data.heap_frag == &mp->hfrag) {
ASSERT(ERTS_SIG_IS_EXTERNAL_MSG(mp));
mp->data.heap_frag = ERTS_MSG_COMBINED_HFRAG;
}
}
}
if (ERTS_SIG_IS_MSG(mp) && mp->data.attached != ERTS_MSG_COMBINED_HFRAG) {
bp = mp->data.heap_frag;
} else {
/* All non msg signals are combined HFRAG messages,
but we overwrite the mp->data field with the
nm_signal queue ptr so have to fix that here
before freeing it. */
mp->data.attached = ERTS_MSG_COMBINED_HFRAG;
bp = mp->hfrag.next;
erts_cleanup_offheap(&mp->hfrag.off_heap);
}
if (bp)
free_message_buffer(bp);
}
BIF_RETTYPE port_set_data_2(BIF_ALIST_2)
{
Port* prt;
Eterm portid = BIF_ARG_1;
Eterm data = BIF_ARG_2;
prt = id_or_name2port(BIF_P, portid);
if (!prt) {
BIF_ERROR(BIF_P, BADARG);
}
if (prt->bp != NULL) {
free_message_buffer(prt->bp);
prt->bp = NULL;
}
if (IS_CONST(data)) {
prt->data = data;
} else {
Uint size;
ErlHeapFragment* bp;
Eterm* hp;
size = size_object(data);
prt->bp = bp = new_message_buffer(size);
hp = bp->mem;
prt->data = copy_struct(data, size, &hp, &bp->off_heap);
}
erts_smp_port_unlock(prt);
BIF_RET(am_true);
}
void erts_factory_trim_and_close(ErtsHeapFactory* factory,
Eterm *brefs, Uint brefs_size)
{
ErlHeapFragment *bp;
switch (factory->mode) {
case FACTORY_MESSAGE: {
ErtsMessage *mp = factory->message;
if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG) {
if (!factory->heap_frags) {
Uint sz = factory->hp - factory->hp_start;
mp = erts_shrink_message(mp, sz, brefs, brefs_size);
factory->message = mp;
factory->mode = FACTORY_CLOSED;
return;
}
/*else we don't trim multi fragmented messages for now (off_heap...) */
break;
}
/* Fall through... */
}
case FACTORY_HEAP_FRAGS:
bp = factory->heap_frags;
if (!bp)
break;
if (bp->next == NULL) {
Uint used_sz = factory->hp - bp->mem;
ASSERT(used_sz <= bp->alloc_size);
if (used_sz > 0) {
if (used_sz != bp->alloc_size)
bp = erts_resize_message_buffer(bp, used_sz,
brefs, brefs_size);
}
else {
free_message_buffer(bp);
bp = NULL;
}
factory->heap_frags = bp;
if (factory->mode == FACTORY_MESSAGE)
factory->message->data.heap_frag = bp;
factory->mode = FACTORY_CLOSED;
return;
}
/*else we don't trim multi fragmented messages for now (off_heap...) */
default:
break;
}
erts_factory_close(factory);
}
void enif_clear_env(ErlNifEnv* env)
{
struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)env;
Process* p = &menv->phony_proc;
ASSERT(p == menv->env.proc);
ASSERT(p->id == ERTS_INVALID_PID);
ASSERT(MBUF(p) == menv->env.heap_frag);
if (MBUF(p) != NULL) {
erts_cleanup_offheap(&MSO(p));
clear_offheap(&MSO(p));
free_message_buffer(MBUF(p));
MBUF(p) = NULL;
menv->env.heap_frag = NULL;
}
ASSERT(HEAP_TOP(p) == HEAP_END(p));
menv->env.hp = menv->env.hp_end = HEAP_TOP(p);
ASSERT(!is_offheap(&MSO(p)));
free_tmp_objs(env);
}
ErlHeapFragment*
erts_resize_message_buffer(ErlHeapFragment *bp, Uint size,
Eterm *brefs, Uint brefs_size)
{
#ifdef DEBUG
int i;
#endif
#ifdef HARD_DEBUG
ErlHeapFragment *dbg_bp;
Eterm *dbg_brefs;
Uint dbg_size;
Uint dbg_tot_size;
Eterm *dbg_hp;
#endif
ErlHeapFragment* nbp;
#ifdef DEBUG
{
Uint off_sz = size < bp->used_size ? size : bp->used_size;
for (i = 0; i < brefs_size; i++) {
Eterm *ptr;
if (is_immed(brefs[i]))
continue;
ptr = ptr_val(brefs[i]);
ASSERT(&bp->mem[0] <= ptr && ptr < &bp->mem[0] + off_sz);
}
}
#endif
if (size >= (bp->used_size - bp->used_size / 16)) {
bp->used_size = size;
return bp;
}
#ifdef HARD_DEBUG
dbg_brefs = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(Eterm *)*brefs_size);
dbg_bp = new_message_buffer(bp->used_size);
dbg_hp = dbg_bp->mem;
dbg_tot_size = 0;
for (i = 0; i < brefs_size; i++) {
dbg_size = size_object(brefs[i]);
dbg_tot_size += dbg_size;
dbg_brefs[i] = copy_struct(brefs[i], dbg_size, &dbg_hp,
&dbg_bp->off_heap);
}
ASSERT(dbg_tot_size == (size < bp->used_size ? size : bp->used_size));
#endif
nbp = (ErlHeapFragment*) ERTS_HEAP_REALLOC(ERTS_ALC_T_HEAP_FRAG,
(void *) bp,
ERTS_HEAP_FRAG_SIZE(bp->alloc_size),
ERTS_HEAP_FRAG_SIZE(size));
if (bp != nbp) {
Uint off_sz = size < nbp->used_size ? size : nbp->used_size;
Eterm *sp = &bp->mem[0];
Eterm *ep = sp + off_sz;
Sint offs = &nbp->mem[0] - sp;
erts_offset_off_heap(&nbp->off_heap, offs, sp, ep);
erts_offset_heap(&nbp->mem[0], off_sz, offs, sp, ep);
if (brefs && brefs_size)
erts_offset_heap_ptr(brefs, brefs_size, offs, sp, ep);
#ifdef DEBUG
for (i = 0; i < brefs_size; i++) {
Eterm *ptr;
if (is_immed(brefs[i]))
continue;
ptr = ptr_val(brefs[i]);
ASSERT(&nbp->mem[0] <= ptr && ptr < &nbp->mem[0] + off_sz);
}
#endif
}
nbp->alloc_size = size;
nbp->used_size = size;
#ifdef HARD_DEBUG
for (i = 0; i < brefs_size; i++)
ASSERT(eq(dbg_brefs[i], brefs[i]));
free_message_buffer(dbg_bp);
erts_free(ERTS_ALC_T_UNDEF, dbg_brefs);
#endif
return nbp;
}
/*
* Garbage collect a process.
*
* p: Pointer to the process structure.
* need: Number of Eterm words needed on the heap.
* objv: Array of terms to add to rootset; that is to preserve.
* nobj: Number of objects in objv.
*/
int
erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
{
int ret;
struct slave_syscall_gc *cmd = erts_alloc(ERTS_ALC_T_TMP, sizeof(*cmd));
Eterm *dram_objv;
int copy_objv = nobj != 0 && !epiphany_in_dram(objv);
if (copy_objv) {
dram_objv = erts_alloc(ERTS_ALC_T_TMP, nobj*sizeof(Eterm));
memcpy(dram_objv, objv, nobj*sizeof(Eterm));
} else {
dram_objv = objv;
}
#ifdef DEBUG
{
int i;
for (i = 0; i < nobj; i++)
ASSERT(is_immed(objv[i]) || epiphany_in_dram(ptr_val(objv[i])));
}
#endif
cmd->need = need;
cmd->objv = dram_objv;
cmd->nobj = nobj;
slave_state_swapout(p, &cmd->state);
erts_master_syscall(SLAVE_SYSCALL_GC, cmd);
if (copy_objv) {
memcpy(objv, dram_objv, nobj*sizeof(Eterm));
erts_free(ERTS_ALC_T_TMP, dram_objv);
}
/*
* The garbage collector will have set mbuf to NULL without freeing it. We
* do so here. See remove_message_buffers in the master.
*/
if (MBUF(p) != NULL) {
free_message_buffer(MBUF(p));
ASSERT(cmd->state.mbuf == NULL);
}
slave_state_swapin(p, &cmd->state);
#ifdef CHECK_FOR_HOLES
/*
* We intentionally do not rescan the areas copied by the GC.
* We trust the GC not to leave any holes.
*/
p->last_htop = p->htop;
p->last_mbuf = 0;
#endif
#ifdef DEBUG
/*
* The scanning for pointers from the old_heap into the new_heap or
* heap fragments turned out to be costly, so we remember how far we
* have scanned this time and will start scanning there next time.
* (We will not detect wild writes into the old heap, or modifications
* of the old heap in-between garbage collections.)
*/
p->last_old_htop = p->old_htop;
#endif
ret = cmd->ret;
erts_free(ERTS_ALC_T_TMP, cmd);
return ret;
}
/* send object-communication packet */
void
SendMessage(MessageBuffer mb, long long destination)
{
ExecTable et;
SendPort sp,spe,free_sp;
MessageBuffer mbp;
int i,flag,rval;
unsigned int least_access;
#if 0
printf("SendMessage enter\n");
#endif
/* find send port already assigned */
for(i=0,flag=0,free_sp=(SendPort)0,sp=apgw_send_tbl;i<MAX_APGW_SEND;i++,sp++)
{
if(sp->exid == destination)
{
flag=1;
break;}
if(sp->exid==0LL)
{
free_sp =sp;
}
}
if(flag)
{ /* Send Port exists */
if(sp->fd != 0)
{ /* found connected port */
if((i = write(sp->fd, (char *)(&(mb->buffer_size)),sizeof(int))) <sizeof(int))
{ /* error */
perror("SendMessage(write):");
SendError(mb);
free_send_port(sp);
return;
}
if((i = write(sp->fd, mb->buffer, mb->buffer_size)) < mb->buffer_size)
{ /* error */
perror("SendMessage(write):");
SendError(mb);
free_send_port(sp);
return;
}
set_last_access_send(sp);
free_message_buffer(mb);
#if 0
printf("Message sent using connected send port\n");
#endif
return;
}
else if(sp->waiting != (MessageBuffer)0)
{ /* send port is waiting for address resolution */
for(mbp=sp->waiting;
mbp->next != (MessageBuffer)0;mbp=mbp->next)
;
mbp->next=mb;
#if 0
printf("message is appended on waiting port\n");
#endif
return;
}
else
{ /* if reach here, it's bug */
printf("Invalid send port ?? sp(%x) fd(%x) waiting(%x) exid(%08x%08x)\n",
sp,sp->fd,sp->waiting,
(int)(((sp->exid)>>32)&INTEGER_MASK),
(int)((sp->exid)&INTEGER_MASK));
return;
}
}
else
{
if(!free_sp)
/*
* Moves content of message buffer attached to a message into a heap.
* The message buffer is deallocated.
*/
void
erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
{
struct erl_off_heap_header* oh;
Eterm term, token, *fhp, *hp;
Sint offs;
Uint sz;
ErlHeapFragment *bp;
#ifdef USE_VM_PROBES
Eterm utag;
#endif
#ifdef HARD_DEBUG
struct erl_off_heap_header* dbg_oh_start = off_heap->first;
Eterm dbg_term, dbg_token;
ErlHeapFragment *dbg_bp;
Uint *dbg_hp, *dbg_thp_start;
Uint dbg_term_sz, dbg_token_sz;
#ifdef USE_VM_PROBES
Eterm dbg_utag;
Uint dbg_utag_sz;
#endif
#endif
bp = msg->data.heap_frag;
term = ERL_MESSAGE_TERM(msg);
token = ERL_MESSAGE_TOKEN(msg);
#ifdef USE_VM_PROBES
utag = ERL_MESSAGE_DT_UTAG(msg);
#endif
if (!bp) {
#ifdef USE_VM_PROBES
ASSERT(is_immed(term) && is_immed(token) && is_immed(utag));
#else
ASSERT(is_immed(term) && is_immed(token));
#endif
return;
}
#ifdef HARD_DEBUG
dbg_term_sz = size_object(term);
dbg_token_sz = size_object(token);
dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz);
#ifdef USE_VM_PROBES
dbg_utag_sz = size_object(utag);
dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz + dbg_utag_sz );
#endif
/*ASSERT(dbg_term_sz + dbg_token_sz == erts_msg_used_frag_sz(msg));
Copied size may be smaller due to removed SubBins's or garbage.
Copied size may be larger due to duplicated shared terms.
*/
dbg_hp = dbg_bp->mem;
dbg_term = copy_struct(term, dbg_term_sz, &dbg_hp, &dbg_bp->off_heap);
dbg_token = copy_struct(token, dbg_token_sz, &dbg_hp, &dbg_bp->off_heap);
#ifdef USE_VM_PROBES
dbg_utag = copy_struct(utag, dbg_utag_sz, &dbg_hp, &dbg_bp->off_heap);
#endif
dbg_thp_start = *hpp;
#endif
if (bp->next != NULL) {
move_multi_frags(hpp, off_heap, bp, msg->m,
#ifdef USE_VM_PROBES
3
#else
2
#endif
);
goto copy_done;
}
OH_OVERHEAD(off_heap, bp->off_heap.overhead);
sz = bp->used_size;
ASSERT(is_immed(term) || in_heapfrag(ptr_val(term),bp));
ASSERT(is_immed(token) || in_heapfrag(ptr_val(token),bp));
fhp = bp->mem;
hp = *hpp;
offs = hp - fhp;
oh = NULL;
while (sz--) {
Uint cpy_sz;
Eterm val = *fhp++;
switch (primary_tag(val)) {
case TAG_PRIMARY_IMMED1:
*hp++ = val;
break;
case TAG_PRIMARY_LIST:
case TAG_PRIMARY_BOXED:
ASSERT(in_heapfrag(ptr_val(val), bp));
*hp++ = offset_ptr(val, offs);
break;
case TAG_PRIMARY_HEADER:
*hp++ = val;
switch (val & _HEADER_SUBTAG_MASK) {
case ARITYVAL_SUBTAG:
break;
case REFC_BINARY_SUBTAG:
case FUN_SUBTAG:
case EXTERNAL_PID_SUBTAG:
case EXTERNAL_PORT_SUBTAG:
case EXTERNAL_REF_SUBTAG:
oh = (struct erl_off_heap_header*) (hp-1);
cpy_sz = thing_arityval(val);
goto cpy_words;
default:
cpy_sz = header_arity(val);
cpy_words:
ASSERT(sz >= cpy_sz);
sz -= cpy_sz;
while (cpy_sz >= 8) {
cpy_sz -= 8;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
}
switch (cpy_sz) {
case 7: *hp++ = *fhp++;
case 6: *hp++ = *fhp++;
case 5: *hp++ = *fhp++;
case 4: *hp++ = *fhp++;
case 3: *hp++ = *fhp++;
case 2: *hp++ = *fhp++;
case 1: *hp++ = *fhp++;
default: break;
}
if (oh) {
/* Add to offheap list */
oh->next = off_heap->first;
off_heap->first = oh;
ASSERT(*hpp <= (Eterm*)oh);
ASSERT(hp > (Eterm*)oh);
oh = NULL;
}
break;
}
break;
}
}
ASSERT(bp->used_size == hp - *hpp);
*hpp = hp;
if (is_not_immed(token)) {
ASSERT(in_heapfrag(ptr_val(token), bp));
ERL_MESSAGE_TOKEN(msg) = offset_ptr(token, offs);
#ifdef HARD_DEBUG
ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TOKEN(msg)));
ASSERT(hp > ptr_val(ERL_MESSAGE_TOKEN(msg)));
#endif
}
if (is_not_immed(term)) {
ASSERT(in_heapfrag(ptr_val(term),bp));
ERL_MESSAGE_TERM(msg) = offset_ptr(term, offs);
#ifdef HARD_DEBUG
ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TERM(msg)));
ASSERT(hp > ptr_val(ERL_MESSAGE_TERM(msg)));
#endif
}
#ifdef USE_VM_PROBES
if (is_not_immed(utag)) {
ASSERT(in_heapfrag(ptr_val(utag), bp));
ERL_MESSAGE_DT_UTAG(msg) = offset_ptr(utag, offs);
#ifdef HARD_DEBUG
ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_DT_UTAG(msg)));
ASSERT(hp > ptr_val(ERL_MESSAGE_DT_UTAG(msg)));
#endif
}
#endif
copy_done:
#ifdef HARD_DEBUG
{
int i, j;
ErlHeapFragment* frag;
{
struct erl_off_heap_header* dbg_oh = off_heap->first;
i = j = 0;
while (dbg_oh != dbg_oh_start) {
dbg_oh = dbg_oh->next;
i++;
}
for (frag=bp; frag; frag=frag->next) {
dbg_oh = frag->off_heap.first;
while (dbg_oh) {
dbg_oh = dbg_oh->next;
j++;
}
}
ASSERT(i == j);
}
}
#endif
bp->off_heap.first = NULL;
free_message_buffer(bp);
msg->data.heap_frag = NULL;
#ifdef HARD_DEBUG
ASSERT(eq(ERL_MESSAGE_TERM(msg), dbg_term));
ASSERT(eq(ERL_MESSAGE_TOKEN(msg), dbg_token));
#ifdef USE_VM_PROBES
ASSERT(eq(ERL_MESSAGE_DT_UTAG(msg), dbg_utag));
#endif
free_message_buffer(dbg_bp);
#endif
}
/* Add a message last in message queue */
static Sint
queue_message(Process *c_p,
Process* receiver,
ErtsProcLocks *receiver_locks,
erts_aint32_t *receiver_state,
ErlHeapFragment* bp,
Eterm message,
Eterm seq_trace_token
#ifdef USE_VM_PROBES
, Eterm dt_utag
#endif
)
{
Sint res;
ErlMessage* mp;
int locked_msgq = 0;
erts_aint_t state;
#ifndef ERTS_SMP
ASSERT(bp != NULL || receiver->mbuf == NULL);
#endif
ERTS_SMP_LC_ASSERT(*receiver_locks == erts_proc_lc_my_proc_locks(receiver));
mp = message_alloc();
if (receiver_state)
state = *receiver_state;
else
state = erts_smp_atomic32_read_acqb(&receiver->state);
#ifdef ERTS_SMP
if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
goto exiting;
if (!(*receiver_locks & ERTS_PROC_LOCK_MSGQ)) {
if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
if (*receiver_locks & ERTS_PROC_LOCK_STATUS) {
erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS);
need_locks |= ERTS_PROC_LOCK_STATUS;
}
erts_smp_proc_lock(receiver, need_locks);
}
locked_msgq = 1;
state = erts_smp_atomic32_read_nob(&receiver->state);
if (receiver_state)
*receiver_state = state;
}
#endif
if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
#ifdef ERTS_SMP
exiting:
#endif
/* Drop message if receiver is exiting or has a pending exit... */
if (locked_msgq)
erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
if (bp)
free_message_buffer(bp);
message_free(mp);
return 0;
}
ERL_MESSAGE_TERM(mp) = message;
ERL_MESSAGE_TOKEN(mp) = seq_trace_token;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = dt_utag;
#endif
mp->next = NULL;
mp->data.heap_frag = bp;
#ifndef ERTS_SMP
res = receiver->msg.len;
#else
res = receiver->msg_inq.len;
if (*receiver_locks & ERTS_PROC_LOCK_MAIN) {
/*
* We move 'in queue' to 'private queue' and place
* message at the end of 'private queue' in order
* to ensure that the 'in queue' doesn't contain
* references into the heap. By ensuring this,
* we don't need to include the 'in queue' in
* the root set when garbage collecting.
*/
res += receiver->msg.len;
ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver);
LINK_MESSAGE_PRIVQ(receiver, mp);
}
else
#endif
{
LINK_MESSAGE(receiver, mp);
}
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
Sint tok_label = 0;
Sint tok_lastcnt = 0;
Sint tok_serial = 0;
dtrace_proc_str(receiver, receiver_name);
if (seq_trace_token != NIL && is_tuple(seq_trace_token)) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token));
}
DTRACE6(message_queued,
receiver_name, size_object(message), receiver->msg.len,
tok_label, tok_lastcnt, tok_serial);
}
#endif
if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE))
trace_receive(receiver, message);
if (locked_msgq)
erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
erts_proc_notify_new_message(receiver,
#ifdef ERTS_SMP
*receiver_locks
#else
0
#endif
);
#ifndef ERTS_SMP
ERTS_HOLE_CHECK(receiver);
#endif
return res;
}
void erts_reserve_heap__(ErtsHeapFactory* factory, Uint need, Uint xtra)
{
/* internal... */
ErlHeapFragment* bp;
switch (factory->mode) {
case FACTORY_HALLOC:
HRelease(factory->p, factory->hp_end, factory->hp);
factory->hp = HAllocX(factory->p, need, xtra);
factory->hp_end = factory->hp + need;
return;
case FACTORY_MESSAGE: {
int replace_oh;
int replace_msg_hfrag;
if (!factory->heap_frags) {
ASSERT(factory->message->data.attached == ERTS_MSG_COMBINED_HFRAG);
bp = &factory->message->hfrag;
}
else {
/* Fall through */
case FACTORY_HEAP_FRAGS:
case FACTORY_TMP:
bp = factory->heap_frags;
}
replace_oh = 0;
replace_msg_hfrag = 0;
if (bp) {
ASSERT(factory->hp >= bp->mem);
ASSERT(factory->hp <= factory->hp_end);
ASSERT(factory->hp_end == bp->mem + bp->alloc_size);
bp->used_size = factory->hp - bp->mem;
if (!bp->used_size && factory->heap_frags) {
factory->heap_frags = bp->next;
bp->next = NULL;
ASSERT(!bp->off_heap.first);
if (factory->off_heap == &bp->off_heap)
replace_oh = !0;
if (factory->message && factory->message->data.heap_frag == bp)
replace_msg_hfrag = !0;
free_message_buffer(bp);
}
}
bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(factory->alloc_type,
ERTS_HEAP_FRAG_SIZE(need+xtra));
bp->next = factory->heap_frags;
factory->heap_frags = bp;
bp->alloc_size = need + xtra;
bp->used_size = need + xtra;
bp->off_heap.first = NULL;
bp->off_heap.overhead = 0;
if (replace_oh) {
factory->off_heap = &bp->off_heap;
factory->off_heap_saved.first = factory->off_heap->first;
factory->off_heap_saved.overhead = factory->off_heap->overhead;
}
if (replace_msg_hfrag)
factory->message->data.heap_frag = bp;
factory->hp = bp->mem;
factory->hp_end = bp->mem + bp->alloc_size;
return;
}
case FACTORY_STATIC:
case FACTORY_CLOSED:
default:
ASSERT(!"Invalid factory mode");
}
}
/* Add a message last in message queue */
void
erts_queue_message(Process* receiver,
ErtsProcLocks *receiver_locks,
ErlHeapFragment* bp,
Eterm message,
Eterm seq_trace_token
#ifdef USE_VM_PROBES
, Eterm dt_utag
#endif
)
{
ErlMessage* mp;
#ifdef ERTS_SMP
ErtsProcLocks need_locks;
#else
ASSERT(bp != NULL || receiver->mbuf == NULL);
#endif
ERTS_SMP_LC_ASSERT(*receiver_locks == erts_proc_lc_my_proc_locks(receiver));
mp = message_alloc();
#ifdef ERTS_SMP
need_locks = ~(*receiver_locks) & (ERTS_PROC_LOCK_MSGQ
| ERTS_PROC_LOCK_STATUS);
if (need_locks) {
*receiver_locks |= need_locks;
if (erts_smp_proc_trylock(receiver, need_locks) == EBUSY) {
if (need_locks == ERTS_PROC_LOCK_MSGQ) {
erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS);
need_locks = (ERTS_PROC_LOCK_MSGQ
| ERTS_PROC_LOCK_STATUS);
}
erts_smp_proc_lock(receiver, need_locks);
}
}
if (receiver->is_exiting || ERTS_PROC_PENDING_EXIT(receiver)) {
/* Drop message if receiver is exiting or has a pending
* exit ...
*/
if (bp)
free_message_buffer(bp);
message_free(mp);
return;
}
#endif
ERL_MESSAGE_TERM(mp) = message;
ERL_MESSAGE_TOKEN(mp) = seq_trace_token;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = dt_utag;
#endif
mp->next = NULL;
mp->data.heap_frag = bp;
#ifdef ERTS_SMP
if (*receiver_locks & ERTS_PROC_LOCK_MAIN) {
/*
* We move 'in queue' to 'private queue' and place
* message at the end of 'private queue' in order
* to ensure that the 'in queue' doesn't contain
* references into the heap. By ensuring this,
* we don't need to include the 'in queue' in
* the root set when garbage collecting.
*/
ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver);
LINK_MESSAGE_PRIVQ(receiver, mp);
}
else {
LINK_MESSAGE(receiver, mp);
}
#else
LINK_MESSAGE(receiver, mp);
#endif
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
Sint tok_label = 0;
Sint tok_lastcnt = 0;
Sint tok_serial = 0;
dtrace_proc_str(receiver, receiver_name);
if (seq_trace_token != NIL && is_tuple(seq_trace_token)) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token));
}
DTRACE6(message_queued,
receiver_name, size_object(message), receiver->msg.len,
tok_label, tok_lastcnt, tok_serial);
}
#endif
notify_new_message(receiver);
if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) {
trace_receive(receiver, message);
}
#ifndef ERTS_SMP
ERTS_HOLE_CHECK(receiver);
#endif
}
Eterm
erts_msg_distext2heap(Process *pp,
ErtsProcLocks *plcksp,
ErlHeapFragment **bpp,
Eterm *tokenp,
ErtsDistExternal *dist_extp)
{
Eterm msg;
Uint tok_sz = 0;
Eterm *hp = NULL;
Eterm *hp_end = NULL;
ErlOffHeap *ohp;
Sint sz;
*bpp = NULL;
sz = erts_decode_dist_ext_size(dist_extp);
if (sz < 0)
goto decode_error;
if (is_not_nil(*tokenp)) {
ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp);
tok_sz = heap_frag->used_size;
sz += tok_sz;
}
if (pp)
hp = erts_alloc_message_heap(sz, bpp, &ohp, pp, plcksp);
else {
*bpp = new_message_buffer(sz);
hp = (*bpp)->mem;
ohp = &(*bpp)->off_heap;
}
hp_end = hp + sz;
msg = erts_decode_dist_ext(&hp, ohp, dist_extp);
if (is_non_value(msg))
goto decode_error;
if (is_not_nil(*tokenp)) {
ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp);
*tokenp = copy_struct(*tokenp, tok_sz, &hp, ohp);
erts_cleanup_offheap(&heap_frag->off_heap);
}
erts_free_dist_ext_copy(dist_extp);
if (hp_end != hp) {
if (!(*bpp)) {
HRelease(pp, hp_end, hp);
}
else {
Uint final_size = hp - &(*bpp)->mem[0];
Eterm brefs[2] = {msg, *tokenp};
ASSERT(sz - (hp_end - hp) == final_size);
*bpp = erts_resize_message_buffer(*bpp, final_size, &brefs[0], 2);
msg = brefs[0];
*tokenp = brefs[1];
}
}
return msg;
decode_error:
if (is_not_nil(*tokenp)) {
ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp);
erts_cleanup_offheap(&heap_frag->off_heap);
}
erts_free_dist_ext_copy(dist_extp);
if (*bpp) {
free_message_buffer(*bpp);
*bpp = NULL;
}
else if (hp) {
HRelease(pp, hp_end, hp);
}
return THE_NON_VALUE;
}
/*
* Moves content of message buffer attached to a message into a heap.
* The message buffer is deallocated.
*/
void
erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
{
/* Unions for typecasts avoids warnings about type-punned pointers and aliasing */
union {
Uint** upp;
ProcBin **pbpp;
ErlFunThing **efpp;
ExternalThing **etpp;
} oh_list_pp, oh_el_next_pp;
union {
Uint *up;
ProcBin *pbp;
ErlFunThing *efp;
ExternalThing *etp;
} oh_el_p;
Eterm term, token, *fhp, *hp;
Sint offs;
Uint sz;
ErlHeapFragment *bp;
#ifdef HARD_DEBUG
ProcBin *dbg_mso_start = off_heap->mso;
ErlFunThing *dbg_fun_start = off_heap->funs;
ExternalThing *dbg_external_start = off_heap->externals;
Eterm dbg_term, dbg_token;
ErlHeapFragment *dbg_bp;
Uint *dbg_hp, *dbg_thp_start;
Uint dbg_term_sz, dbg_token_sz;
#endif
bp = msg->data.heap_frag;
term = ERL_MESSAGE_TERM(msg);
token = ERL_MESSAGE_TOKEN(msg);
if (!bp) {
ASSERT(is_immed(term) && is_immed(token));
return;
}
#ifdef HARD_DEBUG
dbg_term_sz = size_object(term);
dbg_token_sz = size_object(token);
ASSERT(bp->size == dbg_term_sz + dbg_token_sz);
dbg_bp = new_message_buffer(bp->size);
dbg_hp = dbg_bp->mem;
dbg_term = copy_struct(term, dbg_term_sz, &dbg_hp, &dbg_bp->off_heap);
dbg_token = copy_struct(token, dbg_token_sz, &dbg_hp, &dbg_bp->off_heap);
dbg_thp_start = *hpp;
#endif
ASSERT(bp);
msg->data.attached = NULL;
off_heap->overhead += bp->off_heap.overhead;
sz = bp->size;
#ifdef DEBUG
if (is_not_immed(term)) {
ASSERT(bp->mem <= ptr_val(term));
ASSERT(bp->mem + bp->size > ptr_val(term));
}
if (is_not_immed(token)) {
ASSERT(bp->mem <= ptr_val(token));
ASSERT(bp->mem + bp->size > ptr_val(token));
}
#endif
fhp = bp->mem;
hp = *hpp;
offs = hp - fhp;
oh_list_pp.upp = NULL;
oh_el_next_pp.upp = NULL; /* Shut up compiler warning */
oh_el_p.up = NULL; /* Shut up compiler warning */
while (sz--) {
Uint cpy_sz;
Eterm val = *fhp++;
switch (primary_tag(val)) {
case TAG_PRIMARY_IMMED1:
*hp++ = val;
break;
case TAG_PRIMARY_LIST:
case TAG_PRIMARY_BOXED:
ASSERT(bp->mem <= ptr_val(val));
ASSERT(bp->mem + bp->size > ptr_val(val));
*hp++ = offset_ptr(val, offs);
break;
case TAG_PRIMARY_HEADER:
*hp++ = val;
switch (val & _HEADER_SUBTAG_MASK) {
case ARITYVAL_SUBTAG:
break;
case REFC_BINARY_SUBTAG:
oh_list_pp.pbpp = &off_heap->mso;
oh_el_p.up = (hp-1);
oh_el_next_pp.pbpp = &(oh_el_p.pbp)->next;
cpy_sz = thing_arityval(val);
goto cpy_words;
case FUN_SUBTAG:
#ifndef HYBRID
oh_list_pp.efpp = &off_heap->funs;
oh_el_p.up = (hp-1);
oh_el_next_pp.efpp = &(oh_el_p.efp)->next;
#endif
cpy_sz = thing_arityval(val);
goto cpy_words;
case EXTERNAL_PID_SUBTAG:
case EXTERNAL_PORT_SUBTAG:
case EXTERNAL_REF_SUBTAG:
oh_list_pp.etpp = &off_heap->externals;
oh_el_p.up = (hp-1);
oh_el_next_pp.etpp = &(oh_el_p.etp)->next;
cpy_sz = thing_arityval(val);
goto cpy_words;
default:
cpy_sz = header_arity(val);
cpy_words:
sz -= cpy_sz;
while (cpy_sz >= 8) {
cpy_sz -= 8;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
*hp++ = *fhp++;
}
switch (cpy_sz) {
case 7: *hp++ = *fhp++;
case 6: *hp++ = *fhp++;
case 5: *hp++ = *fhp++;
case 4: *hp++ = *fhp++;
case 3: *hp++ = *fhp++;
case 2: *hp++ = *fhp++;
case 1: *hp++ = *fhp++;
default: break;
}
if (oh_list_pp.upp) {
#ifdef HARD_DEBUG
Uint *dbg_old_oh_list_p = *oh_list_pp.upp;
#endif
/* Add to offheap list */
*oh_el_next_pp.upp = *oh_list_pp.upp;
*oh_list_pp.upp = oh_el_p.up;
ASSERT(*hpp <= oh_el_p.up);
ASSERT(hp > oh_el_p.up);
#ifdef HARD_DEBUG
switch (val & _HEADER_SUBTAG_MASK) {
case REFC_BINARY_SUBTAG:
ASSERT(off_heap->mso == *oh_list_pp.pbpp);
ASSERT(off_heap->mso->next
== (ProcBin *) dbg_old_oh_list_p);
break;
#ifndef HYBRID
case FUN_SUBTAG:
ASSERT(off_heap->funs == *oh_list_pp.efpp);
ASSERT(off_heap->funs->next
== (ErlFunThing *) dbg_old_oh_list_p);
break;
#endif
case EXTERNAL_PID_SUBTAG:
case EXTERNAL_PORT_SUBTAG:
case EXTERNAL_REF_SUBTAG:
ASSERT(off_heap->externals
== *oh_list_pp.etpp);
ASSERT(off_heap->externals->next
== (ExternalThing *) dbg_old_oh_list_p);
break;
default:
ASSERT(0);
}
#endif
oh_list_pp.upp = NULL;
}
break;
}
break;
}
}
ASSERT(bp->size == hp - *hpp);
*hpp = hp;
if (is_not_immed(token)) {
ASSERT(bp->mem <= ptr_val(token));
ASSERT(bp->mem + bp->size > ptr_val(token));
ERL_MESSAGE_TOKEN(msg) = offset_ptr(token, offs);
#ifdef HARD_DEBUG
ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TOKEN(msg)));
ASSERT(hp > ptr_val(ERL_MESSAGE_TOKEN(msg)));
#endif
}
if (is_not_immed(term)) {
ASSERT(bp->mem <= ptr_val(term));
ASSERT(bp->mem + bp->size > ptr_val(term));
ERL_MESSAGE_TERM(msg) = offset_ptr(term, offs);
#ifdef HARD_DEBUG
ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TERM(msg)));
ASSERT(hp > ptr_val(ERL_MESSAGE_TERM(msg)));
#endif
}
#ifdef HARD_DEBUG
{
int i, j;
{
ProcBin *mso = off_heap->mso;
i = j = 0;
while (mso != dbg_mso_start) {
mso = mso->next;
i++;
}
mso = bp->off_heap.mso;
while (mso) {
mso = mso->next;
j++;
}
ASSERT(i == j);
}
{
ErlFunThing *fun = off_heap->funs;
i = j = 0;
while (fun != dbg_fun_start) {
fun = fun->next;
i++;
}
fun = bp->off_heap.funs;
while (fun) {
fun = fun->next;
j++;
}
ASSERT(i == j);
}
{
ExternalThing *external = off_heap->externals;
i = j = 0;
while (external != dbg_external_start) {
external = external->next;
i++;
}
external = bp->off_heap.externals;
while (external) {
external = external->next;
j++;
}
ASSERT(i == j);
}
}
#endif
bp->off_heap.mso = NULL;
#ifndef HYBRID
bp->off_heap.funs = NULL;
#endif
bp->off_heap.externals = NULL;
free_message_buffer(bp);
#ifdef HARD_DEBUG
ASSERT(eq(ERL_MESSAGE_TERM(msg), dbg_term));
ASSERT(eq(ERL_MESSAGE_TOKEN(msg), dbg_token));
free_message_buffer(dbg_bp);
#endif
}
/* Add a message last in message queue */
void
erts_queue_message(Process* receiver,
ErtsProcLocks *receiver_locks,
ErlHeapFragment* bp,
Eterm message,
Eterm seq_trace_token)
{
ErlMessage* mp;
#ifdef ERTS_SMP
ErtsProcLocks need_locks;
#else
ASSERT(bp != NULL || receiver->mbuf == NULL);
#endif
ERTS_SMP_LC_ASSERT(*receiver_locks == erts_proc_lc_my_proc_locks(receiver));
mp = message_alloc();
#ifdef ERTS_SMP
need_locks = ~(*receiver_locks) & (ERTS_PROC_LOCK_MSGQ
| ERTS_PROC_LOCK_STATUS);
if (need_locks) {
*receiver_locks |= need_locks;
if (erts_smp_proc_trylock(receiver, need_locks) == EBUSY) {
if (need_locks == ERTS_PROC_LOCK_MSGQ) {
erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS);
need_locks = (ERTS_PROC_LOCK_MSGQ
| ERTS_PROC_LOCK_STATUS);
}
erts_smp_proc_lock(receiver, need_locks);
}
}
if (receiver->is_exiting || ERTS_PROC_PENDING_EXIT(receiver)) {
/* Drop message if receiver is exiting or has a pending
* exit ...
*/
if (bp)
free_message_buffer(bp);
message_free(mp);
return;
}
#endif
ERL_MESSAGE_TERM(mp) = message;
ERL_MESSAGE_TOKEN(mp) = seq_trace_token;
mp->next = NULL;
#ifdef ERTS_SMP
if (*receiver_locks & ERTS_PROC_LOCK_MAIN) {
mp->data.heap_frag = bp;
/*
* We move 'in queue' to 'private queue' and place
* message at the end of 'private queue' in order
* to ensure that the 'in queue' doesn't contain
* references into the heap. By ensuring this,
* we don't need to include the 'in queue' in
* the root set when garbage collecting.
*/
ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver);
LINK_MESSAGE_PRIVQ(receiver, mp);
}
else {
mp->data.heap_frag = bp;
LINK_MESSAGE(receiver, mp);
}
#else
mp->data.heap_frag = bp;
LINK_MESSAGE(receiver, mp);
#endif
notify_new_message(receiver);
if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) {
trace_receive(receiver, message);
}
#ifndef ERTS_SMP
ERTS_HOLE_CHECK(receiver);
#endif
}
