void
erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *first_bp)
{
if (first_bp) {
ErlHeapFragment *bp = first_bp;
while (1) {
/* Move any off_heap's into the process */
if (bp->off_heap.first != NULL) {
struct erl_off_heap_header** next_p = &bp->off_heap.first;
while (*next_p != NULL) {
next_p = &((*next_p)->next);
}
*next_p = MSO(proc).first;
MSO(proc).first = bp->off_heap.first;
bp->off_heap.first = NULL;
OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead);
}
MBUF_SIZE(proc) += bp->used_size;
if (!bp->next)
break;
bp = bp->next;
}
/* Link the message buffer */
bp->next = MBUF(proc);
MBUF(proc) = first_bp;
}
}
BIF_RETTYPE port_get_data_1(BIF_ALIST_1)
{
/*
* This is not a signal. See comment above.
*/
Eterm res;
erts_aint_t data;
Port* prt;
prt = data_lookup_port(BIF_P, BIF_ARG_1);
if (!prt)
BIF_ERROR(BIF_P, BADARG);
data = erts_smp_atomic_read_ddrb(&prt->data);
if (data == (erts_aint_t)NULL)
BIF_ERROR(BIF_P, BADARG); /* Port terminated by racing thread */
if ((data & 0x3) != 0) {
res = (Eterm) (UWord) data;
ASSERT(is_immed(res));
}
else {
ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data;
Eterm *hp = HAlloc(BIF_P, pdhp->hsize);
res = copy_struct(pdhp->data, pdhp->hsize, &hp, &MSO(BIF_P));
}
BIF_RET(res);
}
BIF_RETTYPE
prepare_loading_2(BIF_ALIST_2)
{
byte* temp_alloc = NULL;
byte* code;
Uint sz;
Binary* magic;
Eterm reason;
Eterm* hp;
Eterm res;
if (is_not_atom(BIF_ARG_1)) {
error:
erts_free_aligned_binary_bytes(temp_alloc);
BIF_ERROR(BIF_P, BADARG);
}
if ((code = erts_get_aligned_binary_bytes(BIF_ARG_2, &temp_alloc)) == NULL) {
goto error;
}
magic = erts_alloc_loader_state();
sz = binary_size(BIF_ARG_2);
reason = erts_prepare_loading(magic, BIF_P, BIF_P->group_leader,
&BIF_ARG_1, code, sz);
erts_free_aligned_binary_bytes(temp_alloc);
if (reason != NIL) {
hp = HAlloc(BIF_P, 3);
res = TUPLE2(hp, am_error, reason);
BIF_RET(res);
}
hp = HAlloc(BIF_P, PROC_BIN_SIZE);
res = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), magic);
erts_refc_dec(&magic->refc, 1);
BIF_RET(res);
}
ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj)
{
ErlNifResource* resource = DATA_TO_RESOURCE(obj);
ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource);
Eterm* hp = alloc_heap(env,PROC_BIN_SIZE);
return erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary);
}
ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term)
{
Uint sz;
Eterm* hp;
sz = size_object(src_term);
hp = alloc_heap(dst_env, sz);
return copy_struct(src_term, sz, &hp, &MSO(dst_env->proc));
}
static Eterm
alloc_trap_data(Process* c_p)
{
Binary* mbp = erts_create_magic_binary(sizeof(TrapData),
cleanup_trap_data);
Eterm* hp;
hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE);
return erts_mk_magic_ref(&hp, &MSO(c_p), mbp);
}
static Eterm iol2v_promote_acc(iol2v_state_t *state) {
ProcBin *pb;
state->acc = erts_bin_realloc(state->acc, state->acc_size);
pb = (ProcBin*)HAlloc(state->process, PROC_BIN_SIZE);
pb->thing_word = HEADER_PROC_BIN;
pb->size = state->acc_size;
pb->val = state->acc;
pb->bytes = (byte*)(state->acc)->orig_bytes;
pb->flags = 0;
pb->next = MSO(state->process).first;
OH_OVERHEAD(&(MSO(state->process)), pb->size / sizeof(Eterm));
MSO(state->process).first = (struct erl_off_heap_header*)pb;
state->acc_size = 0;
state->acc = NULL;
return make_binary(pb);
}
void enif_clear_env(ErlNifEnv* env)
{
struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)env;
Process* p = &menv->phony_proc;
ASSERT(p == menv->env.proc);
ASSERT(p->id == ERTS_INVALID_PID);
ASSERT(MBUF(p) == menv->env.heap_frag);
if (MBUF(p) != NULL) {
erts_cleanup_offheap(&MSO(p));
clear_offheap(&MSO(p));
free_message_buffer(MBUF(p));
MBUF(p) = NULL;
menv->env.heap_frag = NULL;
}
ASSERT(HEAP_TOP(p) == HEAP_END(p));
menv->env.hp = menv->env.hp_end = HEAP_TOP(p);
ASSERT(!is_offheap(&MSO(p)));
free_tmp_objs(env);
}
static ERTS_INLINE void
link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp)
{
if (bp) {
/* Link the message buffer */
bp->next = MBUF(proc);
MBUF(proc) = bp;
MBUF_SIZE(proc) += bp->used_size;
FLAGS(proc) |= F_FORCE_GC;
/* Move any off_heap's into the process */
if (bp->off_heap.first != NULL) {
struct erl_off_heap_header** next_p = &bp->off_heap.first;
while (*next_p != NULL) {
next_p = &((*next_p)->next);
}
*next_p = MSO(proc).first;
MSO(proc).first = bp->off_heap.first;
bp->off_heap.first = NULL;
OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead);
}
}
}
static Eterm subtract_create_trap_state(Process *p,
ErtsSubtractContext *context) {
Binary *state_bin;
Eterm *hp;
state_bin = erts_create_magic_binary(sizeof(ErtsSubtractContext),
subtract_ctx_bin_dtor);
subtract_ctx_move(context, ERTS_MAGIC_BIN_DATA(state_bin));
hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
return erts_mk_magic_ref(&hp, &MSO(p), state_bin);
}
static ERTS_INLINE void
link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp)
{
if (bp) {
/* Link the message buffer */
bp->next = MBUF(proc);
MBUF(proc) = bp;
MBUF_SIZE(proc) += bp->size;
MSO(proc).overhead += proc->heap_sz; /* Force GC */
/* Move any binaries into the process */
if (bp->off_heap.mso != NULL) {
ProcBin** next_p = &bp->off_heap.mso;
while (*next_p != NULL) {
next_p = &((*next_p)->next);
}
*next_p = MSO(proc).mso;
MSO(proc).mso = bp->off_heap.mso;
bp->off_heap.mso = NULL;
MSO(proc).overhead += bp->off_heap.overhead;
}
/* Move any funs into the process */
#ifndef HYBRID
if (bp->off_heap.funs != NULL) {
ErlFunThing** next_p = &bp->off_heap.funs;
while (*next_p != NULL) {
next_p = &((*next_p)->next);
}
*next_p = MSO(proc).funs;
MSO(proc).funs = bp->off_heap.funs;
bp->off_heap.funs = NULL;
}
#endif
/* Move any external things into the process */
if (bp->off_heap.externals != NULL) {
ExternalThing** next_p = &bp->off_heap.externals;
while (*next_p != NULL) {
next_p = &((*next_p)->next);
}
*next_p = MSO(proc).externals;
MSO(proc).externals = bp->off_heap.externals;
bp->off_heap.externals = NULL;
}
}
}
Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin)
{
if (bin->bin_term != THE_NON_VALUE) {
return bin->bin_term;
}
else if (bin->ref_bin != NULL) {
Binary* bptr = bin->ref_bin;
ProcBin* pb;
Eterm bin_term;
/* !! Copy-paste from new_binary() !! */
pb = (ProcBin *) alloc_heap(env, PROC_BIN_SIZE);
pb->thing_word = HEADER_PROC_BIN;
pb->size = bptr->orig_size;
pb->next = MSO(env->proc).first;
MSO(env->proc).first = (struct erl_off_heap_header*) pb;
pb->val = bptr;
pb->bytes = (byte*) bptr->orig_bytes;
pb->flags = 0;
OH_OVERHEAD(&(MSO(env->proc)), pb->size / sizeof(Eterm));
bin_term = make_binary(pb);
if (erts_refc_read(&bptr->refc, 1) == 1) {
/* Total ownership transfer */
bin->ref_bin = NULL;
bin->bin_term = bin_term;
}
return bin_term;
}
else {
flush_env(env);
bin->bin_term = new_binary(env->proc, bin->data, bin->size);
cache_env(env);
return bin->bin_term;
}
}
BIF_RETTYPE port_get_data_1(BIF_ALIST_1)
{
BIF_RETTYPE res;
Port* prt;
Eterm portid = BIF_ARG_1;
prt = id_or_name2port(BIF_P, portid);
if (!prt) {
BIF_ERROR(BIF_P, BADARG);
}
if (prt->bp == NULL) { /* MUST be CONST! */
res = prt->data;
} else {
Eterm* hp = HAlloc(BIF_P, prt->bp->used_size);
res = copy_struct(prt->data, prt->bp->used_size, &hp, &MSO(BIF_P));
}
erts_smp_port_unlock(prt);
BIF_RET(res);
}
static BIF_RETTYPE iol2v_yield(iol2v_state_t *state) {
if (is_nil(state->magic_reference)) {
iol2v_state_t *boxed_state;
Binary *magic_binary;
Eterm *hp;
magic_binary = erts_create_magic_binary_x(sizeof(*state),
&iol2v_state_destructor, ERTS_ALC_T_BINARY, 1);
boxed_state = ERTS_MAGIC_BIN_UNALIGNED_DATA(magic_binary);
sys_memcpy(boxed_state, state, sizeof(*state));
hp = HAlloc(boxed_state->process, ERTS_MAGIC_REF_THING_SIZE);
boxed_state->magic_reference =
erts_mk_magic_ref(&hp, &MSO(boxed_state->process), magic_binary);
state = boxed_state;
}
ERTS_BIF_YIELD1(bif_export[BIF_iolist_to_iovec_1],
state->process, state->magic_reference);
}
static Eterm
check_process_code(Process* rp, Module* modp)
{
Eterm* start;
char* mod_start;
Uint mod_size;
Eterm* end;
Eterm* sp;
#ifndef HYBRID /* FIND ME! */
ErlFunThing* funp;
int done_gc = 0;
#endif
#define INSIDE(a) (start <= (a) && (a) < end)
if (modp == NULL) { /* Doesn't exist. */
return am_false;
} else if (modp->old_code == NULL) { /* No old code. */
return am_false;
}
/*
* Pick up limits for the module.
*/
start = modp->old_code;
end = (Eterm *)((char *)start + modp->old_code_length);
mod_start = (char *) start;
mod_size = modp->old_code_length;
/*
* Check if current instruction or continuation pointer points into module.
*/
if (INSIDE(rp->i) || INSIDE(rp->cp)) {
return am_true;
}
/*
* Check all continuation pointers stored on the stack.
*/
for (sp = rp->stop; sp < STACK_START(rp); sp++) {
if (is_CP(*sp) && INSIDE(cp_val(*sp))) {
return am_true;
}
}
/*
* Check all continuation pointers stored in stackdump
* and clear exception stackdump if there is a pointer
* to the module.
*/
if (rp->ftrace != NIL) {
struct StackTrace *s;
ASSERT(is_list(rp->ftrace));
s = (struct StackTrace *) big_val(CDR(list_val(rp->ftrace)));
if ((s->pc && INSIDE(s->pc)) ||
(s->current && INSIDE(s->current))) {
rp->freason = EXC_NULL;
rp->fvalue = NIL;
rp->ftrace = NIL;
} else {
int i;
for (i = 0; i < s->depth; i++) {
if (INSIDE(s->trace[i])) {
rp->freason = EXC_NULL;
rp->fvalue = NIL;
rp->ftrace = NIL;
break;
}
}
}
}
/*
* See if there are funs that refer to the old version of the module.
*/
#ifndef HYBRID /* FIND ME! */
rescan:
for (funp = MSO(rp).funs; funp; funp = funp->next) {
Eterm* fun_code;
fun_code = funp->fe->address;
if (INSIDE((Eterm *) funp->fe->address)) {
if (done_gc) {
return am_true;
} else {
/*
* Try to get rid of this fun by garbage collecting.
* Clear both fvalue and ftrace to make sure they
* don't hold any funs.
*/
rp->freason = EXC_NULL;
rp->fvalue = NIL;
rp->ftrace = NIL;
done_gc = 1;
FLAGS(rp) |= F_NEED_FULLSWEEP;
(void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
goto rescan;
}
}
}
#endif
/*
* See if there are constants inside the module referenced by the process.
*/
done_gc = 0;
for (;;) {
ErlMessage* mp;
if (any_heap_ref_ptrs(&rp->fvalue, &rp->fvalue+1, mod_start, mod_size)) {
rp->freason = EXC_NULL;
rp->fvalue = NIL;
rp->ftrace = NIL;
}
if (any_heap_ref_ptrs(rp->stop, rp->hend, mod_start, mod_size)) {
goto need_gc;
}
if (any_heap_refs(rp->heap, rp->htop, mod_start, mod_size)) {
goto need_gc;
}
if (any_heap_refs(rp->old_heap, rp->old_htop, mod_start, mod_size)) {
goto need_gc;
}
if (rp->dictionary != NULL) {
Eterm* start = rp->dictionary->data;
Eterm* end = start + rp->dictionary->used;
if (any_heap_ref_ptrs(start, end, mod_start, mod_size)) {
goto need_gc;
}
}
for (mp = rp->msg.first; mp != NULL; mp = mp->next) {
if (any_heap_ref_ptrs(mp->m, mp->m+2, mod_start, mod_size)) {
goto need_gc;
}
}
break;
need_gc:
if (done_gc) {
return am_true;
} else {
Eterm* literals;
Uint lit_size;
/*
* Try to get rid of constants by by garbage collecting.
* Clear both fvalue and ftrace.
*/
rp->freason = EXC_NULL;
rp->fvalue = NIL;
rp->ftrace = NIL;
done_gc = 1;
FLAGS(rp) |= F_NEED_FULLSWEEP;
(void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
literals = (Eterm *) modp->old_code[MI_LITERALS_START];
lit_size = (Eterm *) modp->old_code[MI_LITERALS_END] - literals;
erts_garbage_collect_literals(rp, literals, lit_size);
}
}
return am_false;
#undef INSIDE
}
ErtsMessage *
erts_try_alloc_message_on_heap(Process *pp,
erts_aint32_t *psp,
ErtsProcLocks *plp,
Uint sz,
Eterm **hpp,
ErlOffHeap **ohpp,
int *on_heap_p)
{
int locked_main = 0;
ErtsMessage *mp;
ASSERT(!(*psp & ERTS_PSFLG_OFF_HEAP_MSGQ));
if ((*psp) & ERTS_PSFLGS_VOLATILE_HEAP)
goto in_message_fragment;
else if (
*plp & ERTS_PROC_LOCK_MAIN
) {
try_on_heap:
if (((*psp) & ERTS_PSFLGS_VOLATILE_HEAP)
|| (pp->flags & F_DISABLE_GC)
|| HEAP_LIMIT(pp) - HEAP_TOP(pp) <= sz) {
/*
* The heap is either potentially in an inconsistent
* state, or not large enough.
*/
if (locked_main) {
*plp &= ~ERTS_PROC_LOCK_MAIN;
erts_proc_unlock(pp, ERTS_PROC_LOCK_MAIN);
}
goto in_message_fragment;
}
*hpp = HEAP_TOP(pp);
HEAP_TOP(pp) = *hpp + sz;
*ohpp = &MSO(pp);
mp = erts_alloc_message(0, NULL);
mp->data.attached = NULL;
*on_heap_p = !0;
}
else if (pp && erts_proc_trylock(pp, ERTS_PROC_LOCK_MAIN) == 0) {
locked_main = 1;
*psp = erts_atomic32_read_nob(&pp->state);
*plp |= ERTS_PROC_LOCK_MAIN;
goto try_on_heap;
}
else {
in_message_fragment:
if (!((*psp) & ERTS_PSFLG_ON_HEAP_MSGQ)) {
mp = erts_alloc_message(sz, hpp);
*ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap;
}
else {
mp = erts_alloc_message(0, NULL);
if (!sz) {
*hpp = NULL;
*ohpp = NULL;
}
else {
ErlHeapFragment *bp;
bp = new_message_buffer(sz);
*hpp = &bp->mem[0];
mp->data.heap_frag = bp;
*ohpp = &bp->off_heap;
}
}
*on_heap_p = 0;
}
return mp;
}
static BIF_RETTYPE
port_call(Process* c_p, Eterm arg1, Eterm arg2, Eterm arg3)
{
Uint op;
Port *p;
Uint size;
byte *bytes;
byte *endp;
ErlDrvSizeT real_size;
erts_driver_t *drv;
byte port_input[256]; /* Default input buffer to encode in */
byte port_result[256]; /* Buffer for result from port. */
byte* port_resp; /* Pointer to result buffer. */
char *prc;
ErlDrvSSizeT ret;
Eterm res;
Sint result_size;
Eterm *hp;
Eterm *hp_end; /* To satisfy hybrid heap architecture */
unsigned ret_flags = 0U;
int fpe_was_unmasked;
bytes = &port_input[0];
port_resp = port_result;
/* trace of port scheduling with virtual process descheduling
* lock wait
*/
if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) {
trace_virtual_sched(c_p, am_out);
}
if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
profile_runnable_proc(c_p, am_inactive);
}
p = id_or_name2port(c_p, arg1);
if (!p) {
error:
if (port_resp != port_result &&
!(ret_flags & DRIVER_CALL_KEEP_BUFFER)) {
driver_free(port_resp);
}
if (bytes != &port_input[0])
erts_free(ERTS_ALC_T_PORT_CALL_BUF, bytes);
/* Need to virtual schedule in the process if there
* was an error.
*/
if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) {
trace_virtual_sched(c_p, am_in);
}
if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
profile_runnable_proc(c_p, am_active);
}
if (p)
erts_port_release(p);
#ifdef ERTS_SMP
ERTS_SMP_BIF_CHK_PENDING_EXIT(c_p, ERTS_PROC_LOCK_MAIN);
#else
ERTS_BIF_CHK_EXITED(c_p);
#endif
BIF_ERROR(c_p, BADARG);
}
if ((drv = p->drv_ptr) == NULL) {
goto error;
}
if (drv->call == NULL) {
goto error;
}
if (!term_to_Uint(arg2, &op)) {
goto error;
}
p->caller = c_p->id;
/* Lock taken, virtual schedule of port */
if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) {
trace_sched_ports_where(p, am_in, am_call);
}
if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(p)) {
profile_runnable_port(p, am_active);
}
size = erts_encode_ext_size(arg3);
if (size > sizeof(port_input))
bytes = erts_alloc(ERTS_ALC_T_PORT_CALL_BUF, size);
endp = bytes;
erts_encode_ext(arg3, &endp);
real_size = endp - bytes;
if (real_size > size) {
erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n",
__FILE__, __LINE__, endp - (bytes + size));
}
erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(driver_call)) {
DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE);
DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE);
dtrace_pid_str(p->connected, process_str);
dtrace_port_str(p, port_str);
DTRACE5(driver_call, process_str, port_str, p->name, op, real_size);
}
#endif
prc = (char *) port_resp;
fpe_was_unmasked = erts_block_fpe();
ret = drv->call((ErlDrvData)p->drv_data,
(unsigned) op,
(char *) bytes,
(int) real_size,
&prc,
(int) sizeof(port_result),
&ret_flags);
erts_unblock_fpe(fpe_was_unmasked);
if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) {
trace_sched_ports_where(p, am_out, am_call);
}
if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(p)) {
profile_runnable_port(p, am_inactive);
}
port_resp = (byte *) prc;
p->caller = NIL;
erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
#ifdef HARDDEBUG
{
ErlDrvSizeT z;
printf("real_size = %ld,%d, ret = %ld,%d\r\n", (unsigned long) real_size,
(int) real_size, (unsigned long)ret, (int) ret);
printf("[");
for(z = 0; z < real_size; ++z) {
printf("%d, ",(int) bytes[z]);
}
printf("]\r\n");
printf("[");
for(z = 0; z < ret; ++z) {
printf("%d, ",(int) port_resp[z]);
}
printf("]\r\n");
}
#endif
if (ret <= 0 || port_resp[0] != VERSION_MAGIC) {
/* Error or a binary without magic/ with wrong magic */
goto error;
}
result_size = erts_decode_ext_size(port_resp, ret);
if (result_size < 0) {
goto error;
}
hp = HAlloc(c_p, result_size);
hp_end = hp + result_size;
endp = port_resp;
res = erts_decode_ext(&hp, &MSO(c_p), &endp);
if (res == THE_NON_VALUE) {
goto error;
}
HRelease(c_p, hp_end, hp);
if (port_resp != port_result && !(ret_flags & DRIVER_CALL_KEEP_BUFFER)) {
driver_free(port_resp);
}
if (bytes != &port_input[0])
erts_free(ERTS_ALC_T_PORT_CALL_BUF, bytes);
if (p)
erts_port_release(p);
#ifdef ERTS_SMP
ERTS_SMP_BIF_CHK_PENDING_EXIT(c_p, ERTS_PROC_LOCK_MAIN);
#else
ERTS_BIF_CHK_EXITED(c_p);
#endif
if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) {
trace_virtual_sched(c_p, am_in);
}
if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
profile_runnable_proc(c_p, am_active);
}
return res;
}
BIF_RETTYPE persistent_term_put_2(BIF_ALIST_2)
{
static const Uint ITERATIONS_PER_RED = 32;
ErtsPersistentTermPut2Context* ctx;
Eterm state_mref = THE_NON_VALUE;
long iterations_until_trap;
long max_iterations;
#define PUT_TRAP_CODE \
BIF_TRAP2(bif_export[BIF_persistent_term_put_2], BIF_P, state_mref, BIF_ARG_2)
#define TRAPPING_COPY_TABLE_PUT(TABLE_DEST, OLD_TABLE, NEW_SIZE, COPY_TYPE, LOC_NAME) \
TRAPPING_COPY_TABLE(TABLE_DEST, OLD_TABLE, NEW_SIZE, COPY_TYPE, LOC_NAME, PUT_TRAP_CODE)
#ifdef DEBUG
(void)ITERATIONS_PER_RED;
iterations_until_trap = max_iterations =
GET_SMALL_RANDOM_INT(ERTS_BIF_REDS_LEFT(BIF_P) + (Uint)&ctx);
#else
iterations_until_trap = max_iterations =
ITERATIONS_PER_RED * ERTS_BIF_REDS_LEFT(BIF_P);
#endif
if (is_internal_magic_ref(BIF_ARG_1) &&
(ERTS_MAGIC_BIN_DESTRUCTOR(erts_magic_ref2bin(BIF_ARG_1)) ==
persistent_term_put_2_ctx_bin_dtor)) {
/* Restore state after a trap */
Binary* state_bin;
state_mref = BIF_ARG_1;
state_bin = erts_magic_ref2bin(state_mref);
ctx = ERTS_MAGIC_BIN_DATA(state_bin);
ASSERT(BIF_P->flags & F_DISABLE_GC);
erts_set_gc_state(BIF_P, 1);
switch (ctx->trap_location) {
case PUT2_TRAP_LOCATION_NEW_KEY:
goto L_PUT2_TRAP_LOCATION_NEW_KEY;
case PUT2_TRAP_LOCATION_REPLACE_VALUE:
goto L_PUT2_TRAP_LOCATION_REPLACE_VALUE;
}
} else {
/* Save state in magic bin in case trapping is necessary */
Eterm* hp;
Binary* state_bin = erts_create_magic_binary(sizeof(ErtsPersistentTermPut2Context),
persistent_term_put_2_ctx_bin_dtor);
hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE);
state_mref = erts_mk_magic_ref(&hp, &MSO(BIF_P), state_bin);
ctx = ERTS_MAGIC_BIN_DATA(state_bin);
/*
* IMPORTANT: The following field is used to detect if
* persistent_term_put_2_ctx_bin_dtor needs to free memory
*/
ctx->cpy_ctx.new_table = NULL;
}
if (!try_seize_update_permission(BIF_P)) {
ERTS_BIF_YIELD2(bif_export[BIF_persistent_term_put_2],
BIF_P, BIF_ARG_1, BIF_ARG_2);
}
ctx->hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
ctx->key = BIF_ARG_1;
ctx->term = BIF_ARG_2;
ctx->entry_index = lookup(ctx->hash_table, ctx->key);
ctx->heap[0] = make_arityval(2);
ctx->heap[1] = ctx->key;
ctx->heap[2] = ctx->term;
ctx->tuple = make_tuple(ctx->heap);
if (is_nil(ctx->hash_table->term[ctx->entry_index])) {
Uint new_size = ctx->hash_table->allocated;
if (MUST_GROW(ctx->hash_table)) {
new_size *= 2;
}
TRAPPING_COPY_TABLE_PUT(ctx->hash_table,
ctx->hash_table,
new_size,
ERTS_PERSISTENT_TERM_CPY_NO_REHASH,
PUT2_TRAP_LOCATION_NEW_KEY);
ctx->entry_index = lookup(ctx->hash_table, ctx->key);
ctx->hash_table->num_entries++;
} else {
Eterm tuple = ctx->hash_table->term[ctx->entry_index];
Eterm old_term;
ASSERT(is_tuple_arity(tuple, 2));
old_term = boxed_val(tuple)[2];
if (EQ(ctx->term, old_term)) {
/* Same value. No need to update anything. */
release_update_permission(0);
BIF_RET(am_ok);
} else {
/* Mark the old term for deletion. */
mark_for_deletion(ctx->hash_table, ctx->entry_index);
TRAPPING_COPY_TABLE_PUT(ctx->hash_table,
ctx->hash_table,
ctx->hash_table->allocated,
ERTS_PERSISTENT_TERM_CPY_NO_REHASH,
PUT2_TRAP_LOCATION_REPLACE_VALUE);
}
}
{
Uint term_size;
Uint lit_area_size;
ErlOffHeap code_off_heap;
ErtsLiteralArea* literal_area;
erts_shcopy_t info;
Eterm* ptr;
/*
* Preserve internal sharing in the term by using the
* sharing-preserving functions. However, literals must
* be copied in case the module holding them are unloaded.
*/
INITIALIZE_SHCOPY(info);
info.copy_literals = 1;
term_size = copy_shared_calculate(ctx->tuple, &info);
ERTS_INIT_OFF_HEAP(&code_off_heap);
lit_area_size = ERTS_LITERAL_AREA_ALLOC_SIZE(term_size);
literal_area = erts_alloc(ERTS_ALC_T_LITERAL, lit_area_size);
ptr = &literal_area->start[0];
literal_area->end = ptr + term_size;
ctx->tuple = copy_shared_perform(ctx->tuple, term_size, &info, &ptr, &code_off_heap);
ASSERT(tuple_val(ctx->tuple) == literal_area->start);
literal_area->off_heap = code_off_heap.first;
DESTROY_SHCOPY(info);
erts_set_literal_tag(&ctx->tuple, literal_area->start, term_size);
ctx->hash_table->term[ctx->entry_index] = ctx->tuple;
erts_schedule_thr_prgr_later_op(table_updater, ctx->hash_table, &thr_prog_op);
suspend_updater(BIF_P);
}
BUMP_REDS(BIF_P, (max_iterations - iterations_until_trap) / ITERATIONS_PER_RED);
ERTS_BIF_YIELD_RETURN(BIF_P, am_ok);
}
BIF_RETTYPE persistent_term_erase_1(BIF_ALIST_1)
{
static const Uint ITERATIONS_PER_RED = 32;
ErtsPersistentTermErase1Context* ctx;
Eterm state_mref = THE_NON_VALUE;
long iterations_until_trap;
long max_iterations;
#ifdef DEBUG
(void)ITERATIONS_PER_RED;
iterations_until_trap = max_iterations =
GET_SMALL_RANDOM_INT(ERTS_BIF_REDS_LEFT(BIF_P) + (Uint)&ctx);
#else
iterations_until_trap = max_iterations =
ITERATIONS_PER_RED * ERTS_BIF_REDS_LEFT(BIF_P);
#endif
#define ERASE_TRAP_CODE \
BIF_TRAP1(bif_export[BIF_persistent_term_erase_1], BIF_P, state_mref);
#define TRAPPING_COPY_TABLE_ERASE(TABLE_DEST, OLD_TABLE, NEW_SIZE, REHASH, LOC_NAME) \
TRAPPING_COPY_TABLE(TABLE_DEST, OLD_TABLE, NEW_SIZE, REHASH, LOC_NAME, ERASE_TRAP_CODE)
if (is_internal_magic_ref(BIF_ARG_1) &&
(ERTS_MAGIC_BIN_DESTRUCTOR(erts_magic_ref2bin(BIF_ARG_1)) ==
persistent_term_erase_1_ctx_bin_dtor)) {
/* Restore the state after a trap */
Binary* state_bin;
state_mref = BIF_ARG_1;
state_bin = erts_magic_ref2bin(state_mref);
ctx = ERTS_MAGIC_BIN_DATA(state_bin);
ASSERT(BIF_P->flags & F_DISABLE_GC);
erts_set_gc_state(BIF_P, 1);
switch (ctx->trap_location) {
case ERASE1_TRAP_LOCATION_TMP_COPY:
goto L_ERASE1_TRAP_LOCATION_TMP_COPY;
case ERASE1_TRAP_LOCATION_FINAL_COPY:
goto L_ERASE1_TRAP_LOCATION_FINAL_COPY;
}
} else {
/* Save state in magic bin in case trapping is necessary */
Eterm* hp;
Binary* state_bin = erts_create_magic_binary(sizeof(ErtsPersistentTermErase1Context),
persistent_term_erase_1_ctx_bin_dtor);
hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE);
state_mref = erts_mk_magic_ref(&hp, &MSO(BIF_P), state_bin);
ctx = ERTS_MAGIC_BIN_DATA(state_bin);
/*
* IMPORTANT: The following two fields are used to detect if
* persistent_term_erase_1_ctx_bin_dtor needs to free memory
*/
ctx->cpy_ctx.new_table = NULL;
ctx->tmp_table = NULL;
}
if (!try_seize_update_permission(BIF_P)) {
ERTS_BIF_YIELD1(bif_export[BIF_persistent_term_erase_1],
BIF_P, BIF_ARG_1);
}
ctx->key = BIF_ARG_1;
ctx->old_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
ctx->entry_index = lookup(ctx->old_table, ctx->key);
ctx->old_term = ctx->old_table->term[ctx->entry_index];
if (is_boxed(ctx->old_term)) {
Uint new_size;
/*
* Since we don't use any delete markers, we must rehash
* the table when deleting terms to ensure that all terms
* can still be reached if there are hash collisions.
* We can't rehash in place and it would not be safe to modify
* the old table yet, so we will first need a new
* temporary table copy of the same size as the old one.
*/
ASSERT(is_tuple_arity(ctx->old_term, 2));
TRAPPING_COPY_TABLE_ERASE(ctx->tmp_table,
ctx->old_table,
ctx->old_table->allocated,
ERTS_PERSISTENT_TERM_CPY_TEMP,
ERASE1_TRAP_LOCATION_TMP_COPY);
/*
* Delete the term from the temporary table. Then copy the
* temporary table to a new table, rehashing the entries
* while copying.
*/
ctx->tmp_table->term[ctx->entry_index] = NIL;
ctx->tmp_table->num_entries--;
new_size = ctx->tmp_table->allocated;
if (MUST_SHRINK(ctx->tmp_table)) {
new_size /= 2;
}
TRAPPING_COPY_TABLE_ERASE(ctx->new_table,
ctx->tmp_table,
new_size,
ERTS_PERSISTENT_TERM_CPY_REHASH,
ERASE1_TRAP_LOCATION_FINAL_COPY);
erts_free(ERTS_ALC_T_PERSISTENT_TERM_TMP, ctx->tmp_table);
/*
* IMPORTANT: Memory management depends on that ctx->tmp_table
* is set to NULL on the line below
*/
ctx->tmp_table = NULL;
mark_for_deletion(ctx->old_table, ctx->entry_index);
erts_schedule_thr_prgr_later_op(table_updater, ctx->new_table, &thr_prog_op);
suspend_updater(BIF_P);
BUMP_REDS(BIF_P, (max_iterations - iterations_until_trap) / ITERATIONS_PER_RED);
ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
}
/*
* Key is not present. Nothing to do.
*/
ASSERT(is_nil(ctx->old_term));
release_update_permission(0);
BIF_RET(am_false);
}
int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
ErlNifEnv* msg_env, ERL_NIF_TERM msg)
{
struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)msg_env;
ErtsProcLocks rp_locks = 0;
Process* rp;
Process* c_p;
ErlHeapFragment* frags;
#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
ErtsProcLocks rp_had_locks;
#endif
Eterm receiver = to_pid->pid;
int flush_me = 0;
if (env != NULL) {
c_p = env->proc;
if (receiver == c_p->id) {
rp_locks = ERTS_PROC_LOCK_MAIN;
flush_me = 1;
}
}
else {
#ifdef ERTS_SMP
c_p = NULL;
#else
erl_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM");
#endif
}
#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
rp_had_locks = rp_locks;
#endif
rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN,
receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC);
if (rp == NULL) {
ASSERT(env == NULL || receiver != c_p->id);
return 0;
}
flush_env(msg_env);
frags = menv->env.heap_frag;
ASSERT(frags == MBUF(&menv->phony_proc));
if (frags != NULL) {
/* Move all offheap's from phony proc to the first fragment.
Quick and dirty, but erts_move_msg_mbuf_to_heap doesn't care. */
ASSERT(!is_offheap(&frags->off_heap));
frags->off_heap = MSO(&menv->phony_proc);
clear_offheap(&MSO(&menv->phony_proc));
menv->env.heap_frag = NULL;
MBUF(&menv->phony_proc) = NULL;
}
ASSERT(!is_offheap(&MSO(&menv->phony_proc)));
if (flush_me) {
flush_env(env); /* Needed for ERTS_HOLE_CHECK */
}
erts_queue_message(rp, &rp_locks, frags, msg, am_undefined);
if (rp_locks) {
ERTS_SMP_LC_ASSERT(rp_locks == (rp_had_locks | (ERTS_PROC_LOCK_MSGQ |
ERTS_PROC_LOCK_STATUS)));
erts_smp_proc_unlock(rp, (ERTS_PROC_LOCK_MSGQ | ERTS_PROC_LOCK_STATUS));
}
erts_smp_proc_dec_refc(rp);
if (flush_me) {
cache_env(env);
}
return 1;
}
