void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *first_bp) { if (first_bp) { ErlHeapFragment *bp = first_bp; while (1) { /* Move any off_heap's into the process */ if (bp->off_heap.first != NULL) { struct erl_off_heap_header** next_p = &bp->off_heap.first; while (*next_p != NULL) { next_p = &((*next_p)->next); } *next_p = MSO(proc).first; MSO(proc).first = bp->off_heap.first; bp->off_heap.first = NULL; OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead); } MBUF_SIZE(proc) += bp->used_size; if (!bp->next) break; bp = bp->next; } /* Link the message buffer */ bp->next = MBUF(proc); MBUF(proc) = first_bp; } }
BIF_RETTYPE port_get_data_1(BIF_ALIST_1) { /* * This is not a signal. See comment above. */ Eterm res; erts_aint_t data; Port* prt; prt = data_lookup_port(BIF_P, BIF_ARG_1); if (!prt) BIF_ERROR(BIF_P, BADARG); data = erts_smp_atomic_read_ddrb(&prt->data); if (data == (erts_aint_t)NULL) BIF_ERROR(BIF_P, BADARG); /* Port terminated by racing thread */ if ((data & 0x3) != 0) { res = (Eterm) (UWord) data; ASSERT(is_immed(res)); } else { ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; Eterm *hp = HAlloc(BIF_P, pdhp->hsize); res = copy_struct(pdhp->data, pdhp->hsize, &hp, &MSO(BIF_P)); } BIF_RET(res); }
BIF_RETTYPE prepare_loading_2(BIF_ALIST_2) { byte* temp_alloc = NULL; byte* code; Uint sz; Binary* magic; Eterm reason; Eterm* hp; Eterm res; if (is_not_atom(BIF_ARG_1)) { error: erts_free_aligned_binary_bytes(temp_alloc); BIF_ERROR(BIF_P, BADARG); } if ((code = erts_get_aligned_binary_bytes(BIF_ARG_2, &temp_alloc)) == NULL) { goto error; } magic = erts_alloc_loader_state(); sz = binary_size(BIF_ARG_2); reason = erts_prepare_loading(magic, BIF_P, BIF_P->group_leader, &BIF_ARG_1, code, sz); erts_free_aligned_binary_bytes(temp_alloc); if (reason != NIL) { hp = HAlloc(BIF_P, 3); res = TUPLE2(hp, am_error, reason); BIF_RET(res); } hp = HAlloc(BIF_P, PROC_BIN_SIZE); res = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), magic); erts_refc_dec(&magic->refc, 1); BIF_RET(res); }
ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource); Eterm* hp = alloc_heap(env,PROC_BIN_SIZE); return erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary); }
ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term) { Uint sz; Eterm* hp; sz = size_object(src_term); hp = alloc_heap(dst_env, sz); return copy_struct(src_term, sz, &hp, &MSO(dst_env->proc)); }
static Eterm alloc_trap_data(Process* c_p) { Binary* mbp = erts_create_magic_binary(sizeof(TrapData), cleanup_trap_data); Eterm* hp; hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE); return erts_mk_magic_ref(&hp, &MSO(c_p), mbp); }
static Eterm iol2v_promote_acc(iol2v_state_t *state) { ProcBin *pb; state->acc = erts_bin_realloc(state->acc, state->acc_size); pb = (ProcBin*)HAlloc(state->process, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; pb->size = state->acc_size; pb->val = state->acc; pb->bytes = (byte*)(state->acc)->orig_bytes; pb->flags = 0; pb->next = MSO(state->process).first; OH_OVERHEAD(&(MSO(state->process)), pb->size / sizeof(Eterm)); MSO(state->process).first = (struct erl_off_heap_header*)pb; state->acc_size = 0; state->acc = NULL; return make_binary(pb); }
void enif_clear_env(ErlNifEnv* env) { struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)env; Process* p = &menv->phony_proc; ASSERT(p == menv->env.proc); ASSERT(p->id == ERTS_INVALID_PID); ASSERT(MBUF(p) == menv->env.heap_frag); if (MBUF(p) != NULL) { erts_cleanup_offheap(&MSO(p)); clear_offheap(&MSO(p)); free_message_buffer(MBUF(p)); MBUF(p) = NULL; menv->env.heap_frag = NULL; } ASSERT(HEAP_TOP(p) == HEAP_END(p)); menv->env.hp = menv->env.hp_end = HEAP_TOP(p); ASSERT(!is_offheap(&MSO(p))); free_tmp_objs(env); }
static ERTS_INLINE void link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp) { if (bp) { /* Link the message buffer */ bp->next = MBUF(proc); MBUF(proc) = bp; MBUF_SIZE(proc) += bp->used_size; FLAGS(proc) |= F_FORCE_GC; /* Move any off_heap's into the process */ if (bp->off_heap.first != NULL) { struct erl_off_heap_header** next_p = &bp->off_heap.first; while (*next_p != NULL) { next_p = &((*next_p)->next); } *next_p = MSO(proc).first; MSO(proc).first = bp->off_heap.first; bp->off_heap.first = NULL; OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead); } } }
static Eterm subtract_create_trap_state(Process *p, ErtsSubtractContext *context) { Binary *state_bin; Eterm *hp; state_bin = erts_create_magic_binary(sizeof(ErtsSubtractContext), subtract_ctx_bin_dtor); subtract_ctx_move(context, ERTS_MAGIC_BIN_DATA(state_bin)); hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); return erts_mk_magic_ref(&hp, &MSO(p), state_bin); }
static ERTS_INLINE void link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp) { if (bp) { /* Link the message buffer */ bp->next = MBUF(proc); MBUF(proc) = bp; MBUF_SIZE(proc) += bp->size; MSO(proc).overhead += proc->heap_sz; /* Force GC */ /* Move any binaries into the process */ if (bp->off_heap.mso != NULL) { ProcBin** next_p = &bp->off_heap.mso; while (*next_p != NULL) { next_p = &((*next_p)->next); } *next_p = MSO(proc).mso; MSO(proc).mso = bp->off_heap.mso; bp->off_heap.mso = NULL; MSO(proc).overhead += bp->off_heap.overhead; } /* Move any funs into the process */ #ifndef HYBRID if (bp->off_heap.funs != NULL) { ErlFunThing** next_p = &bp->off_heap.funs; while (*next_p != NULL) { next_p = &((*next_p)->next); } *next_p = MSO(proc).funs; MSO(proc).funs = bp->off_heap.funs; bp->off_heap.funs = NULL; } #endif /* Move any external things into the process */ if (bp->off_heap.externals != NULL) { ExternalThing** next_p = &bp->off_heap.externals; while (*next_p != NULL) { next_p = &((*next_p)->next); } *next_p = MSO(proc).externals; MSO(proc).externals = bp->off_heap.externals; bp->off_heap.externals = NULL; } } }
Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin) { if (bin->bin_term != THE_NON_VALUE) { return bin->bin_term; } else if (bin->ref_bin != NULL) { Binary* bptr = bin->ref_bin; ProcBin* pb; Eterm bin_term; /* !! Copy-paste from new_binary() !! */ pb = (ProcBin *) alloc_heap(env, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; pb->size = bptr->orig_size; pb->next = MSO(env->proc).first; MSO(env->proc).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; OH_OVERHEAD(&(MSO(env->proc)), pb->size / sizeof(Eterm)); bin_term = make_binary(pb); if (erts_refc_read(&bptr->refc, 1) == 1) { /* Total ownership transfer */ bin->ref_bin = NULL; bin->bin_term = bin_term; } return bin_term; } else { flush_env(env); bin->bin_term = new_binary(env->proc, bin->data, bin->size); cache_env(env); return bin->bin_term; } }
BIF_RETTYPE port_get_data_1(BIF_ALIST_1) { BIF_RETTYPE res; Port* prt; Eterm portid = BIF_ARG_1; prt = id_or_name2port(BIF_P, portid); if (!prt) { BIF_ERROR(BIF_P, BADARG); } if (prt->bp == NULL) { /* MUST be CONST! */ res = prt->data; } else { Eterm* hp = HAlloc(BIF_P, prt->bp->used_size); res = copy_struct(prt->data, prt->bp->used_size, &hp, &MSO(BIF_P)); } erts_smp_port_unlock(prt); BIF_RET(res); }
static BIF_RETTYPE iol2v_yield(iol2v_state_t *state) { if (is_nil(state->magic_reference)) { iol2v_state_t *boxed_state; Binary *magic_binary; Eterm *hp; magic_binary = erts_create_magic_binary_x(sizeof(*state), &iol2v_state_destructor, ERTS_ALC_T_BINARY, 1); boxed_state = ERTS_MAGIC_BIN_UNALIGNED_DATA(magic_binary); sys_memcpy(boxed_state, state, sizeof(*state)); hp = HAlloc(boxed_state->process, ERTS_MAGIC_REF_THING_SIZE); boxed_state->magic_reference = erts_mk_magic_ref(&hp, &MSO(boxed_state->process), magic_binary); state = boxed_state; } ERTS_BIF_YIELD1(bif_export[BIF_iolist_to_iovec_1], state->process, state->magic_reference); }
static Eterm check_process_code(Process* rp, Module* modp) { Eterm* start; char* mod_start; Uint mod_size; Eterm* end; Eterm* sp; #ifndef HYBRID /* FIND ME! */ ErlFunThing* funp; int done_gc = 0; #endif #define INSIDE(a) (start <= (a) && (a) < end) if (modp == NULL) { /* Doesn't exist. */ return am_false; } else if (modp->old_code == NULL) { /* No old code. */ return am_false; } /* * Pick up limits for the module. */ start = modp->old_code; end = (Eterm *)((char *)start + modp->old_code_length); mod_start = (char *) start; mod_size = modp->old_code_length; /* * Check if current instruction or continuation pointer points into module. */ if (INSIDE(rp->i) || INSIDE(rp->cp)) { return am_true; } /* * Check all continuation pointers stored on the stack. */ for (sp = rp->stop; sp < STACK_START(rp); sp++) { if (is_CP(*sp) && INSIDE(cp_val(*sp))) { return am_true; } } /* * Check all continuation pointers stored in stackdump * and clear exception stackdump if there is a pointer * to the module. */ if (rp->ftrace != NIL) { struct StackTrace *s; ASSERT(is_list(rp->ftrace)); s = (struct StackTrace *) big_val(CDR(list_val(rp->ftrace))); if ((s->pc && INSIDE(s->pc)) || (s->current && INSIDE(s->current))) { rp->freason = EXC_NULL; rp->fvalue = NIL; rp->ftrace = NIL; } else { int i; for (i = 0; i < s->depth; i++) { if (INSIDE(s->trace[i])) { rp->freason = EXC_NULL; rp->fvalue = NIL; rp->ftrace = NIL; break; } } } } /* * See if there are funs that refer to the old version of the module. */ #ifndef HYBRID /* FIND ME! */ rescan: for (funp = MSO(rp).funs; funp; funp = funp->next) { Eterm* fun_code; fun_code = funp->fe->address; if (INSIDE((Eterm *) funp->fe->address)) { if (done_gc) { return am_true; } else { /* * Try to get rid of this fun by garbage collecting. * Clear both fvalue and ftrace to make sure they * don't hold any funs. */ rp->freason = EXC_NULL; rp->fvalue = NIL; rp->ftrace = NIL; done_gc = 1; FLAGS(rp) |= F_NEED_FULLSWEEP; (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); goto rescan; } } } #endif /* * See if there are constants inside the module referenced by the process. */ done_gc = 0; for (;;) { ErlMessage* mp; if (any_heap_ref_ptrs(&rp->fvalue, &rp->fvalue+1, mod_start, mod_size)) { rp->freason = EXC_NULL; rp->fvalue = NIL; rp->ftrace = NIL; } if (any_heap_ref_ptrs(rp->stop, rp->hend, mod_start, mod_size)) { goto need_gc; } if (any_heap_refs(rp->heap, rp->htop, mod_start, mod_size)) { goto need_gc; } if (any_heap_refs(rp->old_heap, rp->old_htop, mod_start, mod_size)) { goto need_gc; } if (rp->dictionary != NULL) { Eterm* start = rp->dictionary->data; Eterm* end = start + rp->dictionary->used; if (any_heap_ref_ptrs(start, end, mod_start, mod_size)) { goto need_gc; } } for (mp = rp->msg.first; mp != NULL; mp = mp->next) { if (any_heap_ref_ptrs(mp->m, mp->m+2, mod_start, mod_size)) { goto need_gc; } } break; need_gc: if (done_gc) { return am_true; } else { Eterm* literals; Uint lit_size; /* * Try to get rid of constants by by garbage collecting. * Clear both fvalue and ftrace. */ rp->freason = EXC_NULL; rp->fvalue = NIL; rp->ftrace = NIL; done_gc = 1; FLAGS(rp) |= F_NEED_FULLSWEEP; (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); literals = (Eterm *) modp->old_code[MI_LITERALS_START]; lit_size = (Eterm *) modp->old_code[MI_LITERALS_END] - literals; erts_garbage_collect_literals(rp, literals, lit_size); } } return am_false; #undef INSIDE }
ErtsMessage * erts_try_alloc_message_on_heap(Process *pp, erts_aint32_t *psp, ErtsProcLocks *plp, Uint sz, Eterm **hpp, ErlOffHeap **ohpp, int *on_heap_p) { int locked_main = 0; ErtsMessage *mp; ASSERT(!(*psp & ERTS_PSFLG_OFF_HEAP_MSGQ)); if ((*psp) & ERTS_PSFLGS_VOLATILE_HEAP) goto in_message_fragment; else if ( *plp & ERTS_PROC_LOCK_MAIN ) { try_on_heap: if (((*psp) & ERTS_PSFLGS_VOLATILE_HEAP) || (pp->flags & F_DISABLE_GC) || HEAP_LIMIT(pp) - HEAP_TOP(pp) <= sz) { /* * The heap is either potentially in an inconsistent * state, or not large enough. */ if (locked_main) { *plp &= ~ERTS_PROC_LOCK_MAIN; erts_proc_unlock(pp, ERTS_PROC_LOCK_MAIN); } goto in_message_fragment; } *hpp = HEAP_TOP(pp); HEAP_TOP(pp) = *hpp + sz; *ohpp = &MSO(pp); mp = erts_alloc_message(0, NULL); mp->data.attached = NULL; *on_heap_p = !0; } else if (pp && erts_proc_trylock(pp, ERTS_PROC_LOCK_MAIN) == 0) { locked_main = 1; *psp = erts_atomic32_read_nob(&pp->state); *plp |= ERTS_PROC_LOCK_MAIN; goto try_on_heap; } else { in_message_fragment: if (!((*psp) & ERTS_PSFLG_ON_HEAP_MSGQ)) { mp = erts_alloc_message(sz, hpp); *ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap; } else { mp = erts_alloc_message(0, NULL); if (!sz) { *hpp = NULL; *ohpp = NULL; } else { ErlHeapFragment *bp; bp = new_message_buffer(sz); *hpp = &bp->mem[0]; mp->data.heap_frag = bp; *ohpp = &bp->off_heap; } } *on_heap_p = 0; } return mp; }
static BIF_RETTYPE port_call(Process* c_p, Eterm arg1, Eterm arg2, Eterm arg3) { Uint op; Port *p; Uint size; byte *bytes; byte *endp; ErlDrvSizeT real_size; erts_driver_t *drv; byte port_input[256]; /* Default input buffer to encode in */ byte port_result[256]; /* Buffer for result from port. */ byte* port_resp; /* Pointer to result buffer. */ char *prc; ErlDrvSSizeT ret; Eterm res; Sint result_size; Eterm *hp; Eterm *hp_end; /* To satisfy hybrid heap architecture */ unsigned ret_flags = 0U; int fpe_was_unmasked; bytes = &port_input[0]; port_resp = port_result; /* trace of port scheduling with virtual process descheduling * lock wait */ if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) { trace_virtual_sched(c_p, am_out); } if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { profile_runnable_proc(c_p, am_inactive); } p = id_or_name2port(c_p, arg1); if (!p) { error: if (port_resp != port_result && !(ret_flags & DRIVER_CALL_KEEP_BUFFER)) { driver_free(port_resp); } if (bytes != &port_input[0]) erts_free(ERTS_ALC_T_PORT_CALL_BUF, bytes); /* Need to virtual schedule in the process if there * was an error. */ if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) { trace_virtual_sched(c_p, am_in); } if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { profile_runnable_proc(c_p, am_active); } if (p) erts_port_release(p); #ifdef ERTS_SMP ERTS_SMP_BIF_CHK_PENDING_EXIT(c_p, ERTS_PROC_LOCK_MAIN); #else ERTS_BIF_CHK_EXITED(c_p); #endif BIF_ERROR(c_p, BADARG); } if ((drv = p->drv_ptr) == NULL) { goto error; } if (drv->call == NULL) { goto error; } if (!term_to_Uint(arg2, &op)) { goto error; } p->caller = c_p->id; /* Lock taken, virtual schedule of port */ if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) { trace_sched_ports_where(p, am_in, am_call); } if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(p)) { profile_runnable_port(p, am_active); } size = erts_encode_ext_size(arg3); if (size > sizeof(port_input)) bytes = erts_alloc(ERTS_ALC_T_PORT_CALL_BUF, size); endp = bytes; erts_encode_ext(arg3, &endp); real_size = endp - bytes; if (real_size > size) { erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n", __FILE__, __LINE__, endp - (bytes + size)); } erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); #ifdef USE_VM_PROBES if (DTRACE_ENABLED(driver_call)) { DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE); DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); dtrace_pid_str(p->connected, process_str); dtrace_port_str(p, port_str); DTRACE5(driver_call, process_str, port_str, p->name, op, real_size); } #endif prc = (char *) port_resp; fpe_was_unmasked = erts_block_fpe(); ret = drv->call((ErlDrvData)p->drv_data, (unsigned) op, (char *) bytes, (int) real_size, &prc, (int) sizeof(port_result), &ret_flags); erts_unblock_fpe(fpe_was_unmasked); if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) { trace_sched_ports_where(p, am_out, am_call); } if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(p)) { profile_runnable_port(p, am_inactive); } port_resp = (byte *) prc; p->caller = NIL; erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); #ifdef HARDDEBUG { ErlDrvSizeT z; printf("real_size = %ld,%d, ret = %ld,%d\r\n", (unsigned long) real_size, (int) real_size, (unsigned long)ret, (int) ret); printf("["); for(z = 0; z < real_size; ++z) { printf("%d, ",(int) bytes[z]); } printf("]\r\n"); printf("["); for(z = 0; z < ret; ++z) { printf("%d, ",(int) port_resp[z]); } printf("]\r\n"); } #endif if (ret <= 0 || port_resp[0] != VERSION_MAGIC) { /* Error or a binary without magic/ with wrong magic */ goto error; } result_size = erts_decode_ext_size(port_resp, ret); if (result_size < 0) { goto error; } hp = HAlloc(c_p, result_size); hp_end = hp + result_size; endp = port_resp; res = erts_decode_ext(&hp, &MSO(c_p), &endp); if (res == THE_NON_VALUE) { goto error; } HRelease(c_p, hp_end, hp); if (port_resp != port_result && !(ret_flags & DRIVER_CALL_KEEP_BUFFER)) { driver_free(port_resp); } if (bytes != &port_input[0]) erts_free(ERTS_ALC_T_PORT_CALL_BUF, bytes); if (p) erts_port_release(p); #ifdef ERTS_SMP ERTS_SMP_BIF_CHK_PENDING_EXIT(c_p, ERTS_PROC_LOCK_MAIN); #else ERTS_BIF_CHK_EXITED(c_p); #endif if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) { trace_virtual_sched(c_p, am_in); } if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { profile_runnable_proc(c_p, am_active); } return res; }
BIF_RETTYPE persistent_term_put_2(BIF_ALIST_2) { static const Uint ITERATIONS_PER_RED = 32; ErtsPersistentTermPut2Context* ctx; Eterm state_mref = THE_NON_VALUE; long iterations_until_trap; long max_iterations; #define PUT_TRAP_CODE \ BIF_TRAP2(bif_export[BIF_persistent_term_put_2], BIF_P, state_mref, BIF_ARG_2) #define TRAPPING_COPY_TABLE_PUT(TABLE_DEST, OLD_TABLE, NEW_SIZE, COPY_TYPE, LOC_NAME) \ TRAPPING_COPY_TABLE(TABLE_DEST, OLD_TABLE, NEW_SIZE, COPY_TYPE, LOC_NAME, PUT_TRAP_CODE) #ifdef DEBUG (void)ITERATIONS_PER_RED; iterations_until_trap = max_iterations = GET_SMALL_RANDOM_INT(ERTS_BIF_REDS_LEFT(BIF_P) + (Uint)&ctx); #else iterations_until_trap = max_iterations = ITERATIONS_PER_RED * ERTS_BIF_REDS_LEFT(BIF_P); #endif if (is_internal_magic_ref(BIF_ARG_1) && (ERTS_MAGIC_BIN_DESTRUCTOR(erts_magic_ref2bin(BIF_ARG_1)) == persistent_term_put_2_ctx_bin_dtor)) { /* Restore state after a trap */ Binary* state_bin; state_mref = BIF_ARG_1; state_bin = erts_magic_ref2bin(state_mref); ctx = ERTS_MAGIC_BIN_DATA(state_bin); ASSERT(BIF_P->flags & F_DISABLE_GC); erts_set_gc_state(BIF_P, 1); switch (ctx->trap_location) { case PUT2_TRAP_LOCATION_NEW_KEY: goto L_PUT2_TRAP_LOCATION_NEW_KEY; case PUT2_TRAP_LOCATION_REPLACE_VALUE: goto L_PUT2_TRAP_LOCATION_REPLACE_VALUE; } } else { /* Save state in magic bin in case trapping is necessary */ Eterm* hp; Binary* state_bin = erts_create_magic_binary(sizeof(ErtsPersistentTermPut2Context), persistent_term_put_2_ctx_bin_dtor); hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE); state_mref = erts_mk_magic_ref(&hp, &MSO(BIF_P), state_bin); ctx = ERTS_MAGIC_BIN_DATA(state_bin); /* * IMPORTANT: The following field is used to detect if * persistent_term_put_2_ctx_bin_dtor needs to free memory */ ctx->cpy_ctx.new_table = NULL; } if (!try_seize_update_permission(BIF_P)) { ERTS_BIF_YIELD2(bif_export[BIF_persistent_term_put_2], BIF_P, BIF_ARG_1, BIF_ARG_2); } ctx->hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); ctx->key = BIF_ARG_1; ctx->term = BIF_ARG_2; ctx->entry_index = lookup(ctx->hash_table, ctx->key); ctx->heap[0] = make_arityval(2); ctx->heap[1] = ctx->key; ctx->heap[2] = ctx->term; ctx->tuple = make_tuple(ctx->heap); if (is_nil(ctx->hash_table->term[ctx->entry_index])) { Uint new_size = ctx->hash_table->allocated; if (MUST_GROW(ctx->hash_table)) { new_size *= 2; } TRAPPING_COPY_TABLE_PUT(ctx->hash_table, ctx->hash_table, new_size, ERTS_PERSISTENT_TERM_CPY_NO_REHASH, PUT2_TRAP_LOCATION_NEW_KEY); ctx->entry_index = lookup(ctx->hash_table, ctx->key); ctx->hash_table->num_entries++; } else { Eterm tuple = ctx->hash_table->term[ctx->entry_index]; Eterm old_term; ASSERT(is_tuple_arity(tuple, 2)); old_term = boxed_val(tuple)[2]; if (EQ(ctx->term, old_term)) { /* Same value. No need to update anything. */ release_update_permission(0); BIF_RET(am_ok); } else { /* Mark the old term for deletion. */ mark_for_deletion(ctx->hash_table, ctx->entry_index); TRAPPING_COPY_TABLE_PUT(ctx->hash_table, ctx->hash_table, ctx->hash_table->allocated, ERTS_PERSISTENT_TERM_CPY_NO_REHASH, PUT2_TRAP_LOCATION_REPLACE_VALUE); } } { Uint term_size; Uint lit_area_size; ErlOffHeap code_off_heap; ErtsLiteralArea* literal_area; erts_shcopy_t info; Eterm* ptr; /* * Preserve internal sharing in the term by using the * sharing-preserving functions. However, literals must * be copied in case the module holding them are unloaded. */ INITIALIZE_SHCOPY(info); info.copy_literals = 1; term_size = copy_shared_calculate(ctx->tuple, &info); ERTS_INIT_OFF_HEAP(&code_off_heap); lit_area_size = ERTS_LITERAL_AREA_ALLOC_SIZE(term_size); literal_area = erts_alloc(ERTS_ALC_T_LITERAL, lit_area_size); ptr = &literal_area->start[0]; literal_area->end = ptr + term_size; ctx->tuple = copy_shared_perform(ctx->tuple, term_size, &info, &ptr, &code_off_heap); ASSERT(tuple_val(ctx->tuple) == literal_area->start); literal_area->off_heap = code_off_heap.first; DESTROY_SHCOPY(info); erts_set_literal_tag(&ctx->tuple, literal_area->start, term_size); ctx->hash_table->term[ctx->entry_index] = ctx->tuple; erts_schedule_thr_prgr_later_op(table_updater, ctx->hash_table, &thr_prog_op); suspend_updater(BIF_P); } BUMP_REDS(BIF_P, (max_iterations - iterations_until_trap) / ITERATIONS_PER_RED); ERTS_BIF_YIELD_RETURN(BIF_P, am_ok); }
BIF_RETTYPE persistent_term_erase_1(BIF_ALIST_1) { static const Uint ITERATIONS_PER_RED = 32; ErtsPersistentTermErase1Context* ctx; Eterm state_mref = THE_NON_VALUE; long iterations_until_trap; long max_iterations; #ifdef DEBUG (void)ITERATIONS_PER_RED; iterations_until_trap = max_iterations = GET_SMALL_RANDOM_INT(ERTS_BIF_REDS_LEFT(BIF_P) + (Uint)&ctx); #else iterations_until_trap = max_iterations = ITERATIONS_PER_RED * ERTS_BIF_REDS_LEFT(BIF_P); #endif #define ERASE_TRAP_CODE \ BIF_TRAP1(bif_export[BIF_persistent_term_erase_1], BIF_P, state_mref); #define TRAPPING_COPY_TABLE_ERASE(TABLE_DEST, OLD_TABLE, NEW_SIZE, REHASH, LOC_NAME) \ TRAPPING_COPY_TABLE(TABLE_DEST, OLD_TABLE, NEW_SIZE, REHASH, LOC_NAME, ERASE_TRAP_CODE) if (is_internal_magic_ref(BIF_ARG_1) && (ERTS_MAGIC_BIN_DESTRUCTOR(erts_magic_ref2bin(BIF_ARG_1)) == persistent_term_erase_1_ctx_bin_dtor)) { /* Restore the state after a trap */ Binary* state_bin; state_mref = BIF_ARG_1; state_bin = erts_magic_ref2bin(state_mref); ctx = ERTS_MAGIC_BIN_DATA(state_bin); ASSERT(BIF_P->flags & F_DISABLE_GC); erts_set_gc_state(BIF_P, 1); switch (ctx->trap_location) { case ERASE1_TRAP_LOCATION_TMP_COPY: goto L_ERASE1_TRAP_LOCATION_TMP_COPY; case ERASE1_TRAP_LOCATION_FINAL_COPY: goto L_ERASE1_TRAP_LOCATION_FINAL_COPY; } } else { /* Save state in magic bin in case trapping is necessary */ Eterm* hp; Binary* state_bin = erts_create_magic_binary(sizeof(ErtsPersistentTermErase1Context), persistent_term_erase_1_ctx_bin_dtor); hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE); state_mref = erts_mk_magic_ref(&hp, &MSO(BIF_P), state_bin); ctx = ERTS_MAGIC_BIN_DATA(state_bin); /* * IMPORTANT: The following two fields are used to detect if * persistent_term_erase_1_ctx_bin_dtor needs to free memory */ ctx->cpy_ctx.new_table = NULL; ctx->tmp_table = NULL; } if (!try_seize_update_permission(BIF_P)) { ERTS_BIF_YIELD1(bif_export[BIF_persistent_term_erase_1], BIF_P, BIF_ARG_1); } ctx->key = BIF_ARG_1; ctx->old_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); ctx->entry_index = lookup(ctx->old_table, ctx->key); ctx->old_term = ctx->old_table->term[ctx->entry_index]; if (is_boxed(ctx->old_term)) { Uint new_size; /* * Since we don't use any delete markers, we must rehash * the table when deleting terms to ensure that all terms * can still be reached if there are hash collisions. * We can't rehash in place and it would not be safe to modify * the old table yet, so we will first need a new * temporary table copy of the same size as the old one. */ ASSERT(is_tuple_arity(ctx->old_term, 2)); TRAPPING_COPY_TABLE_ERASE(ctx->tmp_table, ctx->old_table, ctx->old_table->allocated, ERTS_PERSISTENT_TERM_CPY_TEMP, ERASE1_TRAP_LOCATION_TMP_COPY); /* * Delete the term from the temporary table. Then copy the * temporary table to a new table, rehashing the entries * while copying. */ ctx->tmp_table->term[ctx->entry_index] = NIL; ctx->tmp_table->num_entries--; new_size = ctx->tmp_table->allocated; if (MUST_SHRINK(ctx->tmp_table)) { new_size /= 2; } TRAPPING_COPY_TABLE_ERASE(ctx->new_table, ctx->tmp_table, new_size, ERTS_PERSISTENT_TERM_CPY_REHASH, ERASE1_TRAP_LOCATION_FINAL_COPY); erts_free(ERTS_ALC_T_PERSISTENT_TERM_TMP, ctx->tmp_table); /* * IMPORTANT: Memory management depends on that ctx->tmp_table * is set to NULL on the line below */ ctx->tmp_table = NULL; mark_for_deletion(ctx->old_table, ctx->entry_index); erts_schedule_thr_prgr_later_op(table_updater, ctx->new_table, &thr_prog_op); suspend_updater(BIF_P); BUMP_REDS(BIF_P, (max_iterations - iterations_until_trap) / ITERATIONS_PER_RED); ERTS_BIF_YIELD_RETURN(BIF_P, am_true); } /* * Key is not present. Nothing to do. */ ASSERT(is_nil(ctx->old_term)); release_update_permission(0); BIF_RET(am_false); }
int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg) { struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)msg_env; ErtsProcLocks rp_locks = 0; Process* rp; Process* c_p; ErlHeapFragment* frags; #if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) ErtsProcLocks rp_had_locks; #endif Eterm receiver = to_pid->pid; int flush_me = 0; if (env != NULL) { c_p = env->proc; if (receiver == c_p->id) { rp_locks = ERTS_PROC_LOCK_MAIN; flush_me = 1; } } else { #ifdef ERTS_SMP c_p = NULL; #else erl_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM"); #endif } #if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) rp_had_locks = rp_locks; #endif rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC); if (rp == NULL) { ASSERT(env == NULL || receiver != c_p->id); return 0; } flush_env(msg_env); frags = menv->env.heap_frag; ASSERT(frags == MBUF(&menv->phony_proc)); if (frags != NULL) { /* Move all offheap's from phony proc to the first fragment. Quick and dirty, but erts_move_msg_mbuf_to_heap doesn't care. */ ASSERT(!is_offheap(&frags->off_heap)); frags->off_heap = MSO(&menv->phony_proc); clear_offheap(&MSO(&menv->phony_proc)); menv->env.heap_frag = NULL; MBUF(&menv->phony_proc) = NULL; } ASSERT(!is_offheap(&MSO(&menv->phony_proc))); if (flush_me) { flush_env(env); /* Needed for ERTS_HOLE_CHECK */ } erts_queue_message(rp, &rp_locks, frags, msg, am_undefined); if (rp_locks) { ERTS_SMP_LC_ASSERT(rp_locks == (rp_had_locks | (ERTS_PROC_LOCK_MSGQ | ERTS_PROC_LOCK_STATUS))); erts_smp_proc_unlock(rp, (ERTS_PROC_LOCK_MSGQ | ERTS_PROC_LOCK_STATUS)); } erts_smp_proc_dec_refc(rp); if (flush_me) { cache_env(env); } return 1; }