BIF_RETTYPE
erts_debug_copy_shared_1(BIF_ALIST_1)
{
Process* p = BIF_P;
Eterm term = BIF_ARG_1;
Uint size;
Eterm* hp;
Eterm copy;
erts_shcopy_t info;
INITIALIZE_SHCOPY(info);
size = copy_shared_calculate(term, &info);
if (size > 0) {
hp = HAlloc(p, size);
}
copy = copy_shared_perform(term, size, &info, &hp, &p->off_heap);
DESTROY_SHCOPY(info);
BIF_RET(copy);
}
void
erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
Eterm reason, Eterm token)
{
Eterm mess;
Eterm save;
Eterm from_copy;
Uint sz_reason;
Uint sz_token;
Uint sz_from;
Eterm* hp;
Eterm temptoken;
ErtsMessage* mp;
ErlOffHeap *ohp;
#ifdef SHCOPY_SEND
erts_shcopy_t info;
#endif
if (have_seqtrace(token)) {
ASSERT(is_tuple(token));
sz_token = size_object(token);
sz_from = size_object(from);
#ifdef SHCOPY_SEND
INITIALIZE_SHCOPY(info);
sz_reason = copy_shared_calculate(reason, &info);
#else
sz_reason = size_object(reason);
#endif
mp = erts_alloc_message_heap(to, to_locksp,
sz_reason + sz_from + sz_token + 4,
&hp, &ohp);
#ifdef SHCOPY_SEND
mess = copy_shared_perform(reason, sz_reason, &info, &hp, ohp);
DESTROY_SHCOPY(info);
#else
mess = copy_struct(reason, sz_reason, &hp, ohp);
#endif
from_copy = copy_struct(from, sz_from, &hp, ohp);
save = TUPLE3(hp, am_EXIT, from_copy, mess);
hp += 4;
/* the trace token must in this case be updated by the caller */
seq_trace_output(token, save, SEQ_TRACE_SEND, to->common.id, NULL);
temptoken = copy_struct(token, sz_token, &hp, ohp);
ERL_MESSAGE_TOKEN(mp) = temptoken;
erts_queue_message(to, *to_locksp, mp, save, am_system);
} else {
sz_from = IS_CONST(from) ? 0 : size_object(from);
#ifdef SHCOPY_SEND
INITIALIZE_SHCOPY(info);
sz_reason = copy_shared_calculate(reason, &info);
#else
sz_reason = size_object(reason);
#endif
mp = erts_alloc_message_heap(to, to_locksp,
sz_reason+sz_from+4, &hp, &ohp);
#ifdef SHCOPY_SEND
mess = copy_shared_perform(reason, sz_reason, &info, &hp, ohp);
DESTROY_SHCOPY(info);
#else
mess = copy_struct(reason, sz_reason, &hp, ohp);
#endif
from_copy = (IS_CONST(from)
? from
: copy_struct(from, sz_from, &hp, ohp));
save = TUPLE3(hp, am_EXIT, from_copy, mess);
erts_queue_message(to, *to_locksp, mp, save, am_system);
}
}
Sint
erts_send_message(Process* sender,
Process* receiver,
ErtsProcLocks *receiver_locks,
Eterm message,
unsigned flags)
{
Uint msize;
ErtsMessage* mp;
ErlOffHeap *ohp;
Eterm token = NIL;
Sint res = 0;
#ifdef USE_VM_PROBES
DTRACE_CHARBUF(sender_name, 64);
DTRACE_CHARBUF(receiver_name, 64);
Sint tok_label = 0;
Sint tok_lastcnt = 0;
Sint tok_serial = 0;
Eterm utag = NIL;
#endif
erts_aint32_t receiver_state;
#ifdef SHCOPY_SEND
erts_shcopy_t info;
#else
erts_literal_area_t litarea;
INITIALIZE_LITERAL_PURGE_AREA(litarea);
#endif
#ifdef USE_VM_PROBES
*sender_name = *receiver_name = '\0';
if (DTRACE_ENABLED(message_send)) {
erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)),
"%T", sender->common.id);
erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)),
"%T", receiver->common.id);
}
#endif
receiver_state = erts_atomic32_read_nob(&receiver->state);
if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) {
Eterm* hp;
Eterm stoken = SEQ_TRACE_TOKEN(sender);
Uint seq_trace_size = 0;
#ifdef USE_VM_PROBES
Uint dt_utag_size = 0;
#endif
/* SHCOPY corrupts the heap between
* copy_shared_calculate, and
* copy_shared_perform. (it inserts move_markers like the gc).
* Make sure we don't use the heap between those instances.
*/
if (have_seqtrace(stoken)) {
seq_trace_update_send(sender);
seq_trace_output(stoken, message, SEQ_TRACE_SEND,
receiver->common.id, sender);
seq_trace_size = 6; /* TUPLE5 */
}
#ifdef USE_VM_PROBES
if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) {
dt_utag_size = size_object(DT_UTAG(sender));
} else if (stoken == am_have_dt_utag ) {
stoken = NIL;
}
#endif
#ifdef SHCOPY_SEND
INITIALIZE_SHCOPY(info);
msize = copy_shared_calculate(message, &info);
#else
msize = size_object_litopt(message, &litarea);
#endif
mp = erts_alloc_message_heap_state(receiver,
&receiver_state,
receiver_locks,
(msize
#ifdef USE_VM_PROBES
+ dt_utag_size
#endif
+ seq_trace_size),
&hp,
&ohp);
#ifdef SHCOPY_SEND
if (is_not_immed(message))
message = copy_shared_perform(message, msize, &info, &hp, ohp);
DESTROY_SHCOPY(info);
#else
if (is_not_immed(message))
message = copy_struct_litopt(message, msize, &hp, ohp, &litarea);
#endif
if (is_immed(stoken))
token = stoken;
else
token = copy_struct(stoken, seq_trace_size, &hp, ohp);
#ifdef USE_VM_PROBES
if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) {
if (is_immed(DT_UTAG(sender)))
utag = DT_UTAG(sender);
else
utag = copy_struct(DT_UTAG(sender), dt_utag_size, &hp, ohp);
}
if (DTRACE_ENABLED(message_send)) {
if (have_seqtrace(stoken)) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(stoken));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(stoken));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(stoken));
}
DTRACE6(message_send, sender_name, receiver_name,
msize, tok_label, tok_lastcnt, tok_serial);
}
#endif
} else {
Eterm *hp;
if (receiver == sender && !(receiver_state & ERTS_PSFLG_OFF_HEAP_MSGQ)) {
mp = erts_alloc_message(0, NULL);
msize = 0;
}
else {
#ifdef SHCOPY_SEND
INITIALIZE_SHCOPY(info);
msize = copy_shared_calculate(message, &info);
#else
msize = size_object_litopt(message, &litarea);
#endif
mp = erts_alloc_message_heap_state(receiver,
&receiver_state,
receiver_locks,
msize,
&hp,
&ohp);
#ifdef SHCOPY_SEND
if (is_not_immed(message))
message = copy_shared_perform(message, msize, &info, &hp, ohp);
DESTROY_SHCOPY(info);
#else
if (is_not_immed(message))
message = copy_struct_litopt(message, msize, &hp, ohp, &litarea);
#endif
}
#ifdef USE_VM_PROBES
DTRACE6(message_send, sender_name, receiver_name,
msize, tok_label, tok_lastcnt, tok_serial);
#endif
}
ERL_MESSAGE_TOKEN(mp) = token;
#ifdef USE_VM_PROBES
ERL_MESSAGE_DT_UTAG(mp) = utag;
#endif
res = queue_message(receiver,
&receiver_state,
*receiver_locks,
mp, message,
sender->common.id);
return res;
}
BIF_RETTYPE persistent_term_put_2(BIF_ALIST_2)
{
static const Uint ITERATIONS_PER_RED = 32;
ErtsPersistentTermPut2Context* ctx;
Eterm state_mref = THE_NON_VALUE;
long iterations_until_trap;
long max_iterations;
#define PUT_TRAP_CODE \
BIF_TRAP2(bif_export[BIF_persistent_term_put_2], BIF_P, state_mref, BIF_ARG_2)
#define TRAPPING_COPY_TABLE_PUT(TABLE_DEST, OLD_TABLE, NEW_SIZE, COPY_TYPE, LOC_NAME) \
TRAPPING_COPY_TABLE(TABLE_DEST, OLD_TABLE, NEW_SIZE, COPY_TYPE, LOC_NAME, PUT_TRAP_CODE)
#ifdef DEBUG
(void)ITERATIONS_PER_RED;
iterations_until_trap = max_iterations =
GET_SMALL_RANDOM_INT(ERTS_BIF_REDS_LEFT(BIF_P) + (Uint)&ctx);
#else
iterations_until_trap = max_iterations =
ITERATIONS_PER_RED * ERTS_BIF_REDS_LEFT(BIF_P);
#endif
if (is_internal_magic_ref(BIF_ARG_1) &&
(ERTS_MAGIC_BIN_DESTRUCTOR(erts_magic_ref2bin(BIF_ARG_1)) ==
persistent_term_put_2_ctx_bin_dtor)) {
/* Restore state after a trap */
Binary* state_bin;
state_mref = BIF_ARG_1;
state_bin = erts_magic_ref2bin(state_mref);
ctx = ERTS_MAGIC_BIN_DATA(state_bin);
ASSERT(BIF_P->flags & F_DISABLE_GC);
erts_set_gc_state(BIF_P, 1);
switch (ctx->trap_location) {
case PUT2_TRAP_LOCATION_NEW_KEY:
goto L_PUT2_TRAP_LOCATION_NEW_KEY;
case PUT2_TRAP_LOCATION_REPLACE_VALUE:
goto L_PUT2_TRAP_LOCATION_REPLACE_VALUE;
}
} else {
/* Save state in magic bin in case trapping is necessary */
Eterm* hp;
Binary* state_bin = erts_create_magic_binary(sizeof(ErtsPersistentTermPut2Context),
persistent_term_put_2_ctx_bin_dtor);
hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE);
state_mref = erts_mk_magic_ref(&hp, &MSO(BIF_P), state_bin);
ctx = ERTS_MAGIC_BIN_DATA(state_bin);
/*
* IMPORTANT: The following field is used to detect if
* persistent_term_put_2_ctx_bin_dtor needs to free memory
*/
ctx->cpy_ctx.new_table = NULL;
}
if (!try_seize_update_permission(BIF_P)) {
ERTS_BIF_YIELD2(bif_export[BIF_persistent_term_put_2],
BIF_P, BIF_ARG_1, BIF_ARG_2);
}
ctx->hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
ctx->key = BIF_ARG_1;
ctx->term = BIF_ARG_2;
ctx->entry_index = lookup(ctx->hash_table, ctx->key);
ctx->heap[0] = make_arityval(2);
ctx->heap[1] = ctx->key;
ctx->heap[2] = ctx->term;
ctx->tuple = make_tuple(ctx->heap);
if (is_nil(ctx->hash_table->term[ctx->entry_index])) {
Uint new_size = ctx->hash_table->allocated;
if (MUST_GROW(ctx->hash_table)) {
new_size *= 2;
}
TRAPPING_COPY_TABLE_PUT(ctx->hash_table,
ctx->hash_table,
new_size,
ERTS_PERSISTENT_TERM_CPY_NO_REHASH,
PUT2_TRAP_LOCATION_NEW_KEY);
ctx->entry_index = lookup(ctx->hash_table, ctx->key);
ctx->hash_table->num_entries++;
} else {
Eterm tuple = ctx->hash_table->term[ctx->entry_index];
Eterm old_term;
ASSERT(is_tuple_arity(tuple, 2));
old_term = boxed_val(tuple)[2];
if (EQ(ctx->term, old_term)) {
/* Same value. No need to update anything. */
release_update_permission(0);
BIF_RET(am_ok);
} else {
/* Mark the old term for deletion. */
mark_for_deletion(ctx->hash_table, ctx->entry_index);
TRAPPING_COPY_TABLE_PUT(ctx->hash_table,
ctx->hash_table,
ctx->hash_table->allocated,
ERTS_PERSISTENT_TERM_CPY_NO_REHASH,
PUT2_TRAP_LOCATION_REPLACE_VALUE);
}
}
{
Uint term_size;
Uint lit_area_size;
ErlOffHeap code_off_heap;
ErtsLiteralArea* literal_area;
erts_shcopy_t info;
Eterm* ptr;
/*
* Preserve internal sharing in the term by using the
* sharing-preserving functions. However, literals must
* be copied in case the module holding them are unloaded.
*/
INITIALIZE_SHCOPY(info);
info.copy_literals = 1;
term_size = copy_shared_calculate(ctx->tuple, &info);
ERTS_INIT_OFF_HEAP(&code_off_heap);
lit_area_size = ERTS_LITERAL_AREA_ALLOC_SIZE(term_size);
literal_area = erts_alloc(ERTS_ALC_T_LITERAL, lit_area_size);
ptr = &literal_area->start[0];
literal_area->end = ptr + term_size;
ctx->tuple = copy_shared_perform(ctx->tuple, term_size, &info, &ptr, &code_off_heap);
ASSERT(tuple_val(ctx->tuple) == literal_area->start);
literal_area->off_heap = code_off_heap.first;
DESTROY_SHCOPY(info);
erts_set_literal_tag(&ctx->tuple, literal_area->start, term_size);
ctx->hash_table->term[ctx->entry_index] = ctx->tuple;
erts_schedule_thr_prgr_later_op(table_updater, ctx->hash_table, &thr_prog_op);
suspend_updater(BIF_P);
}
BUMP_REDS(BIF_P, (max_iterations - iterations_until_trap) / ITERATIONS_PER_RED);
ERTS_BIF_YIELD_RETURN(BIF_P, am_ok);
}