BIF_RETTYPE persistent_term_get_0(BIF_ALIST_0)
{
HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
TrapData* trap_data;
Eterm res = NIL;
Eterm magic_ref;
Binary* mbp;
magic_ref = alloc_trap_data(BIF_P);
mbp = erts_magic_ref2bin(magic_ref);
trap_data = ERTS_MAGIC_BIN_DATA(mbp);
trap_data->table = hash_table;
trap_data->idx = 0;
trap_data->remaining = hash_table->num_entries;
res = do_get_all(BIF_P, trap_data, res);
if (trap_data->remaining == 0) {
BUMP_REDS(BIF_P, hash_table->num_entries);
trap_data->table = NULL; /* Prevent refc decrement */
BIF_RET(res);
} else {
/*
* Increment the ref counter to prevent an update operation (by put/2
* or erase/1) to delete this hash table.
*/
erts_atomic_inc_nob(&hash_table->refc);
BUMP_ALL_REDS(BIF_P);
BIF_TRAP2(&persistent_term_get_all_export, BIF_P, magic_ref, res);
}
}
static int persistent_term_put_2_ctx_bin_dtor(Binary *context_bin)
{
ErtsPersistentTermPut2Context* ctx = ERTS_MAGIC_BIN_DATA(context_bin);
if (ctx->cpy_ctx.new_table != NULL) {
erts_free(ERTS_ALC_T_PERSISTENT_TERM, ctx->cpy_ctx.new_table);
release_update_permission(0);
}
return 1;
}
void* enif_alloc_resource(ErlNifResourceType* type, size_t size)
{
Binary* bin = erts_create_magic_binary(SIZEOF_ErlNifResource(size), &nif_resource_dtor);
ErlNifResource* resource = ERTS_MAGIC_BIN_DATA(bin);
resource->type = type;
erts_refc_inc(&bin->refc, 1);
#ifdef DEBUG
erts_refc_init(&resource->nif_refc, 1);
#endif
erts_refc_inc(&resource->type->refc, 2);
return resource->data;
}
static void suspend_until_thr_prg(Process* p)
{
Binary* state_bin;
ErtsFlxCtrWakeUpLaterInfo* info;
state_bin = erts_create_magic_binary(sizeof(ErtsFlxCtrWakeUpLaterInfo),
erts_flxctr_wait_dtor);
info = ERTS_MAGIC_BIN_DATA(state_bin);
info->process = p;
erts_refc_inctest(&state_bin->intern.refc, 1);
erts_suspend(p, ERTS_PROC_LOCK_MAIN, NULL);
erts_proc_inc_refc(p);
ERTS_VBUMP_ALL_REDS(p);
erts_schedule_thr_prgr_later_op(thr_prg_wake_up_later, state_bin, &info->later_op);
}
static int
cleanup_trap_data(Binary *bp)
{
TrapData* trap_data = ERTS_MAGIC_BIN_DATA(bp);
if (trap_data->table) {
/*
* The process has been killed and is now exiting.
* Decrement the reference counter for the table.
*/
dec_table_refc(NULL, trap_data->table);
}
return 1;
}
static Eterm subtract_create_trap_state(Process *p,
ErtsSubtractContext *context) {
Binary *state_bin;
Eterm *hp;
state_bin = erts_create_magic_binary(sizeof(ErtsSubtractContext),
subtract_ctx_bin_dtor);
subtract_ctx_move(context, ERTS_MAGIC_BIN_DATA(state_bin));
hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
return erts_mk_magic_ref(&hp, &MSO(p), state_bin);
}
static void
thr_prg_wake_up_later(void* bin_p)
{
Binary* bin = bin_p;
ErtsFlxCtrWakeUpLaterInfo* info = ERTS_MAGIC_BIN_DATA(bin);
Process* p = info->process;
/* Resume the requesting process */
erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
if (!ERTS_PROC_IS_EXITING(p)) {
erts_resume(p, ERTS_PROC_LOCK_STATUS);
}
erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
/* Free data */
erts_proc_dec_refc(p);
erts_bin_release(bin);
}
static int persistent_term_erase_1_ctx_bin_dtor(Binary *context_bin)
{
ErtsPersistentTermErase1Context* ctx = ERTS_MAGIC_BIN_DATA(context_bin);
if (ctx->cpy_ctx.new_table != NULL) {
if (ctx->cpy_ctx.copy_type == ERTS_PERSISTENT_TERM_CPY_TEMP) {
erts_free(ERTS_ALC_T_PERSISTENT_TERM_TMP, ctx->cpy_ctx.new_table);
} else {
erts_free(ERTS_ALC_T_PERSISTENT_TERM, ctx->cpy_ctx.new_table);
}
if (ctx->tmp_table != NULL) {
erts_free(ERTS_ALC_T_PERSISTENT_TERM_TMP, ctx->tmp_table);
}
release_update_permission(0);
}
return 1;
}
static void
thr_prg_wake_up_and_count(void* bin_p)
{
Binary* bin = bin_p;
DecentralizedReadSnapshotInfo* info = ERTS_MAGIC_BIN_DATA(bin);
Process* p = info->process;
ErtsFlxCtrDecentralizedCtrArray* array = info->array;
ErtsFlxCtrDecentralizedCtrArray* next = info->next_array;
int i, sched;
/* Reset result array */
for (i = 0; i < info->nr_of_counters; i++) {
info->result[i] = 0;
}
/* Read result from snapshot */
for (sched = 0; sched < ERTS_FLXCTR_DECENTRALIZED_NO_SLOTS; sched++) {
for (i = 0; i < info->nr_of_counters; i++) {
info->result[i] = info->result[i] +
erts_atomic_read_nob(&array->array[sched].counters[i]);
}
}
/* Update the next decentralized counter array */
for (i = 0; i < info->nr_of_counters; i++) {
erts_atomic_add_nob(&next->array[0].counters[i], info->result[i]);
}
/* Announce that the snapshot is done */
{
Sint expected = ERTS_FLXCTR_SNAPSHOT_ONGOING;
if (expected != erts_atomic_cmpxchg_mb(&next->snapshot_status,
ERTS_FLXCTR_SNAPSHOT_NOT_ONGOING,
expected)) {
/* The CAS failed which means that this thread need to free the next array. */
erts_free(info->alloc_type, next->block_start);
}
}
/* Resume the process that requested the snapshot */
erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
if (!ERTS_PROC_IS_EXITING(p)) {
erts_resume(p, ERTS_PROC_LOCK_STATUS);
}
/* Free the memory that is no longer needed */
erts_free(info->alloc_type, array->block_start);
erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
erts_proc_dec_refc(p);
erts_bin_release(bin);
}
static void nif_resource_dtor(Binary* bin)
{
ErlNifResource* resource = (ErlNifResource*) ERTS_MAGIC_BIN_DATA(bin);
ErlNifResourceType* type = resource->type;
ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor);
if (type->dtor != NULL) {
ErlNifEnv env;
pre_nif_noproc(&env, type->owner);
type->dtor(&env,resource->data);
post_nif_noproc(&env);
}
if (erts_refc_dectest(&type->refc, 0) == 0) {
ASSERT(type->next == NULL);
ASSERT(type->owner != NULL);
ASSERT(type->owner->mod == NULL);
steal_resource_type(type);
erts_free(ERTS_ALC_T_NIF, type);
}
}
int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* type,
void** objp)
{
ProcBin* pb;
Binary* mbin;
ErlNifResource* resource;
if (!ERTS_TERM_IS_MAGIC_BINARY(term)) {
return 0;
}
pb = (ProcBin*) binary_val(term);
/*if (pb->size != 0) {
return 0; / * Or should we allow "resource binaries" as handles? * /
}*/
mbin = pb->val;
resource = (ErlNifResource*) ERTS_MAGIC_BIN_DATA(mbin);
if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor
|| resource->type != type) {
return 0;
}
*objp = resource->data;
return 1;
}
BIF_RETTYPE persistent_term_erase_1(BIF_ALIST_1)
{
static const Uint ITERATIONS_PER_RED = 32;
ErtsPersistentTermErase1Context* ctx;
Eterm state_mref = THE_NON_VALUE;
long iterations_until_trap;
long max_iterations;
#ifdef DEBUG
(void)ITERATIONS_PER_RED;
iterations_until_trap = max_iterations =
GET_SMALL_RANDOM_INT(ERTS_BIF_REDS_LEFT(BIF_P) + (Uint)&ctx);
#else
iterations_until_trap = max_iterations =
ITERATIONS_PER_RED * ERTS_BIF_REDS_LEFT(BIF_P);
#endif
#define ERASE_TRAP_CODE \
BIF_TRAP1(bif_export[BIF_persistent_term_erase_1], BIF_P, state_mref);
#define TRAPPING_COPY_TABLE_ERASE(TABLE_DEST, OLD_TABLE, NEW_SIZE, REHASH, LOC_NAME) \
TRAPPING_COPY_TABLE(TABLE_DEST, OLD_TABLE, NEW_SIZE, REHASH, LOC_NAME, ERASE_TRAP_CODE)
if (is_internal_magic_ref(BIF_ARG_1) &&
(ERTS_MAGIC_BIN_DESTRUCTOR(erts_magic_ref2bin(BIF_ARG_1)) ==
persistent_term_erase_1_ctx_bin_dtor)) {
/* Restore the state after a trap */
Binary* state_bin;
state_mref = BIF_ARG_1;
state_bin = erts_magic_ref2bin(state_mref);
ctx = ERTS_MAGIC_BIN_DATA(state_bin);
ASSERT(BIF_P->flags & F_DISABLE_GC);
erts_set_gc_state(BIF_P, 1);
switch (ctx->trap_location) {
case ERASE1_TRAP_LOCATION_TMP_COPY:
goto L_ERASE1_TRAP_LOCATION_TMP_COPY;
case ERASE1_TRAP_LOCATION_FINAL_COPY:
goto L_ERASE1_TRAP_LOCATION_FINAL_COPY;
}
} else {
/* Save state in magic bin in case trapping is necessary */
Eterm* hp;
Binary* state_bin = erts_create_magic_binary(sizeof(ErtsPersistentTermErase1Context),
persistent_term_erase_1_ctx_bin_dtor);
hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE);
state_mref = erts_mk_magic_ref(&hp, &MSO(BIF_P), state_bin);
ctx = ERTS_MAGIC_BIN_DATA(state_bin);
/*
* IMPORTANT: The following two fields are used to detect if
* persistent_term_erase_1_ctx_bin_dtor needs to free memory
*/
ctx->cpy_ctx.new_table = NULL;
ctx->tmp_table = NULL;
}
if (!try_seize_update_permission(BIF_P)) {
ERTS_BIF_YIELD1(bif_export[BIF_persistent_term_erase_1],
BIF_P, BIF_ARG_1);
}
ctx->key = BIF_ARG_1;
ctx->old_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
ctx->entry_index = lookup(ctx->old_table, ctx->key);
ctx->old_term = ctx->old_table->term[ctx->entry_index];
if (is_boxed(ctx->old_term)) {
Uint new_size;
/*
* Since we don't use any delete markers, we must rehash
* the table when deleting terms to ensure that all terms
* can still be reached if there are hash collisions.
* We can't rehash in place and it would not be safe to modify
* the old table yet, so we will first need a new
* temporary table copy of the same size as the old one.
*/
ASSERT(is_tuple_arity(ctx->old_term, 2));
TRAPPING_COPY_TABLE_ERASE(ctx->tmp_table,
ctx->old_table,
ctx->old_table->allocated,
ERTS_PERSISTENT_TERM_CPY_TEMP,
ERASE1_TRAP_LOCATION_TMP_COPY);
/*
* Delete the term from the temporary table. Then copy the
* temporary table to a new table, rehashing the entries
* while copying.
*/
ctx->tmp_table->term[ctx->entry_index] = NIL;
ctx->tmp_table->num_entries--;
new_size = ctx->tmp_table->allocated;
if (MUST_SHRINK(ctx->tmp_table)) {
new_size /= 2;
}
TRAPPING_COPY_TABLE_ERASE(ctx->new_table,
ctx->tmp_table,
new_size,
ERTS_PERSISTENT_TERM_CPY_REHASH,
ERASE1_TRAP_LOCATION_FINAL_COPY);
erts_free(ERTS_ALC_T_PERSISTENT_TERM_TMP, ctx->tmp_table);
/*
* IMPORTANT: Memory management depends on that ctx->tmp_table
* is set to NULL on the line below
*/
ctx->tmp_table = NULL;
mark_for_deletion(ctx->old_table, ctx->entry_index);
erts_schedule_thr_prgr_later_op(table_updater, ctx->new_table, &thr_prog_op);
suspend_updater(BIF_P);
BUMP_REDS(BIF_P, (max_iterations - iterations_until_trap) / ITERATIONS_PER_RED);
ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
}
/*
* Key is not present. Nothing to do.
*/
ASSERT(is_nil(ctx->old_term));
release_update_permission(0);
BIF_RET(am_false);
}
BIF_RETTYPE persistent_term_put_2(BIF_ALIST_2)
{
static const Uint ITERATIONS_PER_RED = 32;
ErtsPersistentTermPut2Context* ctx;
Eterm state_mref = THE_NON_VALUE;
long iterations_until_trap;
long max_iterations;
#define PUT_TRAP_CODE \
BIF_TRAP2(bif_export[BIF_persistent_term_put_2], BIF_P, state_mref, BIF_ARG_2)
#define TRAPPING_COPY_TABLE_PUT(TABLE_DEST, OLD_TABLE, NEW_SIZE, COPY_TYPE, LOC_NAME) \
TRAPPING_COPY_TABLE(TABLE_DEST, OLD_TABLE, NEW_SIZE, COPY_TYPE, LOC_NAME, PUT_TRAP_CODE)
#ifdef DEBUG
(void)ITERATIONS_PER_RED;
iterations_until_trap = max_iterations =
GET_SMALL_RANDOM_INT(ERTS_BIF_REDS_LEFT(BIF_P) + (Uint)&ctx);
#else
iterations_until_trap = max_iterations =
ITERATIONS_PER_RED * ERTS_BIF_REDS_LEFT(BIF_P);
#endif
if (is_internal_magic_ref(BIF_ARG_1) &&
(ERTS_MAGIC_BIN_DESTRUCTOR(erts_magic_ref2bin(BIF_ARG_1)) ==
persistent_term_put_2_ctx_bin_dtor)) {
/* Restore state after a trap */
Binary* state_bin;
state_mref = BIF_ARG_1;
state_bin = erts_magic_ref2bin(state_mref);
ctx = ERTS_MAGIC_BIN_DATA(state_bin);
ASSERT(BIF_P->flags & F_DISABLE_GC);
erts_set_gc_state(BIF_P, 1);
switch (ctx->trap_location) {
case PUT2_TRAP_LOCATION_NEW_KEY:
goto L_PUT2_TRAP_LOCATION_NEW_KEY;
case PUT2_TRAP_LOCATION_REPLACE_VALUE:
goto L_PUT2_TRAP_LOCATION_REPLACE_VALUE;
}
} else {
/* Save state in magic bin in case trapping is necessary */
Eterm* hp;
Binary* state_bin = erts_create_magic_binary(sizeof(ErtsPersistentTermPut2Context),
persistent_term_put_2_ctx_bin_dtor);
hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE);
state_mref = erts_mk_magic_ref(&hp, &MSO(BIF_P), state_bin);
ctx = ERTS_MAGIC_BIN_DATA(state_bin);
/*
* IMPORTANT: The following field is used to detect if
* persistent_term_put_2_ctx_bin_dtor needs to free memory
*/
ctx->cpy_ctx.new_table = NULL;
}
if (!try_seize_update_permission(BIF_P)) {
ERTS_BIF_YIELD2(bif_export[BIF_persistent_term_put_2],
BIF_P, BIF_ARG_1, BIF_ARG_2);
}
ctx->hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
ctx->key = BIF_ARG_1;
ctx->term = BIF_ARG_2;
ctx->entry_index = lookup(ctx->hash_table, ctx->key);
ctx->heap[0] = make_arityval(2);
ctx->heap[1] = ctx->key;
ctx->heap[2] = ctx->term;
ctx->tuple = make_tuple(ctx->heap);
if (is_nil(ctx->hash_table->term[ctx->entry_index])) {
Uint new_size = ctx->hash_table->allocated;
if (MUST_GROW(ctx->hash_table)) {
new_size *= 2;
}
TRAPPING_COPY_TABLE_PUT(ctx->hash_table,
ctx->hash_table,
new_size,
ERTS_PERSISTENT_TERM_CPY_NO_REHASH,
PUT2_TRAP_LOCATION_NEW_KEY);
ctx->entry_index = lookup(ctx->hash_table, ctx->key);
ctx->hash_table->num_entries++;
} else {
Eterm tuple = ctx->hash_table->term[ctx->entry_index];
Eterm old_term;
ASSERT(is_tuple_arity(tuple, 2));
old_term = boxed_val(tuple)[2];
if (EQ(ctx->term, old_term)) {
/* Same value. No need to update anything. */
release_update_permission(0);
BIF_RET(am_ok);
} else {
/* Mark the old term for deletion. */
mark_for_deletion(ctx->hash_table, ctx->entry_index);
TRAPPING_COPY_TABLE_PUT(ctx->hash_table,
ctx->hash_table,
ctx->hash_table->allocated,
ERTS_PERSISTENT_TERM_CPY_NO_REHASH,
PUT2_TRAP_LOCATION_REPLACE_VALUE);
}
}
{
Uint term_size;
Uint lit_area_size;
ErlOffHeap code_off_heap;
ErtsLiteralArea* literal_area;
erts_shcopy_t info;
Eterm* ptr;
/*
* Preserve internal sharing in the term by using the
* sharing-preserving functions. However, literals must
* be copied in case the module holding them are unloaded.
*/
INITIALIZE_SHCOPY(info);
info.copy_literals = 1;
term_size = copy_shared_calculate(ctx->tuple, &info);
ERTS_INIT_OFF_HEAP(&code_off_heap);
lit_area_size = ERTS_LITERAL_AREA_ALLOC_SIZE(term_size);
literal_area = erts_alloc(ERTS_ALC_T_LITERAL, lit_area_size);
ptr = &literal_area->start[0];
literal_area->end = ptr + term_size;
ctx->tuple = copy_shared_perform(ctx->tuple, term_size, &info, &ptr, &code_off_heap);
ASSERT(tuple_val(ctx->tuple) == literal_area->start);
literal_area->off_heap = code_off_heap.first;
DESTROY_SHCOPY(info);
erts_set_literal_tag(&ctx->tuple, literal_area->start, term_size);
ctx->hash_table->term[ctx->entry_index] = ctx->tuple;
erts_schedule_thr_prgr_later_op(table_updater, ctx->hash_table, &thr_prog_op);
suspend_updater(BIF_P);
}
BUMP_REDS(BIF_P, (max_iterations - iterations_until_trap) / ITERATIONS_PER_RED);
ERTS_BIF_YIELD_RETURN(BIF_P, am_ok);
}
/* erlang:'--'/2 */
static Eterm subtract(Export *bif_entry, BIF_ALIST_2) {
Eterm lhs = BIF_ARG_1, rhs = BIF_ARG_2;
if ((is_list(lhs) || is_nil(lhs)) && (is_list(rhs) || is_nil(rhs))) {
/* We start with the context on the stack in the hopes that we won't
* have to trap. */
ErtsSubtractContext context;
int res;
res = subtract_start(BIF_P, lhs, rhs, &context);
if (res == 0) {
Eterm state_mref;
state_mref = subtract_create_trap_state(BIF_P, &context);
erts_set_gc_state(BIF_P, 0);
BIF_TRAP2(bif_entry, BIF_P, state_mref, NIL);
}
subtract_ctx_dtor(&context);
if (res < 0) {
BIF_ERROR(BIF_P, BADARG);
}
BIF_RET(context.result);
} else if (is_internal_magic_ref(lhs)) {
ErtsSubtractContext *context;
int (*dtor)(Binary*);
Binary *magic_bin;
int res;
magic_bin = erts_magic_ref2bin(lhs);
dtor = ERTS_MAGIC_BIN_DESTRUCTOR(magic_bin);
if (dtor != subtract_ctx_bin_dtor) {
BIF_ERROR(BIF_P, BADARG);
}
ASSERT(BIF_P->flags & F_DISABLE_GC);
ASSERT(rhs == NIL);
context = ERTS_MAGIC_BIN_DATA(magic_bin);
res = subtract_continue(BIF_P, context);
if (res == 0) {
BIF_TRAP2(bif_entry, BIF_P, lhs, NIL);
}
erts_set_gc_state(BIF_P, 1);
if (res < 0) {
ERTS_BIF_ERROR_TRAPPED2(BIF_P, BADARG, bif_entry,
context->lhs_original,
context->rhs_original);
}
BIF_RET(context->result);
}
ASSERT(!(BIF_P->flags & F_DISABLE_GC));
BIF_ERROR(BIF_P, BADARG);
}
static int subtract_ctx_bin_dtor(Binary *context_bin) {
ErtsSubtractContext *context = ERTS_MAGIC_BIN_DATA(context_bin);
subtract_ctx_dtor(context);
return 1;
}