BIF_RETTYPE persistent_term_get_0(BIF_ALIST_0) { HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); TrapData* trap_data; Eterm res = NIL; Eterm magic_ref; Binary* mbp; magic_ref = alloc_trap_data(BIF_P); mbp = erts_magic_ref2bin(magic_ref); trap_data = ERTS_MAGIC_BIN_DATA(mbp); trap_data->table = hash_table; trap_data->idx = 0; trap_data->remaining = hash_table->num_entries; res = do_get_all(BIF_P, trap_data, res); if (trap_data->remaining == 0) { BUMP_REDS(BIF_P, hash_table->num_entries); trap_data->table = NULL; /* Prevent refc decrement */ BIF_RET(res); } else { /* * Increment the ref counter to prevent an update operation (by put/2 * or erase/1) to delete this hash table. */ erts_atomic_inc_nob(&hash_table->refc); BUMP_ALL_REDS(BIF_P); BIF_TRAP2(&persistent_term_get_all_export, BIF_P, magic_ref, res); } }
static int persistent_term_put_2_ctx_bin_dtor(Binary *context_bin) { ErtsPersistentTermPut2Context* ctx = ERTS_MAGIC_BIN_DATA(context_bin); if (ctx->cpy_ctx.new_table != NULL) { erts_free(ERTS_ALC_T_PERSISTENT_TERM, ctx->cpy_ctx.new_table); release_update_permission(0); } return 1; }
void* enif_alloc_resource(ErlNifResourceType* type, size_t size) { Binary* bin = erts_create_magic_binary(SIZEOF_ErlNifResource(size), &nif_resource_dtor); ErlNifResource* resource = ERTS_MAGIC_BIN_DATA(bin); resource->type = type; erts_refc_inc(&bin->refc, 1); #ifdef DEBUG erts_refc_init(&resource->nif_refc, 1); #endif erts_refc_inc(&resource->type->refc, 2); return resource->data; }
static void suspend_until_thr_prg(Process* p) { Binary* state_bin; ErtsFlxCtrWakeUpLaterInfo* info; state_bin = erts_create_magic_binary(sizeof(ErtsFlxCtrWakeUpLaterInfo), erts_flxctr_wait_dtor); info = ERTS_MAGIC_BIN_DATA(state_bin); info->process = p; erts_refc_inctest(&state_bin->intern.refc, 1); erts_suspend(p, ERTS_PROC_LOCK_MAIN, NULL); erts_proc_inc_refc(p); ERTS_VBUMP_ALL_REDS(p); erts_schedule_thr_prgr_later_op(thr_prg_wake_up_later, state_bin, &info->later_op); }
static int cleanup_trap_data(Binary *bp) { TrapData* trap_data = ERTS_MAGIC_BIN_DATA(bp); if (trap_data->table) { /* * The process has been killed and is now exiting. * Decrement the reference counter for the table. */ dec_table_refc(NULL, trap_data->table); } return 1; }
static Eterm subtract_create_trap_state(Process *p, ErtsSubtractContext *context) { Binary *state_bin; Eterm *hp; state_bin = erts_create_magic_binary(sizeof(ErtsSubtractContext), subtract_ctx_bin_dtor); subtract_ctx_move(context, ERTS_MAGIC_BIN_DATA(state_bin)); hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); return erts_mk_magic_ref(&hp, &MSO(p), state_bin); }
static void thr_prg_wake_up_later(void* bin_p) { Binary* bin = bin_p; ErtsFlxCtrWakeUpLaterInfo* info = ERTS_MAGIC_BIN_DATA(bin); Process* p = info->process; /* Resume the requesting process */ erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); if (!ERTS_PROC_IS_EXITING(p)) { erts_resume(p, ERTS_PROC_LOCK_STATUS); } erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); /* Free data */ erts_proc_dec_refc(p); erts_bin_release(bin); }
static int persistent_term_erase_1_ctx_bin_dtor(Binary *context_bin) { ErtsPersistentTermErase1Context* ctx = ERTS_MAGIC_BIN_DATA(context_bin); if (ctx->cpy_ctx.new_table != NULL) { if (ctx->cpy_ctx.copy_type == ERTS_PERSISTENT_TERM_CPY_TEMP) { erts_free(ERTS_ALC_T_PERSISTENT_TERM_TMP, ctx->cpy_ctx.new_table); } else { erts_free(ERTS_ALC_T_PERSISTENT_TERM, ctx->cpy_ctx.new_table); } if (ctx->tmp_table != NULL) { erts_free(ERTS_ALC_T_PERSISTENT_TERM_TMP, ctx->tmp_table); } release_update_permission(0); } return 1; }
static void thr_prg_wake_up_and_count(void* bin_p) { Binary* bin = bin_p; DecentralizedReadSnapshotInfo* info = ERTS_MAGIC_BIN_DATA(bin); Process* p = info->process; ErtsFlxCtrDecentralizedCtrArray* array = info->array; ErtsFlxCtrDecentralizedCtrArray* next = info->next_array; int i, sched; /* Reset result array */ for (i = 0; i < info->nr_of_counters; i++) { info->result[i] = 0; } /* Read result from snapshot */ for (sched = 0; sched < ERTS_FLXCTR_DECENTRALIZED_NO_SLOTS; sched++) { for (i = 0; i < info->nr_of_counters; i++) { info->result[i] = info->result[i] + erts_atomic_read_nob(&array->array[sched].counters[i]); } } /* Update the next decentralized counter array */ for (i = 0; i < info->nr_of_counters; i++) { erts_atomic_add_nob(&next->array[0].counters[i], info->result[i]); } /* Announce that the snapshot is done */ { Sint expected = ERTS_FLXCTR_SNAPSHOT_ONGOING; if (expected != erts_atomic_cmpxchg_mb(&next->snapshot_status, ERTS_FLXCTR_SNAPSHOT_NOT_ONGOING, expected)) { /* The CAS failed which means that this thread need to free the next array. */ erts_free(info->alloc_type, next->block_start); } } /* Resume the process that requested the snapshot */ erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); if (!ERTS_PROC_IS_EXITING(p)) { erts_resume(p, ERTS_PROC_LOCK_STATUS); } /* Free the memory that is no longer needed */ erts_free(info->alloc_type, array->block_start); erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); erts_proc_dec_refc(p); erts_bin_release(bin); }
static void nif_resource_dtor(Binary* bin) { ErlNifResource* resource = (ErlNifResource*) ERTS_MAGIC_BIN_DATA(bin); ErlNifResourceType* type = resource->type; ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); if (type->dtor != NULL) { ErlNifEnv env; pre_nif_noproc(&env, type->owner); type->dtor(&env,resource->data); post_nif_noproc(&env); } if (erts_refc_dectest(&type->refc, 0) == 0) { ASSERT(type->next == NULL); ASSERT(type->owner != NULL); ASSERT(type->owner->mod == NULL); steal_resource_type(type); erts_free(ERTS_ALC_T_NIF, type); } }
int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* type, void** objp) { ProcBin* pb; Binary* mbin; ErlNifResource* resource; if (!ERTS_TERM_IS_MAGIC_BINARY(term)) { return 0; } pb = (ProcBin*) binary_val(term); /*if (pb->size != 0) { return 0; / * Or should we allow "resource binaries" as handles? * / }*/ mbin = pb->val; resource = (ErlNifResource*) ERTS_MAGIC_BIN_DATA(mbin); if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor || resource->type != type) { return 0; } *objp = resource->data; return 1; }
BIF_RETTYPE persistent_term_erase_1(BIF_ALIST_1) { static const Uint ITERATIONS_PER_RED = 32; ErtsPersistentTermErase1Context* ctx; Eterm state_mref = THE_NON_VALUE; long iterations_until_trap; long max_iterations; #ifdef DEBUG (void)ITERATIONS_PER_RED; iterations_until_trap = max_iterations = GET_SMALL_RANDOM_INT(ERTS_BIF_REDS_LEFT(BIF_P) + (Uint)&ctx); #else iterations_until_trap = max_iterations = ITERATIONS_PER_RED * ERTS_BIF_REDS_LEFT(BIF_P); #endif #define ERASE_TRAP_CODE \ BIF_TRAP1(bif_export[BIF_persistent_term_erase_1], BIF_P, state_mref); #define TRAPPING_COPY_TABLE_ERASE(TABLE_DEST, OLD_TABLE, NEW_SIZE, REHASH, LOC_NAME) \ TRAPPING_COPY_TABLE(TABLE_DEST, OLD_TABLE, NEW_SIZE, REHASH, LOC_NAME, ERASE_TRAP_CODE) if (is_internal_magic_ref(BIF_ARG_1) && (ERTS_MAGIC_BIN_DESTRUCTOR(erts_magic_ref2bin(BIF_ARG_1)) == persistent_term_erase_1_ctx_bin_dtor)) { /* Restore the state after a trap */ Binary* state_bin; state_mref = BIF_ARG_1; state_bin = erts_magic_ref2bin(state_mref); ctx = ERTS_MAGIC_BIN_DATA(state_bin); ASSERT(BIF_P->flags & F_DISABLE_GC); erts_set_gc_state(BIF_P, 1); switch (ctx->trap_location) { case ERASE1_TRAP_LOCATION_TMP_COPY: goto L_ERASE1_TRAP_LOCATION_TMP_COPY; case ERASE1_TRAP_LOCATION_FINAL_COPY: goto L_ERASE1_TRAP_LOCATION_FINAL_COPY; } } else { /* Save state in magic bin in case trapping is necessary */ Eterm* hp; Binary* state_bin = erts_create_magic_binary(sizeof(ErtsPersistentTermErase1Context), persistent_term_erase_1_ctx_bin_dtor); hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE); state_mref = erts_mk_magic_ref(&hp, &MSO(BIF_P), state_bin); ctx = ERTS_MAGIC_BIN_DATA(state_bin); /* * IMPORTANT: The following two fields are used to detect if * persistent_term_erase_1_ctx_bin_dtor needs to free memory */ ctx->cpy_ctx.new_table = NULL; ctx->tmp_table = NULL; } if (!try_seize_update_permission(BIF_P)) { ERTS_BIF_YIELD1(bif_export[BIF_persistent_term_erase_1], BIF_P, BIF_ARG_1); } ctx->key = BIF_ARG_1; ctx->old_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); ctx->entry_index = lookup(ctx->old_table, ctx->key); ctx->old_term = ctx->old_table->term[ctx->entry_index]; if (is_boxed(ctx->old_term)) { Uint new_size; /* * Since we don't use any delete markers, we must rehash * the table when deleting terms to ensure that all terms * can still be reached if there are hash collisions. * We can't rehash in place and it would not be safe to modify * the old table yet, so we will first need a new * temporary table copy of the same size as the old one. */ ASSERT(is_tuple_arity(ctx->old_term, 2)); TRAPPING_COPY_TABLE_ERASE(ctx->tmp_table, ctx->old_table, ctx->old_table->allocated, ERTS_PERSISTENT_TERM_CPY_TEMP, ERASE1_TRAP_LOCATION_TMP_COPY); /* * Delete the term from the temporary table. Then copy the * temporary table to a new table, rehashing the entries * while copying. */ ctx->tmp_table->term[ctx->entry_index] = NIL; ctx->tmp_table->num_entries--; new_size = ctx->tmp_table->allocated; if (MUST_SHRINK(ctx->tmp_table)) { new_size /= 2; } TRAPPING_COPY_TABLE_ERASE(ctx->new_table, ctx->tmp_table, new_size, ERTS_PERSISTENT_TERM_CPY_REHASH, ERASE1_TRAP_LOCATION_FINAL_COPY); erts_free(ERTS_ALC_T_PERSISTENT_TERM_TMP, ctx->tmp_table); /* * IMPORTANT: Memory management depends on that ctx->tmp_table * is set to NULL on the line below */ ctx->tmp_table = NULL; mark_for_deletion(ctx->old_table, ctx->entry_index); erts_schedule_thr_prgr_later_op(table_updater, ctx->new_table, &thr_prog_op); suspend_updater(BIF_P); BUMP_REDS(BIF_P, (max_iterations - iterations_until_trap) / ITERATIONS_PER_RED); ERTS_BIF_YIELD_RETURN(BIF_P, am_true); } /* * Key is not present. Nothing to do. */ ASSERT(is_nil(ctx->old_term)); release_update_permission(0); BIF_RET(am_false); }
BIF_RETTYPE persistent_term_put_2(BIF_ALIST_2) { static const Uint ITERATIONS_PER_RED = 32; ErtsPersistentTermPut2Context* ctx; Eterm state_mref = THE_NON_VALUE; long iterations_until_trap; long max_iterations; #define PUT_TRAP_CODE \ BIF_TRAP2(bif_export[BIF_persistent_term_put_2], BIF_P, state_mref, BIF_ARG_2) #define TRAPPING_COPY_TABLE_PUT(TABLE_DEST, OLD_TABLE, NEW_SIZE, COPY_TYPE, LOC_NAME) \ TRAPPING_COPY_TABLE(TABLE_DEST, OLD_TABLE, NEW_SIZE, COPY_TYPE, LOC_NAME, PUT_TRAP_CODE) #ifdef DEBUG (void)ITERATIONS_PER_RED; iterations_until_trap = max_iterations = GET_SMALL_RANDOM_INT(ERTS_BIF_REDS_LEFT(BIF_P) + (Uint)&ctx); #else iterations_until_trap = max_iterations = ITERATIONS_PER_RED * ERTS_BIF_REDS_LEFT(BIF_P); #endif if (is_internal_magic_ref(BIF_ARG_1) && (ERTS_MAGIC_BIN_DESTRUCTOR(erts_magic_ref2bin(BIF_ARG_1)) == persistent_term_put_2_ctx_bin_dtor)) { /* Restore state after a trap */ Binary* state_bin; state_mref = BIF_ARG_1; state_bin = erts_magic_ref2bin(state_mref); ctx = ERTS_MAGIC_BIN_DATA(state_bin); ASSERT(BIF_P->flags & F_DISABLE_GC); erts_set_gc_state(BIF_P, 1); switch (ctx->trap_location) { case PUT2_TRAP_LOCATION_NEW_KEY: goto L_PUT2_TRAP_LOCATION_NEW_KEY; case PUT2_TRAP_LOCATION_REPLACE_VALUE: goto L_PUT2_TRAP_LOCATION_REPLACE_VALUE; } } else { /* Save state in magic bin in case trapping is necessary */ Eterm* hp; Binary* state_bin = erts_create_magic_binary(sizeof(ErtsPersistentTermPut2Context), persistent_term_put_2_ctx_bin_dtor); hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE); state_mref = erts_mk_magic_ref(&hp, &MSO(BIF_P), state_bin); ctx = ERTS_MAGIC_BIN_DATA(state_bin); /* * IMPORTANT: The following field is used to detect if * persistent_term_put_2_ctx_bin_dtor needs to free memory */ ctx->cpy_ctx.new_table = NULL; } if (!try_seize_update_permission(BIF_P)) { ERTS_BIF_YIELD2(bif_export[BIF_persistent_term_put_2], BIF_P, BIF_ARG_1, BIF_ARG_2); } ctx->hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); ctx->key = BIF_ARG_1; ctx->term = BIF_ARG_2; ctx->entry_index = lookup(ctx->hash_table, ctx->key); ctx->heap[0] = make_arityval(2); ctx->heap[1] = ctx->key; ctx->heap[2] = ctx->term; ctx->tuple = make_tuple(ctx->heap); if (is_nil(ctx->hash_table->term[ctx->entry_index])) { Uint new_size = ctx->hash_table->allocated; if (MUST_GROW(ctx->hash_table)) { new_size *= 2; } TRAPPING_COPY_TABLE_PUT(ctx->hash_table, ctx->hash_table, new_size, ERTS_PERSISTENT_TERM_CPY_NO_REHASH, PUT2_TRAP_LOCATION_NEW_KEY); ctx->entry_index = lookup(ctx->hash_table, ctx->key); ctx->hash_table->num_entries++; } else { Eterm tuple = ctx->hash_table->term[ctx->entry_index]; Eterm old_term; ASSERT(is_tuple_arity(tuple, 2)); old_term = boxed_val(tuple)[2]; if (EQ(ctx->term, old_term)) { /* Same value. No need to update anything. */ release_update_permission(0); BIF_RET(am_ok); } else { /* Mark the old term for deletion. */ mark_for_deletion(ctx->hash_table, ctx->entry_index); TRAPPING_COPY_TABLE_PUT(ctx->hash_table, ctx->hash_table, ctx->hash_table->allocated, ERTS_PERSISTENT_TERM_CPY_NO_REHASH, PUT2_TRAP_LOCATION_REPLACE_VALUE); } } { Uint term_size; Uint lit_area_size; ErlOffHeap code_off_heap; ErtsLiteralArea* literal_area; erts_shcopy_t info; Eterm* ptr; /* * Preserve internal sharing in the term by using the * sharing-preserving functions. However, literals must * be copied in case the module holding them are unloaded. */ INITIALIZE_SHCOPY(info); info.copy_literals = 1; term_size = copy_shared_calculate(ctx->tuple, &info); ERTS_INIT_OFF_HEAP(&code_off_heap); lit_area_size = ERTS_LITERAL_AREA_ALLOC_SIZE(term_size); literal_area = erts_alloc(ERTS_ALC_T_LITERAL, lit_area_size); ptr = &literal_area->start[0]; literal_area->end = ptr + term_size; ctx->tuple = copy_shared_perform(ctx->tuple, term_size, &info, &ptr, &code_off_heap); ASSERT(tuple_val(ctx->tuple) == literal_area->start); literal_area->off_heap = code_off_heap.first; DESTROY_SHCOPY(info); erts_set_literal_tag(&ctx->tuple, literal_area->start, term_size); ctx->hash_table->term[ctx->entry_index] = ctx->tuple; erts_schedule_thr_prgr_later_op(table_updater, ctx->hash_table, &thr_prog_op); suspend_updater(BIF_P); } BUMP_REDS(BIF_P, (max_iterations - iterations_until_trap) / ITERATIONS_PER_RED); ERTS_BIF_YIELD_RETURN(BIF_P, am_ok); }
/* erlang:'--'/2 */ static Eterm subtract(Export *bif_entry, BIF_ALIST_2) { Eterm lhs = BIF_ARG_1, rhs = BIF_ARG_2; if ((is_list(lhs) || is_nil(lhs)) && (is_list(rhs) || is_nil(rhs))) { /* We start with the context on the stack in the hopes that we won't * have to trap. */ ErtsSubtractContext context; int res; res = subtract_start(BIF_P, lhs, rhs, &context); if (res == 0) { Eterm state_mref; state_mref = subtract_create_trap_state(BIF_P, &context); erts_set_gc_state(BIF_P, 0); BIF_TRAP2(bif_entry, BIF_P, state_mref, NIL); } subtract_ctx_dtor(&context); if (res < 0) { BIF_ERROR(BIF_P, BADARG); } BIF_RET(context.result); } else if (is_internal_magic_ref(lhs)) { ErtsSubtractContext *context; int (*dtor)(Binary*); Binary *magic_bin; int res; magic_bin = erts_magic_ref2bin(lhs); dtor = ERTS_MAGIC_BIN_DESTRUCTOR(magic_bin); if (dtor != subtract_ctx_bin_dtor) { BIF_ERROR(BIF_P, BADARG); } ASSERT(BIF_P->flags & F_DISABLE_GC); ASSERT(rhs == NIL); context = ERTS_MAGIC_BIN_DATA(magic_bin); res = subtract_continue(BIF_P, context); if (res == 0) { BIF_TRAP2(bif_entry, BIF_P, lhs, NIL); } erts_set_gc_state(BIF_P, 1); if (res < 0) { ERTS_BIF_ERROR_TRAPPED2(BIF_P, BADARG, bif_entry, context->lhs_original, context->rhs_original); } BIF_RET(context->result); } ASSERT(!(BIF_P->flags & F_DISABLE_GC)); BIF_ERROR(BIF_P, BADARG); }
static int subtract_ctx_bin_dtor(Binary *context_bin) { ErtsSubtractContext *context = ERTS_MAGIC_BIN_DATA(context_bin); subtract_ctx_dtor(context); return 1; }