static HashTable*
next_to_delete(void)
{
HashTable* table;
erts_mtx_lock(&delete_queue_mtx);
table = delete_queue_head;
if (table) {
if (erts_atomic_read_nob(&table->refc)) {
/*
* This hash table is still referenced. Hash tables
* must be deleted in order, so we return a NULL
* pointer.
*/
table = NULL;
} else {
/*
* Remove the first hash table from the queue.
*/
delete_queue_head = table->delete_next;
if (delete_queue_head == NULL) {
delete_queue_tail = &delete_queue_head;
}
}
}
erts_mtx_unlock(&delete_queue_mtx);
return table;
}
BIF_RETTYPE persistent_term_info_0(BIF_ALIST_0)
{
HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
TrapData* trap_data;
Eterm res = NIL;
Eterm magic_ref;
Binary* mbp;
magic_ref = alloc_trap_data(BIF_P);
mbp = erts_magic_ref2bin(magic_ref);
trap_data = ERTS_MAGIC_BIN_DATA(mbp);
trap_data->table = hash_table;
trap_data->idx = 0;
trap_data->remaining = hash_table->num_entries;
trap_data->memory = 0;
res = do_info(BIF_P, trap_data);
if (trap_data->remaining == 0) {
BUMP_REDS(BIF_P, hash_table->num_entries);
trap_data->table = NULL; /* Prevent refc decrement */
BIF_RET(res);
} else {
/*
* Increment the ref counter to prevent an update operation (by put/2
* or erase/1) to delete this hash table.
*/
erts_atomic_inc_nob(&hash_table->refc);
BUMP_ALL_REDS(BIF_P);
BIF_TRAP2(&persistent_term_info_export, BIF_P, magic_ref, res);
}
}
void
erts_thr_progress_unblock(void)
{
erts_tse_t *event;
int id, break_id, sz, wakeup;
ErtsThrPrgrData *tpd = thr_prgr_data(NULL);
ASSERT(tpd->is_blocking);
if (--tpd->is_blocking)
return;
sz = intrnl->managed.no;
wakeup = 1;
if (!tpd->is_managed)
id = break_id = tpd->id < 0 ? 0 : tpd->id % sz;
else {
break_id = tpd->id;
id = break_id + 1;
if (id >= sz)
id = 0;
if (id == break_id)
wakeup = 0;
erts_atomic32_inc_nob(&intrnl->misc.data.block_count);
}
event = ((erts_tse_t *)
erts_atomic_read_nob(&intrnl->misc.data.blocker_event));
ASSERT(event);
erts_atomic_set_nob(&intrnl->misc.data.blocker_event, ERTS_AINT_NULL);
erts_atomic32_read_bor_relb(&intrnl->misc.data.block_count,
ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING);
#if ERTS_THR_PRGR_PRINT_BLOCKERS
erts_fprintf(stderr, "unblock(%d)\n", tpd->id);
#endif
erts_atomic32_read_band_mb(&intrnl->misc.data.lflgs,
~ERTS_THR_PRGR_LFLG_BLOCK);
if (wakeup) {
do {
ErtsThrPrgrVal tmp;
tmp = read_nob(&intrnl->thr[id].data.current);
if (tmp != ERTS_THR_PRGR_VAL_WAITING)
wakeup_managed(id);
if (++id >= sz)
id = 0;
} while (id != break_id);
}
return_tmp_thr_prgr_data(tpd);
erts_tse_return(event);
}
BIF_RETTYPE persistent_term_get_1(BIF_ALIST_1)
{
Eterm key = BIF_ARG_1;
HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
Uint entry_index;
Eterm term;
entry_index = lookup(hash_table, key);
term = hash_table->term[entry_index];
if (is_boxed(term)) {
ASSERT(is_tuple_arity(term, 2));
BIF_RET(tuple_val(term)[2]);
}
BIF_ERROR(BIF_P, BADARG);
}
static ERTS_INLINE int
block_count_dec(void)
{
erts_aint32_t block_count;
block_count = erts_atomic32_dec_read_mb(&intrnl->misc.data.block_count);
if (block_count == 0) {
erts_tse_t *event;
event = ((erts_tse_t*)
erts_atomic_read_nob(&intrnl->misc.data.blocker_event));
if (event)
erts_tse_set(event);
return 1;
}
return (block_count & ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING) == 0;
}
BIF_RETTYPE persistent_term_get_2(BIF_ALIST_2)
{
Eterm key = BIF_ARG_1;
Eterm result = BIF_ARG_2;
HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
Uint entry_index;
Eterm term;
entry_index = lookup(hash_table, key);
term = hash_table->term[entry_index];
if (is_boxed(term)) {
ASSERT(is_tuple_arity(term, 2));
result = tuple_val(term)[2];
}
BIF_RET(result);
}
static void
table_updater(void* data)
{
HashTable* old_table;
HashTable* new_table;
old_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
new_table = (HashTable *) data;
ASSERT(new_table->num_to_delete == 0);
erts_atomic_set_nob(&the_hash_table, (erts_aint_t)new_table);
append_to_delete_queue(old_table);
erts_schedule_thr_prgr_later_op(table_deleter,
old_table,
&old_table->thr_prog_op);
release_update_permission(1);
}
BIF_RETTYPE erts_internal_erase_persistent_terms_0(BIF_ALIST_0)
{
HashTable* old_table;
HashTable* new_table;
if (!try_seize_update_permission(BIF_P)) {
ERTS_BIF_YIELD0(bif_export[BIF_erts_internal_erase_persistent_terms_0],
BIF_P);
}
old_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
old_table->first_to_delete = 0;
old_table->num_to_delete = old_table->allocated;
new_table = create_initial_table();
erts_schedule_thr_prgr_later_op(table_updater, new_table, &thr_prog_op);
suspend_updater(BIF_P);
ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
}
static erts_aint32_t
thr_progress_block(ErtsThrPrgrData *tpd, int wait)
{
erts_tse_t *event = NULL; /* Remove erroneous warning... sigh... */
erts_aint32_t lflgs, bc;
if (tpd->is_blocking++)
return (erts_aint32_t) 0;
while (1) {
lflgs = erts_atomic32_read_bor_nob(&intrnl->misc.data.lflgs,
ERTS_THR_PRGR_LFLG_BLOCK);
if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK)
block_thread(tpd);
else
break;
}
#if ERTS_THR_PRGR_PRINT_BLOCKERS
erts_fprintf(stderr, "block(%d)\n", tpd->id);
#endif
ASSERT(ERTS_AINT_NULL
== erts_atomic_read_nob(&intrnl->misc.data.blocker_event));
if (wait) {
event = erts_tse_fetch();
erts_tse_reset(event);
erts_atomic_set_nob(&intrnl->misc.data.blocker_event,
(erts_aint_t) event);
}
if (tpd->is_managed)
erts_atomic32_dec_nob(&intrnl->misc.data.block_count);
bc = erts_atomic32_read_band_mb(&intrnl->misc.data.block_count,
~ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING);
bc &= ~ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING;
if (wait) {
while (bc != 0) {
erts_tse_wait(event);
erts_tse_reset(event);
bc = erts_atomic32_read_acqb(&intrnl->misc.data.block_count);
}
}
return bc;
}
static void
thr_prg_wake_up_and_count(void* bin_p)
{
Binary* bin = bin_p;
DecentralizedReadSnapshotInfo* info = ERTS_MAGIC_BIN_DATA(bin);
Process* p = info->process;
ErtsFlxCtrDecentralizedCtrArray* array = info->array;
ErtsFlxCtrDecentralizedCtrArray* next = info->next_array;
int i, sched;
/* Reset result array */
for (i = 0; i < info->nr_of_counters; i++) {
info->result[i] = 0;
}
/* Read result from snapshot */
for (sched = 0; sched < ERTS_FLXCTR_DECENTRALIZED_NO_SLOTS; sched++) {
for (i = 0; i < info->nr_of_counters; i++) {
info->result[i] = info->result[i] +
erts_atomic_read_nob(&array->array[sched].counters[i]);
}
}
/* Update the next decentralized counter array */
for (i = 0; i < info->nr_of_counters; i++) {
erts_atomic_add_nob(&next->array[0].counters[i], info->result[i]);
}
/* Announce that the snapshot is done */
{
Sint expected = ERTS_FLXCTR_SNAPSHOT_ONGOING;
if (expected != erts_atomic_cmpxchg_mb(&next->snapshot_status,
ERTS_FLXCTR_SNAPSHOT_NOT_ONGOING,
expected)) {
/* The CAS failed which means that this thread need to free the next array. */
erts_free(info->alloc_type, next->block_start);
}
}
/* Resume the process that requested the snapshot */
erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
if (!ERTS_PROC_IS_EXITING(p)) {
erts_resume(p, ERTS_PROC_LOCK_STATUS);
}
/* Free the memory that is no longer needed */
erts_free(info->alloc_type, array->block_start);
erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
erts_proc_dec_refc(p);
erts_bin_release(bin);
}
Uint erts_process_memory(Process *p, int incl_msg_inq) {
ErtsMessage *mp;
Uint size = 0;
struct saved_calls *scb;
size += sizeof(Process);
if (incl_msg_inq)
ERTS_MSGQ_MV_INQ2PRIVQ(p);
erts_doforall_links(ERTS_P_LINKS(p), &erts_one_link_size, &size);
erts_doforall_monitors(ERTS_P_MONITORS(p), &erts_one_mon_size, &size);
size += (p->heap_sz + p->mbuf_sz) * sizeof(Eterm);
if (p->abandoned_heap)
size += (p->hend - p->heap) * sizeof(Eterm);
if (p->old_hend && p->old_heap)
size += (p->old_hend - p->old_heap) * sizeof(Eterm);
size += p->msg.len * sizeof(ErtsMessage);
for (mp = p->msg.first; mp; mp = mp->next)
if (mp->data.attached)
size += erts_msg_attached_data_size(mp)*sizeof(Eterm);
if (p->arg_reg != p->def_arg_reg) {
size += p->arity * sizeof(p->arg_reg[0]);
}
if (erts_atomic_read_nob(&p->psd) != (erts_aint_t) NULL)
size += sizeof(ErtsPSD);
scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p);
if (scb) {
size += (sizeof(struct saved_calls)
+ (scb->len-1) * sizeof(scb->ct[0]));
}
size += erts_dicts_mem_size(p);
return size;
}
static Range*
find_range(BeamInstr* pc)
{
ErtsCodeIndex active = erts_active_code_ix();
Range* low = r[active].modules;
Range* high = low + r[active].n;
Range* mid = (Range *) erts_atomic_read_nob(&r[active].mid);
CHECK(&r[active]);
while (low < high) {
if (pc < mid->start) {
high = mid;
} else if (pc >= RANGE_END(mid)) {
low = mid + 1;
} else {
erts_atomic_set_nob(&r[active].mid, (erts_aint_t) mid);
return mid;
}
mid = low + (high-low) / 2;
}
return 0;
}
void
erts_init_persistent_dumping(void)
{
HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
ErtsLiteralArea** area_p;
Uint i;
/*
* Overwrite the array of Eterms in the current hash table
* with pointers to literal areas.
*/
erts_persistent_areas = (ErtsLiteralArea **) hash_table->term;
erts_num_persistent_areas = hash_table->num_entries;
area_p = erts_persistent_areas;
for (i = 0; i < hash_table->allocated; i++) {
Eterm term = hash_table->term[i];
if (is_boxed(term)) {
*area_p++ = term_to_area(term);
}
}
}
return data;
}
static ERTS_INLINE void
enqueue_remote_managed_thread(erts_sspa_chunk_header_t *chdr,
erts_sspa_blk_t *this,
int cinit)
{
erts_aint_t itmp;
erts_sspa_blk_t *enq;
erts_atomic_init_nob(&this->next_atmc, ERTS_AINT_NULL);
/* Enqueue at end of list... */
enq = (erts_sspa_blk_t *) erts_atomic_read_nob(&chdr->tail.data.last);
itmp = erts_atomic_cmpxchg_relb(&enq->next_atmc,
(erts_aint_t) this,
ERTS_AINT_NULL);
if (itmp == ERTS_AINT_NULL) {
/* We are required to move last pointer */
#ifdef DEBUG
ASSERT(ERTS_AINT_NULL == erts_atomic_read_nob(&this->next_atmc));
ASSERT(((erts_aint_t) enq)
== erts_atomic_xchg_relb(&chdr->tail.data.last,
(erts_aint_t) this));
#else
erts_atomic_set_relb(&chdr->tail.data.last, (erts_aint_t) this);
#endif
}
else {
return data;
}
static ERTS_INLINE erts_aint_t
enqueue_remote_managed_thread(erts_sspa_chunk_header_t *chdr,
erts_sspa_blk_t *this,
int want_last)
{
erts_aint_t ilast, itmp;
erts_atomic_init_nob(&this->next_atmc, ERTS_AINT_NULL);
/* Enqueue at end of list... */
ilast = erts_atomic_read_nob(&chdr->tail.data.last);
while (1) {
erts_sspa_blk_t *last = (erts_sspa_blk_t *) ilast;
itmp = erts_atomic_cmpxchg_mb(&last->next_atmc,
(erts_aint_t) this,
ERTS_AINT_NULL);
if (itmp == ERTS_AINT_NULL)
break;
ilast = itmp;
}
/* Move last pointer forward... */
while (1) {
erts_aint_t itmp;
if (want_last) {
if (erts_atomic_read_rb(&this->next_atmc) != ERTS_AINT_NULL) {
Uint
erts_persistent_term_count(void)
{
HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
return hash_table->num_entries;
}
UWord
erts_ranges_sz(void)
{
return erts_atomic_read_nob(&mem_used) * sizeof(Range);
}
BIF_RETTYPE persistent_term_erase_1(BIF_ALIST_1)
{
static const Uint ITERATIONS_PER_RED = 32;
ErtsPersistentTermErase1Context* ctx;
Eterm state_mref = THE_NON_VALUE;
long iterations_until_trap;
long max_iterations;
#ifdef DEBUG
(void)ITERATIONS_PER_RED;
iterations_until_trap = max_iterations =
GET_SMALL_RANDOM_INT(ERTS_BIF_REDS_LEFT(BIF_P) + (Uint)&ctx);
#else
iterations_until_trap = max_iterations =
ITERATIONS_PER_RED * ERTS_BIF_REDS_LEFT(BIF_P);
#endif
#define ERASE_TRAP_CODE \
BIF_TRAP1(bif_export[BIF_persistent_term_erase_1], BIF_P, state_mref);
#define TRAPPING_COPY_TABLE_ERASE(TABLE_DEST, OLD_TABLE, NEW_SIZE, REHASH, LOC_NAME) \
TRAPPING_COPY_TABLE(TABLE_DEST, OLD_TABLE, NEW_SIZE, REHASH, LOC_NAME, ERASE_TRAP_CODE)
if (is_internal_magic_ref(BIF_ARG_1) &&
(ERTS_MAGIC_BIN_DESTRUCTOR(erts_magic_ref2bin(BIF_ARG_1)) ==
persistent_term_erase_1_ctx_bin_dtor)) {
/* Restore the state after a trap */
Binary* state_bin;
state_mref = BIF_ARG_1;
state_bin = erts_magic_ref2bin(state_mref);
ctx = ERTS_MAGIC_BIN_DATA(state_bin);
ASSERT(BIF_P->flags & F_DISABLE_GC);
erts_set_gc_state(BIF_P, 1);
switch (ctx->trap_location) {
case ERASE1_TRAP_LOCATION_TMP_COPY:
goto L_ERASE1_TRAP_LOCATION_TMP_COPY;
case ERASE1_TRAP_LOCATION_FINAL_COPY:
goto L_ERASE1_TRAP_LOCATION_FINAL_COPY;
}
} else {
/* Save state in magic bin in case trapping is necessary */
Eterm* hp;
Binary* state_bin = erts_create_magic_binary(sizeof(ErtsPersistentTermErase1Context),
persistent_term_erase_1_ctx_bin_dtor);
hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE);
state_mref = erts_mk_magic_ref(&hp, &MSO(BIF_P), state_bin);
ctx = ERTS_MAGIC_BIN_DATA(state_bin);
/*
* IMPORTANT: The following two fields are used to detect if
* persistent_term_erase_1_ctx_bin_dtor needs to free memory
*/
ctx->cpy_ctx.new_table = NULL;
ctx->tmp_table = NULL;
}
if (!try_seize_update_permission(BIF_P)) {
ERTS_BIF_YIELD1(bif_export[BIF_persistent_term_erase_1],
BIF_P, BIF_ARG_1);
}
ctx->key = BIF_ARG_1;
ctx->old_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
ctx->entry_index = lookup(ctx->old_table, ctx->key);
ctx->old_term = ctx->old_table->term[ctx->entry_index];
if (is_boxed(ctx->old_term)) {
Uint new_size;
/*
* Since we don't use any delete markers, we must rehash
* the table when deleting terms to ensure that all terms
* can still be reached if there are hash collisions.
* We can't rehash in place and it would not be safe to modify
* the old table yet, so we will first need a new
* temporary table copy of the same size as the old one.
*/
ASSERT(is_tuple_arity(ctx->old_term, 2));
TRAPPING_COPY_TABLE_ERASE(ctx->tmp_table,
ctx->old_table,
ctx->old_table->allocated,
ERTS_PERSISTENT_TERM_CPY_TEMP,
ERASE1_TRAP_LOCATION_TMP_COPY);
/*
* Delete the term from the temporary table. Then copy the
* temporary table to a new table, rehashing the entries
* while copying.
*/
ctx->tmp_table->term[ctx->entry_index] = NIL;
ctx->tmp_table->num_entries--;
new_size = ctx->tmp_table->allocated;
if (MUST_SHRINK(ctx->tmp_table)) {
new_size /= 2;
}
TRAPPING_COPY_TABLE_ERASE(ctx->new_table,
ctx->tmp_table,
new_size,
ERTS_PERSISTENT_TERM_CPY_REHASH,
ERASE1_TRAP_LOCATION_FINAL_COPY);
erts_free(ERTS_ALC_T_PERSISTENT_TERM_TMP, ctx->tmp_table);
/*
* IMPORTANT: Memory management depends on that ctx->tmp_table
* is set to NULL on the line below
*/
ctx->tmp_table = NULL;
mark_for_deletion(ctx->old_table, ctx->entry_index);
erts_schedule_thr_prgr_later_op(table_updater, ctx->new_table, &thr_prog_op);
suspend_updater(BIF_P);
BUMP_REDS(BIF_P, (max_iterations - iterations_until_trap) / ITERATIONS_PER_RED);
ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
}
/*
* Key is not present. Nothing to do.
*/
ASSERT(is_nil(ctx->old_term));
release_update_permission(0);
BIF_RET(am_false);
}
BIF_RETTYPE persistent_term_put_2(BIF_ALIST_2)
{
static const Uint ITERATIONS_PER_RED = 32;
ErtsPersistentTermPut2Context* ctx;
Eterm state_mref = THE_NON_VALUE;
long iterations_until_trap;
long max_iterations;
#define PUT_TRAP_CODE \
BIF_TRAP2(bif_export[BIF_persistent_term_put_2], BIF_P, state_mref, BIF_ARG_2)
#define TRAPPING_COPY_TABLE_PUT(TABLE_DEST, OLD_TABLE, NEW_SIZE, COPY_TYPE, LOC_NAME) \
TRAPPING_COPY_TABLE(TABLE_DEST, OLD_TABLE, NEW_SIZE, COPY_TYPE, LOC_NAME, PUT_TRAP_CODE)
#ifdef DEBUG
(void)ITERATIONS_PER_RED;
iterations_until_trap = max_iterations =
GET_SMALL_RANDOM_INT(ERTS_BIF_REDS_LEFT(BIF_P) + (Uint)&ctx);
#else
iterations_until_trap = max_iterations =
ITERATIONS_PER_RED * ERTS_BIF_REDS_LEFT(BIF_P);
#endif
if (is_internal_magic_ref(BIF_ARG_1) &&
(ERTS_MAGIC_BIN_DESTRUCTOR(erts_magic_ref2bin(BIF_ARG_1)) ==
persistent_term_put_2_ctx_bin_dtor)) {
/* Restore state after a trap */
Binary* state_bin;
state_mref = BIF_ARG_1;
state_bin = erts_magic_ref2bin(state_mref);
ctx = ERTS_MAGIC_BIN_DATA(state_bin);
ASSERT(BIF_P->flags & F_DISABLE_GC);
erts_set_gc_state(BIF_P, 1);
switch (ctx->trap_location) {
case PUT2_TRAP_LOCATION_NEW_KEY:
goto L_PUT2_TRAP_LOCATION_NEW_KEY;
case PUT2_TRAP_LOCATION_REPLACE_VALUE:
goto L_PUT2_TRAP_LOCATION_REPLACE_VALUE;
}
} else {
/* Save state in magic bin in case trapping is necessary */
Eterm* hp;
Binary* state_bin = erts_create_magic_binary(sizeof(ErtsPersistentTermPut2Context),
persistent_term_put_2_ctx_bin_dtor);
hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE);
state_mref = erts_mk_magic_ref(&hp, &MSO(BIF_P), state_bin);
ctx = ERTS_MAGIC_BIN_DATA(state_bin);
/*
* IMPORTANT: The following field is used to detect if
* persistent_term_put_2_ctx_bin_dtor needs to free memory
*/
ctx->cpy_ctx.new_table = NULL;
}
if (!try_seize_update_permission(BIF_P)) {
ERTS_BIF_YIELD2(bif_export[BIF_persistent_term_put_2],
BIF_P, BIF_ARG_1, BIF_ARG_2);
}
ctx->hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
ctx->key = BIF_ARG_1;
ctx->term = BIF_ARG_2;
ctx->entry_index = lookup(ctx->hash_table, ctx->key);
ctx->heap[0] = make_arityval(2);
ctx->heap[1] = ctx->key;
ctx->heap[2] = ctx->term;
ctx->tuple = make_tuple(ctx->heap);
if (is_nil(ctx->hash_table->term[ctx->entry_index])) {
Uint new_size = ctx->hash_table->allocated;
if (MUST_GROW(ctx->hash_table)) {
new_size *= 2;
}
TRAPPING_COPY_TABLE_PUT(ctx->hash_table,
ctx->hash_table,
new_size,
ERTS_PERSISTENT_TERM_CPY_NO_REHASH,
PUT2_TRAP_LOCATION_NEW_KEY);
ctx->entry_index = lookup(ctx->hash_table, ctx->key);
ctx->hash_table->num_entries++;
} else {
Eterm tuple = ctx->hash_table->term[ctx->entry_index];
Eterm old_term;
ASSERT(is_tuple_arity(tuple, 2));
old_term = boxed_val(tuple)[2];
if (EQ(ctx->term, old_term)) {
/* Same value. No need to update anything. */
release_update_permission(0);
BIF_RET(am_ok);
} else {
/* Mark the old term for deletion. */
mark_for_deletion(ctx->hash_table, ctx->entry_index);
TRAPPING_COPY_TABLE_PUT(ctx->hash_table,
ctx->hash_table,
ctx->hash_table->allocated,
ERTS_PERSISTENT_TERM_CPY_NO_REHASH,
PUT2_TRAP_LOCATION_REPLACE_VALUE);
}
}
{
Uint term_size;
Uint lit_area_size;
ErlOffHeap code_off_heap;
ErtsLiteralArea* literal_area;
erts_shcopy_t info;
Eterm* ptr;
/*
* Preserve internal sharing in the term by using the
* sharing-preserving functions. However, literals must
* be copied in case the module holding them are unloaded.
*/
INITIALIZE_SHCOPY(info);
info.copy_literals = 1;
term_size = copy_shared_calculate(ctx->tuple, &info);
ERTS_INIT_OFF_HEAP(&code_off_heap);
lit_area_size = ERTS_LITERAL_AREA_ALLOC_SIZE(term_size);
literal_area = erts_alloc(ERTS_ALC_T_LITERAL, lit_area_size);
ptr = &literal_area->start[0];
literal_area->end = ptr + term_size;
ctx->tuple = copy_shared_perform(ctx->tuple, term_size, &info, &ptr, &code_off_heap);
ASSERT(tuple_val(ctx->tuple) == literal_area->start);
literal_area->off_heap = code_off_heap.first;
DESTROY_SHCOPY(info);
erts_set_literal_tag(&ctx->tuple, literal_area->start, term_size);
ctx->hash_table->term[ctx->entry_index] = ctx->tuple;
erts_schedule_thr_prgr_later_op(table_updater, ctx->hash_table, &thr_prog_op);
suspend_updater(BIF_P);
}
BUMP_REDS(BIF_P, (max_iterations - iterations_until_trap) / ITERATIONS_PER_RED);
ERTS_BIF_YIELD_RETURN(BIF_P, am_ok);
}
static ERTS_INLINE int
leader_update(ErtsThrPrgrData *tpd)
{
#ifdef ERTS_ENABLE_LOCK_CHECK
erts_lc_check_exact(NULL, 0);
#endif
if (!tpd->leader) {
/* Probably need to block... */
block_thread(tpd);
}
else {
ErtsThrPrgrVal current;
int ix, chk_next_ix, umrefc_ix, my_ix, no_managed, waiting_unmanaged;
erts_aint32_t lflgs;
ErtsThrPrgrVal next;
erts_aint_t refc;
my_ix = tpd->id;
if (tpd->leader_state.current == ERTS_THR_PRGR_VAL_WAITING) {
/* Took over as leader from another thread */
tpd->leader_state.current = read_nob(&erts_thr_prgr__.current);
tpd->leader_state.next = tpd->leader_state.current;
tpd->leader_state.next++;
if (tpd->leader_state.next == ERTS_THR_PRGR_VAL_WAITING)
tpd->leader_state.next = 0;
tpd->leader_state.chk_next_ix = intrnl->misc.data.chk_next_ix;
tpd->leader_state.umrefc_ix.waiting = intrnl->misc.data.umrefc_ix.waiting;
tpd->leader_state.umrefc_ix.current =
(int) erts_atomic32_read_nob(&intrnl->misc.data.umrefc_ix.current);
if (tpd->confirmed == tpd->leader_state.current) {
ErtsThrPrgrVal val = tpd->leader_state.current + 1;
if (val == ERTS_THR_PRGR_VAL_WAITING)
val = 0;
tpd->confirmed = val;
set_mb(&intrnl->thr[my_ix].data.current, val);
}
}
next = tpd->leader_state.next;
waiting_unmanaged = 0;
umrefc_ix = -1; /* Shut up annoying warning */
chk_next_ix = tpd->leader_state.chk_next_ix;
no_managed = intrnl->managed.no;
ASSERT(0 <= chk_next_ix && chk_next_ix <= no_managed);
/* Check manged threads */
if (chk_next_ix < no_managed) {
for (ix = chk_next_ix; ix < no_managed; ix++) {
ErtsThrPrgrVal tmp;
if (ix == my_ix)
continue;
tmp = read_nob(&intrnl->thr[ix].data.current);
if (tmp != next && tmp != ERTS_THR_PRGR_VAL_WAITING) {
tpd->leader_state.chk_next_ix = ix;
ASSERT(erts_thr_progress_has_passed__(next, tmp));
goto done;
}
}
}
/* Check unmanged threads */
waiting_unmanaged = tpd->leader_state.umrefc_ix.waiting != -1;
umrefc_ix = (waiting_unmanaged
? tpd->leader_state.umrefc_ix.waiting
: tpd->leader_state.umrefc_ix.current);
refc = erts_atomic_read_nob(&intrnl->umrefc[umrefc_ix].refc);
ASSERT(refc >= 0);
if (refc != 0) {
int new_umrefc_ix;
if (waiting_unmanaged)
goto done;
new_umrefc_ix = (umrefc_ix + 1) & 0x1;
tpd->leader_state.umrefc_ix.waiting = umrefc_ix;
tpd->leader_state.chk_next_ix = no_managed;
erts_atomic32_set_nob(&intrnl->misc.data.umrefc_ix.current,
(erts_aint32_t) new_umrefc_ix);
ETHR_MEMBAR(ETHR_StoreLoad);
refc = erts_atomic_read_nob(&intrnl->umrefc[umrefc_ix].refc);
ASSERT(refc >= 0);
waiting_unmanaged = 1;
if (refc != 0)
goto done;
}
/* Make progress */
current = next;
next++;
if (next == ERTS_THR_PRGR_VAL_WAITING)
next = 0;
set_nob(&intrnl->thr[my_ix].data.current, next);
set_mb(&erts_thr_prgr__.current, current);
tpd->confirmed = next;
tpd->leader_state.next = next;
tpd->leader_state.current = current;
#if ERTS_THR_PRGR_PRINT_VAL
if (current % 1000 == 0)
erts_fprintf(stderr, "%b64u\n", current);
#endif
handle_wakeup_requests(current);
if (waiting_unmanaged) {
waiting_unmanaged = 0;
tpd->leader_state.umrefc_ix.waiting = -1;
erts_atomic32_read_band_nob(&intrnl->misc.data.lflgs,
~ERTS_THR_PRGR_LFLG_WAITING_UM);
}
tpd->leader_state.chk_next_ix = 0;
done:
if (tpd->active) {
lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs);
if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK)
(void) block_thread(tpd);
}
else {
int force_wakeup_check = 0;
erts_aint32_t set_flags = ERTS_THR_PRGR_LFLG_NO_LEADER;
tpd->leader = 0;
tpd->leader_state.current = ERTS_THR_PRGR_VAL_WAITING;
#if ERTS_THR_PRGR_PRINT_LEADER
erts_fprintf(stderr, "L <- %d\n", tpd->id);
#endif
ERTS_THR_PROGRESS_STATE_DEBUG_SET_LEADER(tpd->id, 0);
intrnl->misc.data.umrefc_ix.waiting
= tpd->leader_state.umrefc_ix.waiting;
if (waiting_unmanaged)
set_flags |= ERTS_THR_PRGR_LFLG_WAITING_UM;
lflgs = erts_atomic32_read_bor_relb(&intrnl->misc.data.lflgs,
set_flags);
lflgs |= set_flags;
if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK)
lflgs = block_thread(tpd);
if (waiting_unmanaged) {
/* Need to check umrefc again */
ETHR_MEMBAR(ETHR_StoreLoad);
refc = erts_atomic_read_nob(&intrnl->umrefc[umrefc_ix].refc);
if (refc == 0) {
/* Need to force wakeup check */
force_wakeup_check = 1;
}
}
if ((force_wakeup_check
|| ((lflgs & (ERTS_THR_PRGR_LFLG_NO_LEADER
| ERTS_THR_PRGR_LFLG_WAITING_UM
| ERTS_THR_PRGR_LFLG_ACTIVE_MASK))
== ERTS_THR_PRGR_LFLG_NO_LEADER))
&& got_sched_wakeups()) {
/* Someone need to make progress */
wakeup_managed(0);
}
}
}
return tpd->leader;
}
int module_table_sz(void)
{
return erts_atomic_read_nob(&tot_module_bytes);
}
