static ErtsFlxCtrDecentralizedCtrArray*
create_decentralized_ctr_array(ErtsAlcType_t alloc_type, Uint nr_of_counters) {
/* Allocate an ErtsFlxCtrDecentralizedCtrArray and make sure that
the array field is located at the start of a cache line */
char* bytes =
erts_alloc(alloc_type,
sizeof(ErtsFlxCtrDecentralizedCtrArray) +
(sizeof(ErtsFlxCtrDecentralizedCtrArrayElem) *
ERTS_FLXCTR_DECENTRALIZED_NO_SLOTS) +
ERTS_CACHE_LINE_SIZE);
void* block_start = bytes;
int bytes_to_next_cacheline_border;
ErtsFlxCtrDecentralizedCtrArray* array;
int i, sched;
bytes = &bytes[offsetof(ErtsFlxCtrDecentralizedCtrArray, array)];
bytes_to_next_cacheline_border =
ERTS_CACHE_LINE_SIZE - (((Uint)bytes) % ERTS_CACHE_LINE_SIZE);
array = (ErtsFlxCtrDecentralizedCtrArray*)
(&bytes[bytes_to_next_cacheline_border -
(int)offsetof(ErtsFlxCtrDecentralizedCtrArray, array)]);
ASSERT(((Uint)array->array) % ERTS_CACHE_LINE_SIZE == 0);
ASSERT(((Uint)array - (Uint)block_start) <= ERTS_CACHE_LINE_SIZE);
/* Initialize fields */
erts_atomic_init_nob(&array->snapshot_status, ERTS_FLXCTR_SNAPSHOT_ONGOING);
for (sched = 0; sched < ERTS_FLXCTR_DECENTRALIZED_NO_SLOTS; sched++) {
for (i = 0; i < nr_of_counters; i++) {
erts_atomic_init_nob(&array->array[sched].counters[i], 0);
}
}
array->block_start = block_start;
return array;
}
void
erts_update_ranges(BeamInstr* code, Uint size)
{
ErtsCodeIndex dst = erts_staging_code_ix();
ErtsCodeIndex src = erts_active_code_ix();
if (src == dst) {
ASSERT(!erts_initialized);
/*
* During start-up of system, the indices are the same
* and erts_start_staging_ranges() has not been called.
*/
if (r[dst].modules == NULL) {
Sint need = 128;
erts_atomic_add_nob(&mem_used, need);
r[dst].modules = erts_alloc(ERTS_ALC_T_MODULE_REFS,
need * sizeof(Range));
r[dst].allocated = need;
write_ptr = r[dst].modules;
}
}
ASSERT(r[dst].modules);
write_ptr->start = code;
erts_atomic_init_nob(&(write_ptr->end),
(erts_aint_t)(((byte *)code) + size));
write_ptr++;
}
void
erts_thr_q_initialize(ErtsThrQ_t *q, ErtsThrQInit_t *qi)
{
#ifndef USE_THREADS
q->init = *qi;
if (!q->init.notify)
q->init.notify = noop_callback;
q->first = NULL;
q->last = NULL;
q->q.blk = NULL;
#else
erts_atomic_init_nob(&q->tail.data.marker.next, ERTS_AINT_NULL);
q->tail.data.marker.data.ptr = NULL;
erts_atomic_init_nob(&q->tail.data.last,
(erts_aint_t) &q->tail.data.marker);
erts_atomic_init_nob(&q->tail.data.um_refc[0], 0);
erts_atomic_init_nob(&q->tail.data.um_refc[1], 0);
erts_atomic32_init_nob(&q->tail.data.um_refc_ix, 0);
q->tail.data.live = qi->live.objects;
q->tail.data.arg = qi->arg;
q->tail.data.notify = qi->notify;
if (!q->tail.data.notify)
q->tail.data.notify = noop_callback;
erts_atomic_init_nob(&q->head.head, (erts_aint_t) &q->tail.data.marker);
q->head.live = qi->live.objects;
q->head.first = &q->tail.data.marker;
q->head.unref_end = &q->tail.data.marker;
q->head.clean_reached_head_count = 0;
q->head.deq_fini.automatic = qi->auto_finalize_dequeue;
q->head.deq_fini.start = NULL;
q->head.deq_fini.end = NULL;
#ifdef ERTS_SMP
q->head.next.thr_progress = erts_thr_progress_current();
q->head.next.thr_progress_reached = 1;
#endif
q->head.next.um_refc_ix = 1;
q->head.next.unref_end = &q->tail.data.marker;
q->head.used_marker = 1;
q->head.arg = qi->arg;
q->head.notify = q->tail.data.notify;
q->q.finalizing = 0;
q->q.live = qi->live.queue;
q->q.blk = NULL;
#endif
}
void
erts_init_ranges(void)
{
Sint i;
erts_atomic_init_nob(&mem_used, 0);
for (i = 0; i < ERTS_NUM_CODE_IX; i++) {
r[i].modules = 0;
r[i].n = 0;
r[i].allocated = 0;
erts_atomic_init_nob(&r[i].mid, 0);
}
erts_dump_num_lit_areas = 8;
erts_dump_lit_areas = (ErtsLiteralArea **)
erts_alloc(ERTS_ALC_T_CRASH_DUMP,
erts_dump_num_lit_areas * sizeof(ErtsLiteralArea*));
}
void
erts_end_staging_ranges(int commit)
{
if (commit) {
Sint i;
ErtsCodeIndex src = erts_active_code_ix();
ErtsCodeIndex dst = erts_staging_code_ix();
Range* mp;
Sint num_inserted;
mp = r[dst].modules;
num_inserted = write_ptr - mp;
for (i = 0; i < r[src].n; i++) {
Range* rp = r[src].modules+i;
if (rp->start < RANGE_END(rp)) {
/* Only insert a module that has not been purged. */
write_ptr->start = rp->start;
erts_atomic_init_nob(&write_ptr->end,
(erts_aint_t)(RANGE_END(rp)));
write_ptr++;
}
}
/*
* There are num_inserted new range entries (unsorted) at the
* beginning of the modules array, followed by the old entries
* (sorted). We must now sort the entire array.
*/
r[dst].n = write_ptr - mp;
if (num_inserted > 1) {
qsort(mp, r[dst].n, sizeof(Range),
(int (*)(const void *, const void *)) rangecompare);
} else if (num_inserted == 1) {
/* Sift the new range into place. This is faster than qsort(). */
Range t = mp[0];
for (i = 0; i < r[dst].n-1 && t.start > mp[i+1].start; i++) {
mp[i] = mp[i+1];
}
mp[i] = t;
}
r[dst].modules = mp;
CHECK(&r[dst]);
erts_atomic_set_nob(&r[dst].mid,
(erts_aint_t) (r[dst].modules +
r[dst].n / 2));
if (r[dst].allocated * 2 > erts_dump_num_lit_areas) {
erts_dump_num_lit_areas *= 2;
erts_dump_lit_areas = (ErtsLiteralArea **)
erts_realloc(ERTS_ALC_T_CRASH_DUMP,
(void *) erts_dump_lit_areas,
erts_dump_num_lit_areas * sizeof(ErtsLiteralArea*));
}
}
}
void erts_flxctr_init(ErtsFlxCtr* c,
int is_decentralized,
Uint nr_of_counters,
ErtsAlcType_t alloc_type)
{
ASSERT(nr_of_counters <= ERTS_FLXCTR_ATOMICS_PER_CACHE_LINE);
c->is_decentralized = is_decentralized;
c->nr_of_counters = nr_of_counters;
if (c->is_decentralized) {
ErtsFlxCtrDecentralizedCtrArray* array =
create_decentralized_ctr_array(alloc_type, nr_of_counters);
erts_atomic_set_nob(&array->snapshot_status,
ERTS_FLXCTR_SNAPSHOT_NOT_ONGOING);
erts_atomic_init_nob(&c->u.counters_ptr, (Sint)array);
ASSERT(((Uint)array->array) % ERTS_CACHE_LINE_SIZE == 0);
} else {
int i;
for (i = 0; i < nr_of_counters; i++) {
erts_atomic_init_nob(&c->u.counters[i], 0);
}
}
}
void erts_init_bif_persistent_term(void)
{
HashTable* hash_table;
/*
* Initialize the mutex protecting updates.
*/
erts_mtx_init(&update_table_permission_mtx,
"update_persistent_term_permission",
NIL,
ERTS_LOCK_FLAGS_PROPERTY_STATIC |
ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
/*
* Initialize delete queue.
*/
erts_mtx_init(&delete_queue_mtx,
"persistent_term_delete_permission",
NIL,
ERTS_LOCK_FLAGS_PROPERTY_STATIC |
ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
/*
* Allocate a small initial hash table.
*/
hash_table = create_initial_table();
erts_atomic_init_nob(&the_hash_table, (erts_aint_t)hash_table);
/*
* Initialize export entry for traps
*/
erts_init_trap_export(&persistent_term_get_all_export,
am_persistent_term, am_get_all_trap, 2,
&persistent_term_get_all_trap);
erts_init_trap_export(&persistent_term_info_export,
am_persistent_term, am_info_trap, 1,
&persistent_term_info_trap);
}
static HashTable*
create_initial_table(void)
{
HashTable* hash_table;
int i;
hash_table = (HashTable *) erts_alloc(ERTS_ALC_T_PERSISTENT_TERM,
sizeof(HashTable)+sizeof(Eterm) *
(INITIAL_SIZE-1));
hash_table->allocated = INITIAL_SIZE;
hash_table->num_entries = 0;
hash_table->mask = INITIAL_SIZE-1;
hash_table->first_to_delete = 0;
hash_table->num_to_delete = 0;
erts_atomic_init_nob(&hash_table->refc, (erts_aint_t)1);
for (i = 0; i < INITIAL_SIZE; i++) {
hash_table->term[i] = NIL;
}
return hash_table;
}
void init_module_table(void)
{
HashFunctions f;
int i;
f.hash = (H_FUN) module_hash;
f.cmp = (HCMP_FUN) module_cmp;
f.alloc = (HALLOC_FUN) module_alloc;
f.free = (HFREE_FUN) module_free;
f.meta_alloc = (HMALLOC_FUN) erts_alloc;
f.meta_free = (HMFREE_FUN) erts_free;
f.meta_print = (HMPRINT_FUN) erts_print;
for (i = 0; i < ERTS_NUM_CODE_IX; i++) {
erts_index_init(ERTS_ALC_T_MODULE_TABLE, &module_tables[i], "module_code",
MODULE_SIZE, MODULE_LIMIT, f);
}
for (i=0; i<ERTS_NUM_CODE_IX; i++) {
erts_rwmtx_init(&the_old_code_rwlocks[i], "old_code", make_small(i),
ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
}
erts_atomic_init_nob(&tot_module_bytes, 0);
}
static int do_load_driver_entry(DE_Handle *dh, char *path, char *name)
{
void *init_handle;
int res;
ErlDrvEntry *dp;
assert_drv_list_rwlocked();
if ((res = erts_sys_ddll_open(path, &(dh->handle), NULL)) != ERL_DE_NO_ERROR) {
return res;
}
if ((res = erts_sys_ddll_load_driver_init(dh->handle,
&init_handle)) != ERL_DE_NO_ERROR) {
res = ERL_DE_LOAD_ERROR_NO_INIT;
goto error;
}
dp = erts_sys_ddll_call_init(init_handle);
if (dp == NULL) {
res = ERL_DE_LOAD_ERROR_FAILED_INIT;
goto error;
}
switch (dp->extended_marker) {
case ERL_DRV_EXTENDED_MARKER:
if (dp->major_version < ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD
|| (ERL_DRV_EXTENDED_MAJOR_VERSION < dp->major_version
|| (ERL_DRV_EXTENDED_MAJOR_VERSION == dp->major_version
&& ERL_DRV_EXTENDED_MINOR_VERSION < dp->minor_version))) {
/* Incompatible driver version */
res = ERL_DE_LOAD_ERROR_INCORRECT_VERSION;
goto error;
}
break;
default:
/* Old driver; needs to be recompiled... */
res = ERL_DE_LOAD_ERROR_INCORRECT_VERSION;
goto error;
}
if (strcmp(name, dp->driver_name) != 0) {
res = ERL_DE_LOAD_ERROR_BAD_NAME;
goto error;
}
erts_atomic_init_nob(&(dh->refc), (erts_aint_t) 0);
erts_atomic32_init_nob(&dh->port_count, 0);
dh->full_path = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(path) + 1);
sys_strcpy(dh->full_path, path);
dh->flags = 0;
dh->status = ERL_DE_OK;
if (erts_add_driver_entry(dp, dh, 1) != 0 /* io.c */) {
/*
* The init in the driver struct did not return 0
*/
erts_free(ERTS_ALC_T_DDLL_HANDLE, dh->full_path);
dh->full_path = NULL;
res = ERL_DE_LOAD_ERROR_FAILED_INIT;
goto error;
}
return ERL_DE_NO_ERROR;
error:
erts_sys_ddll_close(dh->handle);
return res;
}
erts_sspa_data_t *
erts_sspa_create(size_t blk_sz, int pa_size, int nthreads, const char* name)
{
erts_sspa_data_t *data;
size_t tot_size;
size_t chunk_mem_size;
char *p;
char *chunk_start;
int cix;
int no_blocks = pa_size;
int no_blocks_per_chunk;
size_t aligned_blk_sz;
#if !defined(ERTS_STRUCTURE_ALIGNED_ALLOC)
/* Force 64-bit alignment... */
aligned_blk_sz = ((blk_sz - 1) / 8) * 8 + 8;
#else
/* Alignment of structure is enough... */
aligned_blk_sz = blk_sz;
#endif
if (!name) { /* schedulers only variant */
ASSERT(!nthreads);
nthreads = erts_no_schedulers;
}
else {
ASSERT(nthreads > 0);
}
if (nthreads == 1)
no_blocks_per_chunk = no_blocks;
else {
int extra = (no_blocks - 1)/4 + 1;
if (extra == 0)
extra = 1;
no_blocks_per_chunk = no_blocks;
no_blocks_per_chunk += extra * nthreads;
no_blocks_per_chunk /= nthreads;
}
no_blocks = no_blocks_per_chunk * nthreads;
chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_chunk_header_t));
chunk_mem_size += aligned_blk_sz * no_blocks_per_chunk;
chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(chunk_mem_size);
tot_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t));
tot_size += chunk_mem_size * nthreads;
p = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_PRE_ALLOC_DATA, tot_size);
data = (erts_sspa_data_t *) p;
p += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t));
chunk_start = p;
data->chunks_mem_size = chunk_mem_size;
data->start = chunk_start;
data->end = chunk_start + chunk_mem_size * nthreads;
data->nthreads = nthreads;
if (name) { /* thread variant */
erts_tsd_key_create(&data->tsd_key, (char*)name);
erts_atomic_init_nob(&data->id_generator, 0);
}
/* Initialize all chunks */
for (cix = 0; cix < nthreads; cix++) {
erts_sspa_chunk_t *chnk = erts_sspa_cix2chunk(data, cix);
erts_sspa_chunk_header_t *chdr = &chnk->aligned.header;
erts_sspa_blk_t *blk;
int i;
erts_atomic_init_nob(&chdr->tail.data.last, (erts_aint_t) &chdr->tail.data.marker);
erts_atomic_init_nob(&chdr->tail.data.marker.next_atmc, ERTS_AINT_NULL);
erts_atomic_init_nob(&chdr->tail.data.um_refc[0], 0);
erts_atomic_init_nob(&chdr->tail.data.um_refc[1], 0);
erts_atomic32_init_nob(&chdr->tail.data.um_refc_ix, 0);
chdr->head.no_thr_progress_check = 0;
chdr->head.used_marker = 1;
chdr->head.first = &chdr->tail.data.marker;
chdr->head.unref_end = &chdr->tail.data.marker;
chdr->head.next.thr_progress = erts_thr_progress_current();
chdr->head.next.thr_progress_reached = 1;
chdr->head.next.um_refc_ix = 1;
chdr->head.next.unref_end = &chdr->tail.data.marker;
p = &chnk->data[0];
chdr->local.first = (erts_sspa_blk_t *) p;
blk = (erts_sspa_blk_t *) p;
for (i = 0; i < no_blocks_per_chunk; i++) {
blk = (erts_sspa_blk_t *) p;
p += aligned_blk_sz;
blk->next_ptr = (erts_sspa_blk_t *) p;
}
blk->next_ptr = NULL;
chdr->local.last = blk;
chdr->local.cnt = no_blocks_per_chunk;
chdr->local.lim = no_blocks_per_chunk / 3;
ERTS_SSPA_DBG_CHK_LCL(chdr);
}
return data;
}
erts_sspa_data_t *
erts_sspa_create(size_t blk_sz, int pa_size)
{
erts_sspa_data_t *data;
size_t tot_size;
size_t chunk_mem_size;
char *p;
char *chunk_start;
int cix;
int no_blocks = pa_size;
int no_blocks_per_chunk;
if (erts_no_schedulers == 1)
no_blocks_per_chunk = no_blocks;
else {
int extra = (no_blocks - 1)/4 + 1;
if (extra == 0)
extra = 1;
no_blocks_per_chunk = no_blocks;
no_blocks_per_chunk += extra*erts_no_schedulers;
no_blocks_per_chunk /= erts_no_schedulers;
}
no_blocks = no_blocks_per_chunk * erts_no_schedulers;
chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_chunk_header_t));
chunk_mem_size += blk_sz * no_blocks_per_chunk;
chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(chunk_mem_size);
tot_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t));
tot_size += chunk_mem_size*erts_no_schedulers;
p = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_PRE_ALLOC_DATA, tot_size);
data = (erts_sspa_data_t *) p;
p += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t));
chunk_start = p;
data->chunks_mem_size = chunk_mem_size;
data->start = chunk_start;
data->end = chunk_start + chunk_mem_size*erts_no_schedulers;
/* Initialize all chunks */
for (cix = 0; cix < erts_no_schedulers; cix++) {
erts_sspa_chunk_t *chnk = erts_sspa_cix2chunk(data, cix);
erts_sspa_chunk_header_t *chdr = &chnk->aligned.header;
erts_sspa_blk_t *blk;
int i;
erts_atomic_init_nob(&chdr->tail.data.last, (erts_aint_t) &chdr->tail.data.marker);
erts_atomic_init_nob(&chdr->tail.data.marker.next_atmc, ERTS_AINT_NULL);
erts_atomic_init_nob(&chdr->tail.data.um_refc[0], 0);
erts_atomic_init_nob(&chdr->tail.data.um_refc[1], 0);
erts_atomic32_init_nob(&chdr->tail.data.um_refc_ix, 0);
chdr->head.no_thr_progress_check = 0;
chdr->head.used_marker = 1;
chdr->head.first = &chdr->tail.data.marker;
chdr->head.unref_end = &chdr->tail.data.marker;
chdr->head.next.thr_progress = erts_thr_progress_current();
chdr->head.next.thr_progress_reached = 1;
chdr->head.next.um_refc_ix = 1;
chdr->head.next.unref_end = &chdr->tail.data.marker;
p = &chnk->data[0];
chdr->local.first = (erts_sspa_blk_t *) p;
blk = (erts_sspa_blk_t *) p;
for (i = 0; i < no_blocks_per_chunk; i++) {
blk = (erts_sspa_blk_t *) p;
p += blk_sz;
blk->next_ptr = (erts_sspa_blk_t *) p;
}
blk->next_ptr = NULL;
chdr->local.last = blk;
chdr->local.cnt = no_blocks_per_chunk;
chdr->local.lim = no_blocks_per_chunk / 3;
ERTS_SSPA_DBG_CHK_LCL(chdr);
}
return data;
}
static HashTable*
copy_table(ErtsPersistentTermCpyTableCtx* ctx)
{
Uint old_size = ctx->old_table->allocated;
Uint i;
ErtsAlcType_t alloc_type;
ctx->total_iterations_done = 0;
switch(ctx->location) {
case ERTS_PERSISTENT_TERM_CPY_PLACE_1: goto L_copy_table_place_1;
case ERTS_PERSISTENT_TERM_CPY_PLACE_2: goto L_copy_table_place_2;
case ERTS_PERSISTENT_TERM_CPY_PLACE_3: goto L_copy_table_place_3;
case ERTS_PERSISTENT_TERM_CPY_PLACE_START:
ctx->iterations_done = 0;
}
if (ctx->copy_type == ERTS_PERSISTENT_TERM_CPY_TEMP) {
alloc_type = ERTS_ALC_T_PERSISTENT_TERM_TMP;
} else {
alloc_type = ERTS_ALC_T_PERSISTENT_TERM;
}
ctx->new_table = (HashTable *) erts_alloc(alloc_type,
sizeof(HashTable) +
sizeof(Eterm) * (ctx->new_size-1));
if (ctx->old_table->allocated == ctx->new_size &&
(ctx->copy_type == ERTS_PERSISTENT_TERM_CPY_NO_REHASH ||
ctx->copy_type == ERTS_PERSISTENT_TERM_CPY_TEMP)) {
/*
* Same size and no key deleted. Make an exact copy of the table.
*/
*ctx->new_table = *ctx->old_table;
L_copy_table_place_1:
for (i = ctx->iterations_done;
i < MIN(ctx->iterations_done + ctx->max_iterations,
ctx->new_size);
i++) {
ctx->new_table->term[i] = ctx->old_table->term[i];
}
ctx->total_iterations_done = (i - ctx->iterations_done);
if (i < ctx->new_size) {
ctx->iterations_done = i;
ctx->location = ERTS_PERSISTENT_TERM_CPY_PLACE_1;
return NULL;
}
ctx->iterations_done = 0;
} else {
/*
* The size of the table has changed or an element has been
* deleted. Must rehash, by inserting all old terms into the
* new (empty) table.
*/
ctx->new_table->allocated = ctx->new_size;
ctx->new_table->num_entries = ctx->old_table->num_entries;
ctx->new_table->mask = ctx->new_size - 1;
L_copy_table_place_2:
for (i = ctx->iterations_done;
i < MIN(ctx->iterations_done + ctx->max_iterations,
ctx->new_size);
i++) {
ctx->new_table->term[i] = NIL;
}
ctx->total_iterations_done = (i - ctx->iterations_done);
ctx->max_iterations -= ctx->total_iterations_done;
if (i < ctx->new_size) {
ctx->iterations_done = i;
ctx->location = ERTS_PERSISTENT_TERM_CPY_PLACE_2;
return NULL;
}
ctx->iterations_done = 0;
L_copy_table_place_3:
for (i = ctx->iterations_done;
i < MIN(ctx->iterations_done + ctx->max_iterations,
old_size);
i++) {
if (is_tuple(ctx->old_table->term[i])) {
Eterm key = tuple_val(ctx->old_table->term[i])[1];
Uint entry_index = lookup(ctx->new_table, key);
ASSERT(is_nil(ctx->new_table->term[entry_index]));
ctx->new_table->term[entry_index] = ctx->old_table->term[i];
}
}
ctx->total_iterations_done += (i - ctx->iterations_done);
if (i < old_size) {
ctx->iterations_done = i;
ctx->location = ERTS_PERSISTENT_TERM_CPY_PLACE_3;
return NULL;
}
ctx->iterations_done = 0;
}
ctx->new_table->first_to_delete = 0;
ctx->new_table->num_to_delete = 0;
erts_atomic_init_nob(&ctx->new_table->refc, (erts_aint_t)1);
{
HashTable* new_table = ctx->new_table;
/*
* IMPORTANT: Memory management depends on that ctx->new_table is
* set to NULL on the line below
*/
ctx->new_table = NULL;
return new_table;
}
}
void
erts_init_async(void)
{
async = NULL;
if (erts_async_max_threads > 0) {
#if ERTS_USE_ASYNC_READY_Q
ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT;
#endif
erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER;
char *ptr, thr_name[16];
size_t tot_size = 0;
int i;
tot_size += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncData));
tot_size += sizeof(ErtsAlgndAsyncQ)*erts_async_max_threads;
#if ERTS_USE_ASYNC_READY_Q
tot_size += sizeof(ErtsAlgndAsyncReadyQ)*erts_no_schedulers;
#endif
ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_ASYNC_DATA,
tot_size);
async = (ErtsAsyncData *) ptr;
ptr += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncData));
async->init.data.no_initialized = 0;
erts_mtx_init(&async->init.data.mtx, "async_init_mtx", NIL,
ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER);
erts_cnd_init(&async->init.data.cnd);
erts_atomic_init_nob(&async->init.data.id, 0);
async->queue = (ErtsAlgndAsyncQ *) ptr;
ptr += sizeof(ErtsAlgndAsyncQ)*erts_async_max_threads;
#if ERTS_USE_ASYNC_READY_Q
qinit.live.queue = ERTS_THR_Q_LIVE_LONG;
qinit.live.objects = ERTS_THR_Q_LIVE_SHORT;
qinit.notify = erts_notify_check_async_ready_queue;
async->ready_queue = (ErtsAlgndAsyncReadyQ *) ptr;
ptr += sizeof(ErtsAlgndAsyncReadyQ)*erts_no_schedulers;
for (i = 1; i <= erts_no_schedulers; i++) {
ErtsAsyncReadyQ *arq = async_ready_q(i);
#if ERTS_USE_ASYNC_READY_ENQ_MTX
erts_mtx_init(&arq->x.data.enq_mtx, "async_enq_mtx", make_small(i),
ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER);
#endif
erts_thr_q_finalize_dequeue_state_init(&arq->fin_deq);
qinit.arg = (void *) (SWord) i;
erts_thr_q_initialize(&arq->thr_q, &qinit);
}
#endif
/* Create async threads... */
thr_opts.detached = 0;
thr_opts.suggested_stack_size
= erts_async_thread_suggested_stack_size;
thr_opts.name = thr_name;
for (i = 0; i < erts_async_max_threads; i++) {
ErtsAsyncQ *aq = async_q(i);
erts_snprintf(thr_opts.name, 16, "async_%d", i+1);
erts_thr_create(&aq->thr_id, async_main, (void*) aq, &thr_opts);
}
/* Wait for async threads to initialize... */
erts_mtx_lock(&async->init.data.mtx);
while (async->init.data.no_initialized != erts_async_max_threads)
erts_cnd_wait(&async->init.data.cnd, &async->init.data.mtx);
erts_mtx_unlock(&async->init.data.mtx);
erts_mtx_destroy(&async->init.data.mtx);
erts_cnd_destroy(&async->init.data.cnd);
}
}
void
erts_sys_pre_init(void)
{
erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER;
erts_printf_add_cr_to_stdout = 1;
erts_printf_add_cr_to_stderr = 1;
eid.thread_create_child_func = thr_create_prepare_child;
/* Before creation in parent */
eid.thread_create_prepare_func = thr_create_prepare;
/* After creation in parent */
eid.thread_create_parent_func = thr_create_cleanup,
#ifdef ERTS_ENABLE_LOCK_COUNT
erts_lcnt_pre_thr_init();
#endif
erts_thr_init(&eid);
#ifdef ERTS_ENABLE_LOCK_COUNT
erts_lcnt_post_thr_init();
#endif
#ifdef ERTS_ENABLE_LOCK_CHECK
erts_lc_init();
#endif
erts_init_sys_time_sup();
erts_atomic32_init_nob(&erts_break_requested, 0);
erts_atomic32_init_nob(&have_prepared_crash_dump, 0);
erts_atomic_init_nob(&sys_misc_mem_sz, 0);
{
/*
* Unfortunately we depend on fd 0,1,2 in the old shell code.
* So if for some reason we do not have those open when we start
* we have to open them here. Not doing this can cause the emulator
* to deadlock when reaping the fd_driver ports :(
*/
int fd;
/* Make sure fd 0 is open */
if ((fd = open("/dev/null", O_RDONLY)) != 0)
close(fd);
/* Make sure fds 1 and 2 are open */
while (fd < 3) {
fd = open("/dev/null", O_WRONLY);
}
close(fd);
}
/* We need a file descriptor to close in the crashdump creation.
* We close this one to be sure we can get a fd for our real file ...
* so, we create one here ... a stone to carry all the way home.
*/
crashdump_companion_cube_fd = open("/dev/null", O_RDONLY);
/* don't lose it, there will be cake */
}
void
erts_thr_progress_init(int no_schedulers, int managed, int unmanaged)
{
int i, j, um_low, um_high;
char *ptr;
size_t cb_sz, intrnl_sz, thr_arr_sz, m_wakeup_size, um_wakeup_size,
tot_size;
intrnl_sz = sizeof(ErtsThrPrgrInternalData);
intrnl_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(intrnl_sz);
cb_sz = sizeof(ErtsThrPrgrCallbacks)*(managed+unmanaged);
cb_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(cb_sz);
thr_arr_sz = sizeof(ErtsThrPrgrArray)*managed;
ASSERT(thr_arr_sz == ERTS_ALC_CACHE_LINE_ALIGN_SIZE(thr_arr_sz));
m_wakeup_size = sizeof(ErtsThrPrgrManagedWakeupData);
m_wakeup_size += (managed - 1)*sizeof(int);
m_wakeup_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(m_wakeup_size);
um_low = (unmanaged - 1)/ERTS_THR_PRGR_BM_BITS + 1;
um_high = (um_low - 1)/ERTS_THR_PRGR_BM_BITS + 1;
um_wakeup_size = sizeof(ErtsThrPrgrUnmanagedWakeupData);
um_wakeup_size += (um_high + um_low)*sizeof(erts_atomic32_t);
um_wakeup_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(um_wakeup_size);
tot_size = intrnl_sz;
tot_size += cb_sz;
tot_size += thr_arr_sz;
tot_size += m_wakeup_size*ERTS_THR_PRGR_WAKEUP_DATA_SIZE;
tot_size += um_wakeup_size*ERTS_THR_PRGR_WAKEUP_DATA_SIZE;
ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_THR_PRGR_IDATA,
tot_size);
intrnl = (ErtsThrPrgrInternalData *) ptr;
ptr += intrnl_sz;
erts_atomic32_init_nob(&intrnl->misc.data.lflgs,
ERTS_THR_PRGR_LFLG_NO_LEADER);
erts_atomic32_init_nob(&intrnl->misc.data.block_count,
(ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING
| (erts_aint32_t) managed));
erts_atomic_init_nob(&intrnl->misc.data.blocker_event, ERTS_AINT_NULL);
erts_atomic32_init_nob(&intrnl->misc.data.pref_wakeup_used, 0);
erts_atomic32_init_nob(&intrnl->misc.data.managed_count, 0);
erts_atomic32_init_nob(&intrnl->misc.data.managed_id, no_schedulers);
erts_atomic32_init_nob(&intrnl->misc.data.unmanaged_id, -1);
intrnl->misc.data.chk_next_ix = 0;
intrnl->misc.data.umrefc_ix.waiting = -1;
erts_atomic32_init_nob(&intrnl->misc.data.umrefc_ix.current, 0);
erts_atomic_init_nob(&intrnl->umrefc[0].refc, (erts_aint_t) 0);
erts_atomic_init_nob(&intrnl->umrefc[1].refc, (erts_aint_t) 0);
intrnl->thr = (ErtsThrPrgrArray *) ptr;
ptr += thr_arr_sz;
for (i = 0; i < managed; i++)
init_nob(&intrnl->thr[i].data.current, 0);
intrnl->managed.callbacks = (ErtsThrPrgrCallbacks *) ptr;
intrnl->unmanaged.callbacks = &intrnl->managed.callbacks[managed];
ptr += cb_sz;
intrnl->managed.no = managed;
for (i = 0; i < managed; i++) {
intrnl->managed.callbacks[i].arg = NULL;
intrnl->managed.callbacks[i].wakeup = NULL;
}
intrnl->unmanaged.no = unmanaged;
for (i = 0; i < unmanaged; i++) {
intrnl->unmanaged.callbacks[i].arg = NULL;
intrnl->unmanaged.callbacks[i].wakeup = NULL;
}
for (i = 0; i < ERTS_THR_PRGR_WAKEUP_DATA_SIZE; i++) {
intrnl->managed.data[i] = (ErtsThrPrgrManagedWakeupData *) ptr;
erts_atomic32_init_nob(&intrnl->managed.data[i]->len, 0);
ptr += m_wakeup_size;
}
for (i = 0; i < ERTS_THR_PRGR_WAKEUP_DATA_SIZE; i++) {
erts_atomic32_t *bm;
intrnl->unmanaged.data[i] = (ErtsThrPrgrUnmanagedWakeupData *) ptr;
erts_atomic32_init_nob(&intrnl->unmanaged.data[i]->len, 0);
bm = (erts_atomic32_t *) (ptr + sizeof(ErtsThrPrgrUnmanagedWakeupData));
intrnl->unmanaged.data[i]->high = bm;
intrnl->unmanaged.data[i]->high_sz = um_high;
for (j = 0; j < um_high; j++)
erts_atomic32_init_nob(&intrnl->unmanaged.data[i]->high[j], 0);
intrnl->unmanaged.data[i]->low
= &intrnl->unmanaged.data[i]->high[um_high];
intrnl->unmanaged.data[i]->low_sz = um_low;
for (j = 0; j < um_low; j++)
erts_atomic32_init_nob(&intrnl->unmanaged.data[i]->low[j], 0);
ptr += um_wakeup_size;
}
ERTS_THR_MEMORY_BARRIER;
}
static int
early_init(int *argc, char **argv) /*
* Only put things here which are
* really important initialize
* early!
*/
{
ErtsAllocInitOpts alloc_opts = ERTS_ALLOC_INIT_DEF_OPTS_INITER;
int ncpu;
int ncpuonln;
int ncpuavail;
int schdlrs;
int schdlrs_onln;
int max_main_threads;
int max_reader_groups;
int reader_groups;
char envbuf[21]; /* enough for any 64-bit integer */
size_t envbufsz;
erts_sched_compact_load = 1;
erts_printf_eterm_func = erts_printf_term;
erts_disable_tolerant_timeofday = 0;
display_items = 200;
erts_proc.max = ERTS_DEFAULT_MAX_PROCESSES;
erts_backtrace_depth = DEFAULT_BACKTRACE_SIZE;
erts_async_max_threads = 0;
erts_async_thread_suggested_stack_size = ERTS_ASYNC_THREAD_MIN_STACK_SIZE;
H_MIN_SIZE = H_DEFAULT_SIZE;
BIN_VH_MIN_SIZE = VH_DEFAULT_SIZE;
erts_initialized = 0;
erts_use_sender_punish = 1;
erts_pre_early_init_cpu_topology(&max_reader_groups,
&ncpu,
&ncpuonln,
&ncpuavail);
#ifndef ERTS_SMP
ncpu = 1;
ncpuonln = 1;
ncpuavail = 1;
#endif
ignore_break = 0;
replace_intr = 0;
program = argv[0];
erts_modified_timing_level = -1;
erts_compat_rel = this_rel_num();
erts_use_r9_pids_ports = 0;
erts_sys_pre_init();
erts_atomic_init_nob(&exiting, 0);
#ifdef ERTS_SMP
erts_thr_progress_pre_init();
#endif
#ifdef ERTS_ENABLE_LOCK_CHECK
erts_lc_init();
#endif
#ifdef ERTS_SMP
erts_smp_atomic32_init_nob(&erts_writing_erl_crash_dump, 0L);
erts_tsd_key_create(&erts_is_crash_dumping_key);
#else
erts_writing_erl_crash_dump = 0;
#endif
erts_smp_atomic32_init_nob(&erts_max_gen_gcs,
(erts_aint32_t) ((Uint16) -1));
erts_pre_init_process();
#if defined(USE_THREADS) && !defined(ERTS_SMP)
main_thread = erts_thr_self();
#endif
/*
* We need to know the number of schedulers to use before we
* can initialize the allocators.
*/
no_schedulers = (Uint) (ncpu > 0 ? ncpu : 1);
no_schedulers_online = (ncpuavail > 0
? ncpuavail
: (ncpuonln > 0 ? ncpuonln : no_schedulers));
schdlrs = no_schedulers;
schdlrs_onln = no_schedulers_online;
envbufsz = sizeof(envbuf);
/* erts_sys_getenv() not initialized yet; need erts_sys_getenv__() */
if (erts_sys_getenv__("ERL_THREAD_POOL_SIZE", envbuf, &envbufsz) == 0)
erts_async_max_threads = atoi(envbuf);
else
erts_async_max_threads = 0;
if (erts_async_max_threads > ERTS_MAX_NO_OF_ASYNC_THREADS)
erts_async_max_threads = ERTS_MAX_NO_OF_ASYNC_THREADS;
if (argc && argv) {
int i = 1;
while (i < *argc) {
if (strcmp(argv[i], "--") == 0) { /* end of emulator options */
i++;
break;
}
if (argv[i][0] == '-') {
switch (argv[i][1]) {
case 'r': {
char *sub_param = argv[i]+2;
if (has_prefix("g", sub_param)) {
char *arg = get_arg(sub_param+1, argv[i+1], &i);
if (sscanf(arg, "%d", &max_reader_groups) != 1) {
erts_fprintf(stderr,
"bad reader groups limit: %s\n", arg);
erts_usage();
}
if (max_reader_groups < 0) {
erts_fprintf(stderr,
"bad reader groups limit: %d\n",
max_reader_groups);
erts_usage();
}
}
break;
}
case 'A': {
/* set number of threads in thread pool */
char *arg = get_arg(argv[i]+2, argv[i+1], &i);
if (((erts_async_max_threads = atoi(arg)) < 0) ||
(erts_async_max_threads > ERTS_MAX_NO_OF_ASYNC_THREADS)) {
erts_fprintf(stderr,
"bad number of async threads %s\n",
arg);
erts_usage();
VERBOSE(DEBUG_SYSTEM, ("using %d async-threads\n",
erts_async_max_threads));
}
break;
}
case 'S' : {
int tot, onln;
char *arg = get_arg(argv[i]+2, argv[i+1], &i);
switch (sscanf(arg, "%d:%d", &tot, &onln)) {
case 0:
switch (sscanf(arg, ":%d", &onln)) {
case 1:
tot = no_schedulers;
goto chk_S;
default:
goto bad_S;
}
case 1:
onln = tot < schdlrs_onln ? tot : schdlrs_onln;
case 2:
chk_S:
if (tot > 0)
schdlrs = tot;
else
schdlrs = no_schedulers + tot;
if (onln > 0)
schdlrs_onln = onln;
else
schdlrs_onln = no_schedulers_online + onln;
if (schdlrs < 1 || ERTS_MAX_NO_OF_SCHEDULERS < schdlrs) {
erts_fprintf(stderr,
"bad amount of schedulers %d\n",
tot);
erts_usage();
}
if (schdlrs_onln < 1 || schdlrs < schdlrs_onln) {
erts_fprintf(stderr,
"bad amount of schedulers online %d "
"(total amount of schedulers %d)\n",
schdlrs_onln, schdlrs);
erts_usage();
}
break;
default:
bad_S:
erts_fprintf(stderr,
"bad amount of schedulers %s\n",
arg);
erts_usage();
break;
}
VERBOSE(DEBUG_SYSTEM,
("using %d:%d scheduler(s)\n", tot, onln));
break;
}
default:
break;
}
}
i++;
}
}
#ifndef USE_THREADS
erts_async_max_threads = 0;
#endif
#ifdef ERTS_SMP
no_schedulers = schdlrs;
no_schedulers_online = schdlrs_onln;
erts_no_schedulers = (Uint) no_schedulers;
#endif
erts_early_init_scheduling(no_schedulers);
alloc_opts.ncpu = ncpu;
erts_alloc_init(argc, argv, &alloc_opts); /* Handles (and removes)
-M flags. */
/* Require allocators */
#ifdef ERTS_SMP
/*
* Thread progress management:
*
* * Managed threads:
* ** Scheduler threads (see erl_process.c)
* ** Aux thread (see erl_process.c)
* ** Sys message dispatcher thread (see erl_trace.c)
*
* * Unmanaged threads that need to register:
* ** Async threads (see erl_async.c)
*/
erts_thr_progress_init(no_schedulers,
no_schedulers+2,
erts_async_max_threads);
#endif
erts_thr_q_init();
erts_init_utils();
erts_early_init_cpu_topology(no_schedulers,
&max_main_threads,
max_reader_groups,
&reader_groups);
#ifdef USE_THREADS
{
erts_thr_late_init_data_t elid = ERTS_THR_LATE_INIT_DATA_DEF_INITER;
elid.mem.std.alloc = ethr_std_alloc;
elid.mem.std.realloc = ethr_std_realloc;
elid.mem.std.free = ethr_std_free;
elid.mem.sl.alloc = ethr_sl_alloc;
elid.mem.sl.realloc = ethr_sl_realloc;
elid.mem.sl.free = ethr_sl_free;
elid.mem.ll.alloc = ethr_ll_alloc;
elid.mem.ll.realloc = ethr_ll_realloc;
elid.mem.ll.free = ethr_ll_free;
elid.main_threads = max_main_threads;
elid.reader_groups = reader_groups;
erts_thr_late_init(&elid);
}
#endif
#ifdef ERTS_ENABLE_LOCK_CHECK
erts_lc_late_init();
#endif
#ifdef ERTS_ENABLE_LOCK_COUNT
erts_lcnt_late_init();
#endif
#if defined(HIPE)
hipe_signal_init(); /* must be done very early */
#endif
erl_sys_args(argc, argv);
/* Creates threads on Windows that depend on the arguments, so has to be after erl_sys_args */
erl_sys_init();
erts_ets_realloc_always_moves = 0;
erts_ets_always_compress = 0;
erts_dist_buf_busy_limit = ERTS_DE_BUSY_LIMIT;
return ncpu;
}
void
erts_init_port_data(Port *prt)
{
erts_atomic_init_nob(&prt->data, (erts_aint_t) am_undefined);
}
