static ErtsFlxCtrDecentralizedCtrArray* create_decentralized_ctr_array(ErtsAlcType_t alloc_type, Uint nr_of_counters) { /* Allocate an ErtsFlxCtrDecentralizedCtrArray and make sure that the array field is located at the start of a cache line */ char* bytes = erts_alloc(alloc_type, sizeof(ErtsFlxCtrDecentralizedCtrArray) + (sizeof(ErtsFlxCtrDecentralizedCtrArrayElem) * ERTS_FLXCTR_DECENTRALIZED_NO_SLOTS) + ERTS_CACHE_LINE_SIZE); void* block_start = bytes; int bytes_to_next_cacheline_border; ErtsFlxCtrDecentralizedCtrArray* array; int i, sched; bytes = &bytes[offsetof(ErtsFlxCtrDecentralizedCtrArray, array)]; bytes_to_next_cacheline_border = ERTS_CACHE_LINE_SIZE - (((Uint)bytes) % ERTS_CACHE_LINE_SIZE); array = (ErtsFlxCtrDecentralizedCtrArray*) (&bytes[bytes_to_next_cacheline_border - (int)offsetof(ErtsFlxCtrDecentralizedCtrArray, array)]); ASSERT(((Uint)array->array) % ERTS_CACHE_LINE_SIZE == 0); ASSERT(((Uint)array - (Uint)block_start) <= ERTS_CACHE_LINE_SIZE); /* Initialize fields */ erts_atomic_init_nob(&array->snapshot_status, ERTS_FLXCTR_SNAPSHOT_ONGOING); for (sched = 0; sched < ERTS_FLXCTR_DECENTRALIZED_NO_SLOTS; sched++) { for (i = 0; i < nr_of_counters; i++) { erts_atomic_init_nob(&array->array[sched].counters[i], 0); } } array->block_start = block_start; return array; }
void erts_update_ranges(BeamInstr* code, Uint size) { ErtsCodeIndex dst = erts_staging_code_ix(); ErtsCodeIndex src = erts_active_code_ix(); if (src == dst) { ASSERT(!erts_initialized); /* * During start-up of system, the indices are the same * and erts_start_staging_ranges() has not been called. */ if (r[dst].modules == NULL) { Sint need = 128; erts_atomic_add_nob(&mem_used, need); r[dst].modules = erts_alloc(ERTS_ALC_T_MODULE_REFS, need * sizeof(Range)); r[dst].allocated = need; write_ptr = r[dst].modules; } } ASSERT(r[dst].modules); write_ptr->start = code; erts_atomic_init_nob(&(write_ptr->end), (erts_aint_t)(((byte *)code) + size)); write_ptr++; }
void erts_thr_q_initialize(ErtsThrQ_t *q, ErtsThrQInit_t *qi) { #ifndef USE_THREADS q->init = *qi; if (!q->init.notify) q->init.notify = noop_callback; q->first = NULL; q->last = NULL; q->q.blk = NULL; #else erts_atomic_init_nob(&q->tail.data.marker.next, ERTS_AINT_NULL); q->tail.data.marker.data.ptr = NULL; erts_atomic_init_nob(&q->tail.data.last, (erts_aint_t) &q->tail.data.marker); erts_atomic_init_nob(&q->tail.data.um_refc[0], 0); erts_atomic_init_nob(&q->tail.data.um_refc[1], 0); erts_atomic32_init_nob(&q->tail.data.um_refc_ix, 0); q->tail.data.live = qi->live.objects; q->tail.data.arg = qi->arg; q->tail.data.notify = qi->notify; if (!q->tail.data.notify) q->tail.data.notify = noop_callback; erts_atomic_init_nob(&q->head.head, (erts_aint_t) &q->tail.data.marker); q->head.live = qi->live.objects; q->head.first = &q->tail.data.marker; q->head.unref_end = &q->tail.data.marker; q->head.clean_reached_head_count = 0; q->head.deq_fini.automatic = qi->auto_finalize_dequeue; q->head.deq_fini.start = NULL; q->head.deq_fini.end = NULL; #ifdef ERTS_SMP q->head.next.thr_progress = erts_thr_progress_current(); q->head.next.thr_progress_reached = 1; #endif q->head.next.um_refc_ix = 1; q->head.next.unref_end = &q->tail.data.marker; q->head.used_marker = 1; q->head.arg = qi->arg; q->head.notify = q->tail.data.notify; q->q.finalizing = 0; q->q.live = qi->live.queue; q->q.blk = NULL; #endif }
void erts_init_ranges(void) { Sint i; erts_atomic_init_nob(&mem_used, 0); for (i = 0; i < ERTS_NUM_CODE_IX; i++) { r[i].modules = 0; r[i].n = 0; r[i].allocated = 0; erts_atomic_init_nob(&r[i].mid, 0); } erts_dump_num_lit_areas = 8; erts_dump_lit_areas = (ErtsLiteralArea **) erts_alloc(ERTS_ALC_T_CRASH_DUMP, erts_dump_num_lit_areas * sizeof(ErtsLiteralArea*)); }
void erts_end_staging_ranges(int commit) { if (commit) { Sint i; ErtsCodeIndex src = erts_active_code_ix(); ErtsCodeIndex dst = erts_staging_code_ix(); Range* mp; Sint num_inserted; mp = r[dst].modules; num_inserted = write_ptr - mp; for (i = 0; i < r[src].n; i++) { Range* rp = r[src].modules+i; if (rp->start < RANGE_END(rp)) { /* Only insert a module that has not been purged. */ write_ptr->start = rp->start; erts_atomic_init_nob(&write_ptr->end, (erts_aint_t)(RANGE_END(rp))); write_ptr++; } } /* * There are num_inserted new range entries (unsorted) at the * beginning of the modules array, followed by the old entries * (sorted). We must now sort the entire array. */ r[dst].n = write_ptr - mp; if (num_inserted > 1) { qsort(mp, r[dst].n, sizeof(Range), (int (*)(const void *, const void *)) rangecompare); } else if (num_inserted == 1) { /* Sift the new range into place. This is faster than qsort(). */ Range t = mp[0]; for (i = 0; i < r[dst].n-1 && t.start > mp[i+1].start; i++) { mp[i] = mp[i+1]; } mp[i] = t; } r[dst].modules = mp; CHECK(&r[dst]); erts_atomic_set_nob(&r[dst].mid, (erts_aint_t) (r[dst].modules + r[dst].n / 2)); if (r[dst].allocated * 2 > erts_dump_num_lit_areas) { erts_dump_num_lit_areas *= 2; erts_dump_lit_areas = (ErtsLiteralArea **) erts_realloc(ERTS_ALC_T_CRASH_DUMP, (void *) erts_dump_lit_areas, erts_dump_num_lit_areas * sizeof(ErtsLiteralArea*)); } } }
void erts_flxctr_init(ErtsFlxCtr* c, int is_decentralized, Uint nr_of_counters, ErtsAlcType_t alloc_type) { ASSERT(nr_of_counters <= ERTS_FLXCTR_ATOMICS_PER_CACHE_LINE); c->is_decentralized = is_decentralized; c->nr_of_counters = nr_of_counters; if (c->is_decentralized) { ErtsFlxCtrDecentralizedCtrArray* array = create_decentralized_ctr_array(alloc_type, nr_of_counters); erts_atomic_set_nob(&array->snapshot_status, ERTS_FLXCTR_SNAPSHOT_NOT_ONGOING); erts_atomic_init_nob(&c->u.counters_ptr, (Sint)array); ASSERT(((Uint)array->array) % ERTS_CACHE_LINE_SIZE == 0); } else { int i; for (i = 0; i < nr_of_counters; i++) { erts_atomic_init_nob(&c->u.counters[i], 0); } } }
void erts_init_bif_persistent_term(void) { HashTable* hash_table; /* * Initialize the mutex protecting updates. */ erts_mtx_init(&update_table_permission_mtx, "update_persistent_term_permission", NIL, ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); /* * Initialize delete queue. */ erts_mtx_init(&delete_queue_mtx, "persistent_term_delete_permission", NIL, ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); /* * Allocate a small initial hash table. */ hash_table = create_initial_table(); erts_atomic_init_nob(&the_hash_table, (erts_aint_t)hash_table); /* * Initialize export entry for traps */ erts_init_trap_export(&persistent_term_get_all_export, am_persistent_term, am_get_all_trap, 2, &persistent_term_get_all_trap); erts_init_trap_export(&persistent_term_info_export, am_persistent_term, am_info_trap, 1, &persistent_term_info_trap); }
static HashTable* create_initial_table(void) { HashTable* hash_table; int i; hash_table = (HashTable *) erts_alloc(ERTS_ALC_T_PERSISTENT_TERM, sizeof(HashTable)+sizeof(Eterm) * (INITIAL_SIZE-1)); hash_table->allocated = INITIAL_SIZE; hash_table->num_entries = 0; hash_table->mask = INITIAL_SIZE-1; hash_table->first_to_delete = 0; hash_table->num_to_delete = 0; erts_atomic_init_nob(&hash_table->refc, (erts_aint_t)1); for (i = 0; i < INITIAL_SIZE; i++) { hash_table->term[i] = NIL; } return hash_table; }
void init_module_table(void) { HashFunctions f; int i; f.hash = (H_FUN) module_hash; f.cmp = (HCMP_FUN) module_cmp; f.alloc = (HALLOC_FUN) module_alloc; f.free = (HFREE_FUN) module_free; f.meta_alloc = (HMALLOC_FUN) erts_alloc; f.meta_free = (HMFREE_FUN) erts_free; f.meta_print = (HMPRINT_FUN) erts_print; for (i = 0; i < ERTS_NUM_CODE_IX; i++) { erts_index_init(ERTS_ALC_T_MODULE_TABLE, &module_tables[i], "module_code", MODULE_SIZE, MODULE_LIMIT, f); } for (i=0; i<ERTS_NUM_CODE_IX; i++) { erts_rwmtx_init(&the_old_code_rwlocks[i], "old_code", make_small(i), ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); } erts_atomic_init_nob(&tot_module_bytes, 0); }
static int do_load_driver_entry(DE_Handle *dh, char *path, char *name) { void *init_handle; int res; ErlDrvEntry *dp; assert_drv_list_rwlocked(); if ((res = erts_sys_ddll_open(path, &(dh->handle), NULL)) != ERL_DE_NO_ERROR) { return res; } if ((res = erts_sys_ddll_load_driver_init(dh->handle, &init_handle)) != ERL_DE_NO_ERROR) { res = ERL_DE_LOAD_ERROR_NO_INIT; goto error; } dp = erts_sys_ddll_call_init(init_handle); if (dp == NULL) { res = ERL_DE_LOAD_ERROR_FAILED_INIT; goto error; } switch (dp->extended_marker) { case ERL_DRV_EXTENDED_MARKER: if (dp->major_version < ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD || (ERL_DRV_EXTENDED_MAJOR_VERSION < dp->major_version || (ERL_DRV_EXTENDED_MAJOR_VERSION == dp->major_version && ERL_DRV_EXTENDED_MINOR_VERSION < dp->minor_version))) { /* Incompatible driver version */ res = ERL_DE_LOAD_ERROR_INCORRECT_VERSION; goto error; } break; default: /* Old driver; needs to be recompiled... */ res = ERL_DE_LOAD_ERROR_INCORRECT_VERSION; goto error; } if (strcmp(name, dp->driver_name) != 0) { res = ERL_DE_LOAD_ERROR_BAD_NAME; goto error; } erts_atomic_init_nob(&(dh->refc), (erts_aint_t) 0); erts_atomic32_init_nob(&dh->port_count, 0); dh->full_path = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(path) + 1); sys_strcpy(dh->full_path, path); dh->flags = 0; dh->status = ERL_DE_OK; if (erts_add_driver_entry(dp, dh, 1) != 0 /* io.c */) { /* * The init in the driver struct did not return 0 */ erts_free(ERTS_ALC_T_DDLL_HANDLE, dh->full_path); dh->full_path = NULL; res = ERL_DE_LOAD_ERROR_FAILED_INIT; goto error; } return ERL_DE_NO_ERROR; error: erts_sys_ddll_close(dh->handle); return res; }
erts_sspa_data_t * erts_sspa_create(size_t blk_sz, int pa_size, int nthreads, const char* name) { erts_sspa_data_t *data; size_t tot_size; size_t chunk_mem_size; char *p; char *chunk_start; int cix; int no_blocks = pa_size; int no_blocks_per_chunk; size_t aligned_blk_sz; #if !defined(ERTS_STRUCTURE_ALIGNED_ALLOC) /* Force 64-bit alignment... */ aligned_blk_sz = ((blk_sz - 1) / 8) * 8 + 8; #else /* Alignment of structure is enough... */ aligned_blk_sz = blk_sz; #endif if (!name) { /* schedulers only variant */ ASSERT(!nthreads); nthreads = erts_no_schedulers; } else { ASSERT(nthreads > 0); } if (nthreads == 1) no_blocks_per_chunk = no_blocks; else { int extra = (no_blocks - 1)/4 + 1; if (extra == 0) extra = 1; no_blocks_per_chunk = no_blocks; no_blocks_per_chunk += extra * nthreads; no_blocks_per_chunk /= nthreads; } no_blocks = no_blocks_per_chunk * nthreads; chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_chunk_header_t)); chunk_mem_size += aligned_blk_sz * no_blocks_per_chunk; chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(chunk_mem_size); tot_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t)); tot_size += chunk_mem_size * nthreads; p = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_PRE_ALLOC_DATA, tot_size); data = (erts_sspa_data_t *) p; p += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t)); chunk_start = p; data->chunks_mem_size = chunk_mem_size; data->start = chunk_start; data->end = chunk_start + chunk_mem_size * nthreads; data->nthreads = nthreads; if (name) { /* thread variant */ erts_tsd_key_create(&data->tsd_key, (char*)name); erts_atomic_init_nob(&data->id_generator, 0); } /* Initialize all chunks */ for (cix = 0; cix < nthreads; cix++) { erts_sspa_chunk_t *chnk = erts_sspa_cix2chunk(data, cix); erts_sspa_chunk_header_t *chdr = &chnk->aligned.header; erts_sspa_blk_t *blk; int i; erts_atomic_init_nob(&chdr->tail.data.last, (erts_aint_t) &chdr->tail.data.marker); erts_atomic_init_nob(&chdr->tail.data.marker.next_atmc, ERTS_AINT_NULL); erts_atomic_init_nob(&chdr->tail.data.um_refc[0], 0); erts_atomic_init_nob(&chdr->tail.data.um_refc[1], 0); erts_atomic32_init_nob(&chdr->tail.data.um_refc_ix, 0); chdr->head.no_thr_progress_check = 0; chdr->head.used_marker = 1; chdr->head.first = &chdr->tail.data.marker; chdr->head.unref_end = &chdr->tail.data.marker; chdr->head.next.thr_progress = erts_thr_progress_current(); chdr->head.next.thr_progress_reached = 1; chdr->head.next.um_refc_ix = 1; chdr->head.next.unref_end = &chdr->tail.data.marker; p = &chnk->data[0]; chdr->local.first = (erts_sspa_blk_t *) p; blk = (erts_sspa_blk_t *) p; for (i = 0; i < no_blocks_per_chunk; i++) { blk = (erts_sspa_blk_t *) p; p += aligned_blk_sz; blk->next_ptr = (erts_sspa_blk_t *) p; } blk->next_ptr = NULL; chdr->local.last = blk; chdr->local.cnt = no_blocks_per_chunk; chdr->local.lim = no_blocks_per_chunk / 3; ERTS_SSPA_DBG_CHK_LCL(chdr); } return data; }
erts_sspa_data_t * erts_sspa_create(size_t blk_sz, int pa_size) { erts_sspa_data_t *data; size_t tot_size; size_t chunk_mem_size; char *p; char *chunk_start; int cix; int no_blocks = pa_size; int no_blocks_per_chunk; if (erts_no_schedulers == 1) no_blocks_per_chunk = no_blocks; else { int extra = (no_blocks - 1)/4 + 1; if (extra == 0) extra = 1; no_blocks_per_chunk = no_blocks; no_blocks_per_chunk += extra*erts_no_schedulers; no_blocks_per_chunk /= erts_no_schedulers; } no_blocks = no_blocks_per_chunk * erts_no_schedulers; chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_chunk_header_t)); chunk_mem_size += blk_sz * no_blocks_per_chunk; chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(chunk_mem_size); tot_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t)); tot_size += chunk_mem_size*erts_no_schedulers; p = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_PRE_ALLOC_DATA, tot_size); data = (erts_sspa_data_t *) p; p += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t)); chunk_start = p; data->chunks_mem_size = chunk_mem_size; data->start = chunk_start; data->end = chunk_start + chunk_mem_size*erts_no_schedulers; /* Initialize all chunks */ for (cix = 0; cix < erts_no_schedulers; cix++) { erts_sspa_chunk_t *chnk = erts_sspa_cix2chunk(data, cix); erts_sspa_chunk_header_t *chdr = &chnk->aligned.header; erts_sspa_blk_t *blk; int i; erts_atomic_init_nob(&chdr->tail.data.last, (erts_aint_t) &chdr->tail.data.marker); erts_atomic_init_nob(&chdr->tail.data.marker.next_atmc, ERTS_AINT_NULL); erts_atomic_init_nob(&chdr->tail.data.um_refc[0], 0); erts_atomic_init_nob(&chdr->tail.data.um_refc[1], 0); erts_atomic32_init_nob(&chdr->tail.data.um_refc_ix, 0); chdr->head.no_thr_progress_check = 0; chdr->head.used_marker = 1; chdr->head.first = &chdr->tail.data.marker; chdr->head.unref_end = &chdr->tail.data.marker; chdr->head.next.thr_progress = erts_thr_progress_current(); chdr->head.next.thr_progress_reached = 1; chdr->head.next.um_refc_ix = 1; chdr->head.next.unref_end = &chdr->tail.data.marker; p = &chnk->data[0]; chdr->local.first = (erts_sspa_blk_t *) p; blk = (erts_sspa_blk_t *) p; for (i = 0; i < no_blocks_per_chunk; i++) { blk = (erts_sspa_blk_t *) p; p += blk_sz; blk->next_ptr = (erts_sspa_blk_t *) p; } blk->next_ptr = NULL; chdr->local.last = blk; chdr->local.cnt = no_blocks_per_chunk; chdr->local.lim = no_blocks_per_chunk / 3; ERTS_SSPA_DBG_CHK_LCL(chdr); } return data; }
static HashTable* copy_table(ErtsPersistentTermCpyTableCtx* ctx) { Uint old_size = ctx->old_table->allocated; Uint i; ErtsAlcType_t alloc_type; ctx->total_iterations_done = 0; switch(ctx->location) { case ERTS_PERSISTENT_TERM_CPY_PLACE_1: goto L_copy_table_place_1; case ERTS_PERSISTENT_TERM_CPY_PLACE_2: goto L_copy_table_place_2; case ERTS_PERSISTENT_TERM_CPY_PLACE_3: goto L_copy_table_place_3; case ERTS_PERSISTENT_TERM_CPY_PLACE_START: ctx->iterations_done = 0; } if (ctx->copy_type == ERTS_PERSISTENT_TERM_CPY_TEMP) { alloc_type = ERTS_ALC_T_PERSISTENT_TERM_TMP; } else { alloc_type = ERTS_ALC_T_PERSISTENT_TERM; } ctx->new_table = (HashTable *) erts_alloc(alloc_type, sizeof(HashTable) + sizeof(Eterm) * (ctx->new_size-1)); if (ctx->old_table->allocated == ctx->new_size && (ctx->copy_type == ERTS_PERSISTENT_TERM_CPY_NO_REHASH || ctx->copy_type == ERTS_PERSISTENT_TERM_CPY_TEMP)) { /* * Same size and no key deleted. Make an exact copy of the table. */ *ctx->new_table = *ctx->old_table; L_copy_table_place_1: for (i = ctx->iterations_done; i < MIN(ctx->iterations_done + ctx->max_iterations, ctx->new_size); i++) { ctx->new_table->term[i] = ctx->old_table->term[i]; } ctx->total_iterations_done = (i - ctx->iterations_done); if (i < ctx->new_size) { ctx->iterations_done = i; ctx->location = ERTS_PERSISTENT_TERM_CPY_PLACE_1; return NULL; } ctx->iterations_done = 0; } else { /* * The size of the table has changed or an element has been * deleted. Must rehash, by inserting all old terms into the * new (empty) table. */ ctx->new_table->allocated = ctx->new_size; ctx->new_table->num_entries = ctx->old_table->num_entries; ctx->new_table->mask = ctx->new_size - 1; L_copy_table_place_2: for (i = ctx->iterations_done; i < MIN(ctx->iterations_done + ctx->max_iterations, ctx->new_size); i++) { ctx->new_table->term[i] = NIL; } ctx->total_iterations_done = (i - ctx->iterations_done); ctx->max_iterations -= ctx->total_iterations_done; if (i < ctx->new_size) { ctx->iterations_done = i; ctx->location = ERTS_PERSISTENT_TERM_CPY_PLACE_2; return NULL; } ctx->iterations_done = 0; L_copy_table_place_3: for (i = ctx->iterations_done; i < MIN(ctx->iterations_done + ctx->max_iterations, old_size); i++) { if (is_tuple(ctx->old_table->term[i])) { Eterm key = tuple_val(ctx->old_table->term[i])[1]; Uint entry_index = lookup(ctx->new_table, key); ASSERT(is_nil(ctx->new_table->term[entry_index])); ctx->new_table->term[entry_index] = ctx->old_table->term[i]; } } ctx->total_iterations_done += (i - ctx->iterations_done); if (i < old_size) { ctx->iterations_done = i; ctx->location = ERTS_PERSISTENT_TERM_CPY_PLACE_3; return NULL; } ctx->iterations_done = 0; } ctx->new_table->first_to_delete = 0; ctx->new_table->num_to_delete = 0; erts_atomic_init_nob(&ctx->new_table->refc, (erts_aint_t)1); { HashTable* new_table = ctx->new_table; /* * IMPORTANT: Memory management depends on that ctx->new_table is * set to NULL on the line below */ ctx->new_table = NULL; return new_table; } }
void erts_init_async(void) { async = NULL; if (erts_async_max_threads > 0) { #if ERTS_USE_ASYNC_READY_Q ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT; #endif erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER; char *ptr, thr_name[16]; size_t tot_size = 0; int i; tot_size += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncData)); tot_size += sizeof(ErtsAlgndAsyncQ)*erts_async_max_threads; #if ERTS_USE_ASYNC_READY_Q tot_size += sizeof(ErtsAlgndAsyncReadyQ)*erts_no_schedulers; #endif ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_ASYNC_DATA, tot_size); async = (ErtsAsyncData *) ptr; ptr += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncData)); async->init.data.no_initialized = 0; erts_mtx_init(&async->init.data.mtx, "async_init_mtx", NIL, ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER); erts_cnd_init(&async->init.data.cnd); erts_atomic_init_nob(&async->init.data.id, 0); async->queue = (ErtsAlgndAsyncQ *) ptr; ptr += sizeof(ErtsAlgndAsyncQ)*erts_async_max_threads; #if ERTS_USE_ASYNC_READY_Q qinit.live.queue = ERTS_THR_Q_LIVE_LONG; qinit.live.objects = ERTS_THR_Q_LIVE_SHORT; qinit.notify = erts_notify_check_async_ready_queue; async->ready_queue = (ErtsAlgndAsyncReadyQ *) ptr; ptr += sizeof(ErtsAlgndAsyncReadyQ)*erts_no_schedulers; for (i = 1; i <= erts_no_schedulers; i++) { ErtsAsyncReadyQ *arq = async_ready_q(i); #if ERTS_USE_ASYNC_READY_ENQ_MTX erts_mtx_init(&arq->x.data.enq_mtx, "async_enq_mtx", make_small(i), ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER); #endif erts_thr_q_finalize_dequeue_state_init(&arq->fin_deq); qinit.arg = (void *) (SWord) i; erts_thr_q_initialize(&arq->thr_q, &qinit); } #endif /* Create async threads... */ thr_opts.detached = 0; thr_opts.suggested_stack_size = erts_async_thread_suggested_stack_size; thr_opts.name = thr_name; for (i = 0; i < erts_async_max_threads; i++) { ErtsAsyncQ *aq = async_q(i); erts_snprintf(thr_opts.name, 16, "async_%d", i+1); erts_thr_create(&aq->thr_id, async_main, (void*) aq, &thr_opts); } /* Wait for async threads to initialize... */ erts_mtx_lock(&async->init.data.mtx); while (async->init.data.no_initialized != erts_async_max_threads) erts_cnd_wait(&async->init.data.cnd, &async->init.data.mtx); erts_mtx_unlock(&async->init.data.mtx); erts_mtx_destroy(&async->init.data.mtx); erts_cnd_destroy(&async->init.data.cnd); } }
void erts_sys_pre_init(void) { erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER; erts_printf_add_cr_to_stdout = 1; erts_printf_add_cr_to_stderr = 1; eid.thread_create_child_func = thr_create_prepare_child; /* Before creation in parent */ eid.thread_create_prepare_func = thr_create_prepare; /* After creation in parent */ eid.thread_create_parent_func = thr_create_cleanup, #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_pre_thr_init(); #endif erts_thr_init(&eid); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_post_thr_init(); #endif #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_init(); #endif erts_init_sys_time_sup(); erts_atomic32_init_nob(&erts_break_requested, 0); erts_atomic32_init_nob(&have_prepared_crash_dump, 0); erts_atomic_init_nob(&sys_misc_mem_sz, 0); { /* * Unfortunately we depend on fd 0,1,2 in the old shell code. * So if for some reason we do not have those open when we start * we have to open them here. Not doing this can cause the emulator * to deadlock when reaping the fd_driver ports :( */ int fd; /* Make sure fd 0 is open */ if ((fd = open("/dev/null", O_RDONLY)) != 0) close(fd); /* Make sure fds 1 and 2 are open */ while (fd < 3) { fd = open("/dev/null", O_WRONLY); } close(fd); } /* We need a file descriptor to close in the crashdump creation. * We close this one to be sure we can get a fd for our real file ... * so, we create one here ... a stone to carry all the way home. */ crashdump_companion_cube_fd = open("/dev/null", O_RDONLY); /* don't lose it, there will be cake */ }
void erts_thr_progress_init(int no_schedulers, int managed, int unmanaged) { int i, j, um_low, um_high; char *ptr; size_t cb_sz, intrnl_sz, thr_arr_sz, m_wakeup_size, um_wakeup_size, tot_size; intrnl_sz = sizeof(ErtsThrPrgrInternalData); intrnl_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(intrnl_sz); cb_sz = sizeof(ErtsThrPrgrCallbacks)*(managed+unmanaged); cb_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(cb_sz); thr_arr_sz = sizeof(ErtsThrPrgrArray)*managed; ASSERT(thr_arr_sz == ERTS_ALC_CACHE_LINE_ALIGN_SIZE(thr_arr_sz)); m_wakeup_size = sizeof(ErtsThrPrgrManagedWakeupData); m_wakeup_size += (managed - 1)*sizeof(int); m_wakeup_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(m_wakeup_size); um_low = (unmanaged - 1)/ERTS_THR_PRGR_BM_BITS + 1; um_high = (um_low - 1)/ERTS_THR_PRGR_BM_BITS + 1; um_wakeup_size = sizeof(ErtsThrPrgrUnmanagedWakeupData); um_wakeup_size += (um_high + um_low)*sizeof(erts_atomic32_t); um_wakeup_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(um_wakeup_size); tot_size = intrnl_sz; tot_size += cb_sz; tot_size += thr_arr_sz; tot_size += m_wakeup_size*ERTS_THR_PRGR_WAKEUP_DATA_SIZE; tot_size += um_wakeup_size*ERTS_THR_PRGR_WAKEUP_DATA_SIZE; ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_THR_PRGR_IDATA, tot_size); intrnl = (ErtsThrPrgrInternalData *) ptr; ptr += intrnl_sz; erts_atomic32_init_nob(&intrnl->misc.data.lflgs, ERTS_THR_PRGR_LFLG_NO_LEADER); erts_atomic32_init_nob(&intrnl->misc.data.block_count, (ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING | (erts_aint32_t) managed)); erts_atomic_init_nob(&intrnl->misc.data.blocker_event, ERTS_AINT_NULL); erts_atomic32_init_nob(&intrnl->misc.data.pref_wakeup_used, 0); erts_atomic32_init_nob(&intrnl->misc.data.managed_count, 0); erts_atomic32_init_nob(&intrnl->misc.data.managed_id, no_schedulers); erts_atomic32_init_nob(&intrnl->misc.data.unmanaged_id, -1); intrnl->misc.data.chk_next_ix = 0; intrnl->misc.data.umrefc_ix.waiting = -1; erts_atomic32_init_nob(&intrnl->misc.data.umrefc_ix.current, 0); erts_atomic_init_nob(&intrnl->umrefc[0].refc, (erts_aint_t) 0); erts_atomic_init_nob(&intrnl->umrefc[1].refc, (erts_aint_t) 0); intrnl->thr = (ErtsThrPrgrArray *) ptr; ptr += thr_arr_sz; for (i = 0; i < managed; i++) init_nob(&intrnl->thr[i].data.current, 0); intrnl->managed.callbacks = (ErtsThrPrgrCallbacks *) ptr; intrnl->unmanaged.callbacks = &intrnl->managed.callbacks[managed]; ptr += cb_sz; intrnl->managed.no = managed; for (i = 0; i < managed; i++) { intrnl->managed.callbacks[i].arg = NULL; intrnl->managed.callbacks[i].wakeup = NULL; } intrnl->unmanaged.no = unmanaged; for (i = 0; i < unmanaged; i++) { intrnl->unmanaged.callbacks[i].arg = NULL; intrnl->unmanaged.callbacks[i].wakeup = NULL; } for (i = 0; i < ERTS_THR_PRGR_WAKEUP_DATA_SIZE; i++) { intrnl->managed.data[i] = (ErtsThrPrgrManagedWakeupData *) ptr; erts_atomic32_init_nob(&intrnl->managed.data[i]->len, 0); ptr += m_wakeup_size; } for (i = 0; i < ERTS_THR_PRGR_WAKEUP_DATA_SIZE; i++) { erts_atomic32_t *bm; intrnl->unmanaged.data[i] = (ErtsThrPrgrUnmanagedWakeupData *) ptr; erts_atomic32_init_nob(&intrnl->unmanaged.data[i]->len, 0); bm = (erts_atomic32_t *) (ptr + sizeof(ErtsThrPrgrUnmanagedWakeupData)); intrnl->unmanaged.data[i]->high = bm; intrnl->unmanaged.data[i]->high_sz = um_high; for (j = 0; j < um_high; j++) erts_atomic32_init_nob(&intrnl->unmanaged.data[i]->high[j], 0); intrnl->unmanaged.data[i]->low = &intrnl->unmanaged.data[i]->high[um_high]; intrnl->unmanaged.data[i]->low_sz = um_low; for (j = 0; j < um_low; j++) erts_atomic32_init_nob(&intrnl->unmanaged.data[i]->low[j], 0); ptr += um_wakeup_size; } ERTS_THR_MEMORY_BARRIER; }
static int early_init(int *argc, char **argv) /* * Only put things here which are * really important initialize * early! */ { ErtsAllocInitOpts alloc_opts = ERTS_ALLOC_INIT_DEF_OPTS_INITER; int ncpu; int ncpuonln; int ncpuavail; int schdlrs; int schdlrs_onln; int max_main_threads; int max_reader_groups; int reader_groups; char envbuf[21]; /* enough for any 64-bit integer */ size_t envbufsz; erts_sched_compact_load = 1; erts_printf_eterm_func = erts_printf_term; erts_disable_tolerant_timeofday = 0; display_items = 200; erts_proc.max = ERTS_DEFAULT_MAX_PROCESSES; erts_backtrace_depth = DEFAULT_BACKTRACE_SIZE; erts_async_max_threads = 0; erts_async_thread_suggested_stack_size = ERTS_ASYNC_THREAD_MIN_STACK_SIZE; H_MIN_SIZE = H_DEFAULT_SIZE; BIN_VH_MIN_SIZE = VH_DEFAULT_SIZE; erts_initialized = 0; erts_use_sender_punish = 1; erts_pre_early_init_cpu_topology(&max_reader_groups, &ncpu, &ncpuonln, &ncpuavail); #ifndef ERTS_SMP ncpu = 1; ncpuonln = 1; ncpuavail = 1; #endif ignore_break = 0; replace_intr = 0; program = argv[0]; erts_modified_timing_level = -1; erts_compat_rel = this_rel_num(); erts_use_r9_pids_ports = 0; erts_sys_pre_init(); erts_atomic_init_nob(&exiting, 0); #ifdef ERTS_SMP erts_thr_progress_pre_init(); #endif #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_init(); #endif #ifdef ERTS_SMP erts_smp_atomic32_init_nob(&erts_writing_erl_crash_dump, 0L); erts_tsd_key_create(&erts_is_crash_dumping_key); #else erts_writing_erl_crash_dump = 0; #endif erts_smp_atomic32_init_nob(&erts_max_gen_gcs, (erts_aint32_t) ((Uint16) -1)); erts_pre_init_process(); #if defined(USE_THREADS) && !defined(ERTS_SMP) main_thread = erts_thr_self(); #endif /* * We need to know the number of schedulers to use before we * can initialize the allocators. */ no_schedulers = (Uint) (ncpu > 0 ? ncpu : 1); no_schedulers_online = (ncpuavail > 0 ? ncpuavail : (ncpuonln > 0 ? ncpuonln : no_schedulers)); schdlrs = no_schedulers; schdlrs_onln = no_schedulers_online; envbufsz = sizeof(envbuf); /* erts_sys_getenv() not initialized yet; need erts_sys_getenv__() */ if (erts_sys_getenv__("ERL_THREAD_POOL_SIZE", envbuf, &envbufsz) == 0) erts_async_max_threads = atoi(envbuf); else erts_async_max_threads = 0; if (erts_async_max_threads > ERTS_MAX_NO_OF_ASYNC_THREADS) erts_async_max_threads = ERTS_MAX_NO_OF_ASYNC_THREADS; if (argc && argv) { int i = 1; while (i < *argc) { if (strcmp(argv[i], "--") == 0) { /* end of emulator options */ i++; break; } if (argv[i][0] == '-') { switch (argv[i][1]) { case 'r': { char *sub_param = argv[i]+2; if (has_prefix("g", sub_param)) { char *arg = get_arg(sub_param+1, argv[i+1], &i); if (sscanf(arg, "%d", &max_reader_groups) != 1) { erts_fprintf(stderr, "bad reader groups limit: %s\n", arg); erts_usage(); } if (max_reader_groups < 0) { erts_fprintf(stderr, "bad reader groups limit: %d\n", max_reader_groups); erts_usage(); } } break; } case 'A': { /* set number of threads in thread pool */ char *arg = get_arg(argv[i]+2, argv[i+1], &i); if (((erts_async_max_threads = atoi(arg)) < 0) || (erts_async_max_threads > ERTS_MAX_NO_OF_ASYNC_THREADS)) { erts_fprintf(stderr, "bad number of async threads %s\n", arg); erts_usage(); VERBOSE(DEBUG_SYSTEM, ("using %d async-threads\n", erts_async_max_threads)); } break; } case 'S' : { int tot, onln; char *arg = get_arg(argv[i]+2, argv[i+1], &i); switch (sscanf(arg, "%d:%d", &tot, &onln)) { case 0: switch (sscanf(arg, ":%d", &onln)) { case 1: tot = no_schedulers; goto chk_S; default: goto bad_S; } case 1: onln = tot < schdlrs_onln ? tot : schdlrs_onln; case 2: chk_S: if (tot > 0) schdlrs = tot; else schdlrs = no_schedulers + tot; if (onln > 0) schdlrs_onln = onln; else schdlrs_onln = no_schedulers_online + onln; if (schdlrs < 1 || ERTS_MAX_NO_OF_SCHEDULERS < schdlrs) { erts_fprintf(stderr, "bad amount of schedulers %d\n", tot); erts_usage(); } if (schdlrs_onln < 1 || schdlrs < schdlrs_onln) { erts_fprintf(stderr, "bad amount of schedulers online %d " "(total amount of schedulers %d)\n", schdlrs_onln, schdlrs); erts_usage(); } break; default: bad_S: erts_fprintf(stderr, "bad amount of schedulers %s\n", arg); erts_usage(); break; } VERBOSE(DEBUG_SYSTEM, ("using %d:%d scheduler(s)\n", tot, onln)); break; } default: break; } } i++; } } #ifndef USE_THREADS erts_async_max_threads = 0; #endif #ifdef ERTS_SMP no_schedulers = schdlrs; no_schedulers_online = schdlrs_onln; erts_no_schedulers = (Uint) no_schedulers; #endif erts_early_init_scheduling(no_schedulers); alloc_opts.ncpu = ncpu; erts_alloc_init(argc, argv, &alloc_opts); /* Handles (and removes) -M flags. */ /* Require allocators */ #ifdef ERTS_SMP /* * Thread progress management: * * * Managed threads: * ** Scheduler threads (see erl_process.c) * ** Aux thread (see erl_process.c) * ** Sys message dispatcher thread (see erl_trace.c) * * * Unmanaged threads that need to register: * ** Async threads (see erl_async.c) */ erts_thr_progress_init(no_schedulers, no_schedulers+2, erts_async_max_threads); #endif erts_thr_q_init(); erts_init_utils(); erts_early_init_cpu_topology(no_schedulers, &max_main_threads, max_reader_groups, &reader_groups); #ifdef USE_THREADS { erts_thr_late_init_data_t elid = ERTS_THR_LATE_INIT_DATA_DEF_INITER; elid.mem.std.alloc = ethr_std_alloc; elid.mem.std.realloc = ethr_std_realloc; elid.mem.std.free = ethr_std_free; elid.mem.sl.alloc = ethr_sl_alloc; elid.mem.sl.realloc = ethr_sl_realloc; elid.mem.sl.free = ethr_sl_free; elid.mem.ll.alloc = ethr_ll_alloc; elid.mem.ll.realloc = ethr_ll_realloc; elid.mem.ll.free = ethr_ll_free; elid.main_threads = max_main_threads; elid.reader_groups = reader_groups; erts_thr_late_init(&elid); } #endif #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_late_init(); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_late_init(); #endif #if defined(HIPE) hipe_signal_init(); /* must be done very early */ #endif erl_sys_args(argc, argv); /* Creates threads on Windows that depend on the arguments, so has to be after erl_sys_args */ erl_sys_init(); erts_ets_realloc_always_moves = 0; erts_ets_always_compress = 0; erts_dist_buf_busy_limit = ERTS_DE_BUSY_LIMIT; return ncpu; }
void erts_init_port_data(Port *prt) { erts_atomic_init_nob(&prt->data, (erts_aint_t) am_undefined); }