void erts_thr_progress_dbg_print_state(void) { int id; int sz = intrnl->managed.no; erts_fprintf(stderr, "--- thread progress ---\n"); erts_fprintf(stderr,"current=%b64u\n", erts_thr_progress_current()); for (id = 0; id < sz; id++) { ErtsThrPrgrVal current = read_nob(&intrnl->thr[id].data.current); #ifdef ERTS_THR_PROGRESS_STATE_DEBUG erts_aint32_t state_debug; char *active, *leader; state_debug = erts_atomic32_read_nob(&intrnl->thr[id].data.state_debug); active = (state_debug & ERTS_THR_PROGRESS_STATE_DEBUG_ACTIVE ? "true" : "false"); leader = (state_debug & ERTS_THR_PROGRESS_STATE_DEBUG_LEADER ? "true" : "false"); #endif if (current == ERTS_THR_PRGR_VAL_WAITING) erts_fprintf(stderr, " id=%d, current=WAITING" #ifdef ERTS_THR_PROGRESS_STATE_DEBUG ", active=%s, leader=%s" #endif "\n", id #ifdef ERTS_THR_PROGRESS_STATE_DEBUG , active, leader #endif ); else erts_fprintf(stderr, " id=%d, current=%b64u" #ifdef ERTS_THR_PROGRESS_STATE_DEBUG ", active=%s, leader=%s" #endif "\n", id, current #ifdef ERTS_THR_PROGRESS_STATE_DEBUG , active, leader #endif ); } erts_fprintf(stderr, "-----------------------\n"); }
void erts_thr_q_initialize(ErtsThrQ_t *q, ErtsThrQInit_t *qi) { #ifndef USE_THREADS q->init = *qi; if (!q->init.notify) q->init.notify = noop_callback; q->first = NULL; q->last = NULL; q->q.blk = NULL; #else erts_atomic_init_nob(&q->tail.data.marker.next, ERTS_AINT_NULL); q->tail.data.marker.data.ptr = NULL; erts_atomic_init_nob(&q->tail.data.last, (erts_aint_t) &q->tail.data.marker); erts_atomic_init_nob(&q->tail.data.um_refc[0], 0); erts_atomic_init_nob(&q->tail.data.um_refc[1], 0); erts_atomic32_init_nob(&q->tail.data.um_refc_ix, 0); q->tail.data.live = qi->live.objects; q->tail.data.arg = qi->arg; q->tail.data.notify = qi->notify; if (!q->tail.data.notify) q->tail.data.notify = noop_callback; erts_atomic_init_nob(&q->head.head, (erts_aint_t) &q->tail.data.marker); q->head.live = qi->live.objects; q->head.first = &q->tail.data.marker; q->head.unref_end = &q->tail.data.marker; q->head.clean_reached_head_count = 0; q->head.deq_fini.automatic = qi->auto_finalize_dequeue; q->head.deq_fini.start = NULL; q->head.deq_fini.end = NULL; #ifdef ERTS_SMP q->head.next.thr_progress = erts_thr_progress_current(); q->head.next.thr_progress_reached = 1; #endif q->head.next.um_refc_ix = 1; q->head.next.unref_end = &q->tail.data.marker; q->head.used_marker = 1; q->head.arg = qi->arg; q->head.notify = q->tail.data.notify; q->q.finalizing = 0; q->q.live = qi->live.queue; q->q.blk = NULL; #endif }
erts_sspa_data_t * erts_sspa_create(size_t blk_sz, int pa_size, int nthreads, const char* name) { erts_sspa_data_t *data; size_t tot_size; size_t chunk_mem_size; char *p; char *chunk_start; int cix; int no_blocks = pa_size; int no_blocks_per_chunk; size_t aligned_blk_sz; #if !defined(ERTS_STRUCTURE_ALIGNED_ALLOC) /* Force 64-bit alignment... */ aligned_blk_sz = ((blk_sz - 1) / 8) * 8 + 8; #else /* Alignment of structure is enough... */ aligned_blk_sz = blk_sz; #endif if (!name) { /* schedulers only variant */ ASSERT(!nthreads); nthreads = erts_no_schedulers; } else { ASSERT(nthreads > 0); } if (nthreads == 1) no_blocks_per_chunk = no_blocks; else { int extra = (no_blocks - 1)/4 + 1; if (extra == 0) extra = 1; no_blocks_per_chunk = no_blocks; no_blocks_per_chunk += extra * nthreads; no_blocks_per_chunk /= nthreads; } no_blocks = no_blocks_per_chunk * nthreads; chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_chunk_header_t)); chunk_mem_size += aligned_blk_sz * no_blocks_per_chunk; chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(chunk_mem_size); tot_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t)); tot_size += chunk_mem_size * nthreads; p = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_PRE_ALLOC_DATA, tot_size); data = (erts_sspa_data_t *) p; p += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t)); chunk_start = p; data->chunks_mem_size = chunk_mem_size; data->start = chunk_start; data->end = chunk_start + chunk_mem_size * nthreads; data->nthreads = nthreads; if (name) { /* thread variant */ erts_tsd_key_create(&data->tsd_key, (char*)name); erts_atomic_init_nob(&data->id_generator, 0); } /* Initialize all chunks */ for (cix = 0; cix < nthreads; cix++) { erts_sspa_chunk_t *chnk = erts_sspa_cix2chunk(data, cix); erts_sspa_chunk_header_t *chdr = &chnk->aligned.header; erts_sspa_blk_t *blk; int i; erts_atomic_init_nob(&chdr->tail.data.last, (erts_aint_t) &chdr->tail.data.marker); erts_atomic_init_nob(&chdr->tail.data.marker.next_atmc, ERTS_AINT_NULL); erts_atomic_init_nob(&chdr->tail.data.um_refc[0], 0); erts_atomic_init_nob(&chdr->tail.data.um_refc[1], 0); erts_atomic32_init_nob(&chdr->tail.data.um_refc_ix, 0); chdr->head.no_thr_progress_check = 0; chdr->head.used_marker = 1; chdr->head.first = &chdr->tail.data.marker; chdr->head.unref_end = &chdr->tail.data.marker; chdr->head.next.thr_progress = erts_thr_progress_current(); chdr->head.next.thr_progress_reached = 1; chdr->head.next.um_refc_ix = 1; chdr->head.next.unref_end = &chdr->tail.data.marker; p = &chnk->data[0]; chdr->local.first = (erts_sspa_blk_t *) p; blk = (erts_sspa_blk_t *) p; for (i = 0; i < no_blocks_per_chunk; i++) { blk = (erts_sspa_blk_t *) p; p += aligned_blk_sz; blk->next_ptr = (erts_sspa_blk_t *) p; } blk->next_ptr = NULL; chdr->local.last = blk; chdr->local.cnt = no_blocks_per_chunk; chdr->local.lim = no_blocks_per_chunk / 3; ERTS_SSPA_DBG_CHK_LCL(chdr); } return data; }
erts_sspa_data_t * erts_sspa_create(size_t blk_sz, int pa_size) { erts_sspa_data_t *data; size_t tot_size; size_t chunk_mem_size; char *p; char *chunk_start; int cix; int no_blocks = pa_size; int no_blocks_per_chunk; if (erts_no_schedulers == 1) no_blocks_per_chunk = no_blocks; else { int extra = (no_blocks - 1)/4 + 1; if (extra == 0) extra = 1; no_blocks_per_chunk = no_blocks; no_blocks_per_chunk += extra*erts_no_schedulers; no_blocks_per_chunk /= erts_no_schedulers; } no_blocks = no_blocks_per_chunk * erts_no_schedulers; chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_chunk_header_t)); chunk_mem_size += blk_sz * no_blocks_per_chunk; chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(chunk_mem_size); tot_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t)); tot_size += chunk_mem_size*erts_no_schedulers; p = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_PRE_ALLOC_DATA, tot_size); data = (erts_sspa_data_t *) p; p += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t)); chunk_start = p; data->chunks_mem_size = chunk_mem_size; data->start = chunk_start; data->end = chunk_start + chunk_mem_size*erts_no_schedulers; /* Initialize all chunks */ for (cix = 0; cix < erts_no_schedulers; cix++) { erts_sspa_chunk_t *chnk = erts_sspa_cix2chunk(data, cix); erts_sspa_chunk_header_t *chdr = &chnk->aligned.header; erts_sspa_blk_t *blk; int i; erts_atomic_init_nob(&chdr->tail.data.last, (erts_aint_t) &chdr->tail.data.marker); erts_atomic_init_nob(&chdr->tail.data.marker.next_atmc, ERTS_AINT_NULL); erts_atomic_init_nob(&chdr->tail.data.um_refc[0], 0); erts_atomic_init_nob(&chdr->tail.data.um_refc[1], 0); erts_atomic32_init_nob(&chdr->tail.data.um_refc_ix, 0); chdr->head.no_thr_progress_check = 0; chdr->head.used_marker = 1; chdr->head.first = &chdr->tail.data.marker; chdr->head.unref_end = &chdr->tail.data.marker; chdr->head.next.thr_progress = erts_thr_progress_current(); chdr->head.next.thr_progress_reached = 1; chdr->head.next.um_refc_ix = 1; chdr->head.next.unref_end = &chdr->tail.data.marker; p = &chnk->data[0]; chdr->local.first = (erts_sspa_blk_t *) p; blk = (erts_sspa_blk_t *) p; for (i = 0; i < no_blocks_per_chunk; i++) { blk = (erts_sspa_blk_t *) p; p += blk_sz; blk->next_ptr = (erts_sspa_blk_t *) p; } blk->next_ptr = NULL; chdr->local.last = blk; chdr->local.cnt = no_blocks_per_chunk; chdr->local.lim = no_blocks_per_chunk / 3; ERTS_SSPA_DBG_CHK_LCL(chdr); } return data; }