static ERTS_INLINE erts_tse_t * tse_fetch(erts_pix_lock_t *pix_lock) { erts_tse_t *tse = erts_tse_fetch(); if (!tse->udata) { erts_proc_lock_queues_t *qs; #if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL if (pix_lock) erts_pix_unlock(pix_lock); #endif erts_smp_spin_lock(&qs_lock); qs = queue_free_list; if (qs) { queue_free_list = queue_free_list->next; erts_smp_spin_unlock(&qs_lock); } else { erts_smp_spin_unlock(&qs_lock); qs = erts_alloc(ERTS_ALC_T_PROC_LCK_QS, sizeof(erts_proc_lock_queues_t)); sys_memcpy((void *) qs, (void *) &zeroqs, sizeof(erts_proc_lock_queues_t)); } tse->udata = qs; #if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL if (pix_lock) erts_pix_lock(pix_lock); #endif } tse->uflgs = 0; return tse; }
static void cleanup_tse(void) { erts_tse_t *tse = erts_tse_fetch(); if (tse) erts_tse_return(tse); }
static ERTS_INLINE erts_tse_t * tse_fetch(erts_pix_lock_t *pix_lock) { erts_tse_t *tse = erts_tse_fetch(); tse->uflgs = 0; return tse; }
static erts_tse_t *async_thread_init(ErtsAsyncQ *aq) { ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT; erts_tse_t *tse = erts_tse_fetch(); #ifdef ERTS_SMP ErtsThrPrgrCallbacks callbacks; callbacks.arg = (void *) tse; callbacks.wakeup = async_wakeup; callbacks.prepare_wait = NULL; callbacks.wait = NULL; erts_thr_progress_register_unmanaged_thread(&callbacks); #endif qinit.live.queue = ERTS_THR_Q_LIVE_LONG; qinit.live.objects = ERTS_THR_Q_LIVE_SHORT; qinit.arg = (void *) tse; qinit.notify = async_wakeup; #if ERTS_USE_ASYNC_READY_Q qinit.auto_finalize_dequeue = 0; #endif erts_thr_q_initialize(&aq->thr_q, &qinit); /* Inform main thread that we are done initializing... */ erts_mtx_lock(&async->init.data.mtx); async->init.data.no_initialized++; erts_cnd_signal(&async->init.data.cnd); erts_mtx_unlock(&async->init.data.mtx); return tse; }
static void cleanup_tse(void) { erts_tse_t *tse = erts_tse_fetch(); if (tse) { if (tse->udata) tse_return(tse, 1); else erts_tse_return(tse); } }
static erts_aint32_t thr_progress_block(ErtsThrPrgrData *tpd, int wait) { erts_tse_t *event = NULL; /* Remove erroneous warning... sigh... */ erts_aint32_t lflgs, bc; if (tpd->is_blocking++) return (erts_aint32_t) 0; while (1) { lflgs = erts_atomic32_read_bor_nob(&intrnl->misc.data.lflgs, ERTS_THR_PRGR_LFLG_BLOCK); if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK) block_thread(tpd); else break; } #if ERTS_THR_PRGR_PRINT_BLOCKERS erts_fprintf(stderr, "block(%d)\n", tpd->id); #endif ASSERT(ERTS_AINT_NULL == erts_atomic_read_nob(&intrnl->misc.data.blocker_event)); if (wait) { event = erts_tse_fetch(); erts_tse_reset(event); erts_atomic_set_nob(&intrnl->misc.data.blocker_event, (erts_aint_t) event); } if (tpd->is_managed) erts_atomic32_dec_nob(&intrnl->misc.data.block_count); bc = erts_atomic32_read_band_mb(&intrnl->misc.data.block_count, ~ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING); bc &= ~ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING; if (wait) { while (bc != 0) { erts_tse_wait(event); erts_tse_reset(event); bc = erts_atomic32_read_acqb(&intrnl->misc.data.block_count); } } return bc; }