void jl_safepoint_wait_gc(void)
{
#ifdef JULIA_ENABLE_THREADING
// The thread should have set this is already
assert(jl_get_ptls_states()->gc_state != 0);
// Use normal volatile load in the loop.
// Use a acquire load to make sure the GC result is visible on this thread.
while (jl_gc_running || jl_atomic_load_acquire(&jl_gc_running)) {
jl_cpu_pause(); // yield?
}
#else
assert(0 && "No one should wait for the GC on another thread");
#endif
}
int ti_threadgroup_join(ti_threadgroup_t *tg, int16_t ext_tid)
{
int i;
tg->thread_sense[tg->tid_map[ext_tid]]->sense
= !tg->thread_sense[tg->tid_map[ext_tid]]->sense;
if (tg->tid_map[ext_tid] == 0) {
for (i = 1; i < tg->num_threads; ++i) {
while (tg->thread_sense[i]->sense == tg->group_sense)
jl_cpu_pause();
}
}
return 0;
}
int ti_threadgroup_fork(ti_threadgroup_t *tg, int16_t ext_tid, void **bcast_val)
{
if (tg->tid_map[ext_tid] == 0) {
tg->envelope = bcast_val ? *bcast_val : NULL;
// synchronize `tg->envelope` and `tg->group_sense`
jl_atomic_store_release(&tg->group_sense, tg->thread_sense[0]->sense);
// if it's possible that threads are sleeping, signal them
if (tg->sleep_threshold) {
uv_mutex_lock(&tg->alarm_lock);
uv_cond_broadcast(&tg->alarm);
uv_mutex_unlock(&tg->alarm_lock);
}
}
else {
// spin up to threshold ns (count sheep), then sleep
uint64_t spin_ns;
uint64_t spin_start = 0;
// synchronize `tg->envelope` and `tg->group_sense`
while (jl_atomic_load_acquire(&tg->group_sense) !=
tg->thread_sense[tg->tid_map[ext_tid]]->sense) {
if (tg->sleep_threshold) {
if (!spin_start) {
// Lazily initialize spin_start since uv_hrtime is expensive
spin_start = uv_hrtime();
continue;
}
spin_ns = uv_hrtime() - spin_start;
// In case uv_hrtime is not monotonic, we'll sleep earlier
if (spin_ns >= tg->sleep_threshold) {
uv_mutex_lock(&tg->alarm_lock);
if (tg->group_sense !=
tg->thread_sense[tg->tid_map[ext_tid]]->sense) {
uv_cond_wait(&tg->alarm, &tg->alarm_lock);
}
uv_mutex_unlock(&tg->alarm_lock);
spin_start = 0;
continue;
}
}
jl_cpu_pause();
}
if (bcast_val)
*bcast_val = tg->envelope;
}
return 0;
}
JL_DLLEXPORT void (jl_cpu_pause)(void)
{
jl_cpu_pause();
}
JL_DLLEXPORT jl_task_t *jl_task_get_next(jl_value_t *getsticky)
{
jl_ptls_t ptls = jl_get_ptls_states();
// spin briefly before blocking when the workqueue is empty
size_t spin_count = 0;
jl_task_t *task;
while (1) {
jl_gc_safepoint();
task = get_next_task(getsticky);
if (task)
return task;
if (!_threadedregion) {
spin_count = 0;
if (ptls->tid == 0) {
if (jl_run_once(jl_global_event_loop()) == 0) {
task = get_next_task(getsticky);
if (task)
return task;
#ifdef _OS_WINDOWS_
Sleep(INFINITE);
#else
pause();
#endif
}
}
else {
int sleepnow = 0;
uv_mutex_lock(&sleep_lock);
if (!_threadedregion) {
sleepnow = 1;
}
else {
uv_mutex_unlock(&sleep_lock);
}
if (sleepnow) {
int8_t gc_state = jl_gc_safe_enter(ptls);
uv_cond_wait(&sleep_alarm, &sleep_lock);
uv_mutex_unlock(&sleep_lock);
jl_gc_safe_leave(ptls, gc_state);
}
}
}
else {
if (++spin_count > 1000 && jl_atomic_load(&jl_uv_n_waiters) == 0 && jl_mutex_trylock(&jl_uv_mutex)) {
task = get_next_task(getsticky);
if (task) {
JL_UV_UNLOCK();
return task;
}
uv_loop_t *loop = jl_global_event_loop();
loop->stop_flag = 0;
uv_run(loop, UV_RUN_ONCE);
JL_UV_UNLOCK();
}
else {
jl_cpu_pause();
}
}
}
}