int jl_safepoint_start_gc(void)
{
#ifdef JULIA_ENABLE_THREADING
// The thread should have set this already
assert(jl_get_ptls_states()->gc_state == JL_GC_STATE_WAITING);
jl_mutex_lock_nogc(&safepoint_lock);
// In case multiple threads enter the GC at the same time, only allow
// one of them to actually run the collection. We can't just let the
// master thread do the GC since it might be running unmanaged code
// and can take arbitrarily long time before hitting a safe point.
if (jl_atomic_compare_exchange(&jl_gc_running, 0, 1) != 0) {
jl_mutex_unlock_nogc(&safepoint_lock);
jl_safepoint_wait_gc();
return 0;
}
jl_safepoint_enable(1);
jl_safepoint_enable(2);
jl_mutex_unlock_nogc(&safepoint_lock);
return 1;
#else
// For single thread, GC should not call itself (in finalizers) before
// setting `jl_gc_running` to false so this should never happen.
assert(!jl_gc_running);
jl_gc_running = 1;
return 1;
#endif
}
/* multiq_insert()
*/
static inline int multiq_insert(jl_task_t *task, int16_t priority)
{
jl_ptls_t ptls = jl_get_ptls_states();
uint64_t rn;
task->prio = priority;
do {
rn = cong(heap_p, cong_unbias, &ptls->rngseed);
} while (!jl_mutex_trylock_nogc(&heaps[rn].lock));
if (heaps[rn].ntasks >= tasks_per_heap) {
jl_mutex_unlock_nogc(&heaps[rn].lock);
jl_error("multiq insertion failed, increase #tasks per heap");
return -1;
}
heaps[rn].tasks[heaps[rn].ntasks++] = task;
sift_up(&heaps[rn], heaps[rn].ntasks-1);
jl_mutex_unlock_nogc(&heaps[rn].lock);
int16_t prio = jl_atomic_load(&heaps[rn].prio);
if (task->prio < prio)
jl_atomic_compare_exchange(&heaps[rn].prio, prio, task->prio);
return 0;
}
void jl_safepoint_defer_sigint(void)
{
jl_mutex_lock_nogc(&safepoint_lock);
// Make sure the GC safepoint is disabled for SIGINT.
if (jl_signal_pending == 2) {
jl_safepoint_disable(1);
jl_signal_pending = 1;
}
jl_mutex_unlock_nogc(&safepoint_lock);
}
/* multiq_deletemin()
*/
static inline jl_task_t *multiq_deletemin(void)
{
jl_ptls_t ptls = jl_get_ptls_states();
uint64_t rn1 = 0, rn2;
int16_t i, prio1, prio2;
jl_task_t *task;
for (i = 0; i < heap_p; ++i) {
rn1 = cong(heap_p, cong_unbias, &ptls->rngseed);
rn2 = cong(heap_p, cong_unbias, &ptls->rngseed);
prio1 = jl_atomic_load(&heaps[rn1].prio);
prio2 = jl_atomic_load(&heaps[rn2].prio);
if (prio1 > prio2) {
prio1 = prio2;
rn1 = rn2;
}
else if (prio1 == prio2 && prio1 == INT16_MAX)
continue;
if (jl_mutex_trylock_nogc(&heaps[rn1].lock)) {
if (prio1 == heaps[rn1].prio)
break;
jl_mutex_unlock_nogc(&heaps[rn1].lock);
}
}
if (i == heap_p)
return NULL;
task = heaps[rn1].tasks[0];
heaps[rn1].tasks[0] = heaps[rn1].tasks[--heaps[rn1].ntasks];
heaps[rn1].tasks[heaps[rn1].ntasks] = NULL;
prio1 = INT16_MAX;
if (heaps[rn1].ntasks > 0) {
sift_down(&heaps[rn1], 0);
prio1 = heaps[rn1].tasks[0]->prio;
}
jl_atomic_store(&heaps[rn1].prio, prio1);
jl_mutex_unlock_nogc(&heaps[rn1].lock);
return task;
}
void jl_safepoint_enable_sigint(void)
{
jl_mutex_lock_nogc(&safepoint_lock);
// Make sure both safepoints are enabled exactly once for SIGINT.
switch (jl_signal_pending) {
default:
assert(0 && "Shouldn't happen.");
case 0:
// Enable SIGINT page
jl_safepoint_enable(0);
// fall through
case 1:
// SIGINT page is enabled, enable GC page
jl_safepoint_enable(1);
// fall through
case 2:
jl_signal_pending = 2;
}
jl_mutex_unlock_nogc(&safepoint_lock);
}
void jl_safepoint_end_gc(void)
{
assert(jl_gc_running);
#ifdef JULIA_ENABLE_THREADING
jl_mutex_lock_nogc(&safepoint_lock);
// Need to reset the page protection before resetting the flag since
// the thread will trigger a segfault immediately after returning from
// the signal handler.
jl_safepoint_disable(2);
jl_safepoint_disable(1);
jl_atomic_store_release(&jl_gc_running, 0);
# ifdef __APPLE__
// This wakes up other threads on mac.
jl_mach_gc_end();
# endif
jl_mutex_unlock_nogc(&safepoint_lock);
#else
jl_gc_running = 0;
#endif
}
int jl_safepoint_consume_sigint(void)
{
int has_signal = 0;
jl_mutex_lock_nogc(&safepoint_lock);
// Make sure both safepoints are disabled for SIGINT.
switch (jl_signal_pending) {
default:
assert(0 && "Shouldn't happen.");
case 2:
// Disable gc page
jl_safepoint_disable(1);
// fall through
case 1:
// GC page is disabled, disable SIGINT page
jl_safepoint_disable(0);
has_signal = 1;
// fall through
case 0:
jl_signal_pending = 0;
}
jl_mutex_unlock_nogc(&safepoint_lock);
return has_signal;
}
// Restore thread local state to saved state in error handler `eh`.
// This is executed in two circumstances:
// * We leave a try block through normal control flow
// * An exception causes a nonlocal jump to the catch block. In this case
// there's additional cleanup required, eg pushing the exception stack.
JL_DLLEXPORT void jl_eh_restore_state(jl_handler_t *eh)
{
jl_ptls_t ptls = jl_get_ptls_states();
#ifdef _OS_WINDOWS_
if (ptls->needs_resetstkoflw) {
_resetstkoflw();
ptls->needs_resetstkoflw = 0;
}
#endif
jl_task_t *current_task = ptls->current_task;
// `eh` may be not equal to `ptls->current_task->eh`. See `jl_pop_handler`
// This function should **NOT** have any safepoint before the ones at the
// end.
sig_atomic_t old_defer_signal = ptls->defer_signal;
int8_t old_gc_state = ptls->gc_state;
current_task->eh = eh->prev;
ptls->pgcstack = eh->gcstack;
#ifdef JULIA_ENABLE_THREADING
arraylist_t *locks = ¤t_task->locks;
if (locks->len > eh->locks_len) {
for (size_t i = locks->len;i > eh->locks_len;i--)
jl_mutex_unlock_nogc((jl_mutex_t*)locks->items[i - 1]);
locks->len = eh->locks_len;
}
#endif
ptls->world_age = eh->world_age;
ptls->defer_signal = eh->defer_signal;
ptls->gc_state = eh->gc_state;
ptls->finalizers_inhibited = eh->finalizers_inhibited;
if (old_gc_state && !eh->gc_state) {
jl_gc_safepoint_(ptls);
}
if (old_defer_signal && !eh->defer_signal) {
jl_sigint_safepoint(ptls);
}
}
