static void
workers_wait (void)
{
State old_state, new_state;
++stat_workers_num_waited;
do {
old_state = new_state = workers_state;
/*
* Only the last worker thread awake can set the done
* posted flag, and since we're awake and haven't set
* it yet, it cannot be set.
*/
g_assert (!old_state.data.done_posted);
++new_state.data.num_waiting;
/*
* This is the only place where we use
* workers_gc_in_progress in the worker threads.
*/
if (new_state.data.num_waiting == workers_num && !old_state.data.gc_in_progress)
new_state.data.done_posted = 1;
} while (!set_state (old_state, new_state));
mono_memory_barrier ();
if (new_state.data.done_posted)
MONO_SEM_POST (&workers_done_sem);
MONO_SEM_WAIT (&workers_waiting_sem);
}
static void _wapi_thread_suspend (struct _WapiHandle_thread *thread)
{
g_assert (pthread_equal (thread->id, pthread_self ()));
while (MONO_SEM_WAIT (&thread->suspend_sem) != 0 &&
errno == EINTR);
}
void
mono_thread_info_self_suspend (void)
{
gboolean ret;
MonoThreadInfo *info = mono_thread_info_current ();
if (!info)
return;
EnterCriticalSection (&info->suspend_lock);
THREADS_DEBUG ("self suspend IN COUNT %d\n", info->suspend_count);
g_assert (info->suspend_count == 0);
++info->suspend_count;
info->thread_state |= STATE_SELF_SUSPENDED;
ret = mono_threads_get_runtime_callbacks ()->thread_state_init_from_sigctx (&info->suspend_state, NULL);
g_assert (ret);
LeaveCriticalSection (&info->suspend_lock);
while (MONO_SEM_WAIT (&info->resume_semaphore) != 0) {
/*if (EINTR != errno) ABORT("sem_wait failed"); */
}
g_assert (!info->async_target); /*FIXME this should happen normally for suspend. */
MONO_SEM_POST (&info->finish_resume_semaphore);
}
void
sgen_workers_signal_start_nursery_collection_and_wait (void)
{
State old_state, new_state;
do {
new_state = old_state = workers_state;
new_state.data.state = STATE_NURSERY_COLLECTION;
if (old_state.data.state == STATE_NOT_WORKING) {
assert_not_working (old_state);
} else {
assert_working (old_state, FALSE);
SGEN_ASSERT (0, !old_state.data.post_done, "We are not waiting for the workers");
new_state.data.post_done = 1;
}
} while (!set_state (old_state, new_state));
if (new_state.data.post_done)
MONO_SEM_WAIT (&workers_done_sem);
old_state = workers_state;
assert_nursery_collection (old_state, FALSE);
SGEN_ASSERT (0, !old_state.data.post_done, "We got the semaphore, so it must have been posted");
}
static void
suspend_signal_handler (int _dummy, siginfo_t *info, void *context)
{
MonoThreadInfo *current = mono_thread_info_current ();
gboolean ret;
if (current->syscall_break_signal) {
current->syscall_break_signal = FALSE;
return;
}
ret = mono_threads_get_runtime_callbacks ()->thread_state_init_from_sigctx (¤t->suspend_state, context);
g_assert (ret);
MONO_SEM_POST (¤t->suspend_semaphore);
while (MONO_SEM_WAIT (¤t->resume_semaphore) != 0) {
/*if (EINTR != errno) ABORT("sem_wait failed"); */
}
if (current->async_target) {
#if MONO_ARCH_HAS_MONO_CONTEXT
MonoContext tmp = current->suspend_state.ctx;
mono_threads_get_runtime_callbacks ()->setup_async_callback (&tmp, current->async_target, current->user_data);
current->async_target = current->user_data = NULL;
mono_monoctx_to_sigctx (&tmp, context);
#else
g_error ("The new interruption machinery requires a working mono-context");
#endif
}
MONO_SEM_POST (¤t->finish_resume_semaphore);
}
HANDLE
mono_threads_core_create_thread (LPTHREAD_START_ROUTINE start_routine, gpointer arg, guint32 stack_size, guint32 creation_flags, MonoNativeThreadId *out_tid)
{
pthread_attr_t attr;
int res;
pthread_t thread;
StartInfo start_info;
res = pthread_attr_init (&attr);
g_assert (!res);
if (stack_size == 0) {
#if HAVE_VALGRIND_MEMCHECK_H
if (RUNNING_ON_VALGRIND)
stack_size = 1 << 20;
else
stack_size = (SIZEOF_VOID_P / 4) * 1024 * 1024;
#else
stack_size = (SIZEOF_VOID_P / 4) * 1024 * 1024;
#endif
}
#ifdef PTHREAD_STACK_MIN
if (stack_size < PTHREAD_STACK_MIN)
stack_size = PTHREAD_STACK_MIN;
#endif
#ifdef HAVE_PTHREAD_ATTR_SETSTACKSIZE
res = pthread_attr_setstacksize (&attr, stack_size);
g_assert (!res);
#endif
memset (&start_info, 0, sizeof (StartInfo));
start_info.start_routine = (void *(*)(void *)) start_routine;
start_info.arg = arg;
start_info.flags = creation_flags;
MONO_SEM_INIT (&(start_info.registered), 0);
/* Actually start the thread */
res = mono_gc_pthread_create (&thread, &attr, inner_start_thread, &start_info);
if (res) {
MONO_SEM_DESTROY (&(start_info.registered));
return NULL;
}
MONO_TRY_BLOCKING;
/* Wait until the thread register itself in various places */
while (MONO_SEM_WAIT (&(start_info.registered)) != 0) {
/*if (EINTR != errno) ABORT("sem_wait failed"); */
}
MONO_FINISH_TRY_BLOCKING;
MONO_SEM_DESTROY (&(start_info.registered));
if (out_tid)
*out_tid = thread;
return start_info.handle;
}
gboolean
mono_threads_core_suspend (MonoThreadInfo *info)
{
/*FIXME, check return value*/
mono_threads_pthread_kill (info, mono_thread_get_abort_signal ());
while (MONO_SEM_WAIT (&info->begin_suspend_semaphore) != 0) {
/* g_assert (errno == EINTR); */
}
return info->suspend_can_continue;
}
gboolean
mono_threads_core_resume (MonoThreadInfo *info)
{
MONO_SEM_POST (&info->resume_semaphore);
while (MONO_SEM_WAIT (&info->finish_resume_semaphore) != 0) {
/* g_assert (errno == EINTR); */
}
return TRUE;
}
void
sgen_workers_join (void)
{
State old_state;
int i;
if (!collection_needs_workers ())
return;
for (;;) {
old_state = workers_state;
SGEN_ASSERT (0, old_state.data.state != STATE_NURSERY_COLLECTION, "Can't be in nursery collection when joining");
if (old_state.data.state == STATE_WORKING) {
State new_state = old_state;
SGEN_ASSERT (0, !old_state.data.post_done, "Why is post_done already set?");
new_state.data.post_done = 1;
if (!set_state (old_state, new_state))
continue;
MONO_SEM_WAIT (&workers_done_sem);
old_state = workers_state;
}
assert_not_working (old_state);
/*
* Checking whether there is still work left and, if not, going to sleep,
* are two separate actions that are not performed atomically by the
* workers. Therefore there's a race condition where work can be added
* after they've checked for work, and before they've gone to sleep.
*/
if (!workers_job_queue_num_entries && sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue))
break;
workers_signal_enqueue_work (workers_num, FALSE);
}
/* At this point all the workers have stopped. */
if (sgen_get_major_collector ()->reset_worker_data) {
for (i = 0; i < workers_num; ++i)
sgen_get_major_collector ()->reset_worker_data (workers_data [i].major_collector_data);
}
g_assert (workers_job_queue_num_entries == 0);
g_assert (sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue));
for (i = 0; i < workers_num; ++i)
g_assert (sgen_gray_object_queue_is_empty (&workers_data [i].private_gray_queue));
}
void
sgen_wait_for_suspend_ack (int count)
{
int i, result;
for (i = 0; i < count; ++i) {
while ((result = MONO_SEM_WAIT (suspend_ack_semaphore_ptr)) != 0) {
if (errno != EINTR) {
g_error ("MONO_SEM_WAIT FAILED with %d errno %d (%s)", result, errno, strerror (errno));
}
}
}
}
void
mono_sgen_wait_for_suspend_ack (int count)
{
int i, result;
for (i = 0; i < count; ++i) {
while ((result = MONO_SEM_WAIT (suspend_ack_semaphore_ptr)) != 0) {
if (errno != EINTR) {
g_error ("sem_wait ()");
}
}
}
}
static void*
inner_start_thread (void *arg)
{
StartInfo *start_info = (StartInfo *) arg;
void *t_arg = start_info->arg;
int res;
void *(*start_func)(void*) = start_info->start_routine;
guint32 flags = start_info->flags;
void *result;
HANDLE handle;
MonoThreadInfo *info;
/* Register the thread with the io-layer */
handle = wapi_create_thread_handle ();
if (!handle) {
res = MONO_SEM_POST (&(start_info->registered));
g_assert (!res);
return NULL;
}
start_info->handle = handle;
info = mono_thread_info_attach (&result);
MONO_PREPARE_BLOCKING
info->runtime_thread = TRUE;
info->handle = handle;
if (flags & CREATE_SUSPENDED) {
info->create_suspended = TRUE;
MONO_SEM_INIT (&info->create_suspended_sem, 0);
}
/* start_info is not valid after this */
res = MONO_SEM_POST (&(start_info->registered));
g_assert (!res);
start_info = NULL;
if (flags & CREATE_SUSPENDED) {
while (MONO_SEM_WAIT (&info->create_suspended_sem) != 0 &&
errno == EINTR);
MONO_SEM_DESTROY (&info->create_suspended_sem);
}
MONO_FINISH_BLOCKING
/* Run the actual main function of the thread */
result = start_func (t_arg);
mono_threads_core_exit (GPOINTER_TO_UINT (result));
g_assert_not_reached ();
}
static void
tp_poll_modify (gpointer event_data, int fd, int operation, int events, gboolean is_new)
{
tp_poll_data *data = event_data;
char msg [1];
MONO_SEM_WAIT (&data->new_sem);
INIT_POLLFD (&data->newpfd, GPOINTER_TO_INT (fd), events);
*msg = (char) operation;
#ifndef HOST_WIN32
if (write (data->pipe [1], msg, 1));
#else
send ((SOCKET) data->pipe [1], msg, 1, 0);
#endif
}
static gboolean
mono_thread_info_resume_internal (MonoThreadInfo *info)
{
gboolean result;
if (mono_thread_info_suspend_state (info) == STATE_SELF_SUSPENDED) {
MONO_SEM_POST (&info->resume_semaphore);
while (MONO_SEM_WAIT (&info->finish_resume_semaphore) != 0) {
/* g_assert (errno == EINTR); */
}
result = TRUE;
} else {
result = mono_threads_core_resume (info);
}
info->thread_state &= ~SUSPEND_STATE_MASK;
return result;
}
static void
workers_wait (void)
{
State old_state, new_state;
gboolean post_done;
++stat_workers_num_waited;
do {
new_state = old_state = workers_state;
assert_working_or_nursery_collection (old_state);
--new_state.data.num_awake;
post_done = FALSE;
if (!new_state.data.num_awake && !new_state.data.num_posted) {
/* We are the last thread to go to sleep. */
if (old_state.data.state == STATE_WORKING)
new_state.data.state = STATE_NOT_WORKING;
new_state.data.post_done = 0;
if (old_state.data.post_done)
post_done = TRUE;
}
} while (!set_state (old_state, new_state));
if (post_done)
MONO_SEM_POST (&workers_done_sem);
MONO_SEM_WAIT (&workers_waiting_sem);
do {
new_state = old_state = workers_state;
SGEN_ASSERT (0, old_state.data.num_posted > 0, "How can we be awake without the semaphore having been posted?");
SGEN_ASSERT (0, old_state.data.num_awake < workers_num, "There are too many worker threads awake");
--new_state.data.num_posted;
++new_state.data.num_awake;
assert_working_or_nursery_collection (new_state);
} while (!set_state (old_state, new_state));
}
static void*
ms_sweep_thread_func (void *dummy)
{
g_assert (concurrent_sweep);
for (;;) {
int result;
while ((result = MONO_SEM_WAIT (&ms_sweep_cmd_semaphore)) != 0) {
if (errno != EINTR)
g_error ("MONO_SEM_WAIT");
}
ms_sweep ();
ms_signal_sweep_done ();
}
return NULL;
}
void
sgen_workers_join (void)
{
int i;
if (!sgen_collection_is_parallel ())
return;
g_assert (sgen_gray_object_queue_is_empty (&workers_gc_thread_data.private_gray_queue));
g_assert (sgen_gray_object_queue_is_empty (&workers_distribute_gray_queue));
g_assert (workers_gc_in_progress);
workers_gc_in_progress = FALSE;
if (workers_num_waiting == workers_num) {
/*
* All the workers might have shut down at this point
* and posted the done semaphore but we don't know it
* yet. It's not a big deal to wake them up again -
* they'll just do one iteration of their loop trying to
* find something to do and then go back to waiting
* again.
*/
workers_wake_up_all ();
}
MONO_SEM_WAIT (&workers_done_sem);
workers_marking = FALSE;
if (sgen_get_major_collector ()->reset_worker_data) {
for (i = 0; i < workers_num; ++i)
sgen_get_major_collector ()->reset_worker_data (workers_data [i].major_collector_data);
}
g_assert (workers_done_posted);
g_assert (!workers_gc_thread_data.stealable_stack_fill);
g_assert (sgen_gray_object_queue_is_empty (&workers_gc_thread_data.private_gray_queue));
for (i = 0; i < workers_num; ++i) {
g_assert (!workers_data [i].stealable_stack_fill);
g_assert (sgen_gray_object_queue_is_empty (&workers_data [i].private_gray_queue));
}
}
static void
workers_wait (void)
{
int num;
++stat_workers_num_waited;
do {
num = workers_num_waiting;
} while (InterlockedCompareExchange (&workers_num_waiting, num + 1, num) != num);
if (num + 1 == workers_num && !workers_gc_in_progress) {
/* Make sure the done semaphore is only posted once. */
int posted;
do {
posted = workers_done_posted;
if (posted)
break;
} while (InterlockedCompareExchange (&workers_done_posted, 1, 0) != 0);
if (!posted)
MONO_SEM_POST (&workers_done_sem);
}
MONO_SEM_WAIT (&workers_waiting_sem);
}
HANDLE
mono_threads_core_create_thread (LPTHREAD_START_ROUTINE start_routine, gpointer arg, guint32 stack_size, guint32 creation_flags, MonoNativeThreadId *out_tid)
{
ThreadStartInfo *start_info;
HANDLE result;
DWORD thread_id;
start_info = g_malloc0 (sizeof (ThreadStartInfo));
if (!start_info)
return NULL;
MONO_SEM_INIT (&(start_info->registered), 0);
start_info->arg = arg;
start_info->start_routine = start_routine;
start_info->suspend = creation_flags & CREATE_SUSPENDED;
creation_flags &= ~CREATE_SUSPENDED;
if (start_info->suspend) {
start_info->suspend_event = CreateEvent (NULL, TRUE, FALSE, NULL);
if (!start_info->suspend_event)
return NULL;
}
result = CreateThread (NULL, stack_size, inner_start_thread, start_info, creation_flags, &thread_id);
if (result) {
while (MONO_SEM_WAIT (&(start_info->registered)) != 0) {
/*if (EINTR != errno) ABORT("sem_wait failed"); */
}
if (start_info->suspend) {
g_assert (SuspendThread (result) != (DWORD)-1);
SetEvent (start_info->suspend_event);
}
} else if (start_info->suspend) {
CloseHandle (start_info->suspend_event);
}
if (out_tid)
*out_tid = thread_id;
MONO_SEM_DESTROY (&(start_info->registered));
g_free (start_info);
return result;
}
int
mono_threads_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
{
ThreadStartInfo *start_info;
int result;
start_info = g_malloc0 (sizeof (ThreadStartInfo));
if (!start_info)
return ENOMEM;
MONO_SEM_INIT (&(start_info->registered), 0);
start_info->arg = arg;
start_info->start_routine = start_routine;
result = mono_threads_get_callbacks ()->mono_gc_pthread_create (new_thread, attr, inner_start_thread, start_info);
if (result == 0) {
while (MONO_SEM_WAIT (&(start_info->registered)) != 0) {
/*if (EINTR != errno) ABORT("sem_wait failed"); */
}
}
MONO_SEM_DESTROY (&(start_info->registered));
g_free (start_info);
return result;
}
static void
ms_wait_for_sweep_done (void)
{
SGEN_TV_DECLARE (atv);
SGEN_TV_DECLARE (btv);
int result;
if (!concurrent_sweep)
return;
if (!ms_sweep_in_progress)
return;
SGEN_TV_GETTIME (atv);
while ((result = MONO_SEM_WAIT (&ms_sweep_done_semaphore)) != 0) {
if (errno != EINTR)
g_error ("MONO_SEM_WAIT");
}
SGEN_TV_GETTIME (btv);
stat_time_wait_for_sweep += SGEN_TV_ELAPSED_MS (atv, btv);
g_assert (ms_sweep_in_progress);
ms_sweep_in_progress = FALSE;
}
void
sgen_workers_join (void)
{
State old_state, new_state;
int i;
if (!collection_needs_workers ())
return;
do {
old_state = new_state = workers_state;
g_assert (old_state.data.gc_in_progress);
g_assert (!old_state.data.done_posted);
new_state.data.gc_in_progress = 0;
} while (!set_state (old_state, new_state));
if (new_state.data.num_waiting == workers_num) {
/*
* All the workers have shut down but haven't posted
* the done semaphore yet, or, if we come from below,
* haven't done all their work yet.
*
* It's not a big deal to wake them up again - they'll
* just do one iteration of their loop trying to find
* something to do and then go back to waiting again.
*/
reawaken:
workers_wake_up_all ();
}
MONO_SEM_WAIT (&workers_done_sem);
old_state = new_state = workers_state;
g_assert (old_state.data.num_waiting == workers_num);
g_assert (old_state.data.done_posted);
if (workers_job_queue_num_entries || !sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue)) {
/*
* There's a small race condition that we avoid here.
* It's possible that a worker thread runs out of
* things to do, so it goes to sleep. Right at that
* moment a new job is enqueued, but the thread is
* still registered as running. Now the threads are
* joined, and we wait for the semaphore. Only at
* this point does the worker go to sleep, and posts
* the semaphore, because workers_gc_in_progress is
* already FALSE. The job is still in the queue,
* though.
*
* Clear the done posted flag.
*/
new_state.data.done_posted = 0;
if (!set_state (old_state, new_state))
g_assert_not_reached ();
goto reawaken;
}
/* At this point all the workers have stopped. */
workers_marking = FALSE;
if (sgen_get_major_collector ()->reset_worker_data) {
for (i = 0; i < workers_num; ++i)
sgen_get_major_collector ()->reset_worker_data (workers_data [i].major_collector_data);
}
g_assert (workers_job_queue_num_entries == 0);
g_assert (sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue));
for (i = 0; i < workers_num; ++i) {
g_assert (!workers_data [i].stealable_stack_fill);
g_assert (sgen_gray_object_queue_is_empty (&workers_data [i].private_gray_queue));
}
}
static void
monitor_thread (gpointer unused)
{
ThreadPool *pools [2];
MonoInternalThread *thread;
int i;
guint32 ms;
gint8 num_waiting_iterations = 0;
gint16 history_size = 0, current = -1;
SamplesHistory *history = malloc (sizeof (SamplesHistory) * HISTORY_SIZE);
pools [0] = &async_tp;
pools [1] = &async_io_tp;
thread = mono_thread_internal_current ();
ves_icall_System_Threading_Thread_SetName_internal (thread, mono_string_new (mono_domain_get (), "Threadpool monitor"));
while (1) {
ms = SAMPLES_PERIOD;
i = 10; //number of spurious awakes we tolerate before doing a round of rebalancing.
do {
guint32 ts;
ts = mono_msec_ticks ();
if (SleepEx (ms, TRUE) == 0)
break;
ms -= (mono_msec_ticks () - ts);
if (mono_runtime_is_shutting_down ())
break;
if (THREAD_WANTS_A_BREAK (thread))
mono_thread_interruption_checkpoint ();
} while (ms > 0 && i--);
if (mono_runtime_is_shutting_down ())
break;
if (suspended)
continue;
/* threadpool is cleaning up */
if (async_tp.pool_status == 2 || async_io_tp.pool_status == 2)
break;
switch (monitor_state) {
case MONITOR_STATE_AWAKE:
num_waiting_iterations = 0;
break;
case MONITOR_STATE_FALLING_ASLEEP:
if (++num_waiting_iterations == NUM_WAITING_ITERATIONS) {
if (monitor_state == MONITOR_STATE_FALLING_ASLEEP && InterlockedCompareExchange (&monitor_state, MONITOR_STATE_SLEEPING, MONITOR_STATE_FALLING_ASLEEP) == MONITOR_STATE_FALLING_ASLEEP) {
MONO_SEM_WAIT (&monitor_sem);
num_waiting_iterations = 0;
current = -1;
history_size = 0;
}
}
break;
case MONITOR_STATE_SLEEPING:
g_assert_not_reached ();
}
for (i = 0; i < 2; i++) {
ThreadPool *tp;
tp = pools [i];
if (tp->is_io) {
if (!tp->waiting && mono_cq_count (tp->queue) > 0)
threadpool_start_thread (tp);
} else {
gint8 nthreads_diff = monitor_heuristic (¤t, &history_size, history, tp);
if (nthreads_diff == 1)
threadpool_start_thread (tp);
else if (nthreads_diff == -1)
threadpool_kill_thread (tp);
}
}
}
}
static void*
inner_start_thread (void *arg)
{
StartInfo *start_info = arg;
void *t_arg = start_info->arg;
int res;
void *(*start_func)(void*) = start_info->start_routine;
guint32 flags = start_info->flags;
void *result;
HANDLE handle;
MonoThreadInfo *info;
/* Register the thread with the io-layer */
handle = wapi_create_thread_handle ();
if (!handle) {
res = MONO_SEM_POST (&(start_info->registered));
g_assert (!res);
return NULL;
}
start_info->handle = handle;
info = mono_thread_info_attach (&result);
info->runtime_thread = TRUE;
info->handle = handle;
if (flags & CREATE_SUSPENDED) {
info->create_suspended = TRUE;
MONO_SEM_INIT (&info->create_suspended_sem, 0);
}
/* start_info is not valid after this */
res = MONO_SEM_POST (&(start_info->registered));
g_assert (!res);
start_info = NULL;
if (flags & CREATE_SUSPENDED) {
while (MONO_SEM_WAIT (&info->create_suspended_sem) != 0 &&
errno == EINTR);
MONO_SEM_DESTROY (&info->create_suspended_sem);
}
/* Run the actual main function of the thread */
result = start_func (t_arg);
/*
mono_thread_info_detach ();
*/
#if defined(__native_client__)
nacl_shutdown_gc_thread();
#endif
wapi_thread_handle_set_exited (handle, GPOINTER_TO_UINT (result));
/* This is needed by mono_threads_core_unregister () which is called later */
info->handle = NULL;
g_assert (mono_threads_get_callbacks ()->thread_exit);
mono_threads_get_callbacks ()->thread_exit (NULL);
g_assert_not_reached ();
return result;
}
