void
sgen_workers_start_all_workers (void)
{
State old_state, new_state;
int i;
gboolean result;
if (!collection_needs_workers ())
return;
if (sgen_get_major_collector ()->init_worker_thread)
sgen_get_major_collector ()->init_worker_thread (workers_gc_thread_major_collector_data);
old_state = new_state = workers_state;
assert_not_working (old_state);
g_assert (workers_job_queue_num_entries == 0);
workers_num_jobs_enqueued = 0;
workers_num_jobs_finished = 0;
if (workers_started) {
workers_signal_enqueue_work (workers_num, FALSE);
return;
}
new_state.data.state = STATE_WORKING;
new_state.data.num_awake = workers_num;
result = set_state (old_state, new_state);
SGEN_ASSERT (0, result, "Nobody else should have modified the state - workers have not been started yet");
for (i = 0; i < workers_num; ++i)
workers_start_worker (i);
workers_started = TRUE;
}
void
sgen_workers_start_all_workers (void)
{
int i;
if (!sgen_collection_is_parallel ())
return;
if (sgen_get_major_collector ()->init_worker_thread)
sgen_get_major_collector ()->init_worker_thread (workers_gc_thread_data.major_collector_data);
g_assert (!workers_gc_in_progress);
workers_gc_in_progress = TRUE;
workers_marking = FALSE;
workers_done_posted = 0;
if (workers_started) {
if (workers_num_waiting != workers_num)
g_error ("Expecting all %d sgen workers to be parked, but only %d are", workers_num, workers_num_waiting);
workers_wake_up_all ();
return;
}
for (i = 0; i < workers_num; ++i)
workers_start_worker (i);
workers_started = TRUE;
}
void
sgen_workers_reset_data (void)
{
if (sgen_get_major_collector ()->reset_worker_data)
sgen_get_major_collector ()->reset_worker_data (workers_gc_thread_data.major_collector_data);
}
void
sgen_workers_init_distribute_gray_queue (void)
{
if (!collection_needs_workers ())
return;
init_distribute_gray_queue (sgen_get_major_collector ()->is_concurrent || sgen_get_major_collector ()->is_parallel);
}
void
sgen_workers_join (void)
{
State old_state;
int i;
if (!collection_needs_workers ())
return;
for (;;) {
old_state = workers_state;
SGEN_ASSERT (0, old_state.data.state != STATE_NURSERY_COLLECTION, "Can't be in nursery collection when joining");
if (old_state.data.state == STATE_WORKING) {
State new_state = old_state;
SGEN_ASSERT (0, !old_state.data.post_done, "Why is post_done already set?");
new_state.data.post_done = 1;
if (!set_state (old_state, new_state))
continue;
MONO_SEM_WAIT (&workers_done_sem);
old_state = workers_state;
}
assert_not_working (old_state);
/*
* Checking whether there is still work left and, if not, going to sleep,
* are two separate actions that are not performed atomically by the
* workers. Therefore there's a race condition where work can be added
* after they've checked for work, and before they've gone to sleep.
*/
if (!workers_job_queue_num_entries && sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue))
break;
workers_signal_enqueue_work (workers_num, FALSE);
}
/* At this point all the workers have stopped. */
if (sgen_get_major_collector ()->reset_worker_data) {
for (i = 0; i < workers_num; ++i)
sgen_get_major_collector ()->reset_worker_data (workers_data [i].major_collector_data);
}
g_assert (workers_job_queue_num_entries == 0);
g_assert (sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue));
for (i = 0; i < workers_num; ++i)
g_assert (sgen_gray_object_queue_is_empty (&workers_data [i].private_gray_queue));
}
gboolean
sgen_is_worker_thread (MonoNativeThreadId thread)
{
int i;
if (sgen_get_major_collector ()->is_worker_thread && sgen_get_major_collector ()->is_worker_thread (thread))
return TRUE;
for (i = 0; i < workers_num; ++i) {
if (workers_data [i].thread == thread)
return TRUE;
}
return FALSE;
}
void
sgen_workers_start_all_workers (void)
{
State old_state, new_state;
int i;
if (!collection_needs_workers ())
return;
if (sgen_get_major_collector ()->init_worker_thread)
sgen_get_major_collector ()->init_worker_thread (workers_gc_thread_major_collector_data);
old_state = new_state = workers_state;
g_assert (!old_state.data.gc_in_progress);
new_state.data.gc_in_progress = TRUE;
workers_marking = FALSE;
g_assert (workers_job_queue_num_entries == 0);
workers_num_jobs_enqueued = 0;
workers_num_jobs_finished = 0;
if (workers_started) {
g_assert (old_state.data.done_posted);
if (old_state.data.num_waiting != workers_num) {
g_error ("Expecting all %d sgen workers to be parked, but only %d are",
workers_num, old_state.data.num_waiting);
}
/* Clear the done posted flag */
new_state.data.done_posted = 0;
if (!set_state (old_state, new_state))
g_assert_not_reached ();
workers_wake_up_all ();
return;
}
g_assert (!old_state.data.done_posted);
if (!set_state (old_state, new_state))
g_assert_not_reached ();
for (i = 0; i < workers_num; ++i)
workers_start_worker (i);
workers_started = TRUE;
}
void
sgen_workers_init (int num_workers)
{
int i;
void **workers_data_ptrs = alloca(num_workers * sizeof(void *));
if (!sgen_get_major_collector ()->is_concurrent) {
sgen_thread_pool_init (num_workers, thread_pool_init_func, NULL, NULL, NULL);
return;
}
//g_print ("initing %d workers\n", num_workers);
workers_num = num_workers;
workers_data = sgen_alloc_internal_dynamic (sizeof (WorkerData) * num_workers, INTERNAL_MEM_WORKER_DATA, TRUE);
memset (workers_data, 0, sizeof (WorkerData) * num_workers);
init_distribute_gray_queue ();
for (i = 0; i < workers_num; ++i)
workers_data_ptrs [i] = (void *) &workers_data [i];
sgen_thread_pool_init (num_workers, thread_pool_init_func, marker_idle_func, continue_idle_func, workers_data_ptrs);
mono_counters_register ("# workers finished", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_workers_num_finished);
}
static void
init_private_gray_queue (WorkerData *data)
{
sgen_gray_object_queue_init_with_alloc_prepare (&data->private_gray_queue,
sgen_get_major_collector ()->is_concurrent ? concurrent_enqueue_check : NULL,
workers_gray_queue_share_redirect, data);
}
void
sgen_workers_init_distribute_gray_queue (void)
{
SGEN_ASSERT (0, sgen_get_major_collector ()->is_concurrent,
"Why should we init the distribute gray queue if we don't need it?");
init_distribute_gray_queue ();
}
void
sgen_workers_init (int num_workers)
{
int i;
if (!sgen_get_major_collector ()->is_parallel)
return;
//g_print ("initing %d workers\n", num_workers);
workers_num = num_workers;
workers_data = sgen_alloc_internal_dynamic (sizeof (WorkerData) * num_workers, INTERNAL_MEM_WORKER_DATA, TRUE);
memset (workers_data, 0, sizeof (WorkerData) * num_workers);
MONO_SEM_INIT (&workers_waiting_sem, 0);
MONO_SEM_INIT (&workers_done_sem, 0);
sgen_gray_object_queue_init_with_alloc_prepare (&workers_distribute_gray_queue,
workers_gray_queue_share_redirect, &workers_gc_thread_data);
mono_mutex_init (&workers_gc_thread_data.stealable_stack_mutex, NULL);
workers_gc_thread_data.stealable_stack_fill = 0;
if (sgen_get_major_collector ()->alloc_worker_data)
workers_gc_thread_data.major_collector_data = sgen_get_major_collector ()->alloc_worker_data ();
for (i = 0; i < workers_num; ++i) {
/* private gray queue is inited by the thread itself */
mono_mutex_init (&workers_data [i].stealable_stack_mutex, NULL);
workers_data [i].stealable_stack_fill = 0;
if (sgen_get_major_collector ()->alloc_worker_data)
workers_data [i].major_collector_data = sgen_get_major_collector ()->alloc_worker_data ();
}
LOCK_INIT (workers_job_queue_mutex);
sgen_register_fixed_internal_mem_type (INTERNAL_MEM_JOB_QUEUE_ENTRY, sizeof (JobQueueEntry));
mono_counters_register ("Stolen from self lock", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_workers_stolen_from_self_lock);
mono_counters_register ("Stolen from self no lock", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_workers_stolen_from_self_no_lock);
mono_counters_register ("Stolen from others", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_workers_stolen_from_others);
mono_counters_register ("# workers waited", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_workers_num_waited);
}
void
sgen_memgov_major_collection_end (gboolean forced)
{
last_collection_los_memory_usage = los_memory_usage;
if (forced) {
sgen_get_major_collector ()->finish_sweeping ();
sgen_memgov_calculate_minor_collection_allowance ();
}
}
static mono_native_thread_return_t
workers_thread_func (void *data_untyped)
{
WorkerData *data = data_untyped;
mono_thread_info_register_small_id ();
if (sgen_get_major_collector ()->init_worker_thread)
sgen_get_major_collector ()->init_worker_thread (data->major_collector_data);
init_private_gray_queue (data);
for (;;) {
gboolean did_work = FALSE;
while (workers_dequeue_and_do_job (data)) {
did_work = TRUE;
/* FIXME: maybe distribute the gray queue here? */
}
if (workers_marking && (!sgen_gray_object_queue_is_empty (&data->private_gray_queue) || workers_get_work (data))) {
ScanCopyContext ctx = { sgen_get_major_collector ()->major_ops.scan_object, NULL,
&data->private_gray_queue };
g_assert (!sgen_gray_object_queue_is_empty (&data->private_gray_queue));
while (!sgen_drain_gray_stack (32, ctx))
workers_gray_queue_share_redirect (&data->private_gray_queue);
g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue));
init_private_gray_queue (data);
did_work = TRUE;
}
if (!did_work)
workers_wait ();
}
/* dummy return to make compilers happy */
return NULL;
}
void
sgen_workers_join (void)
{
int i;
if (!sgen_collection_is_parallel ())
return;
g_assert (sgen_gray_object_queue_is_empty (&workers_gc_thread_data.private_gray_queue));
g_assert (sgen_gray_object_queue_is_empty (&workers_distribute_gray_queue));
g_assert (workers_gc_in_progress);
workers_gc_in_progress = FALSE;
if (workers_num_waiting == workers_num) {
/*
* All the workers might have shut down at this point
* and posted the done semaphore but we don't know it
* yet. It's not a big deal to wake them up again -
* they'll just do one iteration of their loop trying to
* find something to do and then go back to waiting
* again.
*/
workers_wake_up_all ();
}
MONO_SEM_WAIT (&workers_done_sem);
workers_marking = FALSE;
if (sgen_get_major_collector ()->reset_worker_data) {
for (i = 0; i < workers_num; ++i)
sgen_get_major_collector ()->reset_worker_data (workers_data [i].major_collector_data);
}
g_assert (workers_done_posted);
g_assert (!workers_gc_thread_data.stealable_stack_fill);
g_assert (sgen_gray_object_queue_is_empty (&workers_gc_thread_data.private_gray_queue));
for (i = 0; i < workers_num; ++i) {
g_assert (!workers_data [i].stealable_stack_fill);
g_assert (sgen_gray_object_queue_is_empty (&workers_data [i].private_gray_queue));
}
}
static void
init_distribute_gray_queue (void)
{
if (workers_distribute_gray_queue_inited) {
g_assert (sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue));
g_assert (workers_distribute_gray_queue.locked);
return;
}
sgen_section_gray_queue_init (&workers_distribute_gray_queue, TRUE,
sgen_get_major_collector ()->is_concurrent ? concurrent_enqueue_check : NULL);
workers_distribute_gray_queue_inited = TRUE;
}
static void
thread_pool_init_func (void *data_untyped)
{
WorkerData *data = data_untyped;
SgenMajorCollector *major = sgen_get_major_collector ();
sgen_client_thread_register_worker ();
if (!major->is_concurrent)
return;
init_private_gray_queue (data);
}
void
sgen_memgov_major_collection_start (void)
{
last_collection_old_num_major_sections = sgen_get_major_collector ()->get_num_major_sections ();
/*
* A domain could have been freed, resulting in
* los_memory_usage being less than last_collection_los_memory_usage.
*/
last_collection_los_memory_alloced = los_memory_usage - MIN (last_collection_los_memory_usage, los_memory_usage);
last_collection_old_los_memory_usage = los_memory_usage;
need_calculate_minor_collection_allowance = TRUE;
}
static mono_native_thread_return_t
workers_thread_func (void *data_untyped)
{
WorkerData *data = data_untyped;
SgenMajorCollector *major = sgen_get_major_collector ();
mono_thread_info_register_small_id ();
if (major->init_worker_thread)
major->init_worker_thread (data->major_collector_data);
init_private_gray_queue (data);
for (;;) {
gboolean did_work = FALSE;
SGEN_ASSERT (0, sgen_get_current_collection_generation () != GENERATION_NURSERY, "Why are we doing work while there's a nursery collection happening?");
while (workers_state.data.state == STATE_WORKING && workers_dequeue_and_do_job (data)) {
did_work = TRUE;
/* FIXME: maybe distribute the gray queue here? */
}
if (!did_work && (!sgen_gray_object_queue_is_empty (&data->private_gray_queue) || workers_get_work (data))) {
SgenObjectOperations *ops = sgen_concurrent_collection_in_progress ()
? &major->major_concurrent_ops
: &major->major_ops;
ScanCopyContext ctx = { ops->scan_object, NULL, &data->private_gray_queue };
g_assert (!sgen_gray_object_queue_is_empty (&data->private_gray_queue));
while (!sgen_drain_gray_stack (32, ctx)) {
if (workers_state.data.state == STATE_NURSERY_COLLECTION)
workers_wait ();
}
g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue));
init_private_gray_queue (data);
did_work = TRUE;
}
if (!did_work)
workers_wait ();
}
/* dummy return to make compilers happy */
return NULL;
}
static gboolean
workers_get_work (WorkerData *data)
{
SgenMajorCollector *major;
g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue));
/* If we're concurrent, steal from the workers distribute gray queue. */
major = sgen_get_major_collector ();
if (major->is_concurrent) {
GrayQueueSection *section = sgen_section_gray_queue_dequeue (&workers_distribute_gray_queue);
if (section) {
sgen_gray_object_enqueue_section (&data->private_gray_queue, section);
return TRUE;
}
}
/* Nobody to steal from */
g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue));
return FALSE;
}
static gboolean
workers_get_work (WorkerData *data)
{
SgenMajorCollector *major;
int i;
g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue));
/* Try to steal from our own stack. */
if (workers_steal (data, data, TRUE))
return TRUE;
/* From another worker. */
for (i = 0; i < workers_num; ++i) {
WorkerData *victim_data = &workers_data [i];
if (data == victim_data)
continue;
if (workers_steal (data, victim_data, TRUE))
return TRUE;
}
/*
* If we're concurrent or parallel, from the workers
* distribute gray queue.
*/
major = sgen_get_major_collector ();
if (major->is_concurrent || major->is_parallel) {
GrayQueueSection *section = sgen_section_gray_queue_dequeue (&workers_distribute_gray_queue);
if (section) {
sgen_gray_object_enqueue_section (&data->private_gray_queue, section);
return TRUE;
}
}
/* Nobody to steal from */
g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue));
return FALSE;
}
void
sgen_workers_init (int num_workers)
{
int i;
if (!sgen_get_major_collector ()->is_concurrent)
return;
//g_print ("initing %d workers\n", num_workers);
workers_num = num_workers;
workers_data = sgen_alloc_internal_dynamic (sizeof (WorkerData) * num_workers, INTERNAL_MEM_WORKER_DATA, TRUE);
memset (workers_data, 0, sizeof (WorkerData) * num_workers);
MONO_SEM_INIT (&workers_waiting_sem, 0);
MONO_SEM_INIT (&workers_done_sem, 0);
init_distribute_gray_queue (sgen_get_major_collector ()->is_concurrent);
if (sgen_get_major_collector ()->alloc_worker_data)
workers_gc_thread_major_collector_data = sgen_get_major_collector ()->alloc_worker_data ();
for (i = 0; i < workers_num; ++i) {
workers_data [i].index = i;
if (sgen_get_major_collector ()->alloc_worker_data)
workers_data [i].major_collector_data = sgen_get_major_collector ()->alloc_worker_data ();
}
LOCK_INIT (workers_job_queue_mutex);
sgen_register_fixed_internal_mem_type (INTERNAL_MEM_JOB_QUEUE_ENTRY, sizeof (JobQueueEntry));
mono_counters_register ("Stolen from self lock", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_workers_stolen_from_self_lock);
mono_counters_register ("Stolen from self no lock", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_workers_stolen_from_self_no_lock);
mono_counters_register ("Stolen from others", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_workers_stolen_from_others);
mono_counters_register ("# workers waited", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_workers_num_waited);
}
static void
init_private_gray_queue (WorkerData *data)
{
sgen_gray_object_queue_init (&data->private_gray_queue,
sgen_get_major_collector ()->is_concurrent ? concurrent_enqueue_check : NULL);
}
void
sgen_workers_join (void)
{
State old_state, new_state;
int i;
if (!collection_needs_workers ())
return;
do {
old_state = new_state = workers_state;
g_assert (old_state.data.gc_in_progress);
g_assert (!old_state.data.done_posted);
new_state.data.gc_in_progress = 0;
} while (!set_state (old_state, new_state));
if (new_state.data.num_waiting == workers_num) {
/*
* All the workers have shut down but haven't posted
* the done semaphore yet, or, if we come from below,
* haven't done all their work yet.
*
* It's not a big deal to wake them up again - they'll
* just do one iteration of their loop trying to find
* something to do and then go back to waiting again.
*/
reawaken:
workers_wake_up_all ();
}
MONO_SEM_WAIT (&workers_done_sem);
old_state = new_state = workers_state;
g_assert (old_state.data.num_waiting == workers_num);
g_assert (old_state.data.done_posted);
if (workers_job_queue_num_entries || !sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue)) {
/*
* There's a small race condition that we avoid here.
* It's possible that a worker thread runs out of
* things to do, so it goes to sleep. Right at that
* moment a new job is enqueued, but the thread is
* still registered as running. Now the threads are
* joined, and we wait for the semaphore. Only at
* this point does the worker go to sleep, and posts
* the semaphore, because workers_gc_in_progress is
* already FALSE. The job is still in the queue,
* though.
*
* Clear the done posted flag.
*/
new_state.data.done_posted = 0;
if (!set_state (old_state, new_state))
g_assert_not_reached ();
goto reawaken;
}
/* At this point all the workers have stopped. */
workers_marking = FALSE;
if (sgen_get_major_collector ()->reset_worker_data) {
for (i = 0; i < workers_num; ++i)
sgen_get_major_collector ()->reset_worker_data (workers_data [i].major_collector_data);
}
g_assert (workers_job_queue_num_entries == 0);
g_assert (sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue));
for (i = 0; i < workers_num; ++i) {
g_assert (!workers_data [i].stealable_stack_fill);
g_assert (sgen_gray_object_queue_is_empty (&workers_data [i].private_gray_queue));
}
}
static void
count_cards (long long *major_total, long long *major_marked, long long *los_total, long long *los_marked)
{
sgen_get_major_collector ()->count_cards (major_total, major_marked);
sgen_los_count_cards (los_total, los_marked);
}
