static gboolean
workers_get_work (WorkerData *data)
{
int i;
g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue));
/* Try to steal from our own stack. */
if (workers_steal (data, data, TRUE))
return TRUE;
/* Then from the GC thread's stack. */
if (workers_steal (data, &workers_gc_thread_data, TRUE))
return TRUE;
/* Finally, from another worker. */
for (i = 0; i < workers_num; ++i) {
WorkerData *victim_data = &workers_data [i];
if (data == victim_data)
continue;
if (workers_steal (data, victim_data, TRUE))
return TRUE;
}
/* Nobody to steal from */
g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue));
return FALSE;
}
static mono_native_thread_return_t
workers_thread_func (void *data_untyped)
{
WorkerData *data = data_untyped;
SgenMajorCollector *major = sgen_get_major_collector ();
mono_thread_info_register_small_id ();
if (major->init_worker_thread)
major->init_worker_thread (data->major_collector_data);
init_private_gray_queue (data);
for (;;) {
gboolean did_work = FALSE;
SGEN_ASSERT (0, sgen_get_current_collection_generation () != GENERATION_NURSERY, "Why are we doing work while there's a nursery collection happening?");
while (workers_state.data.state == STATE_WORKING && workers_dequeue_and_do_job (data)) {
did_work = TRUE;
/* FIXME: maybe distribute the gray queue here? */
}
if (!did_work && (!sgen_gray_object_queue_is_empty (&data->private_gray_queue) || workers_get_work (data))) {
SgenObjectOperations *ops = sgen_concurrent_collection_in_progress ()
? &major->major_concurrent_ops
: &major->major_ops;
ScanCopyContext ctx = { ops->scan_object, NULL, &data->private_gray_queue };
g_assert (!sgen_gray_object_queue_is_empty (&data->private_gray_queue));
while (!sgen_drain_gray_stack (32, ctx)) {
if (workers_state.data.state == STATE_NURSERY_COLLECTION)
workers_wait ();
}
g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue));
init_private_gray_queue (data);
did_work = TRUE;
}
if (!did_work)
workers_wait ();
}
/* dummy return to make compilers happy */
return NULL;
}
GrayQueueEntry
sgen_gray_object_dequeue (SgenGrayQueue *queue)
{
GrayQueueEntry entry;
HEAVY_STAT (stat_gray_queue_dequeue_slow_path ++);
if (sgen_gray_object_queue_is_empty (queue)) {
entry.obj = NULL;
return entry;
}
STATE_ASSERT (queue->first, GRAY_QUEUE_SECTION_STATE_ENQUEUED);
SGEN_ASSERT (9, queue->cursor >= GRAY_FIRST_CURSOR_POSITION (queue->first), "gray queue %p underflow", queue);
entry = *queue->cursor--;
#ifdef SGEN_HEAVY_BINARY_PROTOCOL
binary_protocol_gray_dequeue (queue, queue->cursor + 1, entry.obj);
#endif
if (G_UNLIKELY (queue->cursor < GRAY_FIRST_CURSOR_POSITION (queue->first))) {
GrayQueueSection *section = queue->first;
queue->first = section->next;
section->next = queue->free_list;
STATE_TRANSITION (section, GRAY_QUEUE_SECTION_STATE_ENQUEUED, GRAY_QUEUE_SECTION_STATE_FREE_LIST);
queue->free_list = section;
queue->cursor = queue->first ? queue->first->entries + queue->first->size - 1 : NULL;
}
return entry;
}
static mono_native_thread_return_t
workers_thread_func (void *data_untyped)
{
WorkerData *data = data_untyped;
SgenMajorCollector *major = sgen_get_major_collector ();
mono_thread_info_register_small_id ();
if (major->init_worker_thread)
major->init_worker_thread (data->major_collector_data);
init_private_gray_queue (data);
for (;;) {
gboolean did_work = FALSE;
while (workers_dequeue_and_do_job (data)) {
did_work = TRUE;
/* FIXME: maybe distribute the gray queue here? */
}
if (workers_marking && (!sgen_gray_object_queue_is_empty (&data->private_gray_queue) || workers_get_work (data))) {
SgenObjectOperations *ops = sgen_concurrent_collection_in_progress ()
? &major->major_concurrent_ops
: &major->major_ops;
ScanCopyContext ctx = { ops->scan_object, NULL, &data->private_gray_queue };
g_assert (!sgen_gray_object_queue_is_empty (&data->private_gray_queue));
while (!sgen_drain_gray_stack (32, ctx))
workers_gray_queue_share_redirect (&data->private_gray_queue);
g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue));
init_private_gray_queue (data);
did_work = TRUE;
}
if (!did_work)
workers_wait ();
}
/* dummy return to make compilers happy */
return NULL;
}
GrayQueueEntry
sgen_gray_object_dequeue (SgenGrayQueue *queue, gboolean is_parallel)
{
GrayQueueEntry entry;
HEAVY_STAT (stat_gray_queue_dequeue_slow_path ++);
if (sgen_gray_object_queue_is_empty (queue)) {
entry.obj = NULL;
return entry;
}
STATE_ASSERT (queue->first, GRAY_QUEUE_SECTION_STATE_ENQUEUED);
SGEN_ASSERT (9, queue->cursor >= GRAY_FIRST_CURSOR_POSITION (queue->first), "gray queue %p underflow", queue);
entry = *queue->cursor--;
#ifdef SGEN_HEAVY_BINARY_PROTOCOL
binary_protocol_gray_dequeue (queue, queue->cursor + 1, entry.obj);
#endif
if (G_UNLIKELY (queue->cursor < GRAY_FIRST_CURSOR_POSITION (queue->first))) {
GrayQueueSection *section;
gint32 old_num_sections = 0;
if (is_parallel)
old_num_sections = mono_atomic_dec_i32 (&queue->num_sections);
else
queue->num_sections--;
if (is_parallel && old_num_sections <= 0) {
mono_os_mutex_lock (&queue->steal_mutex);
}
section = queue->first;
queue->first = section->next;
if (queue->first) {
queue->first->prev = NULL;
} else {
queue->last = NULL;
SGEN_ASSERT (0, !old_num_sections, "Why do we have an inconsistent number of sections ?");
}
section->next = queue->free_list;
STATE_TRANSITION (section, GRAY_QUEUE_SECTION_STATE_ENQUEUED, GRAY_QUEUE_SECTION_STATE_FREE_LIST);
queue->free_list = section;
queue->cursor = queue->first ? queue->first->entries + queue->first->size - 1 : NULL;
if (is_parallel && old_num_sections <= 0) {
mono_os_mutex_unlock (&queue->steal_mutex);
}
}
return entry;
}
void
sgen_workers_join (void)
{
int i;
if (!sgen_collection_is_parallel ())
return;
g_assert (sgen_gray_object_queue_is_empty (&workers_gc_thread_data.private_gray_queue));
g_assert (sgen_gray_object_queue_is_empty (&workers_distribute_gray_queue));
g_assert (workers_gc_in_progress);
workers_gc_in_progress = FALSE;
if (workers_num_waiting == workers_num) {
/*
* All the workers might have shut down at this point
* and posted the done semaphore but we don't know it
* yet. It's not a big deal to wake them up again -
* they'll just do one iteration of their loop trying to
* find something to do and then go back to waiting
* again.
*/
workers_wake_up_all ();
}
MONO_SEM_WAIT (&workers_done_sem);
workers_marking = FALSE;
if (sgen_get_major_collector ()->reset_worker_data) {
for (i = 0; i < workers_num; ++i)
sgen_get_major_collector ()->reset_worker_data (workers_data [i].major_collector_data);
}
g_assert (workers_done_posted);
g_assert (!workers_gc_thread_data.stealable_stack_fill);
g_assert (sgen_gray_object_queue_is_empty (&workers_gc_thread_data.private_gray_queue));
for (i = 0; i < workers_num; ++i) {
g_assert (!workers_data [i].stealable_stack_fill);
g_assert (sgen_gray_object_queue_is_empty (&workers_data [i].private_gray_queue));
}
}
static gboolean
workers_get_work (WorkerData *data)
{
SgenMajorCollector *major;
g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue));
/* If we're concurrent, steal from the workers distribute gray queue. */
major = sgen_get_major_collector ();
if (major->is_concurrent) {
GrayQueueSection *section = sgen_section_gray_queue_dequeue (&workers_distribute_gray_queue);
if (section) {
sgen_gray_object_enqueue_section (&data->private_gray_queue, section);
return TRUE;
}
}
/* Nobody to steal from */
g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue));
return FALSE;
}
static mono_native_thread_return_t
workers_thread_func (void *data_untyped)
{
WorkerData *data = data_untyped;
mono_thread_info_register_small_id ();
if (sgen_get_major_collector ()->init_worker_thread)
sgen_get_major_collector ()->init_worker_thread (data->major_collector_data);
sgen_gray_object_queue_init_with_alloc_prepare (&data->private_gray_queue,
workers_gray_queue_share_redirect, data);
for (;;) {
gboolean did_work = FALSE;
while (workers_dequeue_and_do_job (data)) {
did_work = TRUE;
/* FIXME: maybe distribute the gray queue here? */
}
if (workers_marking && (!sgen_gray_object_queue_is_empty (&data->private_gray_queue) || workers_get_work (data))) {
g_assert (!sgen_gray_object_queue_is_empty (&data->private_gray_queue));
while (!sgen_drain_gray_stack (&data->private_gray_queue, 32))
workers_gray_queue_share_redirect (&data->private_gray_queue);
g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue));
sgen_gray_object_queue_init (&data->private_gray_queue);
did_work = TRUE;
}
if (!did_work)
workers_wait ();
}
/* dummy return to make compilers happy */
return NULL;
}
void
sgen_workers_join (void)
{
State old_state;
int i;
if (!collection_needs_workers ())
return;
for (;;) {
old_state = workers_state;
SGEN_ASSERT (0, old_state.data.state != STATE_NURSERY_COLLECTION, "Can't be in nursery collection when joining");
if (old_state.data.state == STATE_WORKING) {
State new_state = old_state;
SGEN_ASSERT (0, !old_state.data.post_done, "Why is post_done already set?");
new_state.data.post_done = 1;
if (!set_state (old_state, new_state))
continue;
MONO_SEM_WAIT (&workers_done_sem);
old_state = workers_state;
}
assert_not_working (old_state);
/*
* Checking whether there is still work left and, if not, going to sleep,
* are two separate actions that are not performed atomically by the
* workers. Therefore there's a race condition where work can be added
* after they've checked for work, and before they've gone to sleep.
*/
if (!workers_job_queue_num_entries && sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue))
break;
workers_signal_enqueue_work (workers_num, FALSE);
}
/* At this point all the workers have stopped. */
if (sgen_get_major_collector ()->reset_worker_data) {
for (i = 0; i < workers_num; ++i)
sgen_get_major_collector ()->reset_worker_data (workers_data [i].major_collector_data);
}
g_assert (workers_job_queue_num_entries == 0);
g_assert (sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue));
for (i = 0; i < workers_num; ++i)
g_assert (sgen_gray_object_queue_is_empty (&workers_data [i].private_gray_queue));
}
static gboolean
workers_get_work (WorkerData *data)
{
SgenMajorCollector *major;
int i;
g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue));
/* Try to steal from our own stack. */
if (workers_steal (data, data, TRUE))
return TRUE;
/* From another worker. */
for (i = 0; i < workers_num; ++i) {
WorkerData *victim_data = &workers_data [i];
if (data == victim_data)
continue;
if (workers_steal (data, victim_data, TRUE))
return TRUE;
}
/*
* If we're concurrent or parallel, from the workers
* distribute gray queue.
*/
major = sgen_get_major_collector ();
if (major->is_concurrent || major->is_parallel) {
GrayQueueSection *section = sgen_section_gray_queue_dequeue (&workers_distribute_gray_queue);
if (section) {
sgen_gray_object_enqueue_section (&data->private_gray_queue, section);
return TRUE;
}
}
/* Nobody to steal from */
g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue));
return FALSE;
}
void
sgen_gray_object_queue_init (SgenGrayQueue *queue, GrayQueueEnqueueCheckFunc enqueue_check_func)
{
g_assert (sgen_gray_object_queue_is_empty (queue));
queue->alloc_prepare_func = NULL;
queue->alloc_prepare_data = NULL;
#ifdef SGEN_CHECK_GRAY_OBJECT_ENQUEUE
queue->enqueue_check_func = enqueue_check_func;
#endif
/* Free the extra sections allocated during the last collection */
sgen_gray_object_queue_trim_free_list (queue);
}
static void
marker_idle_func (void *data_untyped)
{
WorkerData *data = data_untyped;
SGEN_ASSERT (0, continue_idle_func (), "Why are we called when we're not supposed to work?");
SGEN_ASSERT (0, sgen_concurrent_collection_in_progress (), "The worker should only mark in concurrent collections.");
if (workers_state == STATE_WORK_ENQUEUED) {
set_state (STATE_WORK_ENQUEUED, STATE_WORKING);
SGEN_ASSERT (0, workers_state != STATE_NOT_WORKING, "How did we get from WORK ENQUEUED to NOT WORKING?");
}
if (!sgen_gray_object_queue_is_empty (&data->private_gray_queue) || workers_get_work (data)) {
ScanCopyContext ctx = CONTEXT_FROM_OBJECT_OPERATIONS (idle_func_object_ops, &data->private_gray_queue);
SGEN_ASSERT (0, !sgen_gray_object_queue_is_empty (&data->private_gray_queue), "How is our gray queue empty if we just got work?");
sgen_drain_gray_stack (ctx);
} else {
worker_try_finish ();
}
}
void
sgen_gray_object_queue_dispose (SgenGrayQueue *queue)
{
SGEN_ASSERT (0, sgen_gray_object_queue_is_empty (queue), "Why are we disposing a gray queue that's not empty?");
/* Free the extra sections allocated during the last collection */
sgen_gray_object_queue_trim_free_list (queue);
SGEN_ASSERT (0, !last_gray_queue_free_list, "Are we disposing two gray queues after another?");
last_gray_queue_free_list = queue->free_list;
/* just to make sure */
memset (queue, 0, sizeof (SgenGrayQueue));
}
void
sgen_workers_join (void)
{
int i;
sgen_thread_pool_wait_for_all_jobs ();
sgen_thread_pool_idle_wait ();
SGEN_ASSERT (0, workers_state == STATE_NOT_WORKING, "Can only signal enqueue work when in no work state");
/* At this point all the workers have stopped. */
SGEN_ASSERT (0, sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue), "Why is there still work left to do?");
for (i = 0; i < workers_num; ++i)
SGEN_ASSERT (0, sgen_gray_object_queue_is_empty (&workers_data [i].private_gray_queue), "Why is there still work left to do?");
}
/*
* Can only be called if the workers are stopped.
* If we're stopped, there are also no pending jobs.
*/
gboolean
sgen_workers_have_idle_work (void)
{
int i;
SGEN_ASSERT (0, forced_stop && sgen_workers_all_done (), "Checking for idle work should only happen if the workers are stopped.");
if (!sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue))
return TRUE;
for (i = 0; i < workers_num; ++i) {
if (!sgen_gray_object_queue_is_empty (&workers_data [i].private_gray_queue))
return TRUE;
}
return FALSE;
}
static void
workers_gray_queue_share_redirect (SgenGrayQueue *queue)
{
GrayQueueSection *section;
WorkerData *data = queue->alloc_prepare_data;
if (data->stealable_stack_fill) {
/*
* There are still objects in the stealable stack, so
* wake up any workers that might be sleeping
*/
if (workers_state.data.gc_in_progress)
workers_wake_up_all ();
return;
}
/* The stealable stack is empty, so fill it. */
mono_mutex_lock (&data->stealable_stack_mutex);
while (data->stealable_stack_fill < STEALABLE_STACK_SIZE &&
(section = sgen_gray_object_dequeue_section (queue))) {
int num = MIN (section->size, STEALABLE_STACK_SIZE - data->stealable_stack_fill);
memcpy (data->stealable_stack + data->stealable_stack_fill,
section->entries + section->size - num,
sizeof (GrayQueueEntry) * num);
section->size -= num;
data->stealable_stack_fill += num;
if (section->size)
sgen_gray_object_enqueue_section (queue, section);
else
sgen_gray_object_free_queue_section (section);
}
if (sgen_gray_object_queue_is_empty (queue))
workers_steal (data, data, FALSE);
mono_mutex_unlock (&data->stealable_stack_mutex);
if (workers_state.data.gc_in_progress)
workers_wake_up_all ();
}
void
sgen_gray_object_queue_init (SgenGrayQueue *queue)
{
GrayQueueSection *section, *next;
int i;
g_assert (sgen_gray_object_queue_is_empty (queue));
SGEN_ASSERT (9, queue->balance == 0, "unbalanced queue on init %d", queue->balance);
/* Free the extra sections allocated during the last collection */
i = 0;
for (section = queue->free_list; section && i < GRAY_QUEUE_LENGTH_LIMIT - 1; section = section->next)
i ++;
if (!section)
return;
while (section->next) {
next = section->next;
section->next = next->next;
sgen_gray_object_free_queue_section (next);
}
}
char*
sgen_gray_object_dequeue (SgenGrayQueue *queue)
{
char *obj;
if (sgen_gray_object_queue_is_empty (queue))
return NULL;
SGEN_ASSERT (9, queue->first->end, "gray queue %p underflow, first %p, end %d", queue, queue->first, queue->first->end);
obj = queue->first->objects [--queue->first->end];
if (G_UNLIKELY (queue->first->end == 0)) {
GrayQueueSection *section = queue->first;
queue->first = section->next;
section->next = queue->free_list;
queue->free_list = section;
}
SGEN_LOG_DO (9, --queue->balance);
return obj;
}
void
sgen_workers_join (void)
{
State old_state, new_state;
int i;
if (!collection_needs_workers ())
return;
do {
old_state = new_state = workers_state;
g_assert (old_state.data.gc_in_progress);
g_assert (!old_state.data.done_posted);
new_state.data.gc_in_progress = 0;
} while (!set_state (old_state, new_state));
if (new_state.data.num_waiting == workers_num) {
/*
* All the workers have shut down but haven't posted
* the done semaphore yet, or, if we come from below,
* haven't done all their work yet.
*
* It's not a big deal to wake them up again - they'll
* just do one iteration of their loop trying to find
* something to do and then go back to waiting again.
*/
reawaken:
workers_wake_up_all ();
}
MONO_SEM_WAIT (&workers_done_sem);
old_state = new_state = workers_state;
g_assert (old_state.data.num_waiting == workers_num);
g_assert (old_state.data.done_posted);
if (workers_job_queue_num_entries || !sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue)) {
/*
* There's a small race condition that we avoid here.
* It's possible that a worker thread runs out of
* things to do, so it goes to sleep. Right at that
* moment a new job is enqueued, but the thread is
* still registered as running. Now the threads are
* joined, and we wait for the semaphore. Only at
* this point does the worker go to sleep, and posts
* the semaphore, because workers_gc_in_progress is
* already FALSE. The job is still in the queue,
* though.
*
* Clear the done posted flag.
*/
new_state.data.done_posted = 0;
if (!set_state (old_state, new_state))
g_assert_not_reached ();
goto reawaken;
}
/* At this point all the workers have stopped. */
workers_marking = FALSE;
if (sgen_get_major_collector ()->reset_worker_data) {
for (i = 0; i < workers_num; ++i)
sgen_get_major_collector ()->reset_worker_data (workers_data [i].major_collector_data);
}
g_assert (workers_job_queue_num_entries == 0);
g_assert (sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue));
for (i = 0; i < workers_num; ++i) {
g_assert (!workers_data [i].stealable_stack_fill);
g_assert (sgen_gray_object_queue_is_empty (&workers_data [i].private_gray_queue));
}
}
