void sgen_workers_start_all_workers (void) { State old_state, new_state; int i; gboolean result; if (!collection_needs_workers ()) return; if (sgen_get_major_collector ()->init_worker_thread) sgen_get_major_collector ()->init_worker_thread (workers_gc_thread_major_collector_data); old_state = new_state = workers_state; assert_not_working (old_state); g_assert (workers_job_queue_num_entries == 0); workers_num_jobs_enqueued = 0; workers_num_jobs_finished = 0; if (workers_started) { workers_signal_enqueue_work (workers_num, FALSE); return; } new_state.data.state = STATE_WORKING; new_state.data.num_awake = workers_num; result = set_state (old_state, new_state); SGEN_ASSERT (0, result, "Nobody else should have modified the state - workers have not been started yet"); for (i = 0; i < workers_num; ++i) workers_start_worker (i); workers_started = TRUE; }
void sgen_workers_start_all_workers (void) { int i; if (!sgen_collection_is_parallel ()) return; if (sgen_get_major_collector ()->init_worker_thread) sgen_get_major_collector ()->init_worker_thread (workers_gc_thread_data.major_collector_data); g_assert (!workers_gc_in_progress); workers_gc_in_progress = TRUE; workers_marking = FALSE; workers_done_posted = 0; if (workers_started) { if (workers_num_waiting != workers_num) g_error ("Expecting all %d sgen workers to be parked, but only %d are", workers_num, workers_num_waiting); workers_wake_up_all (); return; } for (i = 0; i < workers_num; ++i) workers_start_worker (i); workers_started = TRUE; }
void sgen_workers_reset_data (void) { if (sgen_get_major_collector ()->reset_worker_data) sgen_get_major_collector ()->reset_worker_data (workers_gc_thread_data.major_collector_data); }
void sgen_workers_init_distribute_gray_queue (void) { if (!collection_needs_workers ()) return; init_distribute_gray_queue (sgen_get_major_collector ()->is_concurrent || sgen_get_major_collector ()->is_parallel); }
void sgen_workers_join (void) { State old_state; int i; if (!collection_needs_workers ()) return; for (;;) { old_state = workers_state; SGEN_ASSERT (0, old_state.data.state != STATE_NURSERY_COLLECTION, "Can't be in nursery collection when joining"); if (old_state.data.state == STATE_WORKING) { State new_state = old_state; SGEN_ASSERT (0, !old_state.data.post_done, "Why is post_done already set?"); new_state.data.post_done = 1; if (!set_state (old_state, new_state)) continue; MONO_SEM_WAIT (&workers_done_sem); old_state = workers_state; } assert_not_working (old_state); /* * Checking whether there is still work left and, if not, going to sleep, * are two separate actions that are not performed atomically by the * workers. Therefore there's a race condition where work can be added * after they've checked for work, and before they've gone to sleep. */ if (!workers_job_queue_num_entries && sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue)) break; workers_signal_enqueue_work (workers_num, FALSE); } /* At this point all the workers have stopped. */ if (sgen_get_major_collector ()->reset_worker_data) { for (i = 0; i < workers_num; ++i) sgen_get_major_collector ()->reset_worker_data (workers_data [i].major_collector_data); } g_assert (workers_job_queue_num_entries == 0); g_assert (sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue)); for (i = 0; i < workers_num; ++i) g_assert (sgen_gray_object_queue_is_empty (&workers_data [i].private_gray_queue)); }
gboolean sgen_is_worker_thread (MonoNativeThreadId thread) { int i; if (sgen_get_major_collector ()->is_worker_thread && sgen_get_major_collector ()->is_worker_thread (thread)) return TRUE; for (i = 0; i < workers_num; ++i) { if (workers_data [i].thread == thread) return TRUE; } return FALSE; }
void sgen_workers_start_all_workers (void) { State old_state, new_state; int i; if (!collection_needs_workers ()) return; if (sgen_get_major_collector ()->init_worker_thread) sgen_get_major_collector ()->init_worker_thread (workers_gc_thread_major_collector_data); old_state = new_state = workers_state; g_assert (!old_state.data.gc_in_progress); new_state.data.gc_in_progress = TRUE; workers_marking = FALSE; g_assert (workers_job_queue_num_entries == 0); workers_num_jobs_enqueued = 0; workers_num_jobs_finished = 0; if (workers_started) { g_assert (old_state.data.done_posted); if (old_state.data.num_waiting != workers_num) { g_error ("Expecting all %d sgen workers to be parked, but only %d are", workers_num, old_state.data.num_waiting); } /* Clear the done posted flag */ new_state.data.done_posted = 0; if (!set_state (old_state, new_state)) g_assert_not_reached (); workers_wake_up_all (); return; } g_assert (!old_state.data.done_posted); if (!set_state (old_state, new_state)) g_assert_not_reached (); for (i = 0; i < workers_num; ++i) workers_start_worker (i); workers_started = TRUE; }
void sgen_workers_init (int num_workers) { int i; void **workers_data_ptrs = alloca(num_workers * sizeof(void *)); if (!sgen_get_major_collector ()->is_concurrent) { sgen_thread_pool_init (num_workers, thread_pool_init_func, NULL, NULL, NULL); return; } //g_print ("initing %d workers\n", num_workers); workers_num = num_workers; workers_data = sgen_alloc_internal_dynamic (sizeof (WorkerData) * num_workers, INTERNAL_MEM_WORKER_DATA, TRUE); memset (workers_data, 0, sizeof (WorkerData) * num_workers); init_distribute_gray_queue (); for (i = 0; i < workers_num; ++i) workers_data_ptrs [i] = (void *) &workers_data [i]; sgen_thread_pool_init (num_workers, thread_pool_init_func, marker_idle_func, continue_idle_func, workers_data_ptrs); mono_counters_register ("# workers finished", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_workers_num_finished); }
static void init_private_gray_queue (WorkerData *data) { sgen_gray_object_queue_init_with_alloc_prepare (&data->private_gray_queue, sgen_get_major_collector ()->is_concurrent ? concurrent_enqueue_check : NULL, workers_gray_queue_share_redirect, data); }
void sgen_workers_init_distribute_gray_queue (void) { SGEN_ASSERT (0, sgen_get_major_collector ()->is_concurrent, "Why should we init the distribute gray queue if we don't need it?"); init_distribute_gray_queue (); }
void sgen_workers_init (int num_workers) { int i; if (!sgen_get_major_collector ()->is_parallel) return; //g_print ("initing %d workers\n", num_workers); workers_num = num_workers; workers_data = sgen_alloc_internal_dynamic (sizeof (WorkerData) * num_workers, INTERNAL_MEM_WORKER_DATA, TRUE); memset (workers_data, 0, sizeof (WorkerData) * num_workers); MONO_SEM_INIT (&workers_waiting_sem, 0); MONO_SEM_INIT (&workers_done_sem, 0); sgen_gray_object_queue_init_with_alloc_prepare (&workers_distribute_gray_queue, workers_gray_queue_share_redirect, &workers_gc_thread_data); mono_mutex_init (&workers_gc_thread_data.stealable_stack_mutex, NULL); workers_gc_thread_data.stealable_stack_fill = 0; if (sgen_get_major_collector ()->alloc_worker_data) workers_gc_thread_data.major_collector_data = sgen_get_major_collector ()->alloc_worker_data (); for (i = 0; i < workers_num; ++i) { /* private gray queue is inited by the thread itself */ mono_mutex_init (&workers_data [i].stealable_stack_mutex, NULL); workers_data [i].stealable_stack_fill = 0; if (sgen_get_major_collector ()->alloc_worker_data) workers_data [i].major_collector_data = sgen_get_major_collector ()->alloc_worker_data (); } LOCK_INIT (workers_job_queue_mutex); sgen_register_fixed_internal_mem_type (INTERNAL_MEM_JOB_QUEUE_ENTRY, sizeof (JobQueueEntry)); mono_counters_register ("Stolen from self lock", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_workers_stolen_from_self_lock); mono_counters_register ("Stolen from self no lock", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_workers_stolen_from_self_no_lock); mono_counters_register ("Stolen from others", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_workers_stolen_from_others); mono_counters_register ("# workers waited", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_workers_num_waited); }
void sgen_memgov_major_collection_end (gboolean forced) { last_collection_los_memory_usage = los_memory_usage; if (forced) { sgen_get_major_collector ()->finish_sweeping (); sgen_memgov_calculate_minor_collection_allowance (); } }
static mono_native_thread_return_t workers_thread_func (void *data_untyped) { WorkerData *data = data_untyped; mono_thread_info_register_small_id (); if (sgen_get_major_collector ()->init_worker_thread) sgen_get_major_collector ()->init_worker_thread (data->major_collector_data); init_private_gray_queue (data); for (;;) { gboolean did_work = FALSE; while (workers_dequeue_and_do_job (data)) { did_work = TRUE; /* FIXME: maybe distribute the gray queue here? */ } if (workers_marking && (!sgen_gray_object_queue_is_empty (&data->private_gray_queue) || workers_get_work (data))) { ScanCopyContext ctx = { sgen_get_major_collector ()->major_ops.scan_object, NULL, &data->private_gray_queue }; g_assert (!sgen_gray_object_queue_is_empty (&data->private_gray_queue)); while (!sgen_drain_gray_stack (32, ctx)) workers_gray_queue_share_redirect (&data->private_gray_queue); g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue)); init_private_gray_queue (data); did_work = TRUE; } if (!did_work) workers_wait (); } /* dummy return to make compilers happy */ return NULL; }
void sgen_workers_join (void) { int i; if (!sgen_collection_is_parallel ()) return; g_assert (sgen_gray_object_queue_is_empty (&workers_gc_thread_data.private_gray_queue)); g_assert (sgen_gray_object_queue_is_empty (&workers_distribute_gray_queue)); g_assert (workers_gc_in_progress); workers_gc_in_progress = FALSE; if (workers_num_waiting == workers_num) { /* * All the workers might have shut down at this point * and posted the done semaphore but we don't know it * yet. It's not a big deal to wake them up again - * they'll just do one iteration of their loop trying to * find something to do and then go back to waiting * again. */ workers_wake_up_all (); } MONO_SEM_WAIT (&workers_done_sem); workers_marking = FALSE; if (sgen_get_major_collector ()->reset_worker_data) { for (i = 0; i < workers_num; ++i) sgen_get_major_collector ()->reset_worker_data (workers_data [i].major_collector_data); } g_assert (workers_done_posted); g_assert (!workers_gc_thread_data.stealable_stack_fill); g_assert (sgen_gray_object_queue_is_empty (&workers_gc_thread_data.private_gray_queue)); for (i = 0; i < workers_num; ++i) { g_assert (!workers_data [i].stealable_stack_fill); g_assert (sgen_gray_object_queue_is_empty (&workers_data [i].private_gray_queue)); } }
static void init_distribute_gray_queue (void) { if (workers_distribute_gray_queue_inited) { g_assert (sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue)); g_assert (workers_distribute_gray_queue.locked); return; } sgen_section_gray_queue_init (&workers_distribute_gray_queue, TRUE, sgen_get_major_collector ()->is_concurrent ? concurrent_enqueue_check : NULL); workers_distribute_gray_queue_inited = TRUE; }
static void thread_pool_init_func (void *data_untyped) { WorkerData *data = data_untyped; SgenMajorCollector *major = sgen_get_major_collector (); sgen_client_thread_register_worker (); if (!major->is_concurrent) return; init_private_gray_queue (data); }
void sgen_memgov_major_collection_start (void) { last_collection_old_num_major_sections = sgen_get_major_collector ()->get_num_major_sections (); /* * A domain could have been freed, resulting in * los_memory_usage being less than last_collection_los_memory_usage. */ last_collection_los_memory_alloced = los_memory_usage - MIN (last_collection_los_memory_usage, los_memory_usage); last_collection_old_los_memory_usage = los_memory_usage; need_calculate_minor_collection_allowance = TRUE; }
static mono_native_thread_return_t workers_thread_func (void *data_untyped) { WorkerData *data = data_untyped; SgenMajorCollector *major = sgen_get_major_collector (); mono_thread_info_register_small_id (); if (major->init_worker_thread) major->init_worker_thread (data->major_collector_data); init_private_gray_queue (data); for (;;) { gboolean did_work = FALSE; SGEN_ASSERT (0, sgen_get_current_collection_generation () != GENERATION_NURSERY, "Why are we doing work while there's a nursery collection happening?"); while (workers_state.data.state == STATE_WORKING && workers_dequeue_and_do_job (data)) { did_work = TRUE; /* FIXME: maybe distribute the gray queue here? */ } if (!did_work && (!sgen_gray_object_queue_is_empty (&data->private_gray_queue) || workers_get_work (data))) { SgenObjectOperations *ops = sgen_concurrent_collection_in_progress () ? &major->major_concurrent_ops : &major->major_ops; ScanCopyContext ctx = { ops->scan_object, NULL, &data->private_gray_queue }; g_assert (!sgen_gray_object_queue_is_empty (&data->private_gray_queue)); while (!sgen_drain_gray_stack (32, ctx)) { if (workers_state.data.state == STATE_NURSERY_COLLECTION) workers_wait (); } g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue)); init_private_gray_queue (data); did_work = TRUE; } if (!did_work) workers_wait (); } /* dummy return to make compilers happy */ return NULL; }
static gboolean workers_get_work (WorkerData *data) { SgenMajorCollector *major; g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue)); /* If we're concurrent, steal from the workers distribute gray queue. */ major = sgen_get_major_collector (); if (major->is_concurrent) { GrayQueueSection *section = sgen_section_gray_queue_dequeue (&workers_distribute_gray_queue); if (section) { sgen_gray_object_enqueue_section (&data->private_gray_queue, section); return TRUE; } } /* Nobody to steal from */ g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue)); return FALSE; }
static gboolean workers_get_work (WorkerData *data) { SgenMajorCollector *major; int i; g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue)); /* Try to steal from our own stack. */ if (workers_steal (data, data, TRUE)) return TRUE; /* From another worker. */ for (i = 0; i < workers_num; ++i) { WorkerData *victim_data = &workers_data [i]; if (data == victim_data) continue; if (workers_steal (data, victim_data, TRUE)) return TRUE; } /* * If we're concurrent or parallel, from the workers * distribute gray queue. */ major = sgen_get_major_collector (); if (major->is_concurrent || major->is_parallel) { GrayQueueSection *section = sgen_section_gray_queue_dequeue (&workers_distribute_gray_queue); if (section) { sgen_gray_object_enqueue_section (&data->private_gray_queue, section); return TRUE; } } /* Nobody to steal from */ g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue)); return FALSE; }
void sgen_workers_init (int num_workers) { int i; if (!sgen_get_major_collector ()->is_concurrent) return; //g_print ("initing %d workers\n", num_workers); workers_num = num_workers; workers_data = sgen_alloc_internal_dynamic (sizeof (WorkerData) * num_workers, INTERNAL_MEM_WORKER_DATA, TRUE); memset (workers_data, 0, sizeof (WorkerData) * num_workers); MONO_SEM_INIT (&workers_waiting_sem, 0); MONO_SEM_INIT (&workers_done_sem, 0); init_distribute_gray_queue (sgen_get_major_collector ()->is_concurrent); if (sgen_get_major_collector ()->alloc_worker_data) workers_gc_thread_major_collector_data = sgen_get_major_collector ()->alloc_worker_data (); for (i = 0; i < workers_num; ++i) { workers_data [i].index = i; if (sgen_get_major_collector ()->alloc_worker_data) workers_data [i].major_collector_data = sgen_get_major_collector ()->alloc_worker_data (); } LOCK_INIT (workers_job_queue_mutex); sgen_register_fixed_internal_mem_type (INTERNAL_MEM_JOB_QUEUE_ENTRY, sizeof (JobQueueEntry)); mono_counters_register ("Stolen from self lock", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_workers_stolen_from_self_lock); mono_counters_register ("Stolen from self no lock", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_workers_stolen_from_self_no_lock); mono_counters_register ("Stolen from others", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_workers_stolen_from_others); mono_counters_register ("# workers waited", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_workers_num_waited); }
static void init_private_gray_queue (WorkerData *data) { sgen_gray_object_queue_init (&data->private_gray_queue, sgen_get_major_collector ()->is_concurrent ? concurrent_enqueue_check : NULL); }
void sgen_workers_join (void) { State old_state, new_state; int i; if (!collection_needs_workers ()) return; do { old_state = new_state = workers_state; g_assert (old_state.data.gc_in_progress); g_assert (!old_state.data.done_posted); new_state.data.gc_in_progress = 0; } while (!set_state (old_state, new_state)); if (new_state.data.num_waiting == workers_num) { /* * All the workers have shut down but haven't posted * the done semaphore yet, or, if we come from below, * haven't done all their work yet. * * It's not a big deal to wake them up again - they'll * just do one iteration of their loop trying to find * something to do and then go back to waiting again. */ reawaken: workers_wake_up_all (); } MONO_SEM_WAIT (&workers_done_sem); old_state = new_state = workers_state; g_assert (old_state.data.num_waiting == workers_num); g_assert (old_state.data.done_posted); if (workers_job_queue_num_entries || !sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue)) { /* * There's a small race condition that we avoid here. * It's possible that a worker thread runs out of * things to do, so it goes to sleep. Right at that * moment a new job is enqueued, but the thread is * still registered as running. Now the threads are * joined, and we wait for the semaphore. Only at * this point does the worker go to sleep, and posts * the semaphore, because workers_gc_in_progress is * already FALSE. The job is still in the queue, * though. * * Clear the done posted flag. */ new_state.data.done_posted = 0; if (!set_state (old_state, new_state)) g_assert_not_reached (); goto reawaken; } /* At this point all the workers have stopped. */ workers_marking = FALSE; if (sgen_get_major_collector ()->reset_worker_data) { for (i = 0; i < workers_num; ++i) sgen_get_major_collector ()->reset_worker_data (workers_data [i].major_collector_data); } g_assert (workers_job_queue_num_entries == 0); g_assert (sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue)); for (i = 0; i < workers_num; ++i) { g_assert (!workers_data [i].stealable_stack_fill); g_assert (sgen_gray_object_queue_is_empty (&workers_data [i].private_gray_queue)); } }
static void count_cards (long long *major_total, long long *major_marked, long long *los_total, long long *los_marked) { sgen_get_major_collector ()->count_cards (major_total, major_marked); sgen_los_count_cards (los_total, los_marked); }