void sgen_gray_object_enqueue (SgenGrayQueue *queue, GCObject *obj, SgenDescriptor desc, gboolean is_parallel) { GrayQueueEntry entry = SGEN_GRAY_QUEUE_ENTRY (obj, desc); HEAVY_STAT (stat_gray_queue_enqueue_slow_path ++); SGEN_ASSERT (9, obj, "enqueueing a null object"); //sgen_check_objref (obj); #ifdef SGEN_CHECK_GRAY_OBJECT_ENQUEUE if (queue->enqueue_check_func) queue->enqueue_check_func (obj); #endif if (G_UNLIKELY (!queue->first || queue->cursor == GRAY_LAST_CURSOR_POSITION (queue->first))) { if (queue->first) { /* * We don't actively update the section size with each push/pop. For the first * section we determine the size from the cursor position. For the reset of the * sections we need to have the size set. */ queue->first->size = SGEN_GRAY_QUEUE_SECTION_SIZE; } sgen_gray_object_alloc_queue_section (queue, is_parallel); } STATE_ASSERT (queue->first, GRAY_QUEUE_SECTION_STATE_ENQUEUED); SGEN_ASSERT (9, queue->cursor <= GRAY_LAST_CURSOR_POSITION (queue->first), "gray queue %p overflow, first %p, cursor %p", queue, queue->first, queue->cursor); *++queue->cursor = entry; #ifdef SGEN_HEAVY_BINARY_PROTOCOL binary_protocol_gray_enqueue (queue, queue->cursor, obj); #endif }
void sgen_gray_object_enqueue (SgenGrayQueue *queue, GCObject *obj, SgenDescriptor desc) { GrayQueueEntry entry = SGEN_GRAY_QUEUE_ENTRY (obj, desc); HEAVY_STAT (stat_gray_queue_enqueue_slow_path ++); SGEN_ASSERT (9, obj, "enqueueing a null object"); //sgen_check_objref (obj); #ifdef SGEN_CHECK_GRAY_OBJECT_ENQUEUE if (queue->enqueue_check_func) queue->enqueue_check_func (obj); #endif if (G_UNLIKELY (!queue->first || queue->cursor == GRAY_LAST_CURSOR_POSITION (queue->first))) { if (queue->first) { /* Set the current section size back to default, might have been changed by sgen_gray_object_dequeue_section */ queue->first->size = SGEN_GRAY_QUEUE_SECTION_SIZE; } sgen_gray_object_alloc_queue_section (queue); } STATE_ASSERT (queue->first, GRAY_QUEUE_SECTION_STATE_ENQUEUED); SGEN_ASSERT (9, queue->cursor <= GRAY_LAST_CURSOR_POSITION (queue->first), "gray queue %p overflow, first %p, cursor %p", queue, queue->first, queue->cursor); *++queue->cursor = entry; #ifdef SGEN_HEAVY_BINARY_PROTOCOL binary_protocol_gray_enqueue (queue, queue->cursor, obj); #endif }
/* * We attempt to spread the objects in the gray queue across a number * of sections. If the queue has more sections, then it's already spread, * if it doesn't have enough sections, then we allocate as many as we * can. */ void sgen_gray_object_spread (SgenGrayQueue *queue, int num_sections) { GrayQueueSection *section_start, *section_end; int total_entries = 0, num_entries_per_section; int num_sections_final; if (queue->num_sections >= num_sections) return; if (!queue->first) return; /* Compute number of elements in the gray queue */ queue->first->size = queue->cursor - queue->first->entries + 1; total_entries = queue->first->size; for (section_start = queue->first->next; section_start != NULL; section_start = section_start->next) { SGEN_ASSERT (0, section_start->size == SGEN_GRAY_QUEUE_SECTION_SIZE, "We expect all section aside from the first one to be full"); total_entries += section_start->size; } /* Compute how many sections we should have and elements per section */ num_sections_final = (total_entries > num_sections) ? num_sections : total_entries; num_entries_per_section = total_entries / num_sections_final; /* Allocate all needed sections */ while (queue->num_sections < num_sections_final) sgen_gray_object_alloc_queue_section (queue, TRUE); /* Spread out the elements in the sections. By design, sections at the end are fuller. */ section_start = queue->first; section_end = queue->last; while (section_start != section_end) { /* We move entries from end to start, until they meet */ while (section_start->size < num_entries_per_section) { GrayQueueEntry entry; if (section_end->size <= num_entries_per_section) { section_end = section_end->prev; if (section_end == section_start) break; } if (section_end->size <= num_entries_per_section) break; section_end->size--; entry = section_end->entries [section_end->size]; section_start->entries [section_start->size] = entry; section_start->size++; } section_start = section_start->next; } queue->cursor = queue->first->entries + queue->first->size - 1; queue->num_sections = num_sections_final; }
static gboolean workers_steal (WorkerData *data, WorkerData *victim_data, gboolean lock) { SgenGrayQueue *queue = &data->private_gray_queue; int num, n; g_assert (!queue->first); if (!victim_data->stealable_stack_fill) return FALSE; if (lock && mono_mutex_trylock (&victim_data->stealable_stack_mutex)) return FALSE; n = num = (victim_data->stealable_stack_fill + 1) / 2; /* We're stealing num entries. */ while (n > 0) { int m = MIN (SGEN_GRAY_QUEUE_SECTION_SIZE, n); n -= m; sgen_gray_object_alloc_queue_section (queue); memcpy (queue->first->entries, victim_data->stealable_stack + victim_data->stealable_stack_fill - num + n, sizeof (GrayQueueEntry) * m); queue->first->size = m; /* * DO NOT move outside this loop * Doing so trigger "assert not reached" in sgen-scan-object.h : we use the queue->cursor * to compute the size of the first section during section allocation (via alloc_prepare_func * -> workers_gray_queue_share_redirect -> sgen_gray_object_dequeue_section) which will be then * set to 0, because queue->cursor is still pointing to queue->first->entries [-1], thus * losing objects in the gray queue. */ queue->cursor = queue->first->entries + queue->first->size - 1; } victim_data->stealable_stack_fill -= num; if (lock) mono_mutex_unlock (&victim_data->stealable_stack_mutex); if (data == victim_data) { if (lock) stat_workers_stolen_from_self_lock += num; else stat_workers_stolen_from_self_no_lock += num; } else { stat_workers_stolen_from_others += num; } return num != 0; }
void sgen_gray_object_enqueue (SgenGrayQueue *queue, char *obj) { SGEN_ASSERT (9, obj, "enqueueing a null object"); //sgen_check_objref (obj); if (G_UNLIKELY (!queue->first || queue->first->end == SGEN_GRAY_QUEUE_SECTION_SIZE)) sgen_gray_object_alloc_queue_section (queue); SGEN_ASSERT (9, queue->first->end < SGEN_GRAY_QUEUE_SECTION_SIZE, "gray queue %p overflow, first %p, end %d", queue, queue->first, queue->first->end); queue->first->objects [queue->first->end++] = obj; SGEN_LOG_DO (9, ++queue->balance); }
static gboolean workers_steal (WorkerData *data, WorkerData *victim_data, gboolean lock) { SgenGrayQueue *queue = &data->private_gray_queue; int num, n; g_assert (!queue->first); if (!victim_data->stealable_stack_fill) return FALSE; if (lock && mono_mutex_trylock (&victim_data->stealable_stack_mutex)) return FALSE; n = num = (victim_data->stealable_stack_fill + 1) / 2; /* We're stealing num entries. */ while (n > 0) { int m = MIN (SGEN_GRAY_QUEUE_SECTION_SIZE, n); n -= m; sgen_gray_object_alloc_queue_section (queue); memcpy (queue->first->objects, victim_data->stealable_stack + victim_data->stealable_stack_fill - num + n, sizeof (char*) * m); queue->first->end = m; } victim_data->stealable_stack_fill -= num; if (lock) mono_mutex_unlock (&victim_data->stealable_stack_mutex); if (data == victim_data) { if (lock) stat_workers_stolen_from_self_lock += num; else stat_workers_stolen_from_self_no_lock += num; } else { stat_workers_stolen_from_others += num; } return num != 0; }