void
sgen_gray_object_enqueue (SgenGrayQueue *queue, GCObject *obj, SgenDescriptor desc, gboolean is_parallel)
{
GrayQueueEntry entry = SGEN_GRAY_QUEUE_ENTRY (obj, desc);
HEAVY_STAT (stat_gray_queue_enqueue_slow_path ++);
SGEN_ASSERT (9, obj, "enqueueing a null object");
//sgen_check_objref (obj);
#ifdef SGEN_CHECK_GRAY_OBJECT_ENQUEUE
if (queue->enqueue_check_func)
queue->enqueue_check_func (obj);
#endif
if (G_UNLIKELY (!queue->first || queue->cursor == GRAY_LAST_CURSOR_POSITION (queue->first))) {
if (queue->first) {
/*
* We don't actively update the section size with each push/pop. For the first
* section we determine the size from the cursor position. For the reset of the
* sections we need to have the size set.
*/
queue->first->size = SGEN_GRAY_QUEUE_SECTION_SIZE;
}
sgen_gray_object_alloc_queue_section (queue, is_parallel);
}
STATE_ASSERT (queue->first, GRAY_QUEUE_SECTION_STATE_ENQUEUED);
SGEN_ASSERT (9, queue->cursor <= GRAY_LAST_CURSOR_POSITION (queue->first), "gray queue %p overflow, first %p, cursor %p", queue, queue->first, queue->cursor);
*++queue->cursor = entry;
#ifdef SGEN_HEAVY_BINARY_PROTOCOL
binary_protocol_gray_enqueue (queue, queue->cursor, obj);
#endif
}
void
sgen_gray_object_enqueue (SgenGrayQueue *queue, GCObject *obj, SgenDescriptor desc)
{
GrayQueueEntry entry = SGEN_GRAY_QUEUE_ENTRY (obj, desc);
HEAVY_STAT (stat_gray_queue_enqueue_slow_path ++);
SGEN_ASSERT (9, obj, "enqueueing a null object");
//sgen_check_objref (obj);
#ifdef SGEN_CHECK_GRAY_OBJECT_ENQUEUE
if (queue->enqueue_check_func)
queue->enqueue_check_func (obj);
#endif
if (G_UNLIKELY (!queue->first || queue->cursor == GRAY_LAST_CURSOR_POSITION (queue->first))) {
if (queue->first) {
/* Set the current section size back to default, might have been changed by sgen_gray_object_dequeue_section */
queue->first->size = SGEN_GRAY_QUEUE_SECTION_SIZE;
}
sgen_gray_object_alloc_queue_section (queue);
}
STATE_ASSERT (queue->first, GRAY_QUEUE_SECTION_STATE_ENQUEUED);
SGEN_ASSERT (9, queue->cursor <= GRAY_LAST_CURSOR_POSITION (queue->first), "gray queue %p overflow, first %p, cursor %p", queue, queue->first, queue->cursor);
*++queue->cursor = entry;
#ifdef SGEN_HEAVY_BINARY_PROTOCOL
binary_protocol_gray_enqueue (queue, queue->cursor, obj);
#endif
}
/*
* We attempt to spread the objects in the gray queue across a number
* of sections. If the queue has more sections, then it's already spread,
* if it doesn't have enough sections, then we allocate as many as we
* can.
*/
void
sgen_gray_object_spread (SgenGrayQueue *queue, int num_sections)
{
GrayQueueSection *section_start, *section_end;
int total_entries = 0, num_entries_per_section;
int num_sections_final;
if (queue->num_sections >= num_sections)
return;
if (!queue->first)
return;
/* Compute number of elements in the gray queue */
queue->first->size = queue->cursor - queue->first->entries + 1;
total_entries = queue->first->size;
for (section_start = queue->first->next; section_start != NULL; section_start = section_start->next) {
SGEN_ASSERT (0, section_start->size == SGEN_GRAY_QUEUE_SECTION_SIZE, "We expect all section aside from the first one to be full");
total_entries += section_start->size;
}
/* Compute how many sections we should have and elements per section */
num_sections_final = (total_entries > num_sections) ? num_sections : total_entries;
num_entries_per_section = total_entries / num_sections_final;
/* Allocate all needed sections */
while (queue->num_sections < num_sections_final)
sgen_gray_object_alloc_queue_section (queue, TRUE);
/* Spread out the elements in the sections. By design, sections at the end are fuller. */
section_start = queue->first;
section_end = queue->last;
while (section_start != section_end) {
/* We move entries from end to start, until they meet */
while (section_start->size < num_entries_per_section) {
GrayQueueEntry entry;
if (section_end->size <= num_entries_per_section) {
section_end = section_end->prev;
if (section_end == section_start)
break;
}
if (section_end->size <= num_entries_per_section)
break;
section_end->size--;
entry = section_end->entries [section_end->size];
section_start->entries [section_start->size] = entry;
section_start->size++;
}
section_start = section_start->next;
}
queue->cursor = queue->first->entries + queue->first->size - 1;
queue->num_sections = num_sections_final;
}
static gboolean
workers_steal (WorkerData *data, WorkerData *victim_data, gboolean lock)
{
SgenGrayQueue *queue = &data->private_gray_queue;
int num, n;
g_assert (!queue->first);
if (!victim_data->stealable_stack_fill)
return FALSE;
if (lock && mono_mutex_trylock (&victim_data->stealable_stack_mutex))
return FALSE;
n = num = (victim_data->stealable_stack_fill + 1) / 2;
/* We're stealing num entries. */
while (n > 0) {
int m = MIN (SGEN_GRAY_QUEUE_SECTION_SIZE, n);
n -= m;
sgen_gray_object_alloc_queue_section (queue);
memcpy (queue->first->entries,
victim_data->stealable_stack + victim_data->stealable_stack_fill - num + n,
sizeof (GrayQueueEntry) * m);
queue->first->size = m;
/*
* DO NOT move outside this loop
* Doing so trigger "assert not reached" in sgen-scan-object.h : we use the queue->cursor
* to compute the size of the first section during section allocation (via alloc_prepare_func
* -> workers_gray_queue_share_redirect -> sgen_gray_object_dequeue_section) which will be then
* set to 0, because queue->cursor is still pointing to queue->first->entries [-1], thus
* losing objects in the gray queue.
*/
queue->cursor = queue->first->entries + queue->first->size - 1;
}
victim_data->stealable_stack_fill -= num;
if (lock)
mono_mutex_unlock (&victim_data->stealable_stack_mutex);
if (data == victim_data) {
if (lock)
stat_workers_stolen_from_self_lock += num;
else
stat_workers_stolen_from_self_no_lock += num;
} else {
stat_workers_stolen_from_others += num;
}
return num != 0;
}
void
sgen_gray_object_enqueue (SgenGrayQueue *queue, char *obj)
{
SGEN_ASSERT (9, obj, "enqueueing a null object");
//sgen_check_objref (obj);
if (G_UNLIKELY (!queue->first || queue->first->end == SGEN_GRAY_QUEUE_SECTION_SIZE))
sgen_gray_object_alloc_queue_section (queue);
SGEN_ASSERT (9, queue->first->end < SGEN_GRAY_QUEUE_SECTION_SIZE, "gray queue %p overflow, first %p, end %d", queue, queue->first, queue->first->end);
queue->first->objects [queue->first->end++] = obj;
SGEN_LOG_DO (9, ++queue->balance);
}
static gboolean
workers_steal (WorkerData *data, WorkerData *victim_data, gboolean lock)
{
SgenGrayQueue *queue = &data->private_gray_queue;
int num, n;
g_assert (!queue->first);
if (!victim_data->stealable_stack_fill)
return FALSE;
if (lock && mono_mutex_trylock (&victim_data->stealable_stack_mutex))
return FALSE;
n = num = (victim_data->stealable_stack_fill + 1) / 2;
/* We're stealing num entries. */
while (n > 0) {
int m = MIN (SGEN_GRAY_QUEUE_SECTION_SIZE, n);
n -= m;
sgen_gray_object_alloc_queue_section (queue);
memcpy (queue->first->objects,
victim_data->stealable_stack + victim_data->stealable_stack_fill - num + n,
sizeof (char*) * m);
queue->first->end = m;
}
victim_data->stealable_stack_fill -= num;
if (lock)
mono_mutex_unlock (&victim_data->stealable_stack_mutex);
if (data == victim_data) {
if (lock)
stat_workers_stolen_from_self_lock += num;
else
stat_workers_stolen_from_self_no_lock += num;
} else {
stat_workers_stolen_from_others += num;
}
return num != 0;
}
