void* sgen_fragment_allocator_serial_range_alloc (SgenFragmentAllocator *allocator, size_t desired_size, size_t minimum_size, size_t *out_alloc_size) { SgenFragment *frag, **previous, *min_frag = NULL, **prev_min_frag = NULL; size_t current_minimum = minimum_size; #ifdef NALLOC_DEBUG InterlockedIncrement (&alloc_count); #endif previous = &allocator->alloc_head; for (frag = *previous; frag; frag = *previous) { size_t frag_size = frag->fragment_end - frag->fragment_next; HEAVY_STAT (++stat_alloc_range_iterations); if (desired_size <= frag_size) { void *p; *out_alloc_size = desired_size; p = serial_alloc_from_fragment (previous, frag, desired_size); #ifdef NALLOC_DEBUG add_alloc_record (p, desired_size, RANGE_ALLOC); #endif return p; } if (current_minimum <= frag_size) { min_frag = frag; prev_min_frag = previous; current_minimum = frag_size; } previous = &frag->next; } if (min_frag) { void *p; size_t frag_size = min_frag->fragment_end - min_frag->fragment_next; *out_alloc_size = frag_size; p = serial_alloc_from_fragment (prev_min_frag, min_frag, frag_size); #ifdef NALLOC_DEBUG add_alloc_record (p, frag_size, RANGE_ALLOC); #endif return p; } return NULL; }
void* sgen_fragment_allocator_par_alloc (SgenFragmentAllocator *allocator, size_t size) { SgenFragment *frag; #ifdef NALLOC_DEBUG InterlockedIncrement (&alloc_count); #endif restart: for (frag = (SgenFragment *)unmask (allocator->alloc_head); unmask (frag); frag = (SgenFragment *)unmask (frag->next)) { HEAVY_STAT (++stat_alloc_iterations); if (size <= (size_t)(frag->fragment_end - frag->fragment_next)) { void *p = par_alloc_from_fragment (allocator, frag, size); if (!p) { HEAVY_STAT (++stat_alloc_retries); goto restart; } #ifdef NALLOC_DEBUG add_alloc_record (p, size, FIXED_ALLOC); #endif return p; } } return NULL; }
void* sgen_fragment_allocator_serial_alloc (SgenFragmentAllocator *allocator, size_t size) { SgenFragment *frag; SgenFragment **previous; #ifdef NALLOC_DEBUG InterlockedIncrement (&alloc_count); #endif previous = &allocator->alloc_head; for (frag = *previous; frag; frag = *previous) { char *p = (char *)serial_alloc_from_fragment (previous, frag, size); HEAVY_STAT (++stat_alloc_iterations); if (p) { #ifdef NALLOC_DEBUG add_alloc_record (p, size, FIXED_ALLOC); #endif return p; } previous = &frag->next; } return NULL; }
void* mono_sgen_nursery_alloc (size_t size) { Fragment *frag; DEBUG (4, fprintf (gc_debug_file, "Searching nursery for size: %zd\n", size)); size = SGEN_ALIGN_UP (size); HEAVY_STAT (InterlockedIncrement (&stat_nursery_alloc_requests)); #ifdef NALLOC_DEBUG InterlockedIncrement (&alloc_count); #endif restart: for (frag = unmask (nursery_fragments); frag; frag = unmask (frag->next)) { HEAVY_STAT (InterlockedIncrement (&stat_alloc_iterations)); if (size <= (frag->fragment_end - frag->fragment_next)) { void *p = alloc_from_fragment (frag, size); if (!p) { HEAVY_STAT (InterlockedIncrement (&stat_alloc_retries)); goto restart; } #ifdef NALLOC_DEBUG add_alloc_record (p, size, FIXED_ALLOC); #endif return p; } } return NULL; }
mword mono_sgen_build_nursery_fragments (GCMemSection *nursery_section, void **start, int num_entries) { char *frag_start, *frag_end; size_t frag_size; int i; #ifdef NALLOC_DEBUG reset_alloc_records (); #endif while (unmask (nursery_fragments)) { Fragment *nf = unmask (nursery_fragments); Fragment *next = unmask (nf->next); nf->next_free = fragment_freelist; fragment_freelist = nf; nursery_fragments = next; } frag_start = nursery_start; fragment_total = 0; /* clear scan starts */ memset (nursery_section->scan_starts, 0, nursery_section->num_scan_start * sizeof (gpointer)); for (i = 0; i < num_entries; ++i) { frag_end = start [i]; /* remove the pin bit from pinned objects */ SGEN_UNPIN_OBJECT (frag_end); nursery_section->scan_starts [((char*)frag_end - (char*)nursery_section->data)/SGEN_SCAN_START_SIZE] = frag_end; frag_size = frag_end - frag_start; if (frag_size) add_nursery_frag (frag_size, frag_start, frag_end); frag_size = SGEN_ALIGN_UP (mono_sgen_safe_object_get_size ((MonoObject*)start [i])); #ifdef NALLOC_DEBUG add_alloc_record (start [i], frag_size, PINNING); #endif frag_start = (char*)start [i] + frag_size; } nursery_last_pinned_end = frag_start; frag_end = nursery_end; frag_size = frag_end - frag_start; if (frag_size) add_nursery_frag (frag_size, frag_start, frag_end); if (!unmask (nursery_fragments)) { DEBUG (1, fprintf (gc_debug_file, "Nursery fully pinned (%d)\n", num_entries)); for (i = 0; i < num_entries; ++i) { DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", start [i], mono_sgen_safe_name (start [i]), mono_sgen_safe_object_get_size (start [i]))); } } return fragment_total; }
void sgen_clear_allocator_fragments (SgenFragmentAllocator *allocator) { SgenFragment *frag; for (frag = (SgenFragment *)unmask (allocator->alloc_head); frag; frag = (SgenFragment *)unmask (frag->next)) { SGEN_LOG (4, "Clear nursery frag %p-%p", frag->fragment_next, frag->fragment_end); sgen_clear_range (frag->fragment_next, frag->fragment_end); #ifdef NALLOC_DEBUG add_alloc_record (frag->fragment_next, frag->fragment_end - frag->fragment_next, CLEAR_NURSERY_FRAGS); #endif } }
/* Clear all remaining nursery fragments */ void mono_sgen_clear_nursery_fragments (void) { Fragment *frag; if (mono_sgen_get_nursery_clear_policy () == CLEAR_AT_TLAB_CREATION) { mono_sgen_clear_current_nursery_fragment (); for (frag = unmask (nursery_fragments); frag; frag = unmask (frag->next)) { DEBUG (4, fprintf (gc_debug_file, "Clear nursery frag %p-%p\n", frag->fragment_next, frag->fragment_end)); memset (frag->fragment_next, 0, frag->fragment_end - frag->fragment_next); #ifdef NALLOC_DEBUG add_alloc_record (frag->fragment_next, frag->fragment_end - frag->fragment_next, CLEAR_NURSERY_FRAGS); #endif } } }
mword sgen_build_nursery_fragments (GCMemSection *nursery_section, SgenGrayQueue *unpin_queue) { char *frag_start, *frag_end; size_t frag_size; SgenFragment *frags_ranges; void **pin_start, **pin_entry, **pin_end; #ifdef NALLOC_DEBUG reset_alloc_records (); #endif /*The mutator fragments are done. We no longer need them. */ sgen_fragment_allocator_release (&mutator_allocator); frag_start = sgen_nursery_start; fragment_total = 0; /* The current nursery might give us a fragment list to exclude [start, next[*/ frags_ranges = sgen_minor_collector.build_fragments_get_exclude_head (); /* clear scan starts */ memset (nursery_section->scan_starts, 0, nursery_section->num_scan_start * sizeof (gpointer)); pin_start = pin_entry = sgen_pinning_get_entry (nursery_section->pin_queue_first_entry); pin_end = sgen_pinning_get_entry (nursery_section->pin_queue_last_entry); while (pin_entry < pin_end || frags_ranges) { char *addr0, *addr1; size_t size; addr0 = addr1 = sgen_nursery_end; if (pin_entry < pin_end) addr0 = (char *)*pin_entry; if (frags_ranges) addr1 = frags_ranges->fragment_start; if (addr0 < addr1) { if (unpin_queue) GRAY_OBJECT_ENQUEUE (unpin_queue, (GCObject*)addr0, sgen_obj_get_descriptor_safe ((GCObject*)addr0)); else SGEN_UNPIN_OBJECT (addr0); size = SGEN_ALIGN_UP (sgen_safe_object_get_size ((GCObject*)addr0)); CANARIFY_SIZE (size); sgen_set_nursery_scan_start (addr0); frag_end = addr0; ++pin_entry; } else { frag_end = addr1; size = frags_ranges->fragment_next - addr1; frags_ranges = frags_ranges->next_in_order; } frag_size = frag_end - frag_start; if (size == 0) continue; g_assert (frag_size >= 0); g_assert (size > 0); if (frag_size && size) add_nursery_frag (&mutator_allocator, frag_size, frag_start, frag_end); frag_size = size; #ifdef NALLOC_DEBUG add_alloc_record (*pin_entry, frag_size, PINNING); #endif frag_start = frag_end + frag_size; } nursery_last_pinned_end = frag_start; frag_end = sgen_nursery_end; frag_size = frag_end - frag_start; if (frag_size) add_nursery_frag (&mutator_allocator, frag_size, frag_start, frag_end); /* Now it's safe to release the fragments exclude list. */ sgen_minor_collector.build_fragments_release_exclude_head (); /* First we reorder the fragment list to be in ascending address order. This makes H/W prefetchers happier. */ fragment_list_reverse (&mutator_allocator); /*The collector might want to do something with the final nursery fragment list.*/ sgen_minor_collector.build_fragments_finish (&mutator_allocator); if (!unmask (mutator_allocator.alloc_head)) { SGEN_LOG (1, "Nursery fully pinned"); for (pin_entry = pin_start; pin_entry < pin_end; ++pin_entry) { GCObject *p = (GCObject *)*pin_entry; SGEN_LOG (3, "Bastard pinning obj %p (%s), size: %zd", p, sgen_client_vtable_get_name (SGEN_LOAD_VTABLE (p)), sgen_safe_object_get_size (p)); } } return fragment_total; }
void* sgen_fragment_allocator_par_range_alloc (SgenFragmentAllocator *allocator, size_t desired_size, size_t minimum_size, size_t *out_alloc_size) { SgenFragment *frag, *min_frag; size_t current_minimum; restart: min_frag = NULL; current_minimum = minimum_size; #ifdef NALLOC_DEBUG InterlockedIncrement (&alloc_count); #endif for (frag = (SgenFragment *)unmask (allocator->alloc_head); frag; frag = (SgenFragment *)unmask (frag->next)) { size_t frag_size = frag->fragment_end - frag->fragment_next; HEAVY_STAT (++stat_alloc_range_iterations); if (desired_size <= frag_size) { void *p; *out_alloc_size = desired_size; p = par_alloc_from_fragment (allocator, frag, desired_size); if (!p) { HEAVY_STAT (++stat_alloc_range_retries); goto restart; } #ifdef NALLOC_DEBUG add_alloc_record (p, desired_size, RANGE_ALLOC); #endif return p; } if (current_minimum <= frag_size) { min_frag = frag; current_minimum = frag_size; } } /* The second fragment_next read should be ordered in respect to the first code block */ mono_memory_barrier (); if (min_frag) { void *p; size_t frag_size; frag_size = min_frag->fragment_end - min_frag->fragment_next; if (frag_size < minimum_size) goto restart; *out_alloc_size = frag_size; mono_memory_barrier (); p = par_alloc_from_fragment (allocator, min_frag, frag_size); /*XXX restarting here is quite dubious given this is already second chance allocation. */ if (!p) { HEAVY_STAT (++stat_alloc_retries); goto restart; } #ifdef NALLOC_DEBUG add_alloc_record (p, frag_size, RANGE_ALLOC); #endif return p; } return NULL; }
static void* par_alloc_from_fragment (SgenFragmentAllocator *allocator, SgenFragment *frag, size_t size) { char *p = frag->fragment_next; char *end = p + size; if (end > frag->fragment_end) return NULL; /* p = frag->fragment_next must happen before */ mono_memory_barrier (); if (InterlockedCompareExchangePointer ((volatile gpointer*)&frag->fragment_next, end, p) != p) return NULL; if (frag->fragment_end - end < SGEN_MAX_NURSERY_WASTE) { SgenFragment *next, **prev_ptr; /* * Before we clean the remaining nursery, we must claim the remaining space * as it could end up been used by the range allocator since it can end up * allocating from this dying fragment as it doesn't respect SGEN_MAX_NURSERY_WASTE * when doing second chance allocation. */ if ((sgen_get_nursery_clear_policy () == CLEAR_AT_TLAB_CREATION || sgen_get_nursery_clear_policy () == CLEAR_AT_TLAB_CREATION_DEBUG) && claim_remaining_size (frag, end)) { sgen_clear_range (end, frag->fragment_end); HEAVY_STAT (stat_wasted_bytes_trailer += frag->fragment_end - end); #ifdef NALLOC_DEBUG add_alloc_record (end, frag->fragment_end - end, BLOCK_ZEROING); #endif } prev_ptr = find_previous_pointer_fragment (allocator, frag); /*Use Michaels linked list remove*/ /*prev_ptr will be null if the fragment was removed concurrently */ while (prev_ptr) { next = frag->next; /*already deleted*/ if (!get_mark (next)) { /*frag->next read must happen before the first CAS*/ mono_memory_write_barrier (); /*Fail if the next node is removed concurrently and its CAS wins */ if (InterlockedCompareExchangePointer ((volatile gpointer*)&frag->next, mask (next, 1), next) != next) { continue; } } /* The second CAS must happen after the first CAS or frag->next. */ mono_memory_write_barrier (); /* Fail if the previous node was deleted and its CAS wins */ if (InterlockedCompareExchangePointer ((volatile gpointer*)prev_ptr, unmask (next), frag) != frag) { prev_ptr = find_previous_pointer_fragment (allocator, frag); continue; } break; } } return p; }
mword sgen_build_nursery_fragments (GCMemSection *nursery_section, void **start, size_t num_entries, SgenGrayQueue *unpin_queue) { char *frag_start, *frag_end; size_t frag_size; size_t i = 0; SgenFragment *frags_ranges; #ifdef NALLOC_DEBUG reset_alloc_records (); #endif /*The mutator fragments are done. We no longer need them. */ sgen_fragment_allocator_release (&mutator_allocator); frag_start = sgen_nursery_start; fragment_total = 0; /* The current nursery might give us a fragment list to exclude [start, next[*/ frags_ranges = sgen_minor_collector.build_fragments_get_exclude_head (); /* clear scan starts */ memset (nursery_section->scan_starts, 0, nursery_section->num_scan_start * sizeof (gpointer)); while (i < num_entries || frags_ranges) { char *addr0, *addr1; size_t size; SgenFragment *last_frag = NULL; addr0 = addr1 = sgen_nursery_end; if (i < num_entries) addr0 = start [i]; if (frags_ranges) addr1 = frags_ranges->fragment_start; if (addr0 < addr1) { if (unpin_queue) GRAY_OBJECT_ENQUEUE (unpin_queue, addr0); else SGEN_UNPIN_OBJECT (addr0); sgen_set_nursery_scan_start (addr0); frag_end = addr0; size = SGEN_ALIGN_UP (sgen_safe_object_get_size ((MonoObject*)addr0)); ++i; } else { frag_end = addr1; size = frags_ranges->fragment_next - addr1; last_frag = frags_ranges; frags_ranges = frags_ranges->next_in_order; } frag_size = frag_end - frag_start; if (size == 0) continue; g_assert (frag_size >= 0); g_assert (size > 0); if (frag_size && size) add_nursery_frag (&mutator_allocator, frag_size, frag_start, frag_end); frag_size = size; #ifdef NALLOC_DEBUG add_alloc_record (start [i], frag_size, PINNING); #endif frag_start = frag_end + frag_size; } nursery_last_pinned_end = frag_start; frag_end = sgen_nursery_end; frag_size = frag_end - frag_start; if (frag_size) add_nursery_frag (&mutator_allocator, frag_size, frag_start, frag_end); /* Now it's safe to release the fragments exclude list. */ sgen_minor_collector.build_fragments_release_exclude_head (); /* First we reorder the fragment list to be in ascending address order. This makes H/W prefetchers happier. */ fragment_list_reverse (&mutator_allocator); /*The collector might want to do something with the final nursery fragment list.*/ sgen_minor_collector.build_fragments_finish (&mutator_allocator); if (!unmask (mutator_allocator.alloc_head)) { SGEN_LOG (1, "Nursery fully pinned (%zd)", num_entries); for (i = 0; i < num_entries; ++i) { SGEN_LOG (3, "Bastard pinning obj %p (%s), size: %zd", start [i], sgen_safe_name (start [i]), sgen_safe_object_get_size (start [i])); } } return fragment_total; }
void* mono_sgen_nursery_alloc_range (size_t desired_size, size_t minimum_size, int *out_alloc_size) { Fragment *frag, *min_frag; DEBUG (4, fprintf (gc_debug_file, "Searching for byte range desired size: %zd minimum size %zd\n", desired_size, minimum_size)); HEAVY_STAT (InterlockedIncrement (&stat_nursery_alloc_range_requests)); restart: min_frag = NULL; #ifdef NALLOC_DEBUG InterlockedIncrement (&alloc_count); #endif for (frag = unmask (nursery_fragments); frag; frag = unmask (frag->next)) { int frag_size = frag->fragment_end - frag->fragment_next; HEAVY_STAT (InterlockedIncrement (&stat_alloc_range_iterations)); if (desired_size <= frag_size) { void *p; *out_alloc_size = desired_size; p = alloc_from_fragment (frag, desired_size); if (!p) { HEAVY_STAT (InterlockedIncrement (&stat_alloc_range_retries)); goto restart; } #ifdef NALLOC_DEBUG add_alloc_record (p, desired_size, RANGE_ALLOC); #endif return p; } if (minimum_size <= frag_size) min_frag = frag; } /* The second fragment_next read should be ordered in respect to the first code block */ mono_memory_barrier (); if (min_frag) { void *p; int frag_size; frag_size = min_frag->fragment_end - min_frag->fragment_next; if (frag_size < minimum_size) goto restart; *out_alloc_size = frag_size; mono_memory_barrier (); p = alloc_from_fragment (min_frag, frag_size); /*XXX restarting here is quite dubious given this is already second chance allocation. */ if (!p) { HEAVY_STAT (InterlockedIncrement (&stat_alloc_retries)); goto restart; } #ifdef NALLOC_DEBUG add_alloc_record (p, frag_size, RANGE_ALLOC); #endif return p; } return NULL; }
static void* alloc_from_fragment (Fragment *frag, size_t size) { char *p = frag->fragment_next; char *end = p + size; if (end > frag->fragment_end) return NULL; /* p = frag->fragment_next must happen before */ mono_memory_barrier (); if (InterlockedCompareExchangePointer ((volatile gpointer*)&frag->fragment_next, end, p) != p) return NULL; if (frag->fragment_end - end < SGEN_MAX_NURSERY_WASTE) { Fragment *next, **prev_ptr; /* * Before we clean the remaining nursery, we must claim the remaining space * as it could end up been used by the range allocator since it can end up * allocating from this dying fragment as it doesn't respect SGEN_MAX_NURSERY_WASTE * when doing second chance allocation. */ if (mono_sgen_get_nursery_clear_policy () == CLEAR_AT_TLAB_CREATION && claim_remaining_size (frag, end)) { /* Clear the remaining space, pinning depends on this. FIXME move this to use phony arrays */ memset (end, 0, frag->fragment_end - end); HEAVY_STAT (InterlockedExchangeAdd (&stat_wasted_bytes_trailer, frag->fragment_end - end)); #ifdef NALLOC_DEBUG add_alloc_record (end, frag->fragment_end - end, BLOCK_ZEROING); #endif } prev_ptr = find_previous_pointer_fragment (frag); /*Use Michaels linked list remove*/ /*prev_ptr will be null is the fragment was removed concurrently */ while (prev_ptr) { next = frag->next; /*already deleted*/ if (!get_mark (next)) { /*frag->next read must happen before the first CAS*/ mono_memory_write_barrier (); /*Fail if the next done is removed concurrently and its CAS wins */ if (InterlockedCompareExchangePointer ((volatile gpointer*)&frag->next, mask (next, 1), next) != next) { continue; } } /* The second CAS must happen after the first CAS or frag->next. */ mono_memory_write_barrier (); /* Fail if the previous node was deleted and its CAS wins */ if (InterlockedCompareExchangePointer ((volatile gpointer*)prev_ptr, next, frag) != frag) { prev_ptr = find_previous_pointer_fragment (frag); continue; } /* No need to membar here since the worst that can happen is a CAS failure. */ do { frag->next_free = fragment_freelist; } while (InterlockedCompareExchangePointer ((volatile gpointer*)&fragment_freelist, frag, frag->next_free) != frag->next_free); break; } } return p; }