static void
mono_sgen_ssb_cleanup_thread (SgenThreadInfo *p)
{
RememberedSet *rset;
if (p->remset) {
if (freed_thread_remsets) {
for (rset = p->remset; rset->next; rset = rset->next)
;
rset->next = freed_thread_remsets;
freed_thread_remsets = p->remset;
} else {
freed_thread_remsets = p->remset;
}
}
if (*p->store_remset_buffer_index_addr)
add_generic_store_remset_from_buffer (*p->store_remset_buffer_addr);
mono_sgen_free_internal (*p->store_remset_buffer_addr, INTERNAL_MEM_STORE_REMSET);
/*
* This is currently not strictly required, but we do it
* anyway in case we change thread unregistering:
* If the thread is removed from the thread list after
* unregistering (this is currently not the case), and a
* collection occurs, clear_remsets() would want to memset
* this buffer, which would either clobber memory or crash.
*/
*p->store_remset_buffer_addr = NULL;
}
/* LOCKING: requires that the GC lock is held */
static void
null_links_for_domain (MonoDomain *domain, int generation)
{
DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
DisappearingLink **disappearing_link_hash = hash->table;
int disappearing_link_hash_size = hash->size;
DisappearingLink *entry, *prev;
int i;
for (i = 0; i < disappearing_link_hash_size; ++i) {
prev = NULL;
for (entry = disappearing_link_hash [i]; entry; ) {
char *object = DISLINK_OBJECT (entry);
if (object && !((MonoObject*)object)->vtable) {
DisappearingLink *next = entry->next;
if (prev)
prev->next = next;
else
disappearing_link_hash [i] = next;
if (*(entry->link)) {
*(entry->link) = NULL;
g_warning ("Disappearing link %p not freed", entry->link);
} else {
mono_sgen_free_internal (entry, INTERNAL_MEM_DISLINK);
}
entry = next;
continue;
}
prev = entry;
entry = entry->next;
}
}
}
static gboolean
workers_dequeue_and_do_job (WorkerData *data)
{
JobQueueEntry *entry;
g_assert (collection_is_parallel ());
if (!workers_job_queue_num_entries)
return FALSE;
pthread_mutex_lock (&workers_job_queue_mutex);
entry = (JobQueueEntry*)workers_job_queue;
if (entry) {
workers_job_queue = entry->next;
--workers_job_queue_num_entries;
}
pthread_mutex_unlock (&workers_job_queue_mutex);
if (!entry)
return FALSE;
entry->func (data, entry->data);
mono_sgen_free_internal (entry, INTERNAL_MEM_JOB_QUEUE_ENTRY);
return TRUE;
}
/*
* Clear the info in the remembered sets: we're doing a major collection, so
* the per-thread ones are not needed and the global ones will be reconstructed
* during the copy.
*/
static void
mono_sgen_ssb_prepare_for_major_collection (void)
{
SgenThreadInfo *info;
RememberedSet *remset, *next;
mono_sgen_ssb_prepare_for_minor_collection ();
/* the global list */
for (remset = global_remset; remset; remset = next) {
remset->store_next = remset->data;
next = remset->next;
remset->next = NULL;
if (remset != global_remset) {
DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
mono_sgen_free_internal_dynamic (remset, remset_byte_size (remset), INTERNAL_MEM_REMSET);
}
}
/* the generic store ones */
while (generic_store_remsets) {
GenericStoreRememberedSet *gs_next = generic_store_remsets->next;
mono_sgen_free_internal (generic_store_remsets, INTERNAL_MEM_STORE_REMSET);
generic_store_remsets = gs_next;
}
/* the per-thread ones */
FOREACH_THREAD (info) {
for (remset = info->remset; remset; remset = next) {
remset->store_next = remset->data;
next = remset->next;
remset->next = NULL;
if (remset != info->remset) {
DEBUG (3, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
mono_sgen_free_internal_dynamic (remset, remset_byte_size (remset), INTERNAL_MEM_REMSET);
}
}
clear_thread_store_remset_buffer (info);
} END_FOREACH_THREAD
/* the freed thread ones */
while (freed_thread_remsets) {
next = freed_thread_remsets->next;
DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", freed_thread_remsets->data));
mono_sgen_free_internal_dynamic (freed_thread_remsets, remset_byte_size (freed_thread_remsets), INTERNAL_MEM_REMSET);
freed_thread_remsets = next;
}
}
/* LOCKING: assumes the GC lock is held */
static void
add_or_remove_disappearing_link (MonoObject *obj, void **link, int generation)
{
DisappearingLinkHashTable *hash_table = get_dislink_hash_table (generation);
DisappearingLink *entry, *prev;
unsigned int hash;
DisappearingLink **disappearing_link_hash = hash_table->table;
int disappearing_link_hash_size = hash_table->size;
if (hash_table->num_links >= disappearing_link_hash_size * 2) {
rehash_dislink (hash_table);
disappearing_link_hash = hash_table->table;
disappearing_link_hash_size = hash_table->size;
}
/* FIXME: add check that link is not in the heap */
hash = mono_aligned_addr_hash (link) % disappearing_link_hash_size;
entry = disappearing_link_hash [hash];
prev = NULL;
for (; entry; entry = entry->next) {
/* link already added */
if (link == entry->link) {
/* NULL obj means remove */
if (obj == NULL) {
if (prev)
prev->next = entry->next;
else
disappearing_link_hash [hash] = entry->next;
hash_table->num_links--;
DEBUG (5, fprintf (gc_debug_file, "Removed dislink %p (%d) from %s table\n", entry, hash_table->num_links, generation_name (generation)));
mono_sgen_free_internal (entry, INTERNAL_MEM_DISLINK);
}
return;
}
prev = entry;
}
if (obj == NULL)
return;
entry = mono_sgen_alloc_internal (INTERNAL_MEM_DISLINK);
entry->link = link;
entry->next = disappearing_link_hash [hash];
disappearing_link_hash [hash] = entry;
hash_table->num_links++;
DEBUG (5, fprintf (gc_debug_file, "Added dislink %p for object: %p (%s) at %p to %s table\n", entry, obj, obj->vtable->klass->name, link, generation_name (generation)));
}
/* LOCKING: requires that the GC lock is held */
static void
register_for_finalization (MonoObject *obj, void *user_data, int generation)
{
FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
FinalizeEntry **finalizable_hash;
mword finalizable_hash_size;
FinalizeEntry *entry, *prev;
unsigned int hash;
if (no_finalize)
return;
g_assert (user_data == NULL || user_data == mono_gc_run_finalize);
hash = mono_object_hash (obj);
rehash_fin_table_if_necessary (hash_table);
finalizable_hash = hash_table->table;
finalizable_hash_size = hash_table->size;
hash %= finalizable_hash_size;
prev = NULL;
for (entry = finalizable_hash [hash]; entry; entry = entry->next) {
if (entry->object == obj) {
if (!user_data) {
/* remove from the list */
if (prev)
prev->next = entry->next;
else
finalizable_hash [hash] = entry->next;
hash_table->num_registered--;
DEBUG (5, fprintf (gc_debug_file, "Removed finalizer %p for object: %p (%s) (%d)\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered));
mono_sgen_free_internal (entry, INTERNAL_MEM_FINALIZE_ENTRY);
}
return;
}
prev = entry;
}
if (!user_data) {
/* request to deregister, but already out of the list */
return;
}
entry = mono_sgen_alloc_internal (INTERNAL_MEM_FINALIZE_ENTRY);
entry->object = obj;
entry->next = finalizable_hash [hash];
finalizable_hash [hash] = entry;
hash_table->num_registered++;
DEBUG (5, fprintf (gc_debug_file, "Added finalizer %p for object: %p (%s) (%d) to %s table\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered, generation_name (generation)));
}
static void
mono_sgen_ssb_finish_scan_remsets (void *start_nursery, void *end_nursery, SgenGrayQueue *queue)
{
int i;
SgenThreadInfo *info;
RememberedSet *remset;
GenericStoreRememberedSet *store_remset;
mword *p;
#ifdef HEAVY_STATISTICS
remset_stats ();
#endif
/* the generic store ones */
store_remset = generic_store_remsets;
while (store_remset) {
GenericStoreRememberedSet *next = store_remset->next;
for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
gpointer addr = store_remset->data [i];
if (addr)
handle_remset ((mword*)&addr, start_nursery, end_nursery, FALSE, queue);
}
mono_sgen_free_internal (store_remset, INTERNAL_MEM_STORE_REMSET);
store_remset = next;
}
generic_store_remsets = NULL;
/* the per-thread ones */
FOREACH_THREAD (info) {
RememberedSet *next;
int j;
for (remset = info->remset; remset; remset = next) {
DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %td\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
for (p = remset->data; p < remset->store_next;)
p = handle_remset (p, start_nursery, end_nursery, FALSE, queue);
remset->store_next = remset->data;
next = remset->next;
remset->next = NULL;
if (remset != info->remset) {
DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
mono_sgen_free_internal_dynamic (remset, remset_byte_size (remset), INTERNAL_MEM_REMSET);
}
}
for (j = 0; j < *info->store_remset_buffer_index_addr; ++j)
handle_remset ((mword*)*info->store_remset_buffer_addr + j + 1, start_nursery, end_nursery, FALSE, queue);
clear_thread_store_remset_buffer (info);
} END_FOREACH_THREAD
/* the freed thread ones */
while (freed_thread_remsets) {
RememberedSet *next;
remset = freed_thread_remsets;
DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
for (p = remset->data; p < remset->store_next;)
p = handle_remset (p, start_nursery, end_nursery, FALSE, queue);
next = remset->next;
DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
mono_sgen_free_internal_dynamic (remset, remset_byte_size (remset), INTERNAL_MEM_REMSET);
freed_thread_remsets = next;
}
}
static void
gray_object_free_queue_section (GrayQueueSection *section)
{
mono_sgen_free_internal (section, INTERNAL_MEM_GRAY_QUEUE);
}
static void
major_sweep (void)
{
int i;
#ifdef FIXED_HEAP
int j;
#else
MSBlockInfo **iter;
#endif
/* clear all the free lists */
for (i = 0; i < MS_BLOCK_TYPE_MAX; ++i) {
MSBlockInfo **free_blocks = free_block_lists [i];
int j;
for (j = 0; j < num_block_obj_sizes; ++j)
free_blocks [j] = NULL;
}
/* traverse all blocks, free and zero unmarked objects */
#ifdef FIXED_HEAP
for (j = 0; j < ms_heap_num_blocks; ++j) {
MSBlockInfo *block = &block_infos [j];
#else
iter = &all_blocks;
while (*iter) {
MSBlockInfo *block = *iter;
#endif
int count;
gboolean have_live = FALSE;
int obj_index;
#ifdef FIXED_HEAP
if (!block->used)
continue;
#endif
count = MS_BLOCK_FREE / block->obj_size;
block->free_list = NULL;
for (obj_index = 0; obj_index < count; ++obj_index) {
int word, bit;
void *obj = MS_BLOCK_OBJ (block, obj_index);
MS_CALC_MARK_BIT (word, bit, obj);
if (MS_MARK_BIT (block, word, bit)) {
DEBUG (9, g_assert (MS_OBJ_ALLOCED (obj, block)));
have_live = TRUE;
} else {
/* an unmarked object */
if (MS_OBJ_ALLOCED (obj, block)) {
binary_protocol_empty (obj, block->obj_size);
memset (obj, 0, block->obj_size);
}
*(void**)obj = block->free_list;
block->free_list = obj;
}
}
/* reset mark bits */
memset (block->mark_words, 0, sizeof (mword) * MS_NUM_MARK_WORDS);
/*
* FIXME: reverse free list so that it's in address
* order
*/
if (have_live) {
#ifndef FIXED_HEAP
iter = &block->next;
#endif
/*
* If there are free slots in the block, add
* the block to the corresponding free list.
*/
if (block->free_list) {
MSBlockInfo **free_blocks = FREE_BLOCKS (block->pinned, block->has_references);
int index = MS_BLOCK_OBJ_SIZE_INDEX (block->obj_size);
block->next_free = free_blocks [index];
free_blocks [index] = block;
}
} else {
/*
* Blocks without live objects are removed from the
* block list and freed.
*/
#ifdef FIXED_HEAP
ms_free_block (block);
#else
*iter = block->next;
ms_free_block (block->block);
mono_sgen_free_internal (block, INTERNAL_MEM_MS_BLOCK_INFO);
#endif
--num_major_sections;
}
}
}
static int count_pinned_ref;
static int count_pinned_nonref;
static int count_nonpinned_ref;
static int count_nonpinned_nonref;
static void
count_nonpinned_callback (char *obj, size_t size, void *data)
{
MonoVTable *vtable = (MonoVTable*)LOAD_VTABLE (obj);
if (vtable->klass->has_references)
++count_nonpinned_ref;
else
++count_nonpinned_nonref;
}
/* LOCKING: requires that the GC lock is held */
static void
null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, gboolean before_finalization, GrayQueue *queue)
{
DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
DisappearingLink **disappearing_link_hash = hash->table;
int disappearing_link_hash_size = hash->size;
DisappearingLink *entry, *prev;
int i;
if (!hash->num_links)
return;
for (i = 0; i < disappearing_link_hash_size; ++i) {
prev = NULL;
for (entry = disappearing_link_hash [i]; entry;) {
char *object;
gboolean track = DISLINK_TRACK (entry);
/*
* Tracked references are processed after
* finalization handling whereas standard weak
* references are processed before. If an
* object is still not marked after finalization
* handling it means that it either doesn't have
* a finalizer or the finalizer has already run,
* so we must null a tracking reference.
*/
if (track == before_finalization) {
prev = entry;
entry = entry->next;
continue;
}
object = DISLINK_OBJECT (entry);
if (object >= start && object < end && !major_collector.is_object_live (object)) {
if (object_is_fin_ready (object)) {
void **p = entry->link;
DisappearingLink *old;
*p = NULL;
/* remove from list */
if (prev)
prev->next = entry->next;
else
disappearing_link_hash [i] = entry->next;
DEBUG (5, fprintf (gc_debug_file, "Dislink nullified at %p to GCed object %p\n", p, object));
old = entry->next;
mono_sgen_free_internal (entry, INTERNAL_MEM_DISLINK);
entry = old;
hash->num_links--;
continue;
} else {
char *copy = object;
copy_func ((void**)©, queue);
/* Update pointer if it's moved. If the object
* has been moved out of the nursery, we need to
* remove the link from the minor hash table to
* the major one.
*
* FIXME: what if an object is moved earlier?
*/
if (hash == &minor_disappearing_link_hash && !ptr_in_nursery (copy)) {
void **link = entry->link;
DisappearingLink *old;
/* remove from list */
if (prev)
prev->next = entry->next;
else
disappearing_link_hash [i] = entry->next;
old = entry->next;
mono_sgen_free_internal (entry, INTERNAL_MEM_DISLINK);
entry = old;
hash->num_links--;
g_assert (copy);
*link = HIDE_POINTER (copy, track);
add_or_remove_disappearing_link ((MonoObject*)copy, link, GENERATION_OLD);
DEBUG (5, fprintf (gc_debug_file, "Upgraded dislink at %p to major because object %p moved to %p\n", link, object, copy));
continue;
} else {
*entry->link = HIDE_POINTER (copy, track);
DEBUG (5, fprintf (gc_debug_file, "Updated dislink at %p to %p\n", entry->link, DISLINK_OBJECT (entry)));
}
}
}
prev = entry;
entry = entry->next;
}
}
}
static void
ms_sweep (void)
{
int i;
MSBlockInfo **iter;
/* statistics for evacuation */
int *slots_available = alloca (sizeof (int) * num_block_obj_sizes);
int *slots_used = alloca (sizeof (int) * num_block_obj_sizes);
int *num_blocks = alloca (sizeof (int) * num_block_obj_sizes);
for (i = 0; i < num_block_obj_sizes; ++i)
slots_available [i] = slots_used [i] = num_blocks [i] = 0;
/* clear all the free lists */
for (i = 0; i < MS_BLOCK_TYPE_MAX; ++i) {
MSBlockInfo **free_blocks = free_block_lists [i];
int j;
for (j = 0; j < num_block_obj_sizes; ++j)
free_blocks [j] = NULL;
}
/* traverse all blocks, free and zero unmarked objects */
iter = &all_blocks;
while (*iter) {
MSBlockInfo *block = *iter;
int count;
gboolean have_live = FALSE;
gboolean has_pinned;
int obj_index;
int obj_size_index;
obj_size_index = block->obj_size_index;
has_pinned = block->has_pinned;
block->has_pinned = block->pinned;
block->is_to_space = FALSE;
count = MS_BLOCK_FREE / block->obj_size;
block->free_list = NULL;
for (obj_index = 0; obj_index < count; ++obj_index) {
int word, bit;
void *obj = MS_BLOCK_OBJ (block, obj_index);
MS_CALC_MARK_BIT (word, bit, obj);
if (MS_MARK_BIT (block, word, bit)) {
DEBUG (9, g_assert (MS_OBJ_ALLOCED (obj, block)));
have_live = TRUE;
if (!has_pinned)
++slots_used [obj_size_index];
} else {
/* an unmarked object */
if (MS_OBJ_ALLOCED (obj, block)) {
binary_protocol_empty (obj, block->obj_size);
memset (obj, 0, block->obj_size);
}
*(void**)obj = block->free_list;
block->free_list = obj;
}
}
/* reset mark bits */
memset (block->mark_words, 0, sizeof (mword) * MS_NUM_MARK_WORDS);
/*
* FIXME: reverse free list so that it's in address
* order
*/
if (have_live) {
if (!has_pinned) {
++num_blocks [obj_size_index];
slots_available [obj_size_index] += count;
}
iter = &block->next;
/*
* If there are free slots in the block, add
* the block to the corresponding free list.
*/
if (block->free_list) {
MSBlockInfo **free_blocks = FREE_BLOCKS (block->pinned, block->has_references);
int index = MS_BLOCK_OBJ_SIZE_INDEX (block->obj_size);
block->next_free = free_blocks [index];
free_blocks [index] = block;
}
update_heap_boundaries_for_block (block);
} else {
/*
* Blocks without live objects are removed from the
* block list and freed.
*/
*iter = block->next;
#ifdef FIXED_HEAP
ms_free_block (block);
#else
ms_free_block (block->block);
mono_sgen_free_internal (block, INTERNAL_MEM_MS_BLOCK_INFO);
#endif
--num_major_sections;
}
}
for (i = 0; i < num_block_obj_sizes; ++i) {
float usage = (float)slots_used [i] / (float)slots_available [i];
if (num_blocks [i] > 5 && usage < evacuation_threshold) {
evacuate_block_obj_sizes [i] = TRUE;
/*
g_print ("slot size %d - %d of %d used\n",
block_obj_sizes [i], slots_used [i], slots_available [i]);
*/
} else {
evacuate_block_obj_sizes [i] = FALSE;
}
}
have_swept = TRUE;
}
