/* Does work in a thread's in-tray, if any. Returns a non-zero value if work
* was found and done, and zero otherwise. */
static int process_in_tray(MVMThreadContext *tc, MVMuint8 gen) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Considering extra work\n");
if (MVM_load(&tc->gc_in_tray)) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : Was given extra work by another thread; doing it\n");
MVM_gc_collect(tc, MVMGCWhatToDo_InTray, gen);
return 1;
}
return 0;
}
static void run_gc(MVMThreadContext *tc, MVMuint8 what_to_do) {
MVMuint8 gen;
MVMuint32 i, n;
unsigned int interval_id;
#if MVM_GC_DEBUG
if (tc->in_spesh)
MVM_panic(1, "Must not GC when in the specializer/JIT\n");
#endif
/* Decide nursery or full collection. */
gen = tc->instance->gc_full_collect ? MVMGCGenerations_Both : MVMGCGenerations_Nursery;
if (tc->instance->gc_full_collect) {
interval_id = MVM_telemetry_interval_start(tc, "start full collection");
} else {
interval_id = MVM_telemetry_interval_start(tc, "start minor collection");
}
/* Do GC work for ourselves and any work threads. */
for (i = 0, n = tc->gc_work_count ; i < n; i++) {
MVMThreadContext *other = tc->gc_work[i].tc;
tc->gc_work[i].limit = other->nursery_alloc;
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : starting collection for thread %d\n",
other->thread_id);
other->gc_promoted_bytes = 0;
MVM_gc_collect(other, (other == tc ? what_to_do : MVMGCWhatToDo_NoInstance), gen);
}
/* Wait for everybody to agree we're done. */
finish_gc(tc, gen, what_to_do == MVMGCWhatToDo_All);
MVM_telemetry_interval_stop(tc, interval_id, "finished run_gc");
}
static void run_gc(MVMThreadContext *tc, MVMuint8 what_to_do) {
MVMuint8 gen;
MVMuint32 i, n;
/* Decide nursery or full collection. */
gen = is_full_collection(tc) ? MVMGCGenerations_Both : MVMGCGenerations_Nursery;
/* Do GC work for ourselves and any work threads. */
for (i = 0, n = tc->gc_work_count ; i < n; i++) {
MVMThreadContext *other = tc->gc_work[i].tc;
tc->gc_work[i].limit = other->nursery_alloc;
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : starting collection for thread %d\n",
other->thread_id);
other->gc_promoted_bytes = 0;
MVM_gc_collect(other, (other == tc ? what_to_do : MVMGCWhatToDo_NoInstance), gen);
}
/* Wait for everybody to agree we're done. */
finish_gc(tc, gen, what_to_do == MVMGCWhatToDo_All);
/* Now we're all done, it's safe to finalize any objects that need it. */
/* XXX TODO explore the feasability of doing this in a background
* finalizer/destructor thread and letting the main thread(s) continue
* on their merry way(s). */
for (i = 0, n = tc->gc_work_count ; i < n; i++) {
MVMThreadContext *other = tc->gc_work[i].tc;
/* The thread might've been destroyed */
if (!other)
continue;
/* Contribute this thread's promoted bytes. */
MVM_add(&tc->instance->gc_promoted_bytes_since_last_full, other->gc_promoted_bytes);
/* Collect nursery and gen2 as needed. */
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : collecting nursery uncopied of thread %d\n",
other->thread_id);
MVM_gc_collect_free_nursery_uncopied(other, tc->gc_work[i].limit);
if (gen == MVMGCGenerations_Both) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : freeing gen2 of thread %d\n",
other->thread_id);
MVM_gc_collect_free_gen2_unmarked(other);
}
}
}
static MVMuint32 signal_all_but(MVMThreadContext *tc, MVMThread *t, MVMThread *tail) {
MVMuint32 count = 0;
MVMThread *next;
if (!t) {
return 0;
}
do {
next = t->body.next;
switch (MVM_load(&t->body.stage)) {
case MVM_thread_stage_starting:
case MVM_thread_stage_waiting:
case MVM_thread_stage_started:
if (t->body.tc != tc) {
count += signal_one_thread(tc, t->body.tc);
}
break;
case MVM_thread_stage_exited:
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : queueing to clear nursery of thread %d\n", t->body.tc->thread_id);
add_work(tc, t->body.tc);
break;
case MVM_thread_stage_clearing_nursery:
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : queueing to destroy thread %d\n", t->body.tc->thread_id);
/* last GC run for this thread */
add_work(tc, t->body.tc);
break;
case MVM_thread_stage_destroyed:
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : found a destroyed thread\n");
/* will be cleaned up (removed from the lists) shortly */
break;
default:
MVM_panic(MVM_exitcode_gcorch, "Corrupted MVMThread or running threads list: invalid thread stage %"MVM_PRSz"", MVM_load(&t->body.stage));
}
} while (next && (t = next));
if (tail)
MVM_gc_write_barrier(tc, (MVMCollectable *)t, (MVMCollectable *)tail);
t->body.next = tail;
return count;
}
/* This is called when a thread hits an interrupt at a GC safe point. This means
* that another thread is already trying to start a GC run, so we don't need to
* try and do that, just enlist in the run. */
void MVM_gc_enter_from_interrupt(MVMThreadContext *tc) {
AO_t curr;
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Entered from interrupt\n");
MVM_telemetry_timestamp(tc, "gc_enter_from_interrupt");
/* If profiling, record that GC is starting. */
if (tc->instance->profiling)
MVM_profiler_log_gc_start(tc, is_full_collection(tc));
/* We'll certainly take care of our own work. */
tc->gc_work_count = 0;
add_work(tc, tc);
/* Indicate that we're ready to GC. Only want to decrement it if it's 2 or
* greater (0 should never happen; 1 means the coordinator is still counting
* up how many threads will join in, so we should wait until it decides to
* decrement.) */
while ((curr = MVM_load(&tc->instance->gc_start)) < 2
|| !MVM_trycas(&tc->instance->gc_start, curr, curr - 1)) {
/* MVM_platform_thread_yield();*/
}
/* Wait for all threads to indicate readiness to collect. */
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Waiting for other threads\n");
while (MVM_load(&tc->instance->gc_start)) {
/* MVM_platform_thread_yield();*/
}
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Entering run_gc\n");
run_gc(tc, MVMGCWhatToDo_NoInstance);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : GC complete\n");
/* If profiling, record that GC is over. */
if (tc->instance->profiling)
MVM_profiler_log_gc_end(tc);
}
/* Goes through all threads but the current one and notifies them that a
* GC run is starting. Those that are blocked are considered excluded from
* the run, and are not counted. Returns the count of threads that should be
* added to the finished countdown. */
static MVMuint32 signal_one_thread(MVMThreadContext *tc, MVMThreadContext *to_signal) {
/* Loop here since we may not succeed first time (e.g. the status of the
* thread may change between the two ways we try to twiddle it). */
while (1) {
switch (MVM_load(&to_signal->gc_status)) {
case MVMGCStatus_NONE:
/* Try to set it from running to interrupted - the common case. */
if (MVM_cas(&to_signal->gc_status, MVMGCStatus_NONE,
MVMGCStatus_INTERRUPT) == MVMGCStatus_NONE) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Signalled thread %d to interrupt\n", to_signal->thread_id);
return 1;
}
break;
case MVMGCStatus_INTERRUPT:
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : thread %d already interrupted\n", to_signal->thread_id);
return 0;
case MVMGCStatus_UNABLE:
/* Otherwise, it's blocked; try to set it to work Stolen. */
if (MVM_cas(&to_signal->gc_status, MVMGCStatus_UNABLE,
MVMGCStatus_STOLEN) == MVMGCStatus_UNABLE) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : A blocked thread %d spotted; work stolen\n", to_signal->thread_id);
add_work(tc, to_signal);
return 0;
}
break;
/* this case occurs if a child thread is Stolen by its parent
* before we get to it in the chain. */
case MVMGCStatus_STOLEN:
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : thread %d already stolen (it was a spawning child)\n", to_signal->thread_id);
return 0;
default:
MVM_panic(MVM_exitcode_gcorch, "invalid status %"MVM_PRSz" in GC orchestrate\n", MVM_load(&to_signal->gc_status));
return 0;
}
}
}
/* Some objects, having been copied, need no further attention. Others
* need to do some additional freeing, however. This goes through the
* fromspace and does any needed work to free uncopied things (this may
* run in parallel with the mutator, which will be operating on tospace). */
void MVM_gc_collect_free_nursery_uncopied(MVMThreadContext *tc, void *limit) {
/* We start scanning the fromspace, and keep going until we hit
* the end of the area allocated in it. */
void *scan = tc->nursery_fromspace;
while (scan < limit) {
/* The object here is dead if it never got a forwarding pointer
* written in to it. */
MVMCollectable *item = (MVMCollectable *)scan;
MVMuint8 dead = item->forwarder == NULL;
/* Now go by collectable type. */
if (!(item->flags & (MVM_CF_TYPE_OBJECT | MVM_CF_STABLE))) {
/* Object instance. If dead, call gc_free if needed. Scan is
* incremented by object size. */
MVMObject *obj = (MVMObject *)item;
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : collecting an object %p in the nursery with reprid %d\n", item, REPR(obj)->ID);
if (dead && REPR(obj)->gc_free)
REPR(obj)->gc_free(tc, obj);
}
else if (item->flags & MVM_CF_TYPE_OBJECT) {
/* Type object; doesn't have anything extra that needs freeing. */
}
else if (item->flags & MVM_CF_STABLE) {
MVMSTable *st = (MVMSTable *)item;
if (dead) {
/* GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : enqueuing an STable %d in the nursery to be freed\n", item);*/
MVM_gc_collect_enqueue_stable_for_deletion(tc, st);
}
}
else {
printf("item flags: %d\n", item->flags);
MVM_panic(MVM_exitcode_gcnursery, "Internal error: impossible case encountered in GC free");
}
/* Go to the next item. */
scan = (char *)scan + item->size;
}
}
/* This is called when the allocator finds it has run out of memory and wants
* to trigger a GC run. In this case, it's possible (probable, really) that it
* will need to do that triggering, notifying other running threads that the
* time has come to GC. */
void MVM_gc_enter_from_allocator(MVMThreadContext *tc) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Entered from allocate\n");
/* Try to start the GC run. */
if (MVM_trycas(&tc->instance->gc_start, 0, 1)) {
MVMThread *last_starter = NULL;
MVMuint32 num_threads = 0;
MVMuint32 is_full;
/* Need to wait for other threads to reset their gc_status. */
while (MVM_load(&tc->instance->gc_ack)) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : waiting for other thread's gc_ack\n");
MVM_platform_thread_yield();
}
/* We are the winner of the GC starting race. This gives us some
* extra responsibilities as well as doing the usual things.
* First, increment GC sequence number. */
MVM_incr(&tc->instance->gc_seq_number);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : GC thread elected coordinator: starting gc seq %d\n",
(int)MVM_load(&tc->instance->gc_seq_number));
/* Decide if it will be a full collection. */
is_full = is_full_collection(tc);
/* If profiling, record that GC is starting. */
if (tc->instance->profiling)
MVM_profiler_log_gc_start(tc, is_full);
/* Ensure our stolen list is empty. */
tc->gc_work_count = 0;
/* Flag that we didn't agree on this run that all the in-trays are
* cleared (a responsibility of the co-ordinator. */
MVM_store(&tc->instance->gc_intrays_clearing, 1);
/* We'll take care of our own work. */
add_work(tc, tc);
/* Find other threads, and signal or steal. */
do {
MVMThread *threads = (MVMThread *)MVM_load(&tc->instance->threads);
if (threads && threads != last_starter) {
MVMThread *head = threads;
MVMuint32 add;
while ((threads = (MVMThread *)MVM_casptr(&tc->instance->threads, head, NULL)) != head) {
head = threads;
}
add = signal_all_but(tc, head, last_starter);
last_starter = head;
if (add) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Found %d other threads\n", add);
MVM_add(&tc->instance->gc_start, add);
num_threads += add;
}
}
/* If there's an event loop thread, wake it up to participate. */
if (tc->instance->event_loop_wakeup)
uv_async_send(tc->instance->event_loop_wakeup);
} while (MVM_load(&tc->instance->gc_start) > 1);
/* Sanity checks. */
if (!MVM_trycas(&tc->instance->threads, NULL, last_starter))
MVM_panic(MVM_exitcode_gcorch, "threads list corrupted\n");
if (MVM_load(&tc->instance->gc_finish) != 0)
MVM_panic(MVM_exitcode_gcorch, "Finish votes was %"MVM_PRSz"\n", MVM_load(&tc->instance->gc_finish));
/* gc_ack gets an extra so the final acknowledger
* can also free the STables. */
MVM_store(&tc->instance->gc_finish, num_threads + 1);
MVM_store(&tc->instance->gc_ack, num_threads + 2);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : finish votes is %d\n",
(int)MVM_load(&tc->instance->gc_finish));
/* Now we're ready to start, zero promoted since last full collection
* counter if this is a full collect. */
if (is_full)
MVM_store(&tc->instance->gc_promoted_bytes_since_last_full, 0);
/* Signal to the rest to start */
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : coordinator signalling start\n");
if (MVM_decr(&tc->instance->gc_start) != 1)
MVM_panic(MVM_exitcode_gcorch, "Start votes was %"MVM_PRSz"\n", MVM_load(&tc->instance->gc_start));
/* Start collecting. */
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : coordinator entering run_gc\n");
run_gc(tc, MVMGCWhatToDo_All);
/* Free any STables that have been marked for deletion. It's okay for
* us to muck around in another thread's fromspace while it's mutating
* tospace, really. */
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Freeing STables if needed\n");
MVM_gc_collect_free_stables(tc);
/* If profiling, record that GC is over. */
if (tc->instance->profiling)
MVM_profiler_log_gc_end(tc);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : GC complete (cooridnator)\n");
}
else {
/* Another thread beat us to starting the GC sync process. Thus, act as
* if we were interrupted to GC. */
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Lost coordinator election\n");
MVM_gc_enter_from_interrupt(tc);
}
}
static void finish_gc(MVMThreadContext *tc, MVMuint8 gen, MVMuint8 is_coordinator) {
MVMuint32 i, did_work;
/* Do any extra work that we have been passed. */
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : doing any work in thread in-trays\n");
did_work = 1;
while (did_work) {
did_work = 0;
for (i = 0; i < tc->gc_work_count; i++)
did_work += process_in_tray(tc->gc_work[i].tc, gen);
}
/* Decrement gc_finish to say we're done, and wait for termination. */
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Voting to finish\n");
MVM_decr(&tc->instance->gc_finish);
while (MVM_load(&tc->instance->gc_finish)) {
for (i = 0; i < 1000; i++)
; /* XXX Something HT-efficienter. */
/* XXX Here we can look to see if we got passed any work, and if so
* try to un-vote. */
}
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Termination agreed\n");
/* Co-ordinator should do final check over all the in-trays, and trigger
* collection until all is settled. Rest should wait. Additionally, after
* in-trays are settled, coordinator walks threads looking for anything
* that needs adding to the finalize queue. It then will make another
* iteration over in-trays to handle cross-thread references to objects
* needing finalization. For full collections, collected objects are then
* cleaned from all inter-generational sets, and finally any objects to
* be freed at the fixed size allocator's next safepoint are freed. */
if (is_coordinator) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : Co-ordinator handling in-tray clearing completion\n");
clear_intrays(tc, gen);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : Co-ordinator handling finalizers\n");
MVM_finalize_walk_queues(tc, gen);
clear_intrays(tc, gen);
if (gen == MVMGCGenerations_Both) {
MVMThread *cur_thread = (MVMThread *)MVM_load(&tc->instance->threads);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : Co-ordinator handling inter-gen root cleanup\n");
while (cur_thread) {
if (cur_thread->body.tc)
MVM_gc_root_gen2_cleanup(cur_thread->body.tc);
cur_thread = cur_thread->body.next;
}
}
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : Co-ordinator handling fixed-size allocator safepoint frees\n");
MVM_fixed_size_safepoint(tc, tc->instance->fsa);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : Co-ordinator signalling in-trays clear\n");
MVM_store(&tc->instance->gc_intrays_clearing, 0);
}
else {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : Waiting for in-tray clearing completion\n");
while (MVM_load(&tc->instance->gc_intrays_clearing))
for (i = 0; i < 1000; i++)
; /* XXX Something HT-efficienter. */
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : Got in-tray clearing complete notice\n");
}
/* Reset GC status flags. This is also where thread destruction happens,
* and it needs to happen before we acknowledge this GC run is finished. */
for (i = 0; i < tc->gc_work_count; i++) {
MVMThreadContext *other = tc->gc_work[i].tc;
MVMThread *thread_obj = other->thread_obj;
if (MVM_load(&thread_obj->body.stage) == MVM_thread_stage_clearing_nursery) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : transferring gen2 of thread %d\n", other->thread_id);
MVM_gc_gen2_transfer(other, tc);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : destroying thread %d\n", other->thread_id);
MVM_tc_destroy(other);
tc->gc_work[i].tc = thread_obj->body.tc = NULL;
MVM_store(&thread_obj->body.stage, MVM_thread_stage_destroyed);
}
else {
if (MVM_load(&thread_obj->body.stage) == MVM_thread_stage_exited) {
/* Don't bother freeing gen2; we'll do it next time */
MVM_store(&thread_obj->body.stage, MVM_thread_stage_clearing_nursery);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : set thread %d clearing nursery stage to %d\n",
other->thread_id, (int)MVM_load(&thread_obj->body.stage));
}
MVM_cas(&other->gc_status, MVMGCStatus_STOLEN, MVMGCStatus_UNABLE);
MVM_cas(&other->gc_status, MVMGCStatus_INTERRUPT, MVMGCStatus_NONE);
}
}
/* Signal acknowledgement of completing the cleanup,
* except for STables, and if we're the final to do
* so, free the STables, which have been linked. */
if (MVM_decr(&tc->instance->gc_ack) == 2) {
/* Set it to zero (we're guaranteed the only ones trying to write to
* it here). Actual STable free in MVM_gc_enter_from_allocator. */
MVM_store(&tc->instance->gc_ack, 0);
}
}
/* Goes through the unmarked objects in the second generation heap and builds
* free lists out of them. Also does any required finalization. */
void MVM_gc_collect_free_gen2_unmarked(MVMThreadContext *tc, MVMint32 global_destruction) {
/* Visit each of the size class bins. */
MVMGen2Allocator *gen2 = tc->gen2;
MVMuint32 bin, obj_size, page, i;
char ***freelist_insert_pos;
for (bin = 0; bin < MVM_GEN2_BINS; bin++) {
/* If we've nothing allocated in this size class, skip it. */
if (gen2->size_classes[bin].pages == NULL)
continue;
/* Calculate object size for this bin. */
obj_size = (bin + 1) << MVM_GEN2_BIN_BITS;
/* freelist_insert_pos is a pointer to a memory location that
* stores the address of the last traversed free list node (char **). */
/* Initialize freelist insertion position to free list head. */
freelist_insert_pos = &gen2->size_classes[bin].free_list;
/* Visit each page. */
for (page = 0; page < gen2->size_classes[bin].num_pages; page++) {
/* Visit all the objects, looking for dead ones and reset the
* mark for each of them. */
char *cur_ptr = gen2->size_classes[bin].pages[page];
char *end_ptr = page + 1 == gen2->size_classes[bin].num_pages
? gen2->size_classes[bin].alloc_pos
: cur_ptr + obj_size * MVM_GEN2_PAGE_ITEMS;
while (cur_ptr < end_ptr) {
MVMCollectable *col = (MVMCollectable *)cur_ptr;
/* Is this already a free list slot? If so, it becomes the
* new free list insert position. */
if (*freelist_insert_pos == (char **)cur_ptr) {
freelist_insert_pos = (char ***)cur_ptr;
}
/* Otherwise, it must be a collectable of some kind. Is it
* live? */
else if (col->flags & MVM_CF_GEN2_LIVE) {
/* Yes; clear the mark. */
col->flags &= ~MVM_CF_GEN2_LIVE;
}
else {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : collecting an object %p in the gen2\n", col);
/* No, it's dead. Do any cleanup. */
if (!(col->flags & (MVM_CF_TYPE_OBJECT | MVM_CF_STABLE))) {
/* Object instance; call gc_free if needed. */
MVMObject *obj = (MVMObject *)col;
if (STABLE(obj) && REPR(obj)->gc_free)
REPR(obj)->gc_free(tc, obj);
#ifdef MVM_USE_OVERFLOW_SERIALIZATION_INDEX
if (col->flags & MVM_CF_SERIALZATION_INDEX_ALLOCATED)
MVM_free(col->sc_forward_u.sci);
#endif
}
else if (col->flags & MVM_CF_TYPE_OBJECT) {
#ifdef MVM_USE_OVERFLOW_SERIALIZATION_INDEX
if (col->flags & MVM_CF_SERIALZATION_INDEX_ALLOCATED)
MVM_free(col->sc_forward_u.sci);
#endif
}
else if (col->flags & MVM_CF_STABLE) {
if (
#ifdef MVM_USE_OVERFLOW_SERIALIZATION_INDEX
!(col->flags & MVM_CF_SERIALZATION_INDEX_ALLOCATED) &&
#endif
col->sc_forward_u.sc.sc_idx == 0
&& col->sc_forward_u.sc.idx == MVM_DIRECT_SC_IDX_SENTINEL) {
/* We marked it dead last time, kill it. */
MVM_6model_stable_gc_free(tc, (MVMSTable *)col);
}
else {
#ifdef MVM_USE_OVERFLOW_SERIALIZATION_INDEX
if (col->flags & MVM_CF_SERIALZATION_INDEX_ALLOCATED) {
/* Whatever happens next, we can free this
memory immediately, because no-one will be
serializing a dead STable. */
assert(!(col->sc_forward_u.sci->sc_idx == 0
&& col->sc_forward_u.sci->idx
== MVM_DIRECT_SC_IDX_SENTINEL));
MVM_free(col->sc_forward_u.sci);
col->flags &= ~MVM_CF_SERIALZATION_INDEX_ALLOCATED;
}
#endif
if (global_destruction) {
/* We're in global destruction, so enqueue to the end
* like we do in the nursery */
MVM_gc_collect_enqueue_stable_for_deletion(tc, (MVMSTable *)col);
} else {
/* There will definitely be another gc run, so mark it as "died last time". */
col->sc_forward_u.sc.sc_idx = 0;
col->sc_forward_u.sc.idx = MVM_DIRECT_SC_IDX_SENTINEL;
}
/* Skip the freelist updating. */
cur_ptr += obj_size;
continue;
}
}
else {
printf("item flags: %d\n", col->flags);
MVM_panic(MVM_exitcode_gcnursery, "Internal error: impossible case encountered in gen2 GC free");
}
/* Chain in to the free list. */
*((char **)cur_ptr) = (char *)*freelist_insert_pos;
*freelist_insert_pos = (char **)cur_ptr;
/* Update the pointer to the insert position to point to us */
freelist_insert_pos = (char ***)cur_ptr;
}
/* Move to the next object. */
cur_ptr += obj_size;
}
}
}
/* Also need to consider overflows. */
for (i = 0; i < gen2->num_overflows; i++) {
if (gen2->overflows[i]) {
MVMCollectable *col = gen2->overflows[i];
if (col->flags & MVM_CF_GEN2_LIVE) {
/* A living over-sized object; just clear the mark. */
col->flags &= ~MVM_CF_GEN2_LIVE;
}
else {
/* Dead over-sized object. We know if it's this big it cannot
* be a type object or STable, so only need handle the simple
* object case. */
if (!(col->flags & (MVM_CF_TYPE_OBJECT | MVM_CF_STABLE))) {
MVMObject *obj = (MVMObject *)col;
if (REPR(obj)->gc_free)
REPR(obj)->gc_free(tc, obj);
#ifdef MVM_USE_OVERFLOW_SERIALIZATION_INDEX
if (col->flags & MVM_CF_SERIALZATION_INDEX_ALLOCATED)
MVM_free(col->sc_forward_u.sci);
#endif
}
else {
MVM_panic(MVM_exitcode_gcnursery, "Internal error: gen2 overflow contains non-object");
}
MVM_free(col);
gen2->overflows[i] = NULL;
}
}
}
/* And finally compact the overflow list */
MVM_gc_gen2_compact_overflows(gen2);
}
/* Some objects, having been copied, need no further attention. Others
* need to do some additional freeing, however. This goes through the
* fromspace and does any needed work to free uncopied things (this may
* run in parallel with the mutator, which will be operating on tospace). */
void MVM_gc_collect_free_nursery_uncopied(MVMThreadContext *tc, void *limit) {
/* We start scanning the fromspace, and keep going until we hit
* the end of the area allocated in it. */
void *scan = tc->nursery_fromspace;
while (scan < limit) {
/* The object here is dead if it never got a forwarding pointer
* written in to it. */
MVMCollectable *item = (MVMCollectable *)scan;
MVMuint8 dead = !(item->flags & MVM_CF_FORWARDER_VALID);
if (!dead)
assert(item->sc_forward_u.forwarder != NULL);
/* Now go by collectable type. */
if (!(item->flags & (MVM_CF_TYPE_OBJECT | MVM_CF_STABLE))) {
/* Object instance. If dead, call gc_free if needed. Scan is
* incremented by object size. */
MVMObject *obj = (MVMObject *)item;
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : collecting an object %p in the nursery with reprid %d\n", item, REPR(obj)->ID);
if (dead && REPR(obj)->gc_free)
REPR(obj)->gc_free(tc, obj);
#ifdef MVM_USE_OVERFLOW_SERIALIZATION_INDEX
if (dead && item->flags & MVM_CF_SERIALZATION_INDEX_ALLOCATED)
MVM_free(item->sc_forward_u.sci);
#endif
if (dead && item->flags & MVM_CF_HAS_OBJECT_ID)
MVM_gc_object_id_clear(tc, item);
}
else if (item->flags & MVM_CF_TYPE_OBJECT) {
/* Type object */
#ifdef MVM_USE_OVERFLOW_SERIALIZATION_INDEX
if (dead && item->flags & MVM_CF_SERIALZATION_INDEX_ALLOCATED)
MVM_free(item->sc_forward_u.sci);
#endif
if (dead && item->flags & MVM_CF_HAS_OBJECT_ID)
MVM_gc_object_id_clear(tc, item);
}
else if (item->flags & MVM_CF_STABLE) {
MVMSTable *st = (MVMSTable *)item;
if (dead) {
/* GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : enqueuing an STable %d in the nursery to be freed\n", item);*/
#ifdef MVM_USE_OVERFLOW_SERIALIZATION_INDEX
if (item->flags & MVM_CF_SERIALZATION_INDEX_ALLOCATED) {
MVM_free(item->sc_forward_u.sci);
/* Arguably we don't need to do this, if we're always
consistent about what we put on the stable queue. */
item->flags &= ~MVM_CF_SERIALZATION_INDEX_ALLOCATED;
}
#endif
MVM_gc_collect_enqueue_stable_for_deletion(tc, st);
}
}
else {
printf("item flags: %d\n", item->flags);
MVM_panic(MVM_exitcode_gcnursery, "Internal error: impossible case encountered in GC free");
}
/* Go to the next item. */
scan = (char *)scan + item->size;
}
}
/* Does a garbage collection run. Exactly what it does is configured by the
* couple of arguments that it takes.
*
* The what_to_do argument specifies where it should look for things to add
* to the worklist: everywhere, just at thread local stuff, or just in the
* thread's in-tray.
*
* The gen argument specifies whether to collect the nursery or both of the
* generations. Nursery collection is done by semi-space copying. Once an
* object is seen/copied once in the nursery (may be tuned in the future to
* twice or so - we'll see) then it is not copied to tospace, but instead
* promoted to the second generation. If we are collecting generation 2 also,
* then objects that are alive in the second generation are simply marked.
* Since the second generation is managed as a set of sized pools, there is
* much less motivation for any kind of copying/compaction; the internal
* fragmentation that makes finding a right-sized gap problematic will not
* happen.
*
* Note that it adds the roots and processes them in phases, to try to avoid
* building up a huge worklist. */
void MVM_gc_collect(MVMThreadContext *tc, MVMuint8 what_to_do, MVMuint8 gen) {
/* Create a GC worklist. */
MVMGCWorklist *worklist = MVM_gc_worklist_create(tc, gen != MVMGCGenerations_Nursery, 0);
/* Initialize work passing data structure. */
WorkToPass wtp;
wtp.num_target_threads = 0;
wtp.target_work = NULL;
/* See what we need to work on this time. */
if (what_to_do == MVMGCWhatToDo_InTray) {
/* We just need to process anything in the in-tray. */
add_in_tray_to_worklist(tc, worklist);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from in tray \n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
}
else if (what_to_do == MVMGCWhatToDo_Finalizing) {
/* Need to process the finalizing queue. */
MVMuint32 i;
for (i = 0; i < tc->num_finalizing; i++)
MVM_gc_worklist_add(tc, worklist, &(tc->finalizing[i]));
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from finalizing \n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
}
else {
/* Main collection run. Swap fromspace and tospace, allocating the
* new tospace if that didn't yet happen (we don't allocate it at
* startup, to cut memory use for threads that quit before a GC). */
void *fromspace = tc->nursery_tospace;
void *tospace = tc->nursery_fromspace;
if (!tospace)
tospace = MVM_calloc(1, MVM_NURSERY_SIZE);
tc->nursery_fromspace = fromspace;
tc->nursery_tospace = tospace;
/* Reset nursery allocation pointers to the new tospace. */
tc->nursery_alloc = tospace;
tc->nursery_alloc_limit = (char *)tc->nursery_alloc + MVM_NURSERY_SIZE;
/* Add permanent roots and process them; only one thread will do
* this, since they are instance-wide. */
if (what_to_do != MVMGCWhatToDo_NoInstance) {
MVM_gc_root_add_permanents_to_worklist(tc, worklist, NULL);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from instance permanents\n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
MVM_gc_root_add_instance_roots_to_worklist(tc, worklist, NULL);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from instance roots\n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
}
/* Add per-thread state to worklist and process it. */
MVM_gc_root_add_tc_roots_to_worklist(tc, worklist, NULL);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from TC objects\n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
/* Add current frame to worklist. */
MVM_gc_worklist_add_frame(tc, worklist, tc->cur_frame);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from current frame\n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
/* Add temporary roots and process them (these are per-thread). */
MVM_gc_root_add_temps_to_worklist(tc, worklist, NULL);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from thread temps\n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
/* Add things that are roots for the first generation because they are
* pointed to by objects in the second generation and process them
* (also per-thread). Note we need not do this if we're doing a full
* collection anyway (in fact, we must not for correctness, otherwise
* the gen2 rooting keeps them alive forever). */
if (gen == MVMGCGenerations_Nursery) {
MVM_gc_root_add_gen2s_to_worklist(tc, worklist);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from gen2 \n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
}
/* Process anything in the in-tray. */
add_in_tray_to_worklist(tc, worklist);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from in tray \n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
/* At this point, we have probably done most of the work we will
* need to (only get more if another thread passes us more); zero
* out the remaining tospace. */
memset(tc->nursery_alloc, 0, (char *)tc->nursery_alloc_limit - (char *)tc->nursery_alloc);
}
/* Destroy the worklist. */
MVM_gc_worklist_destroy(tc, worklist);
/* Pass any work for other threads we accumulated but that didn't trigger
* the work passing threshold, then cleanup work passing list. */
if (wtp.num_target_threads) {
pass_leftover_work(tc, &wtp);
MVM_free(wtp.target_work);
}
}
/* Processes the current worklist. */
static void process_worklist(MVMThreadContext *tc, MVMGCWorklist *worklist, WorkToPass *wtp, MVMuint8 gen) {
MVMGen2Allocator *gen2;
MVMCollectable **item_ptr;
MVMCollectable *new_addr;
MVMuint32 gen2count;
/* Grab the second generation allocator; we may move items into the
* old generation. */
gen2 = tc->gen2;
MVM_gc_worklist_mark_frame_roots(tc, worklist);
while ((item_ptr = MVM_gc_worklist_get(tc, worklist))) {
/* Dereference the object we're considering. */
MVMCollectable *item = *item_ptr;
MVMuint8 item_gen2;
MVMuint8 to_gen2 = 0;
/* If the item is NULL, that's fine - it's just a null reference and
* thus we've no object to consider. */
if (item == NULL)
continue;
/* If it's in the second generation and we're only doing a nursery,
* collection, we have nothing to do. */
item_gen2 = item->flags & MVM_CF_SECOND_GEN;
if (item_gen2) {
if (gen == MVMGCGenerations_Nursery)
continue;
if (item->flags & MVM_CF_GEN2_LIVE) {
/* gen2 and marked as live. */
continue;
}
} else if (item->flags & MVM_CF_FORWARDER_VALID) {
/* If the item was already seen and copied, then it will have a
* forwarding address already. Just update this pointer to the
* new address and we're done. */
assert(*item_ptr != item->sc_forward_u.forwarder);
if (MVM_GC_DEBUG_ENABLED(MVM_GC_DEBUG_COLLECT)) {
if (*item_ptr != item->sc_forward_u.forwarder) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : updating handle %p from %p to forwarder %p\n", item_ptr, item, item->sc_forward_u.forwarder);
}
else {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : already visited handle %p to forwarder %p\n", item_ptr, item->sc_forward_u.forwarder);
}
}
*item_ptr = item->sc_forward_u.forwarder;
continue;
} else {
/* If the pointer is already into tospace (the bit we've already
copied into), we already updated it, so we're done. */
if (item >= (MVMCollectable *)tc->nursery_tospace && item < (MVMCollectable *)tc->nursery_alloc) {
continue;
}
}
/* If it's owned by a different thread, we need to pass it over to
* the owning thread. */
if (item->owner != tc->thread_id) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : sending a handle %p to object %p to thread %d\n", item_ptr, item, item->owner);
pass_work_item(tc, wtp, item_ptr);
continue;
}
/* If it's in to-space but *ahead* of our copy offset then it's an
out-of-date pointer and we have some kind of corruption. */
if (item >= (MVMCollectable *)tc->nursery_alloc && item < (MVMCollectable *)tc->nursery_alloc_limit)
MVM_panic(1, "Heap corruption detected: pointer %p to past fromspace", item);
/* At this point, we didn't already see the object, which means we
* need to take some action. Go on the generation... */
if (item_gen2) {
assert(!(item->flags & MVM_CF_FORWARDER_VALID));
/* It's in the second generation. We'll just mark it. */
new_addr = item;
if (MVM_GC_DEBUG_ENABLED(MVM_GC_DEBUG_COLLECT)) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : handle %p was already %p\n", item_ptr, new_addr);
}
item->flags |= MVM_CF_GEN2_LIVE;
assert(*item_ptr == new_addr);
} else {
/* Catch NULL stable (always sign of trouble) in debug mode. */
if (MVM_GC_DEBUG_ENABLED(MVM_GC_DEBUG_COLLECT) && !STABLE(item)) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : found a zeroed handle %p to object %p\n", item_ptr, item);
printf("%d", ((MVMCollectable *)1)->owner);
}
/* Did we see it in the nursery before, or should we move it to
* gen2 anyway since it a persistent ID was requested? */
if (item->flags & (MVM_CF_NURSERY_SEEN | MVM_CF_HAS_OBJECT_ID)) {
/* Yes; we should move it to the second generation. Allocate
* space in the second generation. */
to_gen2 = 1;
new_addr = item->flags & MVM_CF_HAS_OBJECT_ID
? MVM_gc_object_id_use_allocation(tc, item)
: MVM_gc_gen2_allocate(gen2, item->size);
/* Add on to the promoted amount (used by profiler). */
tc->gc_promoted_bytes += item->size;
/* Copy the object to the second generation and mark it as
* living there. */
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : copying an object %p of size %d to gen2 %p\n",
item, item->size, new_addr);
memcpy(new_addr, item, item->size);
new_addr->flags ^= MVM_CF_NURSERY_SEEN;
new_addr->flags |= MVM_CF_SECOND_GEN;
/* If it references frames or static frames, we need to keep
* on visiting it. */
if (!(new_addr->flags & (MVM_CF_TYPE_OBJECT | MVM_CF_STABLE))) {
MVMObject *new_obj_addr = (MVMObject *)new_addr;
if (REPR(new_obj_addr)->refs_frames)
MVM_gc_root_gen2_add(tc, (MVMCollectable *)new_obj_addr);
}
/* If we're going to sweep the second generation, also need
* to mark it as live. */
if (gen == MVMGCGenerations_Both)
new_addr->flags |= MVM_CF_GEN2_LIVE;
}
else {
/* No, so it will live in the nursery for another GC
* iteration. Allocate space in the nursery. */
new_addr = (MVMCollectable *)tc->nursery_alloc;
tc->nursery_alloc = (char *)tc->nursery_alloc + item->size;
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : copying an object %p (reprid %d) of size %d to tospace %p\n",
item, REPR(item)->ID, item->size, new_addr);
/* Copy the object to tospace and mark it as seen in the
* nursery (so the next time around it will move to the
* older generation, if it survives). */
memcpy(new_addr, item, item->size);
new_addr->flags |= MVM_CF_NURSERY_SEEN;
}
/* Store the forwarding pointer and update the original
* reference. */
if (MVM_GC_DEBUG_ENABLED(MVM_GC_DEBUG_COLLECT) && new_addr != item) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : updating handle %p from referent %p (reprid %d) to %p\n", item_ptr, item, REPR(item)->ID, new_addr);
}
*item_ptr = new_addr;
item->sc_forward_u.forwarder = new_addr;
/* Set the flag on the copy of item *in fromspace* to mark that the
forwarder pointer is valid. */
item->flags |= MVM_CF_FORWARDER_VALID;
}
/* make sure any rooted frames mark their stuff */
MVM_gc_worklist_mark_frame_roots(tc, worklist);
/* Finally, we need to mark the collectable (at its moved address).
* Track how many items we had before we mark it, in case we need
* to write barrier them post-move to uphold the generational
* invariant. */
gen2count = worklist->items;
MVM_gc_mark_collectable(tc, worklist, new_addr);
/* make sure any rooted frames mark their stuff */
MVM_gc_worklist_mark_frame_roots(tc, worklist);
/* In moving an object to generation 2, we may have left it pointing
* to nursery objects. If so, make sure it's in the gen2 roots. */
if (to_gen2) {
MVMCollectable **j;
MVMuint32 max = worklist->items, k;
for (k = gen2count; k < max; k++) {
j = worklist->list[k];
if (*j)
MVM_gc_write_barrier(tc, new_addr, *j);
}
}
}
}
/* Goes through the unmarked objects in the second generation heap and builds
* free lists out of them. Also does any required finalization. */
void MVM_gc_collect_free_gen2_unmarked(MVMThreadContext *tc) {
/* Visit each of the size class bins. */
MVMGen2Allocator *gen2 = tc->gen2;
MVMuint32 bin, obj_size, page, i;
char ***freelist_insert_pos;
for (bin = 0; bin < MVM_GEN2_BINS; bin++) {
/* If we've nothing allocated in this size class, skip it. */
if (gen2->size_classes[bin].pages == NULL)
continue;
/* Calculate object size for this bin. */
obj_size = (bin + 1) << MVM_GEN2_BIN_BITS;
/* freelist_insert_pos is a pointer to a memory location that
* stores the address of the last traversed free list node (char **). */
/* Initialize freelist insertion position to free list head. */
freelist_insert_pos = &gen2->size_classes[bin].free_list;
/* Visit each page. */
for (page = 0; page < gen2->size_classes[bin].num_pages; page++) {
/* Visit all the objects, looking for dead ones and reset the
* mark for each of them. */
char *cur_ptr = gen2->size_classes[bin].pages[page];
char *end_ptr = page + 1 == gen2->size_classes[bin].num_pages
? gen2->size_classes[bin].alloc_pos
: cur_ptr + obj_size * MVM_GEN2_PAGE_ITEMS;
char **last_insert_pos = NULL;
while (cur_ptr < end_ptr) {
MVMCollectable *col = (MVMCollectable *)cur_ptr;
/* Is this already a free list slot? If so, it becomes the
* new free list insert position. */
if (*freelist_insert_pos == (char **)cur_ptr) {
freelist_insert_pos = (char ***)cur_ptr;
}
/* Otherwise, it must be a collectable of some kind. Is it
* live? */
else if (col->forwarder) {
/* Yes; clear the mark. */
col->forwarder = NULL;
}
else {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : collecting an object %p in the gen2\n", col);
/* No, it's dead. Do any cleanup. */
if (!(col->flags & (MVM_CF_TYPE_OBJECT | MVM_CF_STABLE))) {
/* Object instance; call gc_free if needed. */
MVMObject *obj = (MVMObject *)col;
if (REPR(obj)->gc_free)
REPR(obj)->gc_free(tc, obj);
}
else if (col->flags & MVM_CF_TYPE_OBJECT) {
/* Type object; doesn't have anything extra that needs freeing. */
}
else if (col->flags & MVM_CF_STABLE) {
if (col->sc == (MVMSerializationContext *)1) {
/* We marked it dead last time, kill it. */
MVM_6model_stable_gc_free(tc, (MVMSTable *)col);
}
else {
if (MVM_load(&tc->gc_status) == MVMGCStatus_NONE) {
/* We're in global destruction, so enqueue to the end
* like we do in the nursery */
MVM_gc_collect_enqueue_stable_for_deletion(tc, (MVMSTable *)col);
} else {
/* There will definitely be another gc run, so mark it as "died last time". */
col->sc = (MVMSerializationContext *)1;
}
/* Skip the freelist updating. */
cur_ptr += obj_size;
continue;
}
}
else {
printf("item flags: %d\n", col->flags);
MVM_panic(MVM_exitcode_gcnursery, "Internal error: impossible case encountered in gen2 GC free");
}
/* Chain in to the free list. */
*((char **)cur_ptr) = (char *)*freelist_insert_pos;
*freelist_insert_pos = (char **)cur_ptr;
/* Update the pointer to the insert position to point to us */
freelist_insert_pos = (char ***)cur_ptr;
}
/* Move to the next object. */
cur_ptr += obj_size;
}
}
}
/* Also need to consider overflows. */
for (i = 0; i < gen2->num_overflows; i++) {
if (gen2->overflows[i]) {
MVMCollectable *col = gen2->overflows[i];
if (col->forwarder) {
/* A living over-sized object; just clear the mark. */
col->forwarder = NULL;
}
else {
/* Dead over-sized object. We know if it's this big it cannot
* be a type object or STable, so only need handle the simple
* object case. */
if (!(col->flags & (MVM_CF_TYPE_OBJECT | MVM_CF_STABLE))) {
MVMObject *obj = (MVMObject *)col;
if (REPR(obj)->gc_free)
REPR(obj)->gc_free(tc, obj);
}
else {
MVM_panic(MVM_exitcode_gcnursery, "Internal error: gen2 overflow contains non-object");
}
free(col);
gen2->overflows[i] = NULL;
}
}
}
}
/* Does a garbage collection run. Exactly what it does is configured by the
* couple of arguments that it takes.
*
* The what_to_do argument specifies where it should look for things to add
* to the worklist: everywhere, just at thread local stuff, or just in the
* thread's in-tray.
*
* The gen argument specifies whether to collect the nursery or both of the
* generations. Nursery collection is done by semi-space copying. Once an
* object is seen/copied once in the nursery (may be tuned in the future to
* twice or so - we'll see) then it is not copied to tospace, but instead
* promoted to the second generation. If we are collecting generation 2 also,
* then objects that are alive in the second generation are simply marked.
* Since the second generation is managed as a set of sized pools, there is
* much less motivation for any kind of copying/compaction; the internal
* fragmentation that makes finding a right-sized gap problematic will not
* happen.
*
* Note that it adds the roots and processes them in phases, to try to avoid
* building up a huge worklist. */
void MVM_gc_collect(MVMThreadContext *tc, MVMuint8 what_to_do, MVMuint8 gen) {
/* Create a GC worklist. */
MVMGCWorklist *worklist = MVM_gc_worklist_create(tc, gen != MVMGCGenerations_Nursery);
/* Initialize work passing data structure. */
WorkToPass wtp;
wtp.num_target_threads = 0;
wtp.target_work = NULL;
/* If we're starting a run (as opposed to just coming back here to do a
* little more work we got after we first thought we were done...) */
if (what_to_do != MVMGCWhatToDo_InTray) {
/* Swap fromspace and tospace. */
void * fromspace = tc->nursery_tospace;
void * tospace = tc->nursery_fromspace;
tc->nursery_fromspace = fromspace;
tc->nursery_tospace = tospace;
/* Reset nursery allocation pointers to the new tospace. */
tc->nursery_alloc = tospace;
tc->nursery_alloc_limit = (char *)tc->nursery_alloc + MVM_NURSERY_SIZE;
MVM_gc_worklist_add(tc, worklist, &tc->thread_obj);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from thread_obj\n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
/* Add permanent roots and process them; only one thread will do
* this, since they are instance-wide. */
if (what_to_do != MVMGCWhatToDo_NoInstance) {
MVM_gc_root_add_permanents_to_worklist(tc, worklist);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from instance permanents\n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
MVM_gc_root_add_instance_roots_to_worklist(tc, worklist);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from instance roots\n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
}
/* Add per-thread state to worklist and process it. */
MVM_gc_root_add_tc_roots_to_worklist(tc, worklist);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from TC objects\n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
/* Add current frame to worklist. */
MVM_gc_worklist_add_frame(tc, worklist, tc->cur_frame);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from current frame\n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
/* Add temporary roots and process them (these are per-thread). */
MVM_gc_root_add_temps_to_worklist(tc, worklist);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from thread temps\n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
/* Add things that are roots for the first generation because they are
* pointed to by objects in the second generation and process them
* (also per-thread). Note we need not do this if we're doing a full
* collection anyway (in fact, we must not for correctness, otherwise
* the gen2 rooting keeps them alive forever). */
if (gen == MVMGCGenerations_Nursery) {
MVM_gc_root_add_gen2s_to_worklist(tc, worklist);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from gen2 \n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
}
/* Find roots in frames and process them. */
if (tc->cur_frame) {
MVM_gc_worklist_add_frame(tc, worklist, tc->cur_frame);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from cur_frame \n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
}
/* Process anything in the in-tray. */
add_in_tray_to_worklist(tc, worklist);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from in tray \n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
/* At this point, we have probably done most of the work we will
* need to (only get more if another thread passes us more); zero
* out the remaining tospace. */
memset(tc->nursery_alloc, 0, (char *)tc->nursery_alloc_limit - (char *)tc->nursery_alloc);
}
else {
/* We just need to process anything in the in-tray. */
add_in_tray_to_worklist(tc, worklist);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from in tray \n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
}
/* Destroy the worklist. */
MVM_gc_worklist_destroy(tc, worklist);
/* Pass any work for other threads we accumulated but that didn't trigger
* the work passing threshold, then cleanup work passing list. */
if (wtp.num_target_threads) {
pass_leftover_work(tc, &wtp);
free(wtp.target_work);
}
/* If it was a full collection, some of the things in gen2 that we root
* due to point to gen1 objects may be dead. */
if (gen != MVMGCGenerations_Nursery)
MVM_gc_root_gen2_cleanup(tc);
}
/* Does a garbage collection run. Exactly what it does is configured by the
* couple of arguments that it takes.
*
* The what_to_do argument specifies where it should look for things to add
* to the worklist: everywhere, just at thread local stuff, or just in the
* thread's in-tray.
*
* The gen argument specifies whether to collect the nursery or both of the
* generations. Nursery collection is done by semi-space copying. Once an
* object is seen/copied once in the nursery (may be tuned in the future to
* twice or so - we'll see) then it is not copied to tospace, but instead
* promoted to the second generation. If we are collecting generation 2 also,
* then objects that are alive in the second generation are simply marked.
* Since the second generation is managed as a set of sized pools, there is
* much less motivation for any kind of copying/compaction; the internal
* fragmentation that makes finding a right-sized gap problematic will not
* happen.
*
* Note that it adds the roots and processes them in phases, to try to avoid
* building up a huge worklist. */
void MVM_gc_collect(MVMThreadContext *tc, MVMuint8 what_to_do, MVMuint8 gen) {
/* Create a GC worklist. */
MVMGCWorklist *worklist = MVM_gc_worklist_create(tc, gen != MVMGCGenerations_Nursery);
/* Initialize work passing data structure. */
WorkToPass wtp;
wtp.num_target_threads = 0;
wtp.target_work = NULL;
/* See what we need to work on this time. */
if (what_to_do == MVMGCWhatToDo_InTray) {
/* We just need to process anything in the in-tray. */
add_in_tray_to_worklist(tc, worklist);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from in tray \n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
}
else if (what_to_do == MVMGCWhatToDo_Finalizing) {
/* Need to process the finalizing queue. */
MVMuint32 i;
for (i = 0; i < tc->num_finalizing; i++)
MVM_gc_worklist_add(tc, worklist, &(tc->finalizing[i]));
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from finalizing \n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
}
else {
/* Main collection run. The current tospace becomes fromspace, with
* the size of the current tospace becoming stashed as the size of
* that fromspace. */
void *old_fromspace = tc->nursery_fromspace;
MVMuint32 old_fromspace_size = tc->nursery_fromspace_size;
tc->nursery_fromspace = tc->nursery_tospace;
tc->nursery_fromspace_size = tc->nursery_tospace_size;
/* Decide on this threads's tospace size. If fromspace was already at
* the maximum nursery size, then that is the new tospace size. If
* not, then see if this thread caused the current GC run, and grant
* it a bigger tospace. Otherwise, new tospace size is left as the
* last tospace size. */
if (tc->nursery_tospace_size < MVM_NURSERY_SIZE) {
if (tc->instance->thread_to_blame_for_gc == tc)
tc->nursery_tospace_size *= 2;
}
/* If the old fromspace matches the target size, just re-use it. If
* not, free it and allocate a new tospace. */
if (old_fromspace_size == tc->nursery_tospace_size) {
tc->nursery_tospace = old_fromspace;
}
else {
MVM_free(old_fromspace);
tc->nursery_tospace = MVM_calloc(1, tc->nursery_tospace_size);
}
/* Reset nursery allocation pointers to the new tospace. */
tc->nursery_alloc = tc->nursery_tospace;
tc->nursery_alloc_limit = (char *)tc->nursery_tospace + tc->nursery_tospace_size;
/* Add permanent roots and process them; only one thread will do
* this, since they are instance-wide. */
if (what_to_do != MVMGCWhatToDo_NoInstance) {
MVM_gc_root_add_permanents_to_worklist(tc, worklist, NULL);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from instance permanents\n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
MVM_gc_root_add_instance_roots_to_worklist(tc, worklist, NULL);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from instance roots\n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
}
/* Add per-thread state to worklist and process it. */
MVM_gc_root_add_tc_roots_to_worklist(tc, worklist, NULL);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from TC objects\n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
/* Walk current call stack, following caller chain until we reach a
* heap-allocated frame. Note that tc->cur_frame may itself be a heap
* frame, in which case we put it directly on the worklist as it can
* move. */
if (tc->cur_frame && MVM_FRAME_IS_ON_CALLSTACK(tc, tc->cur_frame)) {
MVMFrame *cur_frame = tc->cur_frame;
while (cur_frame && MVM_FRAME_IS_ON_CALLSTACK(tc, cur_frame)) {
MVM_gc_root_add_frame_roots_to_worklist(tc, worklist, cur_frame);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from a stack frame\n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
cur_frame = cur_frame->caller;
}
}
else {
MVM_gc_worklist_add(tc, worklist, &tc->cur_frame);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from current frame\n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
}
/* Add temporary roots and process them (these are per-thread). */
MVM_gc_root_add_temps_to_worklist(tc, worklist, NULL);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from thread temps\n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
/* Add things that are roots for the first generation because they are
* pointed to by objects in the second generation and process them
* (also per-thread). Note we need not do this if we're doing a full
* collection anyway (in fact, we must not for correctness, otherwise
* the gen2 rooting keeps them alive forever). */
if (gen == MVMGCGenerations_Nursery) {
MVM_gc_root_add_gen2s_to_worklist(tc, worklist);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from gen2 \n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
}
/* Process anything in the in-tray. */
add_in_tray_to_worklist(tc, worklist);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_COLLECT, "Thread %d run %d : processing %d items from in tray \n", worklist->items);
process_worklist(tc, worklist, &wtp, gen);
/* At this point, we have probably done most of the work we will
* need to (only get more if another thread passes us more); zero
* out the remaining tospace. */
memset(tc->nursery_alloc, 0, (char *)tc->nursery_alloc_limit - (char *)tc->nursery_alloc);
}
/* Destroy the worklist. */
MVM_gc_worklist_destroy(tc, worklist);
/* Pass any work for other threads we accumulated but that didn't trigger
* the work passing threshold, then cleanup work passing list. */
if (wtp.num_target_threads) {
pass_leftover_work(tc, &wtp);
MVM_free(wtp.target_work);
}
}
