static void finish_gc(MVMThreadContext *tc, MVMuint8 gen, MVMuint8 is_coordinator) {
MVMuint32 i, did_work;
/* Do any extra work that we have been passed. */
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : doing any work in thread in-trays\n");
did_work = 1;
while (did_work) {
did_work = 0;
for (i = 0; i < tc->gc_work_count; i++)
did_work += process_in_tray(tc->gc_work[i].tc, gen);
}
/* Decrement gc_finish to say we're done, and wait for termination. */
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Voting to finish\n");
MVM_decr(&tc->instance->gc_finish);
while (MVM_load(&tc->instance->gc_finish)) {
for (i = 0; i < 1000; i++)
; /* XXX Something HT-efficienter. */
/* XXX Here we can look to see if we got passed any work, and if so
* try to un-vote. */
}
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Termination agreed\n");
/* Co-ordinator should do final check over all the in-trays, and trigger
* collection until all is settled. Rest should wait. Additionally, after
* in-trays are settled, coordinator walks threads looking for anything
* that needs adding to the finalize queue. It then will make another
* iteration over in-trays to handle cross-thread references to objects
* needing finalization. For full collections, collected objects are then
* cleaned from all inter-generational sets, and finally any objects to
* be freed at the fixed size allocator's next safepoint are freed. */
if (is_coordinator) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : Co-ordinator handling in-tray clearing completion\n");
clear_intrays(tc, gen);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : Co-ordinator handling finalizers\n");
MVM_finalize_walk_queues(tc, gen);
clear_intrays(tc, gen);
if (gen == MVMGCGenerations_Both) {
MVMThread *cur_thread = (MVMThread *)MVM_load(&tc->instance->threads);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : Co-ordinator handling inter-gen root cleanup\n");
while (cur_thread) {
if (cur_thread->body.tc)
MVM_gc_root_gen2_cleanup(cur_thread->body.tc);
cur_thread = cur_thread->body.next;
}
}
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : Co-ordinator handling fixed-size allocator safepoint frees\n");
MVM_fixed_size_safepoint(tc, tc->instance->fsa);
MVM_profile_heap_take_snapshot(tc);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : Co-ordinator signalling in-trays clear\n");
MVM_store(&tc->instance->gc_intrays_clearing, 0);
}
else {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : Waiting for in-tray clearing completion\n");
while (MVM_load(&tc->instance->gc_intrays_clearing))
for (i = 0; i < 1000; i++)
; /* XXX Something HT-efficienter. */
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : Got in-tray clearing complete notice\n");
}
/* Reset GC status flags. This is also where thread destruction happens,
* and it needs to happen before we acknowledge this GC run is finished. */
for (i = 0; i < tc->gc_work_count; i++) {
MVMThreadContext *other = tc->gc_work[i].tc;
MVMThread *thread_obj = other->thread_obj;
if (MVM_load(&thread_obj->body.stage) == MVM_thread_stage_clearing_nursery) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : transferring gen2 of thread %d\n", other->thread_id);
MVM_gc_gen2_transfer(other, tc);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : destroying thread %d\n", other->thread_id);
MVM_tc_destroy(other);
tc->gc_work[i].tc = thread_obj->body.tc = NULL;
MVM_store(&thread_obj->body.stage, MVM_thread_stage_destroyed);
}
else {
/* Free gen2 unmarked if full collection. */
if (gen == MVMGCGenerations_Both) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : freeing gen2 of thread %d\n",
other->thread_id);
MVM_gc_collect_free_gen2_unmarked(other, 0);
}
/* Contribute this thread's promoted bytes. */
MVM_add(&tc->instance->gc_promoted_bytes_since_last_full, other->gc_promoted_bytes);
/* Collect nursery. */
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : collecting nursery uncopied of thread %d\n",
other->thread_id);
MVM_gc_collect_free_nursery_uncopied(other, tc->gc_work[i].limit);
/* Handle exited threads. */
if (MVM_load(&thread_obj->body.stage) == MVM_thread_stage_exited) {
/* Don't bother freeing gen2; we'll do it next time */
MVM_store(&thread_obj->body.stage, MVM_thread_stage_clearing_nursery);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : set thread %d clearing nursery stage to %d\n",
other->thread_id, (int)MVM_load(&thread_obj->body.stage));
}
/* Mark thread free to continue. */
MVM_cas(&other->gc_status, MVMGCStatus_STOLEN, MVMGCStatus_UNABLE);
MVM_cas(&other->gc_status, MVMGCStatus_INTERRUPT, MVMGCStatus_NONE);
}
}
/* Signal acknowledgement of completing the cleanup,
* except for STables, and if we're the final to do
* so, free the STables, which have been linked. */
if (MVM_decr(&tc->instance->gc_ack) == 2) {
/* Set it to zero (we're guaranteed the only ones trying to write to
* it here). Actual STable free in MVM_gc_enter_from_allocator. */
MVM_store(&tc->instance->gc_ack, 0);
}
}
/* This is called when the allocator finds it has run out of memory and wants
* to trigger a GC run. In this case, it's possible (probable, really) that it
* will need to do that triggering, notifying other running threads that the
* time has come to GC. */
void MVM_gc_enter_from_allocator(MVMThreadContext *tc) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Entered from allocate\n");
MVM_telemetry_timestamp(tc, "gc_enter_from_allocator");
/* Try to start the GC run. */
if (MVM_trycas(&tc->instance->gc_start, 0, 1)) {
MVMThread *last_starter = NULL;
MVMuint32 num_threads = 0;
/* Need to wait for other threads to reset their gc_status. */
while (MVM_load(&tc->instance->gc_ack)) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : waiting for other thread's gc_ack\n");
MVM_platform_thread_yield();
}
/* We are the winner of the GC starting race. This gives us some
* extra responsibilities as well as doing the usual things.
* First, increment GC sequence number. */
MVM_incr(&tc->instance->gc_seq_number);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE,
"Thread %d run %d : GC thread elected coordinator: starting gc seq %d\n",
(int)MVM_load(&tc->instance->gc_seq_number));
/* Decide if it will be a full collection. */
tc->instance->gc_full_collect = is_full_collection(tc);
MVM_telemetry_timestamp(tc, "won the gc starting race");
/* If profiling, record that GC is starting. */
if (tc->instance->profiling)
MVM_profiler_log_gc_start(tc, tc->instance->gc_full_collect);
/* Ensure our stolen list is empty. */
tc->gc_work_count = 0;
/* Flag that we didn't agree on this run that all the in-trays are
* cleared (a responsibility of the co-ordinator. */
MVM_store(&tc->instance->gc_intrays_clearing, 1);
/* We'll take care of our own work. */
add_work(tc, tc);
/* Find other threads, and signal or steal. */
do {
MVMThread *threads = (MVMThread *)MVM_load(&tc->instance->threads);
if (threads && threads != last_starter) {
MVMThread *head = threads;
MVMuint32 add;
while ((threads = (MVMThread *)MVM_casptr(&tc->instance->threads, head, NULL)) != head) {
head = threads;
}
add = signal_all_but(tc, head, last_starter);
last_starter = head;
if (add) {
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Found %d other threads\n", add);
MVM_add(&tc->instance->gc_start, add);
num_threads += add;
}
}
/* If there's an event loop thread, wake it up to participate. */
if (tc->instance->event_loop_wakeup)
uv_async_send(tc->instance->event_loop_wakeup);
} while (MVM_load(&tc->instance->gc_start) > 1);
/* Sanity checks. */
if (!MVM_trycas(&tc->instance->threads, NULL, last_starter))
MVM_panic(MVM_exitcode_gcorch, "threads list corrupted\n");
if (MVM_load(&tc->instance->gc_finish) != 0)
MVM_panic(MVM_exitcode_gcorch, "Finish votes was %"MVM_PRSz"\n", MVM_load(&tc->instance->gc_finish));
/* gc_ack gets an extra so the final acknowledger
* can also free the STables. */
MVM_store(&tc->instance->gc_finish, num_threads + 1);
MVM_store(&tc->instance->gc_ack, num_threads + 2);
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : finish votes is %d\n",
(int)MVM_load(&tc->instance->gc_finish));
/* Now we're ready to start, zero promoted since last full collection
* counter if this is a full collect. */
if (tc->instance->gc_full_collect)
MVM_store(&tc->instance->gc_promoted_bytes_since_last_full, 0);
/* This is a safe point for us to free any STables that have been marked
* for deletion in the previous collection (since we let finalization -
* which appends to this list - happen after we set threads on their
* way again, it's not safe to do it in the previous collection). */
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Freeing STables if needed\n");
MVM_gc_collect_free_stables(tc);
/* Signal to the rest to start */
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : coordinator signalling start\n");
if (MVM_decr(&tc->instance->gc_start) != 1)
MVM_panic(MVM_exitcode_gcorch, "Start votes was %"MVM_PRSz"\n", MVM_load(&tc->instance->gc_start));
/* Start collecting. */
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : coordinator entering run_gc\n");
run_gc(tc, MVMGCWhatToDo_All);
/* If profiling, record that GC is over. */
if (tc->instance->profiling)
MVM_profiler_log_gc_end(tc);
MVM_telemetry_timestamp(tc, "gc finished");
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : GC complete (cooridnator)\n");
}
else {
/* Another thread beat us to starting the GC sync process. Thus, act as
* if we were interrupted to GC. */
GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Lost coordinator election\n");
MVM_gc_enter_from_interrupt(tc);
}
}
MVMInstance * MVM_vm_create_instance(void) {
MVMInstance *instance;
int init_stat;
/* Set up instance data structure. */
instance = calloc(1, sizeof(MVMInstance));
/* Create the main thread's ThreadContext and stash it. */
instance->main_thread = MVM_tc_create(instance);
/* No user threads when we start, and next thread to be created gets ID 1
* (the main thread got ID 0). */
instance->num_user_threads = 0;
MVM_store(&instance->next_user_thread_id, 1);
/* Set up the permanent roots storage. */
instance->num_permroots = 0;
instance->alloc_permroots = 16;
instance->permroots = malloc(sizeof(MVMCollectable **) * instance->alloc_permroots);
init_mutex(instance->mutex_permroots, "permanent roots");
/* Set up REPR registry mutex. */
init_mutex(instance->mutex_repr_registry, "REPR registry");
/* Set up HLL config mutex. */
init_mutex(instance->mutex_hllconfigs, "hll configs");
/* Set up weak reference hash mutex. */
init_mutex(instance->mutex_sc_weakhash, "sc weakhash");
/* Set up loaded compunits hash mutex. */
init_mutex(instance->mutex_loaded_compunits, "loaded compunits");
/* Set up container registry mutex. */
init_mutex(instance->mutex_container_registry, "container registry");
/* Bootstrap 6model. It is assumed the GC will not be called during this. */
MVM_6model_bootstrap(instance->main_thread);
/* Fix up main thread's usecapture. */
instance->main_thread->cur_usecapture = MVM_repr_alloc_init(instance->main_thread, instance->CallCapture);
/* get libuv default event loop. */
instance->default_loop = instance->main_thread->loop;
/* Create main thread object, and also make it the start of the all threads
* linked list. */
MVM_store(&instance->threads,
(instance->main_thread->thread_obj = (MVMThread *)
REPR(instance->boot_types.BOOTThread)->allocate(
instance->main_thread, STABLE(instance->boot_types.BOOTThread))));
instance->threads->body.stage = MVM_thread_stage_started;
instance->threads->body.tc = instance->main_thread;
/* Create compiler registry */
instance->compiler_registry = MVM_repr_alloc_init(instance->main_thread, instance->boot_types.BOOTHash);
/* Set up compiler registr mutex. */
init_mutex(instance->mutex_compiler_registry, "compiler registry");
/* Create hll symbol tables */
instance->hll_syms = MVM_repr_alloc_init(instance->main_thread, instance->boot_types.BOOTHash);
/* Set up hll symbol tables mutex. */
init_mutex(instance->mutex_hll_syms, "hll syms");
/* Initialize string cclass handling. */
MVM_string_cclass_init(instance->main_thread);
/* Set up some string constants commonly used. */
string_consts(instance->main_thread);
return instance;
}
/* Create a new instance of the VM. */
MVMInstance * MVM_vm_create_instance(void) {
MVMInstance *instance;
char *spesh_log, *spesh_disable, *spesh_inline_disable, *spesh_osr_disable;
char *jit_log, *jit_disable, *jit_bytecode_dir;
char *dynvar_log;
int init_stat;
/* Set up instance data structure. */
instance = calloc(1, sizeof(MVMInstance));
/* Create the main thread's ThreadContext and stash it. */
instance->main_thread = MVM_tc_create(instance);
instance->main_thread->thread_id = 1;
/* No user threads when we start, and next thread to be created gets ID 2
* (the main thread got ID 1). */
instance->num_user_threads = 0;
MVM_store(&instance->next_user_thread_id, 2);
/* Set up the permanent roots storage. */
instance->num_permroots = 0;
instance->alloc_permroots = 16;
instance->permroots = MVM_malloc(sizeof(MVMCollectable **) * instance->alloc_permroots);
init_mutex(instance->mutex_permroots, "permanent roots");
/* Create fixed size allocator. */
instance->fsa = MVM_fixed_size_create(instance->main_thread);
/* Set up REPR registry mutex. */
init_mutex(instance->mutex_repr_registry, "REPR registry");
/* Set up HLL config mutex. */
init_mutex(instance->mutex_hllconfigs, "hll configs");
/* Set up DLL registry mutex. */
init_mutex(instance->mutex_dll_registry, "REPR registry");
/* Set up extension registry mutex. */
init_mutex(instance->mutex_ext_registry, "extension registry");
/* Set up extension op registry mutex. */
init_mutex(instance->mutex_extop_registry, "extension op registry");
/* Set up weak reference hash mutex. */
init_mutex(instance->mutex_sc_weakhash, "sc weakhash");
/* Set up loaded compunits hash mutex. */
init_mutex(instance->mutex_loaded_compunits, "loaded compunits");
/* Set up container registry mutex. */
init_mutex(instance->mutex_container_registry, "container registry");
/* Set up persistent object ID hash mutex. */
init_mutex(instance->mutex_object_ids, "object ID hash");
/* Allocate all things during following setup steps directly in gen2, as
* they will have program lifetime. */
MVM_gc_allocate_gen2_default_set(instance->main_thread);
init_mutex(instance->mutex_int_const_cache, "int constant cache");
instance->int_const_cache = calloc(1, sizeof(MVMIntConstCache));
/* Bootstrap 6model. It is assumed the GC will not be called during this. */
MVM_6model_bootstrap(instance->main_thread);
/* Fix up main thread's usecapture. */
instance->main_thread->cur_usecapture = MVM_repr_alloc_init(instance->main_thread, instance->CallCapture);
/* Initialize event loop thread starting mutex. */
init_mutex(instance->mutex_event_loop_start, "event loop thread start");
/* Create main thread object, and also make it the start of the all threads
* linked list. */
MVM_store(&instance->threads,
(instance->main_thread->thread_obj = (MVMThread *)
REPR(instance->boot_types.BOOTThread)->allocate(
instance->main_thread, STABLE(instance->boot_types.BOOTThread))));
instance->threads->body.stage = MVM_thread_stage_started;
instance->threads->body.tc = instance->main_thread;
instance->threads->body.thread_id = uv_thread_self();
/* Create compiler registry */
instance->compiler_registry = MVM_repr_alloc_init(instance->main_thread, instance->boot_types.BOOTHash);
/* Set up compiler registr mutex. */
init_mutex(instance->mutex_compiler_registry, "compiler registry");
/* Create hll symbol tables */
instance->hll_syms = MVM_repr_alloc_init(instance->main_thread, instance->boot_types.BOOTHash);
/* Set up hll symbol tables mutex. */
init_mutex(instance->mutex_hll_syms, "hll syms");
/* Initialize string cclass handling. */
MVM_string_cclass_init(instance->main_thread);
/* Create callsite intern pool. */
instance->callsite_interns = calloc(1, sizeof(MVMCallsiteInterns));
init_mutex(instance->mutex_callsite_interns, "callsite interns");
/* Allocate int to str cache. */
instance->int_to_str_cache = calloc(MVM_INT_TO_STR_CACHE_SIZE, sizeof(MVMString *));
/* Mutex for spesh installations, and check if we've a file we
* should log specializations to. */
init_mutex(instance->mutex_spesh_install, "spesh installations");
spesh_log = getenv("MVM_SPESH_LOG");
if (spesh_log && strlen(spesh_log))
instance->spesh_log_fh = fopen(spesh_log, "w");
spesh_disable = getenv("MVM_SPESH_DISABLE");
if (!spesh_disable || strlen(spesh_disable) == 0) {
instance->spesh_enabled = 1;
spesh_inline_disable = getenv("MVM_SPESH_INLINE_DISABLE");
if (!spesh_inline_disable || strlen(spesh_inline_disable) == 0)
instance->spesh_inline_enabled = 1;
spesh_osr_disable = getenv("MVM_SPESH_OSR_DISABLE");
if (!spesh_osr_disable || strlen(spesh_osr_disable) == 0)
instance->spesh_osr_enabled = 1;
}
jit_disable = getenv("MVM_JIT_DISABLE");
if (!jit_disable || strlen(jit_disable) == 0)
instance->jit_enabled = 1;
jit_log = getenv("MVM_JIT_LOG");
if (jit_log && strlen(jit_log))
instance->jit_log_fh = fopen(jit_log, "w");
jit_bytecode_dir = getenv("MVM_JIT_BYTECODE_DIR");
if (jit_bytecode_dir && strlen(jit_bytecode_dir))
instance->jit_bytecode_dir = jit_bytecode_dir;
dynvar_log = getenv("MVM_DYNVAR_LOG");
if (dynvar_log && strlen(dynvar_log))
instance->dynvar_log_fh = fopen(dynvar_log, "w");
else
instance->dynvar_log_fh = NULL;
/* Create std[in/out/err]. */
setup_std_handles(instance->main_thread);
/* Current instrumentation level starts at 1; used to trigger all frames
* to be verified before their first run. */
instance->instrumentation_level = 1;
/* Back to nursery allocation, now we're set up. */
MVM_gc_allocate_gen2_default_clear(instance->main_thread);
return instance;
}
/* Create a new instance of the VM. */
MVMInstance * MVM_vm_create_instance(void) {
MVMInstance *instance;
char *spesh_log, *spesh_nodelay, *spesh_disable, *spesh_inline_disable, *spesh_osr_disable;
char *jit_log, *jit_disable, *jit_bytecode_dir;
char *dynvar_log;
int init_stat;
/* Set up instance data structure. */
instance = MVM_calloc(1, sizeof(MVMInstance));
/* Create the main thread's ThreadContext and stash it. */
instance->main_thread = MVM_tc_create(instance);
instance->main_thread->thread_id = 1;
/* No user threads when we start, and next thread to be created gets ID 2
* (the main thread got ID 1). */
instance->num_user_threads = 0;
MVM_store(&instance->next_user_thread_id, 2);
/* Set up the permanent roots storage. */
instance->num_permroots = 0;
instance->alloc_permroots = 16;
instance->permroots = MVM_malloc(sizeof(MVMCollectable **) * instance->alloc_permroots);
instance->permroot_descriptions = MVM_malloc(sizeof(char *) * instance->alloc_permroots);
init_mutex(instance->mutex_permroots, "permanent roots");
/* Create fixed size allocator. */
instance->fsa = MVM_fixed_size_create(instance->main_thread);
/* Set up REPR registry mutex. */
init_mutex(instance->mutex_repr_registry, "REPR registry");
/* Set up HLL config mutex. */
init_mutex(instance->mutex_hllconfigs, "hll configs");
/* Set up DLL registry mutex. */
init_mutex(instance->mutex_dll_registry, "REPR registry");
/* Set up extension registry mutex. */
init_mutex(instance->mutex_ext_registry, "extension registry");
/* Set up extension op registry mutex. */
init_mutex(instance->mutex_extop_registry, "extension op registry");
/* Set up weak reference hash mutex. */
init_mutex(instance->mutex_sc_weakhash, "sc weakhash");
/* Set up loaded compunits hash mutex. */
init_mutex(instance->mutex_loaded_compunits, "loaded compunits");
/* Set up container registry mutex. */
init_mutex(instance->mutex_container_registry, "container registry");
/* Set up persistent object ID hash mutex. */
init_mutex(instance->mutex_object_ids, "object ID hash");
/* Allocate all things during following setup steps directly in gen2, as
* they will have program lifetime. */
MVM_gc_allocate_gen2_default_set(instance->main_thread);
/* Set up integer constant and string cache. */
init_mutex(instance->mutex_int_const_cache, "int constant cache");
instance->int_const_cache = MVM_calloc(1, sizeof(MVMIntConstCache));
instance->int_to_str_cache = MVM_calloc(MVM_INT_TO_STR_CACHE_SIZE, sizeof(MVMString *));
/* Bootstrap 6model. It is assumed the GC will not be called during this. */
MVM_6model_bootstrap(instance->main_thread);
/* Fix up main thread's usecapture and last_payload. */
instance->main_thread->cur_usecapture = MVM_repr_alloc_init(instance->main_thread, instance->CallCapture);
instance->main_thread->last_payload = instance->VMNull;
/* Initialize event loop thread starting mutex. */
init_mutex(instance->mutex_event_loop_start, "event loop thread start");
/* Create main thread object, and also make it the start of the all threads
* linked list. */
MVM_store(&instance->threads,
(instance->main_thread->thread_obj = (MVMThread *)
REPR(instance->boot_types.BOOTThread)->allocate(
instance->main_thread, STABLE(instance->boot_types.BOOTThread))));
instance->threads->body.stage = MVM_thread_stage_started;
instance->threads->body.tc = instance->main_thread;
instance->threads->body.native_thread_id = MVM_platform_thread_id();
instance->threads->body.thread_id = instance->main_thread->thread_id;
/* Create compiler registry */
instance->compiler_registry = MVM_repr_alloc_init(instance->main_thread, instance->boot_types.BOOTHash);
/* Set up compiler registr mutex. */
init_mutex(instance->mutex_compiler_registry, "compiler registry");
/* Create hll symbol tables */
instance->hll_syms = MVM_repr_alloc_init(instance->main_thread, instance->boot_types.BOOTHash);
/* Set up hll symbol tables mutex. */
init_mutex(instance->mutex_hll_syms, "hll syms");
/* Initialize Unicode database */
MVM_unicode_init(instance->main_thread);
/* Initialize string cclass handling. */
MVM_string_cclass_init(instance->main_thread);
/* Create callsite intern pool. */
instance->callsite_interns = MVM_calloc(1, sizeof(MVMCallsiteInterns));
init_mutex(instance->mutex_callsite_interns, "callsite interns");
/* There's some callsites we statically use all over the place. Intern
* them, so that spesh may end up optimizing more "internal" stuff. */
MVM_callsite_initialize_common(instance->main_thread);
/* Multi-cache additions mutex. */
init_mutex(instance->mutex_multi_cache_add, "multi-cache addition");
/* Current instrumentation level starts at 1; used to trigger all frames
* to be verified before their first run. */
instance->instrumentation_level = 1;
/* Mutex for spesh installations, and check if we've a file we
* should log specializations to. */
init_mutex(instance->mutex_spesh_install, "spesh installations");
spesh_log = getenv("MVM_SPESH_LOG");
if (spesh_log && strlen(spesh_log))
instance->spesh_log_fh = fopen_perhaps_with_pid(spesh_log, "w");
spesh_disable = getenv("MVM_SPESH_DISABLE");
if (!spesh_disable || strlen(spesh_disable) == 0) {
instance->spesh_enabled = 1;
spesh_inline_disable = getenv("MVM_SPESH_INLINE_DISABLE");
if (!spesh_inline_disable || strlen(spesh_inline_disable) == 0)
instance->spesh_inline_enabled = 1;
spesh_osr_disable = getenv("MVM_SPESH_OSR_DISABLE");
if (!spesh_osr_disable || strlen(spesh_osr_disable) == 0)
instance->spesh_osr_enabled = 1;
}
/* Should we specialize without warm up delays? Used to find bugs in the
* specializer and JIT. */
spesh_nodelay = getenv("MVM_SPESH_NODELAY");
if (spesh_nodelay && strlen(spesh_nodelay)) {
instance->spesh_nodelay = 1;
}
/* JIT environment/logging setup. */
jit_disable = getenv("MVM_JIT_DISABLE");
if (!jit_disable || strlen(jit_disable) == 0)
instance->jit_enabled = 1;
jit_log = getenv("MVM_JIT_LOG");
if (jit_log && strlen(jit_log))
instance->jit_log_fh = fopen_perhaps_with_pid(jit_log, "w");
jit_bytecode_dir = getenv("MVM_JIT_BYTECODE_DIR");
if (jit_bytecode_dir && strlen(jit_bytecode_dir)) {
char *bytecode_map_name = MVM_malloc(strlen(jit_bytecode_dir) + strlen("/jit-map.txt") + 1);
sprintf(bytecode_map_name, "%s/jit-map.txt", jit_bytecode_dir);
instance->jit_bytecode_map = fopen(bytecode_map_name, "w");
instance->jit_bytecode_dir = jit_bytecode_dir;
MVM_free(bytecode_map_name);
}
instance->jit_seq_nr = 0;
/* Various kinds of debugging that can be enabled. */
dynvar_log = getenv("MVM_DYNVAR_LOG");
if (dynvar_log && strlen(dynvar_log)) {
instance->dynvar_log_fh = fopen_perhaps_with_pid(dynvar_log, "w");
fprintf(instance->dynvar_log_fh, "+ x 0 0 0 0 0 %llu\n", uv_hrtime());
fflush(instance->dynvar_log_fh);
instance->dynvar_log_lasttime = uv_hrtime();
}
else
instance->dynvar_log_fh = NULL;
instance->nfa_debug_enabled = getenv("MVM_NFA_DEB") ? 1 : 0;
if (getenv("MVM_CROSS_THREAD_WRITE_LOG")) {
instance->cross_thread_write_logging = 1;
instance->cross_thread_write_logging_include_locked =
getenv("MVM_CROSS_THREAD_WRITE_LOG_INCLUDE_LOCKED") ? 1 : 0;
instance->instrumentation_level++;
init_mutex(instance->mutex_cross_thread_write_logging,
"cross thread write logging output");
}
else {
instance->cross_thread_write_logging = 0;
}
/* Set up NFG state mutation mutex. */
instance->nfg = calloc(1, sizeof(MVMNFGState));
init_mutex(instance->nfg->update_mutex, "NFG update mutex");
/* Create std[in/out/err]. */
setup_std_handles(instance->main_thread);
/* Back to nursery allocation, now we're set up. */
MVM_gc_allocate_gen2_default_clear(instance->main_thread);
return instance;
}
/* Enters the work loop. */
static void worker(MVMThreadContext *tc, MVMCallsite *callsite, MVMRegister *args) {
MVMObject *updated_static_frames = MVM_repr_alloc_init(tc,
tc->instance->boot_types.BOOTArray);
MVMObject *previous_static_frames = MVM_repr_alloc_init(tc,
tc->instance->boot_types.BOOTArray);
tc->instance->speshworker_thread_id = tc->thread_obj->body.thread_id;
MVMROOT2(tc, updated_static_frames, previous_static_frames, {
while (1) {
MVMObject *log_obj;
MVMuint64 start_time;
unsigned int interval_id;
if (MVM_spesh_debug_enabled(tc))
start_time = uv_hrtime();
log_obj = MVM_repr_shift_o(tc, tc->instance->spesh_queue);
if (MVM_spesh_debug_enabled(tc)) {
MVM_spesh_debug_printf(tc,
"Received Logs\n"
"=============\n\n"
"Was waiting %dus for logs on the log queue.\n\n",
(int)((uv_hrtime() - start_time) / 1000));
}
if (tc->instance->main_thread->prof_data)
MVM_profiler_log_spesh_start(tc);
interval_id = MVM_telemetry_interval_start(tc, "spesh worker consuming a log");
uv_mutex_lock(&(tc->instance->mutex_spesh_sync));
tc->instance->spesh_working = 1;
uv_mutex_unlock(&(tc->instance->mutex_spesh_sync));
tc->instance->spesh_stats_version++;
if (log_obj->st->REPR->ID == MVM_REPR_ID_MVMSpeshLog) {
MVMSpeshLog *sl = (MVMSpeshLog *)log_obj;
MVM_telemetry_interval_annotate((uintptr_t)sl->body.thread->body.tc, interval_id, "from this thread");
MVMROOT(tc, sl, {
MVMThreadContext *stc;
MVMuint32 i;
MVMuint32 n;
/* Update stats, and if we're logging dump each of them. */
tc->instance->spesh_stats_version++;
if (MVM_spesh_debug_enabled(tc))
start_time = uv_hrtime();
MVM_spesh_stats_update(tc, sl, updated_static_frames);
n = MVM_repr_elems(tc, updated_static_frames);
if (MVM_spesh_debug_enabled(tc)) {
MVM_spesh_debug_printf(tc,
"Statistics Updated\n"
"==================\n"
"%d frames had their statistics updated in %dus.\n\n",
(int)n, (int)((uv_hrtime() - start_time) / 1000));
for (i = 0; i < n; i++) {
char *dump = MVM_spesh_dump_stats(tc, (MVMStaticFrame* )
MVM_repr_at_pos_o(tc, updated_static_frames, i));
MVM_spesh_debug_printf(tc, "%s==========\n\n", dump);
MVM_free(dump);
}
}
MVM_telemetry_interval_annotate((uintptr_t)n, interval_id, "stats for this many frames");
GC_SYNC_POINT(tc);
/* Form a specialization plan. */
if (MVM_spesh_debug_enabled(tc))
start_time = uv_hrtime();
tc->instance->spesh_plan = MVM_spesh_plan(tc, updated_static_frames);
if (MVM_spesh_debug_enabled(tc)) {
n = tc->instance->spesh_plan->num_planned;
MVM_spesh_debug_printf(tc,
"Specialization Plan\n"
"===================\n"
"%u specialization(s) will be produced (planned in %dus).\n\n",
n, (int)((uv_hrtime() - start_time) / 1000));
for (i = 0; i < n; i++) {
char *dump = MVM_spesh_dump_planned(tc,
&(tc->instance->spesh_plan->planned[i]));
MVM_spesh_debug_printf(tc, "%s==========\n\n", dump);
MVM_free(dump);
}
}
MVM_telemetry_interval_annotate((uintptr_t)tc->instance->spesh_plan->num_planned, interval_id,
"this many specializations planned");
GC_SYNC_POINT(tc);
/* Implement the plan and then discard it. */
n = tc->instance->spesh_plan->num_planned;
for (i = 0; i < n; i++) {
MVM_spesh_candidate_add(tc, &(tc->instance->spesh_plan->planned[i]));
GC_SYNC_POINT(tc);
}
MVM_spesh_plan_destroy(tc, tc->instance->spesh_plan);
tc->instance->spesh_plan = NULL;
/* Clear up stats that didn't get updated for a while,
* then add frames updated this time into the previously
* updated array. */
MVM_spesh_stats_cleanup(tc, previous_static_frames);
n = MVM_repr_elems(tc, updated_static_frames);
for (i = 0; i < n; i++)
MVM_repr_push_o(tc, previous_static_frames,
MVM_repr_at_pos_o(tc, updated_static_frames, i));
/* Clear updated static frames array. */
MVM_repr_pos_set_elems(tc, updated_static_frames, 0);
/* Allow the sending thread to produce more logs again,
* putting a new spesh log in place if needed. */
stc = sl->body.thread->body.tc;
if (stc && !sl->body.was_compunit_bumped)
if (MVM_incr(&(stc->spesh_log_quota)) == 0) {
stc->spesh_log = MVM_spesh_log_create(tc, sl->body.thread);
MVM_telemetry_timestamp(stc, "logging restored after quota had run out");
}
/* If needed, signal sending thread that it can continue. */
if (sl->body.block_mutex) {
uv_mutex_lock(sl->body.block_mutex);
MVM_store(&(sl->body.completed), 1);
uv_cond_signal(sl->body.block_condvar);
uv_mutex_unlock(sl->body.block_mutex);
}
{
MVMSpeshLogEntry *entries = sl->body.entries;
sl->body.entries = NULL;
MVM_free(entries);
}
});
}
else if (MVM_is_null(tc, log_obj)) {
/* Takes a static frame and a thread context. Invokes the static frame. */
void MVM_frame_invoke(MVMThreadContext *tc, MVMStaticFrame *static_frame,
MVMCallsite *callsite, MVMRegister *args,
MVMFrame *outer, MVMObject *code_ref) {
MVMFrame *frame;
MVMuint32 pool_index, found_spesh;
MVMFrame *node;
int fresh = 0;
MVMStaticFrameBody *static_frame_body = &static_frame->body;
/* If the frame was never invoked before, need initial calculations
* and verification. */
if (!static_frame_body->invoked)
prepare_and_verify_static_frame(tc, static_frame);
/* Get frame body from the re-use pool, or allocate it. */
pool_index = static_frame_body->pool_index;
if (pool_index >= tc->frame_pool_table_size)
grow_frame_pool(tc, pool_index);
node = tc->frame_pool_table[pool_index];
if (node == NULL) {
fresh = 1;
frame = malloc(sizeof(MVMFrame));
frame->params.named_used = NULL;
/* Ensure special return pointers and continuation tags are null. */
frame->special_return = NULL;
frame->special_unwind = NULL;
frame->continuation_tags = NULL;
}
else {
tc->frame_pool_table[pool_index] = node->outer;
node->outer = NULL;
frame = node;
}
#if MVM_HLL_PROFILE_CALLS
frame->profile_index = tc->profile_index;
tc->profile_data[frame->profile_index].duration_nanos = MVM_platform_now();
tc->profile_data[frame->profile_index].callsite_id = 0; /* XXX get a real callsite id */
tc->profile_data[frame->profile_index].code_id = 0; /* XXX get a real code id */
/* increment the profile data index */
++tc->profile_index;
if (tc->profile_index == tc->profile_data_size) {
tc->profile_data_size *= 2;
tc->profile_data = realloc(tc->profile_data, tc->profile_data_size);
}
#endif
/* Copy thread context (back?) into the frame. */
frame->tc = tc;
/* Set static frame. */
frame->static_info = static_frame;
/* Store the code ref (NULL at the top-level). */
frame->code_ref = code_ref;
/* Allocate space for lexicals and work area, copying the default lexical
* environment into place. */
if (static_frame_body->env_size) {
if (fresh)
frame->env = malloc(static_frame_body->env_size);
memcpy(frame->env, static_frame_body->static_env, static_frame_body->env_size);
}
else {
frame->env = NULL;
}
if (static_frame_body->work_size) {
if (fresh || !frame->work)
frame->work = malloc(static_frame_body->work_size);
memset(frame->work, 0, static_frame_body->work_size);
}
else {
frame->work = NULL;
}
/* Calculate args buffer position and make sure current call site starts
* empty. */
frame->args = static_frame_body->work_size ?
frame->work + static_frame_body->num_locals :
NULL;
frame->cur_args_callsite = NULL;
/* Outer. */
if (outer) {
/* We were provided with an outer frame; just ensure that it is
* based on the correct static frame (compare on bytecode address
* to come with nqp::freshcoderef). */
if (outer->static_info->body.bytecode == static_frame_body->outer->body.bytecode)
frame->outer = outer;
else
MVM_exception_throw_adhoc(tc,
"When invoking %s, Provided outer frame %p (%s %s) does not match expected static frame type %p (%s %s)",
static_frame_body->name ? MVM_string_utf8_encode_C_string(tc, static_frame_body->name) : "<anonymous static frame>",
outer->static_info,
MVM_repr_get_by_id(tc, REPR(outer->static_info)->ID)->name,
outer->static_info->body.name ? MVM_string_utf8_encode_C_string(tc, outer->static_info->body.name) : "<anonymous static frame>",
static_frame_body->outer,
MVM_repr_get_by_id(tc, REPR(static_frame_body->outer)->ID)->name,
static_frame_body->outer->body.name ? MVM_string_utf8_encode_C_string(tc, static_frame_body->outer->body.name) : "<anonymous static frame>");
}
else if (static_frame_body->static_code && static_frame_body->static_code->body.outer) {
/* We're lacking an outer, but our static code object may have one.
* This comes up in the case of cloned protoregexes, for example. */
frame->outer = static_frame_body->static_code->body.outer;
}
else if (static_frame_body->outer) {
/* Auto-close, and cache it in the static frame. */
frame->outer = autoclose(tc, static_frame_body->outer);
static_frame_body->static_code->body.outer = MVM_frame_inc_ref(tc, frame->outer);
}
else {
frame->outer = NULL;
}
if (frame->outer)
MVM_frame_inc_ref(tc, frame->outer);
/* Caller is current frame in the thread context. */
if (tc->cur_frame)
frame->caller = MVM_frame_inc_ref(tc, tc->cur_frame);
else
frame->caller = NULL;
frame->keep_caller = 0;
frame->in_continuation = 0;
/* Initial reference count is 1 by virtue of it being the currently
* executing frame. */
MVM_store(&frame->ref_count, 1);
MVM_store(&frame->gc_seq_number, 0);
/* Initialize argument processing. */
MVM_args_proc_init(tc, &frame->params, callsite, args);
/* Make sure there's no frame context pointer and special return data
* won't be marked. */
frame->context_object = NULL;
frame->mark_special_return_data = NULL;
/* Clear frame flags. */
frame->flags = 0;
/* See if any specializations apply. */
found_spesh = 0;
if (++static_frame_body->invocations >= 10 && callsite->is_interned) {
/* Look for specialized bytecode. */
MVMint32 num_spesh = static_frame_body->num_spesh_candidates;
MVMint32 i, j;
for (i = 0; i < num_spesh; i++) {
MVMSpeshCandidate *cand = &static_frame_body->spesh_candidates[i];
if (cand->cs == callsite) {
MVMint32 match = 1;
for (j = 0; j < cand->num_guards; j++) {
MVMint32 pos = cand->guards[j].slot;
MVMSTable *st = (MVMSTable *)cand->guards[j].match;
MVMObject *arg = args[pos].o;
if (!arg) {
match = 0;
break;
}
switch (cand->guards[j].kind) {
case MVM_SPESH_GUARD_CONC:
if (!IS_CONCRETE(arg) || STABLE(arg) != st)
match = 0;
break;
case MVM_SPESH_GUARD_TYPE:
if (IS_CONCRETE(arg) || STABLE(arg) != st)
match = 0;
break;
case MVM_SPESH_GUARD_DC_CONC: {
MVMRegister dc;
STABLE(arg)->container_spec->fetch(tc, arg, &dc);
if (!dc.o || !IS_CONCRETE(dc.o) || STABLE(dc.o) != st)
match = 0;
break;
}
case MVM_SPESH_GUARD_DC_TYPE: {
MVMRegister dc;
STABLE(arg)->container_spec->fetch(tc, arg, &dc);
if (!dc.o || IS_CONCRETE(dc.o) || STABLE(dc.o) != st)
match = 0;
break;
}
}
if (!match)
break;
}
if (match) {
frame->effective_bytecode = cand->bytecode;
frame->effective_handlers = cand->handlers;
frame->effective_spesh_slots = cand->spesh_slots;
frame->spesh_cand = cand;
found_spesh = 1;
break;
}
}
}
/* If we didn't find any, and we're below the limit, can generate a
* specialization. */
if (!found_spesh && num_spesh < MVM_SPESH_LIMIT && tc->instance->spesh_enabled) {
MVMSpeshCandidate *cand = MVM_spesh_candidate_generate(tc, static_frame,
callsite, args);
if (cand) {
frame->effective_bytecode = cand->bytecode;
frame->effective_handlers = cand->handlers;
frame->effective_spesh_slots = cand->spesh_slots;
frame->spesh_cand = cand;
found_spesh = 1;
}
}
}
if (!found_spesh) {
frame->effective_bytecode = static_frame_body->bytecode;
frame->effective_handlers = static_frame_body->handlers;
frame->spesh_cand = NULL;
}
/* Update interpreter and thread context, so next execution will use this
* frame. */
tc->cur_frame = frame;
*(tc->interp_cur_op) = frame->effective_bytecode;
*(tc->interp_bytecode_start) = frame->effective_bytecode;
*(tc->interp_reg_base) = frame->work;
*(tc->interp_cu) = static_frame_body->cu;
/* If we need to do so, make clones of things in the lexical environment
* that need it. Note that we do this after tc->cur_frame became the
* current frame, to make sure these new objects will certainly get
* marked if GC is triggered along the way. */
if (static_frame_body->static_env_flags) {
/* Drag everything out of static_frame_body before we start,
* as GC action may invalidate it. */
MVMuint8 *flags = static_frame_body->static_env_flags;
MVMint64 numlex = static_frame_body->num_lexicals;
MVMRegister *state = NULL;
MVMint64 state_act = 0; /* 0 = none so far, 1 = first time, 2 = later */
MVMint64 i;
for (i = 0; i < numlex; i++) {
switch (flags[i]) {
case 0: break;
case 1:
frame->env[i].o = MVM_repr_clone(tc, frame->env[i].o);
break;
case 2:
redo_state:
switch (state_act) {
case 0:
if (!frame->code_ref)
MVM_exception_throw_adhoc(tc,
"Frame must have code-ref to have state variables");
state = ((MVMCode *)frame->code_ref)->body.state_vars;
if (state) {
/* Already have state vars; pull them from this. */
state_act = 2;
}
else {
/* Allocate storage for state vars. */
state = malloc(frame->static_info->body.env_size);
memset(state, 0, frame->static_info->body.env_size);
((MVMCode *)frame->code_ref)->body.state_vars = state;
state_act = 1;
/* Note that this frame should run state init code. */
frame->flags |= MVM_FRAME_FLAG_STATE_INIT;
}
goto redo_state;
case 1:
frame->env[i].o = MVM_repr_clone(tc, frame->env[i].o);
MVM_ASSIGN_REF(tc, &(frame->code_ref->header), state[i].o, frame->env[i].o);
break;
case 2:
frame->env[i].o = state[i].o;
break;
}
break;
default:
MVM_exception_throw_adhoc(tc,
"Unknown lexical environment setup flag");
}
}
}
}
