static void finish_gc(MVMThreadContext *tc, MVMuint8 gen, MVMuint8 is_coordinator) { MVMuint32 i, did_work; /* Do any extra work that we have been passed. */ GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : doing any work in thread in-trays\n"); did_work = 1; while (did_work) { did_work = 0; for (i = 0; i < tc->gc_work_count; i++) did_work += process_in_tray(tc->gc_work[i].tc, gen); } /* Decrement gc_finish to say we're done, and wait for termination. */ GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Voting to finish\n"); MVM_decr(&tc->instance->gc_finish); while (MVM_load(&tc->instance->gc_finish)) { for (i = 0; i < 1000; i++) ; /* XXX Something HT-efficienter. */ /* XXX Here we can look to see if we got passed any work, and if so * try to un-vote. */ } GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Termination agreed\n"); /* Co-ordinator should do final check over all the in-trays, and trigger * collection until all is settled. Rest should wait. Additionally, after * in-trays are settled, coordinator walks threads looking for anything * that needs adding to the finalize queue. It then will make another * iteration over in-trays to handle cross-thread references to objects * needing finalization. For full collections, collected objects are then * cleaned from all inter-generational sets, and finally any objects to * be freed at the fixed size allocator's next safepoint are freed. */ if (is_coordinator) { GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Co-ordinator handling in-tray clearing completion\n"); clear_intrays(tc, gen); GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Co-ordinator handling finalizers\n"); MVM_finalize_walk_queues(tc, gen); clear_intrays(tc, gen); if (gen == MVMGCGenerations_Both) { MVMThread *cur_thread = (MVMThread *)MVM_load(&tc->instance->threads); GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Co-ordinator handling inter-gen root cleanup\n"); while (cur_thread) { if (cur_thread->body.tc) MVM_gc_root_gen2_cleanup(cur_thread->body.tc); cur_thread = cur_thread->body.next; } } GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Co-ordinator handling fixed-size allocator safepoint frees\n"); MVM_fixed_size_safepoint(tc, tc->instance->fsa); MVM_profile_heap_take_snapshot(tc); GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Co-ordinator signalling in-trays clear\n"); MVM_store(&tc->instance->gc_intrays_clearing, 0); } else { GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Waiting for in-tray clearing completion\n"); while (MVM_load(&tc->instance->gc_intrays_clearing)) for (i = 0; i < 1000; i++) ; /* XXX Something HT-efficienter. */ GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Got in-tray clearing complete notice\n"); } /* Reset GC status flags. This is also where thread destruction happens, * and it needs to happen before we acknowledge this GC run is finished. */ for (i = 0; i < tc->gc_work_count; i++) { MVMThreadContext *other = tc->gc_work[i].tc; MVMThread *thread_obj = other->thread_obj; if (MVM_load(&thread_obj->body.stage) == MVM_thread_stage_clearing_nursery) { GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : transferring gen2 of thread %d\n", other->thread_id); MVM_gc_gen2_transfer(other, tc); GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : destroying thread %d\n", other->thread_id); MVM_tc_destroy(other); tc->gc_work[i].tc = thread_obj->body.tc = NULL; MVM_store(&thread_obj->body.stage, MVM_thread_stage_destroyed); } else { /* Free gen2 unmarked if full collection. */ if (gen == MVMGCGenerations_Both) { GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : freeing gen2 of thread %d\n", other->thread_id); MVM_gc_collect_free_gen2_unmarked(other, 0); } /* Contribute this thread's promoted bytes. */ MVM_add(&tc->instance->gc_promoted_bytes_since_last_full, other->gc_promoted_bytes); /* Collect nursery. */ GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : collecting nursery uncopied of thread %d\n", other->thread_id); MVM_gc_collect_free_nursery_uncopied(other, tc->gc_work[i].limit); /* Handle exited threads. */ if (MVM_load(&thread_obj->body.stage) == MVM_thread_stage_exited) { /* Don't bother freeing gen2; we'll do it next time */ MVM_store(&thread_obj->body.stage, MVM_thread_stage_clearing_nursery); GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : set thread %d clearing nursery stage to %d\n", other->thread_id, (int)MVM_load(&thread_obj->body.stage)); } /* Mark thread free to continue. */ MVM_cas(&other->gc_status, MVMGCStatus_STOLEN, MVMGCStatus_UNABLE); MVM_cas(&other->gc_status, MVMGCStatus_INTERRUPT, MVMGCStatus_NONE); } } /* Signal acknowledgement of completing the cleanup, * except for STables, and if we're the final to do * so, free the STables, which have been linked. */ if (MVM_decr(&tc->instance->gc_ack) == 2) { /* Set it to zero (we're guaranteed the only ones trying to write to * it here). Actual STable free in MVM_gc_enter_from_allocator. */ MVM_store(&tc->instance->gc_ack, 0); } }
/* This is called when the allocator finds it has run out of memory and wants * to trigger a GC run. In this case, it's possible (probable, really) that it * will need to do that triggering, notifying other running threads that the * time has come to GC. */ void MVM_gc_enter_from_allocator(MVMThreadContext *tc) { GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Entered from allocate\n"); MVM_telemetry_timestamp(tc, "gc_enter_from_allocator"); /* Try to start the GC run. */ if (MVM_trycas(&tc->instance->gc_start, 0, 1)) { MVMThread *last_starter = NULL; MVMuint32 num_threads = 0; /* Need to wait for other threads to reset their gc_status. */ while (MVM_load(&tc->instance->gc_ack)) { GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : waiting for other thread's gc_ack\n"); MVM_platform_thread_yield(); } /* We are the winner of the GC starting race. This gives us some * extra responsibilities as well as doing the usual things. * First, increment GC sequence number. */ MVM_incr(&tc->instance->gc_seq_number); GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : GC thread elected coordinator: starting gc seq %d\n", (int)MVM_load(&tc->instance->gc_seq_number)); /* Decide if it will be a full collection. */ tc->instance->gc_full_collect = is_full_collection(tc); MVM_telemetry_timestamp(tc, "won the gc starting race"); /* If profiling, record that GC is starting. */ if (tc->instance->profiling) MVM_profiler_log_gc_start(tc, tc->instance->gc_full_collect); /* Ensure our stolen list is empty. */ tc->gc_work_count = 0; /* Flag that we didn't agree on this run that all the in-trays are * cleared (a responsibility of the co-ordinator. */ MVM_store(&tc->instance->gc_intrays_clearing, 1); /* We'll take care of our own work. */ add_work(tc, tc); /* Find other threads, and signal or steal. */ do { MVMThread *threads = (MVMThread *)MVM_load(&tc->instance->threads); if (threads && threads != last_starter) { MVMThread *head = threads; MVMuint32 add; while ((threads = (MVMThread *)MVM_casptr(&tc->instance->threads, head, NULL)) != head) { head = threads; } add = signal_all_but(tc, head, last_starter); last_starter = head; if (add) { GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Found %d other threads\n", add); MVM_add(&tc->instance->gc_start, add); num_threads += add; } } /* If there's an event loop thread, wake it up to participate. */ if (tc->instance->event_loop_wakeup) uv_async_send(tc->instance->event_loop_wakeup); } while (MVM_load(&tc->instance->gc_start) > 1); /* Sanity checks. */ if (!MVM_trycas(&tc->instance->threads, NULL, last_starter)) MVM_panic(MVM_exitcode_gcorch, "threads list corrupted\n"); if (MVM_load(&tc->instance->gc_finish) != 0) MVM_panic(MVM_exitcode_gcorch, "Finish votes was %"MVM_PRSz"\n", MVM_load(&tc->instance->gc_finish)); /* gc_ack gets an extra so the final acknowledger * can also free the STables. */ MVM_store(&tc->instance->gc_finish, num_threads + 1); MVM_store(&tc->instance->gc_ack, num_threads + 2); GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : finish votes is %d\n", (int)MVM_load(&tc->instance->gc_finish)); /* Now we're ready to start, zero promoted since last full collection * counter if this is a full collect. */ if (tc->instance->gc_full_collect) MVM_store(&tc->instance->gc_promoted_bytes_since_last_full, 0); /* This is a safe point for us to free any STables that have been marked * for deletion in the previous collection (since we let finalization - * which appends to this list - happen after we set threads on their * way again, it's not safe to do it in the previous collection). */ GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Freeing STables if needed\n"); MVM_gc_collect_free_stables(tc); /* Signal to the rest to start */ GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : coordinator signalling start\n"); if (MVM_decr(&tc->instance->gc_start) != 1) MVM_panic(MVM_exitcode_gcorch, "Start votes was %"MVM_PRSz"\n", MVM_load(&tc->instance->gc_start)); /* Start collecting. */ GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : coordinator entering run_gc\n"); run_gc(tc, MVMGCWhatToDo_All); /* If profiling, record that GC is over. */ if (tc->instance->profiling) MVM_profiler_log_gc_end(tc); MVM_telemetry_timestamp(tc, "gc finished"); GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : GC complete (cooridnator)\n"); } else { /* Another thread beat us to starting the GC sync process. Thus, act as * if we were interrupted to GC. */ GCDEBUG_LOG(tc, MVM_GC_DEBUG_ORCHESTRATE, "Thread %d run %d : Lost coordinator election\n"); MVM_gc_enter_from_interrupt(tc); } }
MVMInstance * MVM_vm_create_instance(void) { MVMInstance *instance; int init_stat; /* Set up instance data structure. */ instance = calloc(1, sizeof(MVMInstance)); /* Create the main thread's ThreadContext and stash it. */ instance->main_thread = MVM_tc_create(instance); /* No user threads when we start, and next thread to be created gets ID 1 * (the main thread got ID 0). */ instance->num_user_threads = 0; MVM_store(&instance->next_user_thread_id, 1); /* Set up the permanent roots storage. */ instance->num_permroots = 0; instance->alloc_permroots = 16; instance->permroots = malloc(sizeof(MVMCollectable **) * instance->alloc_permroots); init_mutex(instance->mutex_permroots, "permanent roots"); /* Set up REPR registry mutex. */ init_mutex(instance->mutex_repr_registry, "REPR registry"); /* Set up HLL config mutex. */ init_mutex(instance->mutex_hllconfigs, "hll configs"); /* Set up weak reference hash mutex. */ init_mutex(instance->mutex_sc_weakhash, "sc weakhash"); /* Set up loaded compunits hash mutex. */ init_mutex(instance->mutex_loaded_compunits, "loaded compunits"); /* Set up container registry mutex. */ init_mutex(instance->mutex_container_registry, "container registry"); /* Bootstrap 6model. It is assumed the GC will not be called during this. */ MVM_6model_bootstrap(instance->main_thread); /* Fix up main thread's usecapture. */ instance->main_thread->cur_usecapture = MVM_repr_alloc_init(instance->main_thread, instance->CallCapture); /* get libuv default event loop. */ instance->default_loop = instance->main_thread->loop; /* Create main thread object, and also make it the start of the all threads * linked list. */ MVM_store(&instance->threads, (instance->main_thread->thread_obj = (MVMThread *) REPR(instance->boot_types.BOOTThread)->allocate( instance->main_thread, STABLE(instance->boot_types.BOOTThread)))); instance->threads->body.stage = MVM_thread_stage_started; instance->threads->body.tc = instance->main_thread; /* Create compiler registry */ instance->compiler_registry = MVM_repr_alloc_init(instance->main_thread, instance->boot_types.BOOTHash); /* Set up compiler registr mutex. */ init_mutex(instance->mutex_compiler_registry, "compiler registry"); /* Create hll symbol tables */ instance->hll_syms = MVM_repr_alloc_init(instance->main_thread, instance->boot_types.BOOTHash); /* Set up hll symbol tables mutex. */ init_mutex(instance->mutex_hll_syms, "hll syms"); /* Initialize string cclass handling. */ MVM_string_cclass_init(instance->main_thread); /* Set up some string constants commonly used. */ string_consts(instance->main_thread); return instance; }
/* Create a new instance of the VM. */ MVMInstance * MVM_vm_create_instance(void) { MVMInstance *instance; char *spesh_log, *spesh_disable, *spesh_inline_disable, *spesh_osr_disable; char *jit_log, *jit_disable, *jit_bytecode_dir; char *dynvar_log; int init_stat; /* Set up instance data structure. */ instance = calloc(1, sizeof(MVMInstance)); /* Create the main thread's ThreadContext and stash it. */ instance->main_thread = MVM_tc_create(instance); instance->main_thread->thread_id = 1; /* No user threads when we start, and next thread to be created gets ID 2 * (the main thread got ID 1). */ instance->num_user_threads = 0; MVM_store(&instance->next_user_thread_id, 2); /* Set up the permanent roots storage. */ instance->num_permroots = 0; instance->alloc_permroots = 16; instance->permroots = MVM_malloc(sizeof(MVMCollectable **) * instance->alloc_permroots); init_mutex(instance->mutex_permroots, "permanent roots"); /* Create fixed size allocator. */ instance->fsa = MVM_fixed_size_create(instance->main_thread); /* Set up REPR registry mutex. */ init_mutex(instance->mutex_repr_registry, "REPR registry"); /* Set up HLL config mutex. */ init_mutex(instance->mutex_hllconfigs, "hll configs"); /* Set up DLL registry mutex. */ init_mutex(instance->mutex_dll_registry, "REPR registry"); /* Set up extension registry mutex. */ init_mutex(instance->mutex_ext_registry, "extension registry"); /* Set up extension op registry mutex. */ init_mutex(instance->mutex_extop_registry, "extension op registry"); /* Set up weak reference hash mutex. */ init_mutex(instance->mutex_sc_weakhash, "sc weakhash"); /* Set up loaded compunits hash mutex. */ init_mutex(instance->mutex_loaded_compunits, "loaded compunits"); /* Set up container registry mutex. */ init_mutex(instance->mutex_container_registry, "container registry"); /* Set up persistent object ID hash mutex. */ init_mutex(instance->mutex_object_ids, "object ID hash"); /* Allocate all things during following setup steps directly in gen2, as * they will have program lifetime. */ MVM_gc_allocate_gen2_default_set(instance->main_thread); init_mutex(instance->mutex_int_const_cache, "int constant cache"); instance->int_const_cache = calloc(1, sizeof(MVMIntConstCache)); /* Bootstrap 6model. It is assumed the GC will not be called during this. */ MVM_6model_bootstrap(instance->main_thread); /* Fix up main thread's usecapture. */ instance->main_thread->cur_usecapture = MVM_repr_alloc_init(instance->main_thread, instance->CallCapture); /* Initialize event loop thread starting mutex. */ init_mutex(instance->mutex_event_loop_start, "event loop thread start"); /* Create main thread object, and also make it the start of the all threads * linked list. */ MVM_store(&instance->threads, (instance->main_thread->thread_obj = (MVMThread *) REPR(instance->boot_types.BOOTThread)->allocate( instance->main_thread, STABLE(instance->boot_types.BOOTThread)))); instance->threads->body.stage = MVM_thread_stage_started; instance->threads->body.tc = instance->main_thread; instance->threads->body.thread_id = uv_thread_self(); /* Create compiler registry */ instance->compiler_registry = MVM_repr_alloc_init(instance->main_thread, instance->boot_types.BOOTHash); /* Set up compiler registr mutex. */ init_mutex(instance->mutex_compiler_registry, "compiler registry"); /* Create hll symbol tables */ instance->hll_syms = MVM_repr_alloc_init(instance->main_thread, instance->boot_types.BOOTHash); /* Set up hll symbol tables mutex. */ init_mutex(instance->mutex_hll_syms, "hll syms"); /* Initialize string cclass handling. */ MVM_string_cclass_init(instance->main_thread); /* Create callsite intern pool. */ instance->callsite_interns = calloc(1, sizeof(MVMCallsiteInterns)); init_mutex(instance->mutex_callsite_interns, "callsite interns"); /* Allocate int to str cache. */ instance->int_to_str_cache = calloc(MVM_INT_TO_STR_CACHE_SIZE, sizeof(MVMString *)); /* Mutex for spesh installations, and check if we've a file we * should log specializations to. */ init_mutex(instance->mutex_spesh_install, "spesh installations"); spesh_log = getenv("MVM_SPESH_LOG"); if (spesh_log && strlen(spesh_log)) instance->spesh_log_fh = fopen(spesh_log, "w"); spesh_disable = getenv("MVM_SPESH_DISABLE"); if (!spesh_disable || strlen(spesh_disable) == 0) { instance->spesh_enabled = 1; spesh_inline_disable = getenv("MVM_SPESH_INLINE_DISABLE"); if (!spesh_inline_disable || strlen(spesh_inline_disable) == 0) instance->spesh_inline_enabled = 1; spesh_osr_disable = getenv("MVM_SPESH_OSR_DISABLE"); if (!spesh_osr_disable || strlen(spesh_osr_disable) == 0) instance->spesh_osr_enabled = 1; } jit_disable = getenv("MVM_JIT_DISABLE"); if (!jit_disable || strlen(jit_disable) == 0) instance->jit_enabled = 1; jit_log = getenv("MVM_JIT_LOG"); if (jit_log && strlen(jit_log)) instance->jit_log_fh = fopen(jit_log, "w"); jit_bytecode_dir = getenv("MVM_JIT_BYTECODE_DIR"); if (jit_bytecode_dir && strlen(jit_bytecode_dir)) instance->jit_bytecode_dir = jit_bytecode_dir; dynvar_log = getenv("MVM_DYNVAR_LOG"); if (dynvar_log && strlen(dynvar_log)) instance->dynvar_log_fh = fopen(dynvar_log, "w"); else instance->dynvar_log_fh = NULL; /* Create std[in/out/err]. */ setup_std_handles(instance->main_thread); /* Current instrumentation level starts at 1; used to trigger all frames * to be verified before their first run. */ instance->instrumentation_level = 1; /* Back to nursery allocation, now we're set up. */ MVM_gc_allocate_gen2_default_clear(instance->main_thread); return instance; }
/* Create a new instance of the VM. */ MVMInstance * MVM_vm_create_instance(void) { MVMInstance *instance; char *spesh_log, *spesh_nodelay, *spesh_disable, *spesh_inline_disable, *spesh_osr_disable; char *jit_log, *jit_disable, *jit_bytecode_dir; char *dynvar_log; int init_stat; /* Set up instance data structure. */ instance = MVM_calloc(1, sizeof(MVMInstance)); /* Create the main thread's ThreadContext and stash it. */ instance->main_thread = MVM_tc_create(instance); instance->main_thread->thread_id = 1; /* No user threads when we start, and next thread to be created gets ID 2 * (the main thread got ID 1). */ instance->num_user_threads = 0; MVM_store(&instance->next_user_thread_id, 2); /* Set up the permanent roots storage. */ instance->num_permroots = 0; instance->alloc_permroots = 16; instance->permroots = MVM_malloc(sizeof(MVMCollectable **) * instance->alloc_permroots); instance->permroot_descriptions = MVM_malloc(sizeof(char *) * instance->alloc_permroots); init_mutex(instance->mutex_permroots, "permanent roots"); /* Create fixed size allocator. */ instance->fsa = MVM_fixed_size_create(instance->main_thread); /* Set up REPR registry mutex. */ init_mutex(instance->mutex_repr_registry, "REPR registry"); /* Set up HLL config mutex. */ init_mutex(instance->mutex_hllconfigs, "hll configs"); /* Set up DLL registry mutex. */ init_mutex(instance->mutex_dll_registry, "REPR registry"); /* Set up extension registry mutex. */ init_mutex(instance->mutex_ext_registry, "extension registry"); /* Set up extension op registry mutex. */ init_mutex(instance->mutex_extop_registry, "extension op registry"); /* Set up weak reference hash mutex. */ init_mutex(instance->mutex_sc_weakhash, "sc weakhash"); /* Set up loaded compunits hash mutex. */ init_mutex(instance->mutex_loaded_compunits, "loaded compunits"); /* Set up container registry mutex. */ init_mutex(instance->mutex_container_registry, "container registry"); /* Set up persistent object ID hash mutex. */ init_mutex(instance->mutex_object_ids, "object ID hash"); /* Allocate all things during following setup steps directly in gen2, as * they will have program lifetime. */ MVM_gc_allocate_gen2_default_set(instance->main_thread); /* Set up integer constant and string cache. */ init_mutex(instance->mutex_int_const_cache, "int constant cache"); instance->int_const_cache = MVM_calloc(1, sizeof(MVMIntConstCache)); instance->int_to_str_cache = MVM_calloc(MVM_INT_TO_STR_CACHE_SIZE, sizeof(MVMString *)); /* Bootstrap 6model. It is assumed the GC will not be called during this. */ MVM_6model_bootstrap(instance->main_thread); /* Fix up main thread's usecapture and last_payload. */ instance->main_thread->cur_usecapture = MVM_repr_alloc_init(instance->main_thread, instance->CallCapture); instance->main_thread->last_payload = instance->VMNull; /* Initialize event loop thread starting mutex. */ init_mutex(instance->mutex_event_loop_start, "event loop thread start"); /* Create main thread object, and also make it the start of the all threads * linked list. */ MVM_store(&instance->threads, (instance->main_thread->thread_obj = (MVMThread *) REPR(instance->boot_types.BOOTThread)->allocate( instance->main_thread, STABLE(instance->boot_types.BOOTThread)))); instance->threads->body.stage = MVM_thread_stage_started; instance->threads->body.tc = instance->main_thread; instance->threads->body.native_thread_id = MVM_platform_thread_id(); instance->threads->body.thread_id = instance->main_thread->thread_id; /* Create compiler registry */ instance->compiler_registry = MVM_repr_alloc_init(instance->main_thread, instance->boot_types.BOOTHash); /* Set up compiler registr mutex. */ init_mutex(instance->mutex_compiler_registry, "compiler registry"); /* Create hll symbol tables */ instance->hll_syms = MVM_repr_alloc_init(instance->main_thread, instance->boot_types.BOOTHash); /* Set up hll symbol tables mutex. */ init_mutex(instance->mutex_hll_syms, "hll syms"); /* Initialize Unicode database */ MVM_unicode_init(instance->main_thread); /* Initialize string cclass handling. */ MVM_string_cclass_init(instance->main_thread); /* Create callsite intern pool. */ instance->callsite_interns = MVM_calloc(1, sizeof(MVMCallsiteInterns)); init_mutex(instance->mutex_callsite_interns, "callsite interns"); /* There's some callsites we statically use all over the place. Intern * them, so that spesh may end up optimizing more "internal" stuff. */ MVM_callsite_initialize_common(instance->main_thread); /* Multi-cache additions mutex. */ init_mutex(instance->mutex_multi_cache_add, "multi-cache addition"); /* Current instrumentation level starts at 1; used to trigger all frames * to be verified before their first run. */ instance->instrumentation_level = 1; /* Mutex for spesh installations, and check if we've a file we * should log specializations to. */ init_mutex(instance->mutex_spesh_install, "spesh installations"); spesh_log = getenv("MVM_SPESH_LOG"); if (spesh_log && strlen(spesh_log)) instance->spesh_log_fh = fopen_perhaps_with_pid(spesh_log, "w"); spesh_disable = getenv("MVM_SPESH_DISABLE"); if (!spesh_disable || strlen(spesh_disable) == 0) { instance->spesh_enabled = 1; spesh_inline_disable = getenv("MVM_SPESH_INLINE_DISABLE"); if (!spesh_inline_disable || strlen(spesh_inline_disable) == 0) instance->spesh_inline_enabled = 1; spesh_osr_disable = getenv("MVM_SPESH_OSR_DISABLE"); if (!spesh_osr_disable || strlen(spesh_osr_disable) == 0) instance->spesh_osr_enabled = 1; } /* Should we specialize without warm up delays? Used to find bugs in the * specializer and JIT. */ spesh_nodelay = getenv("MVM_SPESH_NODELAY"); if (spesh_nodelay && strlen(spesh_nodelay)) { instance->spesh_nodelay = 1; } /* JIT environment/logging setup. */ jit_disable = getenv("MVM_JIT_DISABLE"); if (!jit_disable || strlen(jit_disable) == 0) instance->jit_enabled = 1; jit_log = getenv("MVM_JIT_LOG"); if (jit_log && strlen(jit_log)) instance->jit_log_fh = fopen_perhaps_with_pid(jit_log, "w"); jit_bytecode_dir = getenv("MVM_JIT_BYTECODE_DIR"); if (jit_bytecode_dir && strlen(jit_bytecode_dir)) { char *bytecode_map_name = MVM_malloc(strlen(jit_bytecode_dir) + strlen("/jit-map.txt") + 1); sprintf(bytecode_map_name, "%s/jit-map.txt", jit_bytecode_dir); instance->jit_bytecode_map = fopen(bytecode_map_name, "w"); instance->jit_bytecode_dir = jit_bytecode_dir; MVM_free(bytecode_map_name); } instance->jit_seq_nr = 0; /* Various kinds of debugging that can be enabled. */ dynvar_log = getenv("MVM_DYNVAR_LOG"); if (dynvar_log && strlen(dynvar_log)) { instance->dynvar_log_fh = fopen_perhaps_with_pid(dynvar_log, "w"); fprintf(instance->dynvar_log_fh, "+ x 0 0 0 0 0 %llu\n", uv_hrtime()); fflush(instance->dynvar_log_fh); instance->dynvar_log_lasttime = uv_hrtime(); } else instance->dynvar_log_fh = NULL; instance->nfa_debug_enabled = getenv("MVM_NFA_DEB") ? 1 : 0; if (getenv("MVM_CROSS_THREAD_WRITE_LOG")) { instance->cross_thread_write_logging = 1; instance->cross_thread_write_logging_include_locked = getenv("MVM_CROSS_THREAD_WRITE_LOG_INCLUDE_LOCKED") ? 1 : 0; instance->instrumentation_level++; init_mutex(instance->mutex_cross_thread_write_logging, "cross thread write logging output"); } else { instance->cross_thread_write_logging = 0; } /* Set up NFG state mutation mutex. */ instance->nfg = calloc(1, sizeof(MVMNFGState)); init_mutex(instance->nfg->update_mutex, "NFG update mutex"); /* Create std[in/out/err]. */ setup_std_handles(instance->main_thread); /* Back to nursery allocation, now we're set up. */ MVM_gc_allocate_gen2_default_clear(instance->main_thread); return instance; }
/* Enters the work loop. */ static void worker(MVMThreadContext *tc, MVMCallsite *callsite, MVMRegister *args) { MVMObject *updated_static_frames = MVM_repr_alloc_init(tc, tc->instance->boot_types.BOOTArray); MVMObject *previous_static_frames = MVM_repr_alloc_init(tc, tc->instance->boot_types.BOOTArray); tc->instance->speshworker_thread_id = tc->thread_obj->body.thread_id; MVMROOT2(tc, updated_static_frames, previous_static_frames, { while (1) { MVMObject *log_obj; MVMuint64 start_time; unsigned int interval_id; if (MVM_spesh_debug_enabled(tc)) start_time = uv_hrtime(); log_obj = MVM_repr_shift_o(tc, tc->instance->spesh_queue); if (MVM_spesh_debug_enabled(tc)) { MVM_spesh_debug_printf(tc, "Received Logs\n" "=============\n\n" "Was waiting %dus for logs on the log queue.\n\n", (int)((uv_hrtime() - start_time) / 1000)); } if (tc->instance->main_thread->prof_data) MVM_profiler_log_spesh_start(tc); interval_id = MVM_telemetry_interval_start(tc, "spesh worker consuming a log"); uv_mutex_lock(&(tc->instance->mutex_spesh_sync)); tc->instance->spesh_working = 1; uv_mutex_unlock(&(tc->instance->mutex_spesh_sync)); tc->instance->spesh_stats_version++; if (log_obj->st->REPR->ID == MVM_REPR_ID_MVMSpeshLog) { MVMSpeshLog *sl = (MVMSpeshLog *)log_obj; MVM_telemetry_interval_annotate((uintptr_t)sl->body.thread->body.tc, interval_id, "from this thread"); MVMROOT(tc, sl, { MVMThreadContext *stc; MVMuint32 i; MVMuint32 n; /* Update stats, and if we're logging dump each of them. */ tc->instance->spesh_stats_version++; if (MVM_spesh_debug_enabled(tc)) start_time = uv_hrtime(); MVM_spesh_stats_update(tc, sl, updated_static_frames); n = MVM_repr_elems(tc, updated_static_frames); if (MVM_spesh_debug_enabled(tc)) { MVM_spesh_debug_printf(tc, "Statistics Updated\n" "==================\n" "%d frames had their statistics updated in %dus.\n\n", (int)n, (int)((uv_hrtime() - start_time) / 1000)); for (i = 0; i < n; i++) { char *dump = MVM_spesh_dump_stats(tc, (MVMStaticFrame* ) MVM_repr_at_pos_o(tc, updated_static_frames, i)); MVM_spesh_debug_printf(tc, "%s==========\n\n", dump); MVM_free(dump); } } MVM_telemetry_interval_annotate((uintptr_t)n, interval_id, "stats for this many frames"); GC_SYNC_POINT(tc); /* Form a specialization plan. */ if (MVM_spesh_debug_enabled(tc)) start_time = uv_hrtime(); tc->instance->spesh_plan = MVM_spesh_plan(tc, updated_static_frames); if (MVM_spesh_debug_enabled(tc)) { n = tc->instance->spesh_plan->num_planned; MVM_spesh_debug_printf(tc, "Specialization Plan\n" "===================\n" "%u specialization(s) will be produced (planned in %dus).\n\n", n, (int)((uv_hrtime() - start_time) / 1000)); for (i = 0; i < n; i++) { char *dump = MVM_spesh_dump_planned(tc, &(tc->instance->spesh_plan->planned[i])); MVM_spesh_debug_printf(tc, "%s==========\n\n", dump); MVM_free(dump); } } MVM_telemetry_interval_annotate((uintptr_t)tc->instance->spesh_plan->num_planned, interval_id, "this many specializations planned"); GC_SYNC_POINT(tc); /* Implement the plan and then discard it. */ n = tc->instance->spesh_plan->num_planned; for (i = 0; i < n; i++) { MVM_spesh_candidate_add(tc, &(tc->instance->spesh_plan->planned[i])); GC_SYNC_POINT(tc); } MVM_spesh_plan_destroy(tc, tc->instance->spesh_plan); tc->instance->spesh_plan = NULL; /* Clear up stats that didn't get updated for a while, * then add frames updated this time into the previously * updated array. */ MVM_spesh_stats_cleanup(tc, previous_static_frames); n = MVM_repr_elems(tc, updated_static_frames); for (i = 0; i < n; i++) MVM_repr_push_o(tc, previous_static_frames, MVM_repr_at_pos_o(tc, updated_static_frames, i)); /* Clear updated static frames array. */ MVM_repr_pos_set_elems(tc, updated_static_frames, 0); /* Allow the sending thread to produce more logs again, * putting a new spesh log in place if needed. */ stc = sl->body.thread->body.tc; if (stc && !sl->body.was_compunit_bumped) if (MVM_incr(&(stc->spesh_log_quota)) == 0) { stc->spesh_log = MVM_spesh_log_create(tc, sl->body.thread); MVM_telemetry_timestamp(stc, "logging restored after quota had run out"); } /* If needed, signal sending thread that it can continue. */ if (sl->body.block_mutex) { uv_mutex_lock(sl->body.block_mutex); MVM_store(&(sl->body.completed), 1); uv_cond_signal(sl->body.block_condvar); uv_mutex_unlock(sl->body.block_mutex); } { MVMSpeshLogEntry *entries = sl->body.entries; sl->body.entries = NULL; MVM_free(entries); } }); } else if (MVM_is_null(tc, log_obj)) {
/* Takes a static frame and a thread context. Invokes the static frame. */ void MVM_frame_invoke(MVMThreadContext *tc, MVMStaticFrame *static_frame, MVMCallsite *callsite, MVMRegister *args, MVMFrame *outer, MVMObject *code_ref) { MVMFrame *frame; MVMuint32 pool_index, found_spesh; MVMFrame *node; int fresh = 0; MVMStaticFrameBody *static_frame_body = &static_frame->body; /* If the frame was never invoked before, need initial calculations * and verification. */ if (!static_frame_body->invoked) prepare_and_verify_static_frame(tc, static_frame); /* Get frame body from the re-use pool, or allocate it. */ pool_index = static_frame_body->pool_index; if (pool_index >= tc->frame_pool_table_size) grow_frame_pool(tc, pool_index); node = tc->frame_pool_table[pool_index]; if (node == NULL) { fresh = 1; frame = malloc(sizeof(MVMFrame)); frame->params.named_used = NULL; /* Ensure special return pointers and continuation tags are null. */ frame->special_return = NULL; frame->special_unwind = NULL; frame->continuation_tags = NULL; } else { tc->frame_pool_table[pool_index] = node->outer; node->outer = NULL; frame = node; } #if MVM_HLL_PROFILE_CALLS frame->profile_index = tc->profile_index; tc->profile_data[frame->profile_index].duration_nanos = MVM_platform_now(); tc->profile_data[frame->profile_index].callsite_id = 0; /* XXX get a real callsite id */ tc->profile_data[frame->profile_index].code_id = 0; /* XXX get a real code id */ /* increment the profile data index */ ++tc->profile_index; if (tc->profile_index == tc->profile_data_size) { tc->profile_data_size *= 2; tc->profile_data = realloc(tc->profile_data, tc->profile_data_size); } #endif /* Copy thread context (back?) into the frame. */ frame->tc = tc; /* Set static frame. */ frame->static_info = static_frame; /* Store the code ref (NULL at the top-level). */ frame->code_ref = code_ref; /* Allocate space for lexicals and work area, copying the default lexical * environment into place. */ if (static_frame_body->env_size) { if (fresh) frame->env = malloc(static_frame_body->env_size); memcpy(frame->env, static_frame_body->static_env, static_frame_body->env_size); } else { frame->env = NULL; } if (static_frame_body->work_size) { if (fresh || !frame->work) frame->work = malloc(static_frame_body->work_size); memset(frame->work, 0, static_frame_body->work_size); } else { frame->work = NULL; } /* Calculate args buffer position and make sure current call site starts * empty. */ frame->args = static_frame_body->work_size ? frame->work + static_frame_body->num_locals : NULL; frame->cur_args_callsite = NULL; /* Outer. */ if (outer) { /* We were provided with an outer frame; just ensure that it is * based on the correct static frame (compare on bytecode address * to come with nqp::freshcoderef). */ if (outer->static_info->body.bytecode == static_frame_body->outer->body.bytecode) frame->outer = outer; else MVM_exception_throw_adhoc(tc, "When invoking %s, Provided outer frame %p (%s %s) does not match expected static frame type %p (%s %s)", static_frame_body->name ? MVM_string_utf8_encode_C_string(tc, static_frame_body->name) : "<anonymous static frame>", outer->static_info, MVM_repr_get_by_id(tc, REPR(outer->static_info)->ID)->name, outer->static_info->body.name ? MVM_string_utf8_encode_C_string(tc, outer->static_info->body.name) : "<anonymous static frame>", static_frame_body->outer, MVM_repr_get_by_id(tc, REPR(static_frame_body->outer)->ID)->name, static_frame_body->outer->body.name ? MVM_string_utf8_encode_C_string(tc, static_frame_body->outer->body.name) : "<anonymous static frame>"); } else if (static_frame_body->static_code && static_frame_body->static_code->body.outer) { /* We're lacking an outer, but our static code object may have one. * This comes up in the case of cloned protoregexes, for example. */ frame->outer = static_frame_body->static_code->body.outer; } else if (static_frame_body->outer) { /* Auto-close, and cache it in the static frame. */ frame->outer = autoclose(tc, static_frame_body->outer); static_frame_body->static_code->body.outer = MVM_frame_inc_ref(tc, frame->outer); } else { frame->outer = NULL; } if (frame->outer) MVM_frame_inc_ref(tc, frame->outer); /* Caller is current frame in the thread context. */ if (tc->cur_frame) frame->caller = MVM_frame_inc_ref(tc, tc->cur_frame); else frame->caller = NULL; frame->keep_caller = 0; frame->in_continuation = 0; /* Initial reference count is 1 by virtue of it being the currently * executing frame. */ MVM_store(&frame->ref_count, 1); MVM_store(&frame->gc_seq_number, 0); /* Initialize argument processing. */ MVM_args_proc_init(tc, &frame->params, callsite, args); /* Make sure there's no frame context pointer and special return data * won't be marked. */ frame->context_object = NULL; frame->mark_special_return_data = NULL; /* Clear frame flags. */ frame->flags = 0; /* See if any specializations apply. */ found_spesh = 0; if (++static_frame_body->invocations >= 10 && callsite->is_interned) { /* Look for specialized bytecode. */ MVMint32 num_spesh = static_frame_body->num_spesh_candidates; MVMint32 i, j; for (i = 0; i < num_spesh; i++) { MVMSpeshCandidate *cand = &static_frame_body->spesh_candidates[i]; if (cand->cs == callsite) { MVMint32 match = 1; for (j = 0; j < cand->num_guards; j++) { MVMint32 pos = cand->guards[j].slot; MVMSTable *st = (MVMSTable *)cand->guards[j].match; MVMObject *arg = args[pos].o; if (!arg) { match = 0; break; } switch (cand->guards[j].kind) { case MVM_SPESH_GUARD_CONC: if (!IS_CONCRETE(arg) || STABLE(arg) != st) match = 0; break; case MVM_SPESH_GUARD_TYPE: if (IS_CONCRETE(arg) || STABLE(arg) != st) match = 0; break; case MVM_SPESH_GUARD_DC_CONC: { MVMRegister dc; STABLE(arg)->container_spec->fetch(tc, arg, &dc); if (!dc.o || !IS_CONCRETE(dc.o) || STABLE(dc.o) != st) match = 0; break; } case MVM_SPESH_GUARD_DC_TYPE: { MVMRegister dc; STABLE(arg)->container_spec->fetch(tc, arg, &dc); if (!dc.o || IS_CONCRETE(dc.o) || STABLE(dc.o) != st) match = 0; break; } } if (!match) break; } if (match) { frame->effective_bytecode = cand->bytecode; frame->effective_handlers = cand->handlers; frame->effective_spesh_slots = cand->spesh_slots; frame->spesh_cand = cand; found_spesh = 1; break; } } } /* If we didn't find any, and we're below the limit, can generate a * specialization. */ if (!found_spesh && num_spesh < MVM_SPESH_LIMIT && tc->instance->spesh_enabled) { MVMSpeshCandidate *cand = MVM_spesh_candidate_generate(tc, static_frame, callsite, args); if (cand) { frame->effective_bytecode = cand->bytecode; frame->effective_handlers = cand->handlers; frame->effective_spesh_slots = cand->spesh_slots; frame->spesh_cand = cand; found_spesh = 1; } } } if (!found_spesh) { frame->effective_bytecode = static_frame_body->bytecode; frame->effective_handlers = static_frame_body->handlers; frame->spesh_cand = NULL; } /* Update interpreter and thread context, so next execution will use this * frame. */ tc->cur_frame = frame; *(tc->interp_cur_op) = frame->effective_bytecode; *(tc->interp_bytecode_start) = frame->effective_bytecode; *(tc->interp_reg_base) = frame->work; *(tc->interp_cu) = static_frame_body->cu; /* If we need to do so, make clones of things in the lexical environment * that need it. Note that we do this after tc->cur_frame became the * current frame, to make sure these new objects will certainly get * marked if GC is triggered along the way. */ if (static_frame_body->static_env_flags) { /* Drag everything out of static_frame_body before we start, * as GC action may invalidate it. */ MVMuint8 *flags = static_frame_body->static_env_flags; MVMint64 numlex = static_frame_body->num_lexicals; MVMRegister *state = NULL; MVMint64 state_act = 0; /* 0 = none so far, 1 = first time, 2 = later */ MVMint64 i; for (i = 0; i < numlex; i++) { switch (flags[i]) { case 0: break; case 1: frame->env[i].o = MVM_repr_clone(tc, frame->env[i].o); break; case 2: redo_state: switch (state_act) { case 0: if (!frame->code_ref) MVM_exception_throw_adhoc(tc, "Frame must have code-ref to have state variables"); state = ((MVMCode *)frame->code_ref)->body.state_vars; if (state) { /* Already have state vars; pull them from this. */ state_act = 2; } else { /* Allocate storage for state vars. */ state = malloc(frame->static_info->body.env_size); memset(state, 0, frame->static_info->body.env_size); ((MVMCode *)frame->code_ref)->body.state_vars = state; state_act = 1; /* Note that this frame should run state init code. */ frame->flags |= MVM_FRAME_FLAG_STATE_INIT; } goto redo_state; case 1: frame->env[i].o = MVM_repr_clone(tc, frame->env[i].o); MVM_ASSIGN_REF(tc, &(frame->code_ref->header), state[i].o, frame->env[i].o); break; case 2: frame->env[i].o = state[i].o; break; } break; default: MVM_exception_throw_adhoc(tc, "Unknown lexical environment setup flag"); } } } }