/* Frees the memory associated with an argument guard. If `safe` is set to a * non-zero value then the memory is freed at the next safepoint. If it is set * to zero, the memory is freed immediately. */ void MVM_spesh_arg_guard_destroy(MVMThreadContext *tc, MVMSpeshArgGuard *ag, MVMuint32 safe) { if (ag) { size_t total_size = sizeof(MVMSpeshArgGuard) + ag->num_nodes * sizeof(MVMSpeshArgGuardNode); if (safe) MVM_fixed_size_free_at_safepoint(tc, tc->instance->fsa, total_size, ag); else MVM_fixed_size_free(tc, tc->instance->fsa, total_size, ag); } }
/* Assumes that we are holding the lock that serializes updates, and already * checked that the synthetic does not exist. Adds it to the lookup trie and * synthetics table, making sure to do enough copy/free-at-safe-point work to * not upset other threads possibly doing concurrent reads. */ static MVMGrapheme32 add_synthetic(MVMThreadContext *tc, MVMCodepoint *codes, MVMint32 num_codes, MVMint32 utf8_c8) { MVMNFGState *nfg = tc->instance->nfg; MVMNFGSynthetic *synth; MVMGrapheme32 result; size_t comb_size; /* Grow the synthetics table if needed. */ if (nfg->num_synthetics % MVM_SYNTHETIC_GROW_ELEMS == 0) { size_t orig_size = nfg->num_synthetics * sizeof(MVMNFGSynthetic); size_t new_size = (nfg->num_synthetics + MVM_SYNTHETIC_GROW_ELEMS) * sizeof(MVMNFGSynthetic); MVMNFGSynthetic *new_synthetics = MVM_fixed_size_alloc(tc, tc->instance->fsa, new_size); if (orig_size) { memcpy(new_synthetics, nfg->synthetics, orig_size); MVM_fixed_size_free_at_safepoint(tc, tc->instance->fsa, orig_size, nfg->synthetics); } nfg->synthetics = new_synthetics; } /* Set up the new synthetic entry. */ synth = &(nfg->synthetics[nfg->num_synthetics]); synth->base = *codes; synth->num_combs = num_codes - 1; comb_size = synth->num_combs * sizeof(MVMCodepoint); synth->combs = MVM_fixed_size_alloc(tc, tc->instance->fsa, comb_size); memcpy(synth->combs, codes + 1, comb_size); synth->case_uc = 0; synth->case_lc = 0; synth->case_tc = 0; synth->case_fc = 0; synth->is_utf8_c8 = utf8_c8; /* Memory barrier to make sure the synthetic is fully in place before we * bump the count. */ MVM_barrier(); nfg->num_synthetics++; /* Give the synthetic an ID by negating the new number of synthetics. */ result = -nfg->num_synthetics; /* Make an entry in the lookup trie for the new synthetic, so we can use * it in the future when seeing the same codepoint sequence. */ add_synthetic_to_trie(tc, codes, num_codes, result); return result; }
/* Produces and installs a specialized version of the code, according to the * specified plan. */ void MVM_spesh_candidate_add(MVMThreadContext *tc, MVMSpeshPlanned *p) { MVMSpeshGraph *sg; MVMSpeshCode *sc; MVMSpeshCandidate *candidate; MVMSpeshCandidate **new_candidate_list; MVMStaticFrameSpesh *spesh; MVMuint64 start_time, spesh_time, jit_time, end_time; /* If we've reached our specialization limit, don't continue. */ MVMint32 spesh_produced = ++tc->instance->spesh_produced; if (tc->instance->spesh_limit) if (spesh_produced > tc->instance->spesh_limit) return; /* Produce the specialization graph and, if we're logging, dump it out * pre-transformation. */ #if MVM_GC_DEBUG tc->in_spesh = 1; #endif sg = MVM_spesh_graph_create(tc, p->sf, 0, 1); if (MVM_spesh_debug_enabled(tc)) { char *c_name = MVM_string_utf8_encode_C_string(tc, p->sf->body.name); char *c_cuid = MVM_string_utf8_encode_C_string(tc, p->sf->body.cuuid); MVMSpeshFacts **facts = sg->facts; char *before; sg->facts = NULL; before = MVM_spesh_dump(tc, sg); sg->facts = facts; MVM_spesh_debug_printf(tc, "Specialization of '%s' (cuid: %s)\n\n", c_name, c_cuid); MVM_spesh_debug_printf(tc, "Before:\n%s", before); MVM_free(c_name); MVM_free(c_cuid); MVM_free(before); fflush(tc->instance->spesh_log_fh); start_time = uv_hrtime(); } /* Attach the graph so we will be able to mark it during optimization, * allowing us to stick GC sync points at various places and so not let * the optimization work block GC for too long. */ tc->spesh_active_graph = sg; spesh_gc_point(tc); /* Perform the optimization and, if we're logging, dump out the result. */ if (p->cs_stats->cs) MVM_spesh_args(tc, sg, p->cs_stats->cs, p->type_tuple); spesh_gc_point(tc); MVM_spesh_facts_discover(tc, sg, p, 0); spesh_gc_point(tc); MVM_spesh_optimize(tc, sg, p); spesh_gc_point(tc); /* Clear active graph; beyond this point, no more GC syncs. */ tc->spesh_active_graph = NULL; if (MVM_spesh_debug_enabled(tc)) spesh_time = uv_hrtime(); /* Generate code and install it into the candidate. */ sc = MVM_spesh_codegen(tc, sg); candidate = MVM_calloc(1, sizeof(MVMSpeshCandidate)); candidate->bytecode = sc->bytecode; candidate->bytecode_size = sc->bytecode_size; candidate->handlers = sc->handlers; candidate->deopt_usage_info = sc->deopt_usage_info; candidate->num_handlers = sg->num_handlers; candidate->num_deopts = sg->num_deopt_addrs; candidate->deopts = sg->deopt_addrs; candidate->deopt_named_used_bit_field = sg->deopt_named_used_bit_field; candidate->deopt_pea = sg->deopt_pea; candidate->num_locals = sg->num_locals; candidate->num_lexicals = sg->num_lexicals; candidate->num_inlines = sg->num_inlines; candidate->inlines = sg->inlines; candidate->local_types = sg->local_types; candidate->lexical_types = sg->lexical_types; MVM_free(sc); /* Try to JIT compile the optimised graph. The JIT graph hangs from * the spesh graph and can safely be deleted with it. */ if (tc->instance->jit_enabled) { MVMJitGraph *jg; if (MVM_spesh_debug_enabled(tc)) jit_time = uv_hrtime(); jg = MVM_jit_try_make_graph(tc, sg); if (jg != NULL) { candidate->jitcode = MVM_jit_compile_graph(tc, jg); MVM_jit_graph_destroy(tc, jg); } } if (MVM_spesh_debug_enabled(tc)) { char *after = MVM_spesh_dump(tc, sg); end_time = uv_hrtime(); MVM_spesh_debug_printf(tc, "After:\n%s", after); MVM_spesh_debug_printf(tc, "Specialization took %" PRIu64 "us (total %" PRIu64"us)\n", (spesh_time - start_time) / 1000, (end_time - start_time) / 1000); if (tc->instance->jit_enabled) { MVM_spesh_debug_printf(tc, "JIT was %ssuccessful and compilation took %" PRIu64 "us\n", candidate->jitcode ? "" : "not ", (end_time - jit_time) / 1000); if (candidate->jitcode) { MVM_spesh_debug_printf(tc, " Bytecode size: %" PRIu64 " byte\n", candidate->jitcode->size); } } MVM_spesh_debug_printf(tc, "\n========\n\n"); MVM_free(after); fflush(tc->instance->spesh_log_fh); } /* calculate work environment taking JIT spill area into account */ calculate_work_env_sizes(tc, sg->sf, candidate); /* Update spesh slots. */ candidate->num_spesh_slots = sg->num_spesh_slots; candidate->spesh_slots = sg->spesh_slots; /* Claim ownership of allocated memory assigned to the candidate */ sg->cand = candidate; MVM_spesh_graph_destroy(tc, sg); /* Create a new candidate list and copy any existing ones. Free memory * using the FSA safepoint mechanism. */ spesh = p->sf->body.spesh; new_candidate_list = MVM_fixed_size_alloc(tc, tc->instance->fsa, (spesh->body.num_spesh_candidates + 1) * sizeof(MVMSpeshCandidate *)); if (spesh->body.num_spesh_candidates) { size_t orig_size = spesh->body.num_spesh_candidates * sizeof(MVMSpeshCandidate *); memcpy(new_candidate_list, spesh->body.spesh_candidates, orig_size); MVM_fixed_size_free_at_safepoint(tc, tc->instance->fsa, orig_size, spesh->body.spesh_candidates); } new_candidate_list[spesh->body.num_spesh_candidates] = candidate; spesh->body.spesh_candidates = new_candidate_list; /* May now be referencing nursery objects, so barrier just in case. */ if (spesh->common.header.flags & MVM_CF_SECOND_GEN) MVM_gc_write_barrier_hit(tc, (MVMCollectable *)spesh); /* Update the guards, and bump the candidate count. This means there is a * period when we can read, in another thread, a candidate ahead of the * count being updated. Since we set it up above, that's fine enough. The * updating of the count *after* this, plus the barrier, is to make sure * the guards are in place before the count is bumped, since OSR will * watch the number of candidates to see if there's one for it to try and * jump in to, and if the guards aren't in place first will see there is * not, and not bother checking again. */ MVM_spesh_arg_guard_add(tc, &(spesh->body.spesh_arg_guard), p->cs_stats->cs, p->type_tuple, spesh->body.num_spesh_candidates); MVM_barrier(); spesh->body.num_spesh_candidates++; /* If we're logging, dump the upadated arg guards also. */ if (MVM_spesh_debug_enabled(tc)) { char *guard_dump = MVM_spesh_dump_arg_guard(tc, p->sf); MVM_spesh_debug_printf(tc, "%s========\n\n", guard_dump); fflush(tc->instance->spesh_log_fh); MVM_free(guard_dump); } #if MVM_GC_DEBUG tc->in_spesh = 0; #endif }
/* Recursive algorithm to add to the trie. Descends existing trie nodes so far * as we have them following the code points, then passes on a NULL for the * levels of current below that do not exist. Once we bottom out, makes a copy * of or creates a node for the synthetic. As we walk back up we create or * copy+tweak nodes until we have produced a new trie, re-using what we can of * the existing one. */ static MVMNFGTrieNode * twiddle_trie_node(MVMThreadContext *tc, MVMNFGTrieNode *current, MVMCodepoint *cur_code, MVMint32 codes_remaining, MVMGrapheme32 synthetic) { /* Make a new empty node, which we'll maybe copy some things from the * current node into. */ MVMNFGTrieNode *new_node = MVM_fixed_size_alloc(tc, tc->instance->fsa, sizeof(MVMNFGTrieNode)); /* If we've more codes remaining... */ if (codes_remaining > 0) { /* Recurse, to get a new child node. */ MVMint32 idx = find_child_node_idx(tc, current, *cur_code); MVMNFGTrieNode *new_child = twiddle_trie_node(tc, idx >= 0 ? current->next_codes[idx].node : NULL, cur_code + 1, codes_remaining - 1, synthetic); /* If we had an existing child node... */ if (idx >= 0) { /* Make a copy of the next_codes list. */ size_t the_size = current->num_entries * sizeof(MVMNFGTrieNodeEntry); MVMNFGTrieNodeEntry *new_next_codes = MVM_fixed_size_alloc(tc, tc->instance->fsa, the_size); memcpy(new_next_codes, current->next_codes, the_size); /* Update the copy to point to the new child. */ new_next_codes[idx].node = new_child; /* Install the new next_codes list in the new node, and free the * existing child list at the next safe point. */ new_node->num_entries = current->num_entries; new_node->next_codes = new_next_codes; MVM_fixed_size_free_at_safepoint(tc, tc->instance->fsa, the_size, current->next_codes); } /* Otherwise, we're going to need to insert the new child into a * (possibly existing) child list. */ else { /* Calculate new child node list size and allocate it. */ MVMint32 orig_entries = current ? current->num_entries : 0; MVMint32 new_entries = orig_entries + 1; size_t new_size = new_entries * sizeof(MVMNFGTrieNodeEntry); MVMNFGTrieNodeEntry *new_next_codes = MVM_fixed_size_alloc(tc, tc->instance->fsa, new_size); /* Go through original entries, copying those that are for a lower * code point than the one we're inserting a child for. */ MVMint32 insert_pos = 0; MVMint32 orig_pos = 0; while (orig_pos < orig_entries && current->next_codes[orig_pos].code < *cur_code) new_next_codes[insert_pos++] = current->next_codes[orig_pos++]; /* Insert the new child. */ new_next_codes[insert_pos].code = *cur_code; new_next_codes[insert_pos].node = new_child; insert_pos++; /* Copy the rest. */ while (orig_pos < orig_entries) new_next_codes[insert_pos++] = current->next_codes[orig_pos++]; /* Install the new next_codes list in the new node, and free any * existing child list at the next safe point. */ new_node->num_entries = new_entries; new_node->next_codes = new_next_codes; if (orig_entries) MVM_fixed_size_free_at_safepoint(tc, tc->instance->fsa, orig_entries * sizeof(MVMNFGTrieNodeEntry), current->next_codes); } /* Always need to copy synthetic set on the existing node also; * otherwise make sure to clear it. */ new_node->graph = current ? current->graph : 0; } /* Otherwise, we reached the point where we need to install the synthetic. * If we already had a node here, we re-use the children of it. */ else { new_node->graph = synthetic; if (current) { new_node->num_entries = current->num_entries; new_node->next_codes = current->next_codes; } else { new_node->num_entries = 0; new_node->next_codes = NULL; } } /* Free any existing node at next safe point, return the new one. */ if (current) MVM_fixed_size_free_at_safepoint(tc, tc->instance->fsa, sizeof(MVMNFGTrieNode), current); return new_node; }
/* Assumes that we are holding the lock that serializes updates, and already * checked that the synthetic does not exist. Adds it to the lookup trie and * synthetics table, making sure to do enough copy/free-at-safe-point work to * not upset other threads possibly doing concurrent reads. */ static MVMGrapheme32 add_synthetic(MVMThreadContext *tc, MVMCodepoint *codes, MVMint32 num_codes, MVMint32 utf8_c8) { MVMNFGState *nfg = tc->instance->nfg; MVMNFGSynthetic *synth; MVMGrapheme32 result; /* Grow the synthetics table if needed. */ if (nfg->num_synthetics % MVM_SYNTHETIC_GROW_ELEMS == 0) { size_t orig_size = nfg->num_synthetics * sizeof(MVMNFGSynthetic); size_t new_size = (nfg->num_synthetics + MVM_SYNTHETIC_GROW_ELEMS) * sizeof(MVMNFGSynthetic); MVMNFGSynthetic *new_synthetics = MVM_fixed_size_alloc(tc, tc->instance->fsa, new_size); if (orig_size) { memcpy(new_synthetics, nfg->synthetics, orig_size); MVM_fixed_size_free_at_safepoint(tc, tc->instance->fsa, orig_size, nfg->synthetics); } nfg->synthetics = new_synthetics; } /* Set up the new synthetic entry. */ synth = &(nfg->synthetics[nfg->num_synthetics]); synth->num_codes = num_codes; /* Find which codepoint is the base codepoint. It is always index 0 unless * there are Prepend codepoints */ if (!utf8_c8 && MVM_unicode_codepoint_get_property_int(tc, codes[0], MVM_UNICODE_PROPERTY_GRAPHEME_CLUSTER_BREAK) == MVM_UNICODE_PVALUE_GCB_PREPEND) { MVMint64 i = 0; MVMCodepoint cached = codes[i++]; MVMint64 cached_GCB = MVM_UNICODE_PVALUE_GCB_PREPEND; while (i < num_codes) { /* If it's the same codepoint as before, don't need to request * the property value again */ if (cached == codes[i] || MVM_UNICODE_PVALUE_GCB_PREPEND == (cached_GCB = MVM_unicode_codepoint_get_property_int(tc, (cached = codes[i]), MVM_UNICODE_PROPERTY_GRAPHEME_CLUSTER_BREAK))) { } else { /* If we see an Extend then this is a degenerate without any * base character, so set i to num_codes so base_index gets set * to 0 */ if (cached_GCB == MVM_UNICODE_PVALUE_GCB_EXTEND) i = num_codes; break; } i++; } /* If all the codepoints were prepend then we need to set it to 0 */ synth->base_index = num_codes == i ? 0 : i; } else { synth->base_index = 0; } synth->codes = MVM_fixed_size_alloc(tc, tc->instance->fsa, num_codes * sizeof(MVMCodepoint)); memcpy(synth->codes, codes, (synth->num_codes * sizeof(MVMCodepoint))); synth->case_uc = 0; synth->case_lc = 0; synth->case_tc = 0; synth->case_fc = 0; synth->is_utf8_c8 = utf8_c8; /* Memory barrier to make sure the synthetic is fully in place before we * bump the count. */ MVM_barrier(); nfg->num_synthetics++; /* Give the synthetic an ID by negating the new number of synthetics. */ result = -(nfg->num_synthetics); /* Make an entry in the lookup trie for the new synthetic, so we can use * it in the future when seeing the same codepoint sequence. */ add_synthetic_to_trie(tc, codes, num_codes, result); return result; }