/* Frees the memory associated with an argument guard. If `safe` is set to a
* non-zero value then the memory is freed at the next safepoint. If it is set
* to zero, the memory is freed immediately. */
void MVM_spesh_arg_guard_destroy(MVMThreadContext *tc, MVMSpeshArgGuard *ag, MVMuint32 safe) {
if (ag) {
size_t total_size = sizeof(MVMSpeshArgGuard) +
ag->num_nodes * sizeof(MVMSpeshArgGuardNode);
if (safe)
MVM_fixed_size_free_at_safepoint(tc, tc->instance->fsa, total_size, ag);
else
MVM_fixed_size_free(tc, tc->instance->fsa, total_size, ag);
}
}
/* Assumes that we are holding the lock that serializes updates, and already
* checked that the synthetic does not exist. Adds it to the lookup trie and
* synthetics table, making sure to do enough copy/free-at-safe-point work to
* not upset other threads possibly doing concurrent reads. */
static MVMGrapheme32 add_synthetic(MVMThreadContext *tc, MVMCodepoint *codes, MVMint32 num_codes, MVMint32 utf8_c8) {
MVMNFGState *nfg = tc->instance->nfg;
MVMNFGSynthetic *synth;
MVMGrapheme32 result;
size_t comb_size;
/* Grow the synthetics table if needed. */
if (nfg->num_synthetics % MVM_SYNTHETIC_GROW_ELEMS == 0) {
size_t orig_size = nfg->num_synthetics * sizeof(MVMNFGSynthetic);
size_t new_size = (nfg->num_synthetics + MVM_SYNTHETIC_GROW_ELEMS) * sizeof(MVMNFGSynthetic);
MVMNFGSynthetic *new_synthetics = MVM_fixed_size_alloc(tc, tc->instance->fsa, new_size);
if (orig_size) {
memcpy(new_synthetics, nfg->synthetics, orig_size);
MVM_fixed_size_free_at_safepoint(tc, tc->instance->fsa, orig_size, nfg->synthetics);
}
nfg->synthetics = new_synthetics;
}
/* Set up the new synthetic entry. */
synth = &(nfg->synthetics[nfg->num_synthetics]);
synth->base = *codes;
synth->num_combs = num_codes - 1;
comb_size = synth->num_combs * sizeof(MVMCodepoint);
synth->combs = MVM_fixed_size_alloc(tc, tc->instance->fsa, comb_size);
memcpy(synth->combs, codes + 1, comb_size);
synth->case_uc = 0;
synth->case_lc = 0;
synth->case_tc = 0;
synth->case_fc = 0;
synth->is_utf8_c8 = utf8_c8;
/* Memory barrier to make sure the synthetic is fully in place before we
* bump the count. */
MVM_barrier();
nfg->num_synthetics++;
/* Give the synthetic an ID by negating the new number of synthetics. */
result = -nfg->num_synthetics;
/* Make an entry in the lookup trie for the new synthetic, so we can use
* it in the future when seeing the same codepoint sequence. */
add_synthetic_to_trie(tc, codes, num_codes, result);
return result;
}
/* Produces and installs a specialized version of the code, according to the
* specified plan. */
void MVM_spesh_candidate_add(MVMThreadContext *tc, MVMSpeshPlanned *p) {
MVMSpeshGraph *sg;
MVMSpeshCode *sc;
MVMSpeshCandidate *candidate;
MVMSpeshCandidate **new_candidate_list;
MVMStaticFrameSpesh *spesh;
MVMuint64 start_time, spesh_time, jit_time, end_time;
/* If we've reached our specialization limit, don't continue. */
MVMint32 spesh_produced = ++tc->instance->spesh_produced;
if (tc->instance->spesh_limit)
if (spesh_produced > tc->instance->spesh_limit)
return;
/* Produce the specialization graph and, if we're logging, dump it out
* pre-transformation. */
#if MVM_GC_DEBUG
tc->in_spesh = 1;
#endif
sg = MVM_spesh_graph_create(tc, p->sf, 0, 1);
if (MVM_spesh_debug_enabled(tc)) {
char *c_name = MVM_string_utf8_encode_C_string(tc, p->sf->body.name);
char *c_cuid = MVM_string_utf8_encode_C_string(tc, p->sf->body.cuuid);
MVMSpeshFacts **facts = sg->facts;
char *before;
sg->facts = NULL;
before = MVM_spesh_dump(tc, sg);
sg->facts = facts;
MVM_spesh_debug_printf(tc,
"Specialization of '%s' (cuid: %s)\n\n", c_name, c_cuid);
MVM_spesh_debug_printf(tc, "Before:\n%s", before);
MVM_free(c_name);
MVM_free(c_cuid);
MVM_free(before);
fflush(tc->instance->spesh_log_fh);
start_time = uv_hrtime();
}
/* Attach the graph so we will be able to mark it during optimization,
* allowing us to stick GC sync points at various places and so not let
* the optimization work block GC for too long. */
tc->spesh_active_graph = sg;
spesh_gc_point(tc);
/* Perform the optimization and, if we're logging, dump out the result. */
if (p->cs_stats->cs)
MVM_spesh_args(tc, sg, p->cs_stats->cs, p->type_tuple);
spesh_gc_point(tc);
MVM_spesh_facts_discover(tc, sg, p, 0);
spesh_gc_point(tc);
MVM_spesh_optimize(tc, sg, p);
spesh_gc_point(tc);
/* Clear active graph; beyond this point, no more GC syncs. */
tc->spesh_active_graph = NULL;
if (MVM_spesh_debug_enabled(tc))
spesh_time = uv_hrtime();
/* Generate code and install it into the candidate. */
sc = MVM_spesh_codegen(tc, sg);
candidate = MVM_calloc(1, sizeof(MVMSpeshCandidate));
candidate->bytecode = sc->bytecode;
candidate->bytecode_size = sc->bytecode_size;
candidate->handlers = sc->handlers;
candidate->deopt_usage_info = sc->deopt_usage_info;
candidate->num_handlers = sg->num_handlers;
candidate->num_deopts = sg->num_deopt_addrs;
candidate->deopts = sg->deopt_addrs;
candidate->deopt_named_used_bit_field = sg->deopt_named_used_bit_field;
candidate->deopt_pea = sg->deopt_pea;
candidate->num_locals = sg->num_locals;
candidate->num_lexicals = sg->num_lexicals;
candidate->num_inlines = sg->num_inlines;
candidate->inlines = sg->inlines;
candidate->local_types = sg->local_types;
candidate->lexical_types = sg->lexical_types;
MVM_free(sc);
/* Try to JIT compile the optimised graph. The JIT graph hangs from
* the spesh graph and can safely be deleted with it. */
if (tc->instance->jit_enabled) {
MVMJitGraph *jg;
if (MVM_spesh_debug_enabled(tc))
jit_time = uv_hrtime();
jg = MVM_jit_try_make_graph(tc, sg);
if (jg != NULL) {
candidate->jitcode = MVM_jit_compile_graph(tc, jg);
MVM_jit_graph_destroy(tc, jg);
}
}
if (MVM_spesh_debug_enabled(tc)) {
char *after = MVM_spesh_dump(tc, sg);
end_time = uv_hrtime();
MVM_spesh_debug_printf(tc, "After:\n%s", after);
MVM_spesh_debug_printf(tc,
"Specialization took %" PRIu64 "us (total %" PRIu64"us)\n",
(spesh_time - start_time) / 1000,
(end_time - start_time) / 1000);
if (tc->instance->jit_enabled) {
MVM_spesh_debug_printf(tc,
"JIT was %ssuccessful and compilation took %" PRIu64 "us\n",
candidate->jitcode ? "" : "not ", (end_time - jit_time) / 1000);
if (candidate->jitcode) {
MVM_spesh_debug_printf(tc, " Bytecode size: %" PRIu64 " byte\n",
candidate->jitcode->size);
}
}
MVM_spesh_debug_printf(tc, "\n========\n\n");
MVM_free(after);
fflush(tc->instance->spesh_log_fh);
}
/* calculate work environment taking JIT spill area into account */
calculate_work_env_sizes(tc, sg->sf, candidate);
/* Update spesh slots. */
candidate->num_spesh_slots = sg->num_spesh_slots;
candidate->spesh_slots = sg->spesh_slots;
/* Claim ownership of allocated memory assigned to the candidate */
sg->cand = candidate;
MVM_spesh_graph_destroy(tc, sg);
/* Create a new candidate list and copy any existing ones. Free memory
* using the FSA safepoint mechanism. */
spesh = p->sf->body.spesh;
new_candidate_list = MVM_fixed_size_alloc(tc, tc->instance->fsa,
(spesh->body.num_spesh_candidates + 1) * sizeof(MVMSpeshCandidate *));
if (spesh->body.num_spesh_candidates) {
size_t orig_size = spesh->body.num_spesh_candidates * sizeof(MVMSpeshCandidate *);
memcpy(new_candidate_list, spesh->body.spesh_candidates, orig_size);
MVM_fixed_size_free_at_safepoint(tc, tc->instance->fsa, orig_size,
spesh->body.spesh_candidates);
}
new_candidate_list[spesh->body.num_spesh_candidates] = candidate;
spesh->body.spesh_candidates = new_candidate_list;
/* May now be referencing nursery objects, so barrier just in case. */
if (spesh->common.header.flags & MVM_CF_SECOND_GEN)
MVM_gc_write_barrier_hit(tc, (MVMCollectable *)spesh);
/* Update the guards, and bump the candidate count. This means there is a
* period when we can read, in another thread, a candidate ahead of the
* count being updated. Since we set it up above, that's fine enough. The
* updating of the count *after* this, plus the barrier, is to make sure
* the guards are in place before the count is bumped, since OSR will
* watch the number of candidates to see if there's one for it to try and
* jump in to, and if the guards aren't in place first will see there is
* not, and not bother checking again. */
MVM_spesh_arg_guard_add(tc, &(spesh->body.spesh_arg_guard),
p->cs_stats->cs, p->type_tuple, spesh->body.num_spesh_candidates);
MVM_barrier();
spesh->body.num_spesh_candidates++;
/* If we're logging, dump the upadated arg guards also. */
if (MVM_spesh_debug_enabled(tc)) {
char *guard_dump = MVM_spesh_dump_arg_guard(tc, p->sf);
MVM_spesh_debug_printf(tc, "%s========\n\n", guard_dump);
fflush(tc->instance->spesh_log_fh);
MVM_free(guard_dump);
}
#if MVM_GC_DEBUG
tc->in_spesh = 0;
#endif
}
/* Recursive algorithm to add to the trie. Descends existing trie nodes so far
* as we have them following the code points, then passes on a NULL for the
* levels of current below that do not exist. Once we bottom out, makes a copy
* of or creates a node for the synthetic. As we walk back up we create or
* copy+tweak nodes until we have produced a new trie, re-using what we can of
* the existing one. */
static MVMNFGTrieNode * twiddle_trie_node(MVMThreadContext *tc, MVMNFGTrieNode *current, MVMCodepoint *cur_code, MVMint32 codes_remaining, MVMGrapheme32 synthetic) {
/* Make a new empty node, which we'll maybe copy some things from the
* current node into. */
MVMNFGTrieNode *new_node = MVM_fixed_size_alloc(tc, tc->instance->fsa, sizeof(MVMNFGTrieNode));
/* If we've more codes remaining... */
if (codes_remaining > 0) {
/* Recurse, to get a new child node. */
MVMint32 idx = find_child_node_idx(tc, current, *cur_code);
MVMNFGTrieNode *new_child = twiddle_trie_node(tc,
idx >= 0 ? current->next_codes[idx].node : NULL,
cur_code + 1, codes_remaining - 1, synthetic);
/* If we had an existing child node... */
if (idx >= 0) {
/* Make a copy of the next_codes list. */
size_t the_size = current->num_entries * sizeof(MVMNFGTrieNodeEntry);
MVMNFGTrieNodeEntry *new_next_codes = MVM_fixed_size_alloc(tc,
tc->instance->fsa, the_size);
memcpy(new_next_codes, current->next_codes, the_size);
/* Update the copy to point to the new child. */
new_next_codes[idx].node = new_child;
/* Install the new next_codes list in the new node, and free the
* existing child list at the next safe point. */
new_node->num_entries = current->num_entries;
new_node->next_codes = new_next_codes;
MVM_fixed_size_free_at_safepoint(tc, tc->instance->fsa, the_size,
current->next_codes);
}
/* Otherwise, we're going to need to insert the new child into a
* (possibly existing) child list. */
else {
/* Calculate new child node list size and allocate it. */
MVMint32 orig_entries = current ? current->num_entries : 0;
MVMint32 new_entries = orig_entries + 1;
size_t new_size = new_entries * sizeof(MVMNFGTrieNodeEntry);
MVMNFGTrieNodeEntry *new_next_codes = MVM_fixed_size_alloc(tc,
tc->instance->fsa, new_size);
/* Go through original entries, copying those that are for a lower
* code point than the one we're inserting a child for. */
MVMint32 insert_pos = 0;
MVMint32 orig_pos = 0;
while (orig_pos < orig_entries && current->next_codes[orig_pos].code < *cur_code)
new_next_codes[insert_pos++] = current->next_codes[orig_pos++];
/* Insert the new child. */
new_next_codes[insert_pos].code = *cur_code;
new_next_codes[insert_pos].node = new_child;
insert_pos++;
/* Copy the rest. */
while (orig_pos < orig_entries)
new_next_codes[insert_pos++] = current->next_codes[orig_pos++];
/* Install the new next_codes list in the new node, and free any
* existing child list at the next safe point. */
new_node->num_entries = new_entries;
new_node->next_codes = new_next_codes;
if (orig_entries)
MVM_fixed_size_free_at_safepoint(tc, tc->instance->fsa,
orig_entries * sizeof(MVMNFGTrieNodeEntry),
current->next_codes);
}
/* Always need to copy synthetic set on the existing node also;
* otherwise make sure to clear it. */
new_node->graph = current ? current->graph : 0;
}
/* Otherwise, we reached the point where we need to install the synthetic.
* If we already had a node here, we re-use the children of it. */
else {
new_node->graph = synthetic;
if (current) {
new_node->num_entries = current->num_entries;
new_node->next_codes = current->next_codes;
}
else {
new_node->num_entries = 0;
new_node->next_codes = NULL;
}
}
/* Free any existing node at next safe point, return the new one. */
if (current)
MVM_fixed_size_free_at_safepoint(tc, tc->instance->fsa,
sizeof(MVMNFGTrieNode), current);
return new_node;
}
/* Assumes that we are holding the lock that serializes updates, and already
* checked that the synthetic does not exist. Adds it to the lookup trie and
* synthetics table, making sure to do enough copy/free-at-safe-point work to
* not upset other threads possibly doing concurrent reads. */
static MVMGrapheme32 add_synthetic(MVMThreadContext *tc, MVMCodepoint *codes, MVMint32 num_codes, MVMint32 utf8_c8) {
MVMNFGState *nfg = tc->instance->nfg;
MVMNFGSynthetic *synth;
MVMGrapheme32 result;
/* Grow the synthetics table if needed. */
if (nfg->num_synthetics % MVM_SYNTHETIC_GROW_ELEMS == 0) {
size_t orig_size = nfg->num_synthetics * sizeof(MVMNFGSynthetic);
size_t new_size = (nfg->num_synthetics + MVM_SYNTHETIC_GROW_ELEMS) * sizeof(MVMNFGSynthetic);
MVMNFGSynthetic *new_synthetics = MVM_fixed_size_alloc(tc, tc->instance->fsa, new_size);
if (orig_size) {
memcpy(new_synthetics, nfg->synthetics, orig_size);
MVM_fixed_size_free_at_safepoint(tc, tc->instance->fsa, orig_size, nfg->synthetics);
}
nfg->synthetics = new_synthetics;
}
/* Set up the new synthetic entry. */
synth = &(nfg->synthetics[nfg->num_synthetics]);
synth->num_codes = num_codes;
/* Find which codepoint is the base codepoint. It is always index 0 unless
* there are Prepend codepoints */
if (!utf8_c8 && MVM_unicode_codepoint_get_property_int(tc, codes[0], MVM_UNICODE_PROPERTY_GRAPHEME_CLUSTER_BREAK)
== MVM_UNICODE_PVALUE_GCB_PREPEND) {
MVMint64 i = 0;
MVMCodepoint cached = codes[i++];
MVMint64 cached_GCB = MVM_UNICODE_PVALUE_GCB_PREPEND;
while (i < num_codes) {
/* If it's the same codepoint as before, don't need to request
* the property value again */
if (cached == codes[i] || MVM_UNICODE_PVALUE_GCB_PREPEND ==
(cached_GCB = MVM_unicode_codepoint_get_property_int(tc, (cached = codes[i]),
MVM_UNICODE_PROPERTY_GRAPHEME_CLUSTER_BREAK))) {
}
else {
/* If we see an Extend then this is a degenerate without any
* base character, so set i to num_codes so base_index gets set
* to 0 */
if (cached_GCB == MVM_UNICODE_PVALUE_GCB_EXTEND)
i = num_codes;
break;
}
i++;
}
/* If all the codepoints were prepend then we need to set it to 0 */
synth->base_index = num_codes == i ? 0 : i;
}
else {
synth->base_index = 0;
}
synth->codes = MVM_fixed_size_alloc(tc, tc->instance->fsa,
num_codes * sizeof(MVMCodepoint));
memcpy(synth->codes, codes, (synth->num_codes * sizeof(MVMCodepoint)));
synth->case_uc = 0;
synth->case_lc = 0;
synth->case_tc = 0;
synth->case_fc = 0;
synth->is_utf8_c8 = utf8_c8;
/* Memory barrier to make sure the synthetic is fully in place before we
* bump the count. */
MVM_barrier();
nfg->num_synthetics++;
/* Give the synthetic an ID by negating the new number of synthetics. */
result = -(nfg->num_synthetics);
/* Make an entry in the lookup trie for the new synthetic, so we can use
* it in the future when seeing the same codepoint sequence. */
add_synthetic_to_trie(tc, codes, num_codes, result);
return result;
}
