/* Merges the inlinee's spesh graph into the inliner. */
static void merge_graph(MVMThreadContext *tc, MVMSpeshGraph *inliner,
MVMSpeshGraph *inlinee, MVMCode *inlinee_code,
MVMSpeshIns *invoke_ins) {
MVMSpeshFacts **merged_facts;
MVMuint16 *merged_fact_counts;
MVMint32 i, total_inlines, orig_deopt_addrs;
MVMSpeshBB *inlinee_first_bb = NULL, *inlinee_last_bb = NULL;
MVMint32 active_handlers_at_invoke = 0;
/* If the inliner and inlinee are from different compilation units, we
* potentially have to fix up extra things. */
MVMint32 same_comp_unit = inliner->sf->body.cu == inlinee->sf->body.cu;
/* Renumber the locals, lexicals, and basic blocks of the inlinee; also
* re-write any indexes in annotations that need it. */
MVMSpeshBB *bb = inlinee->entry;
while (bb) {
MVMSpeshIns *ins = bb->first_ins;
while (ins) {
MVMuint16 opcode = ins->info->opcode;
MVMSpeshAnn *ann = ins->annotations;
while (ann) {
switch (ann->type) {
case MVM_SPESH_ANN_FH_START:
case MVM_SPESH_ANN_FH_END:
case MVM_SPESH_ANN_FH_GOTO:
ann->data.frame_handler_index += inliner->num_handlers;
break;
case MVM_SPESH_ANN_DEOPT_INLINE:
ann->data.deopt_idx += inliner->num_deopt_addrs;
break;
case MVM_SPESH_ANN_INLINE_START:
case MVM_SPESH_ANN_INLINE_END:
ann->data.inline_idx += inliner->num_inlines;
break;
}
ann = ann->next;
}
if (opcode == MVM_SSA_PHI) {
for (i = 0; i < ins->info->num_operands; i++)
ins->operands[i].reg.orig += inliner->num_locals;
}
else {
for (i = 0; i < ins->info->num_operands; i++) {
MVMuint8 flags = ins->info->operands[i];
switch (flags & MVM_operand_rw_mask) {
case MVM_operand_read_reg:
case MVM_operand_write_reg:
ins->operands[i].reg.orig += inliner->num_locals;
break;
case MVM_operand_read_lex:
case MVM_operand_write_lex:
ins->operands[i].lex.idx += inliner->num_lexicals;
break;
default: {
MVMuint32 type = flags & MVM_operand_type_mask;
if (type == MVM_operand_spesh_slot) {
ins->operands[i].lit_i16 += inliner->num_spesh_slots;
}
else if (type == MVM_operand_callsite) {
if (!same_comp_unit)
fix_callsite(tc, inliner, inlinee, &(ins->operands[i]));
}
else if (type == MVM_operand_coderef) {
if (!same_comp_unit)
fix_coderef(tc, inliner, inlinee, &(ins->operands[i]));
}
else if (type == MVM_operand_str) {
if (!same_comp_unit)
fix_str(tc, inliner, inlinee, &(ins->operands[i]));
}
break;
}
}
}
}
ins = ins->next;
}
bb->idx += inliner->num_bbs - 1; /* -1 as we won't include entry */
bb->inlined = 1;
if (!bb->linear_next)
inlinee_last_bb = bb;
bb = bb->linear_next;
}
/* Incorporate the basic blocks by concatening them onto the end of the
* linear_next chain of the inliner; skip the inlinee's fake entry BB. */
bb = inliner->entry;
while (bb) {
if (!bb->linear_next) {
/* Found the end; insert and we're done. */
bb->linear_next = inlinee_first_bb = inlinee->entry->linear_next;
bb = NULL;
}
else {
bb = bb->linear_next;
}
}
/* Merge facts. */
merged_facts = MVM_spesh_alloc(tc, inliner,
(inliner->num_locals + inlinee->num_locals) * sizeof(MVMSpeshFacts *));
memcpy(merged_facts, inliner->facts,
inliner->num_locals * sizeof(MVMSpeshFacts *));
memcpy(merged_facts + inliner->num_locals, inlinee->facts,
inlinee->num_locals * sizeof(MVMSpeshFacts *));
inliner->facts = merged_facts;
merged_fact_counts = MVM_spesh_alloc(tc, inliner,
(inliner->num_locals + inlinee->num_locals) * sizeof(MVMuint16));
memcpy(merged_fact_counts, inliner->fact_counts,
inliner->num_locals * sizeof(MVMuint16));
memcpy(merged_fact_counts + inliner->num_locals, inlinee->fact_counts,
inlinee->num_locals * sizeof(MVMuint16));
inliner->fact_counts = merged_fact_counts;
/* Copy over spesh slots. */
for (i = 0; i < inlinee->num_spesh_slots; i++)
MVM_spesh_add_spesh_slot(tc, inliner, inlinee->spesh_slots[i]);
/* If they are from separate compilation units, make another pass through
* to fix up on wvals. Note we can't do this in the first pass as we must
* not modify the spesh slots once we've got started with the rewrites.
* Now we've resolved all that, we're good to map wvals elsewhere into
* some extra spesh slots. */
if (!same_comp_unit) {
bb = inlinee->entry;
while (bb) {
MVMSpeshIns *ins = bb->first_ins;
while (ins) {
MVMuint16 opcode = ins->info->opcode;
if (opcode == MVM_OP_wval || opcode == MVM_OP_wval_wide)
fix_wval(tc, inliner, inlinee, ins);
ins = ins->next;
}
bb = bb->linear_next;
}
}
/* Merge de-opt tables, if needed. */
orig_deopt_addrs = inliner->num_deopt_addrs;
if (inlinee->num_deopt_addrs) {
assert(inlinee->deopt_addrs != inliner->deopt_addrs);
inliner->alloc_deopt_addrs += inlinee->alloc_deopt_addrs;
if (inliner->deopt_addrs)
inliner->deopt_addrs = MVM_realloc(inliner->deopt_addrs,
inliner->alloc_deopt_addrs * sizeof(MVMint32) * 2);
else
inliner->deopt_addrs = MVM_malloc(inliner->alloc_deopt_addrs * sizeof(MVMint32) * 2);
memcpy(inliner->deopt_addrs + inliner->num_deopt_addrs * 2,
inlinee->deopt_addrs, inlinee->alloc_deopt_addrs * sizeof(MVMint32) * 2);
inliner->num_deopt_addrs += inlinee->num_deopt_addrs;
}
/* Merge inlines table, and add us an entry too. */
total_inlines = inliner->num_inlines + inlinee->num_inlines + 1;
inliner->inlines = inliner->num_inlines
? MVM_realloc(inliner->inlines, total_inlines * sizeof(MVMSpeshInline))
: MVM_malloc(total_inlines * sizeof(MVMSpeshInline));
memcpy(inliner->inlines + inliner->num_inlines, inlinee->inlines,
inlinee->num_inlines * sizeof(MVMSpeshInline));
for (i = inliner->num_inlines; i < total_inlines - 1; i++) {
inliner->inlines[i].locals_start += inliner->num_locals;
inliner->inlines[i].lexicals_start += inliner->num_lexicals;
inliner->inlines[i].return_deopt_idx += orig_deopt_addrs;
}
inliner->inlines[total_inlines - 1].code = inlinee_code;
inliner->inlines[total_inlines - 1].g = inlinee;
inliner->inlines[total_inlines - 1].locals_start = inliner->num_locals;
inliner->inlines[total_inlines - 1].lexicals_start = inliner->num_lexicals;
switch (invoke_ins->info->opcode) {
case MVM_OP_invoke_v:
inliner->inlines[total_inlines - 1].res_type = MVM_RETURN_VOID;
break;
case MVM_OP_invoke_o:
inliner->inlines[total_inlines - 1].res_reg = invoke_ins->operands[0].reg.orig;
inliner->inlines[total_inlines - 1].res_type = MVM_RETURN_OBJ;
break;
case MVM_OP_invoke_i:
inliner->inlines[total_inlines - 1].res_reg = invoke_ins->operands[0].reg.orig;
inliner->inlines[total_inlines - 1].res_type = MVM_RETURN_INT;
break;
case MVM_OP_invoke_n:
inliner->inlines[total_inlines - 1].res_reg = invoke_ins->operands[0].reg.orig;
inliner->inlines[total_inlines - 1].res_type = MVM_RETURN_NUM;
break;
case MVM_OP_invoke_s:
inliner->inlines[total_inlines - 1].res_reg = invoke_ins->operands[0].reg.orig;
inliner->inlines[total_inlines - 1].res_type = MVM_RETURN_STR;
break;
default:
MVM_oops(tc, "Spesh inline: unknown invoke instruction");
}
inliner->inlines[total_inlines - 1].return_deopt_idx = return_deopt_idx(tc, invoke_ins);
inliner->num_inlines = total_inlines;
/* Create/update per-specialization local and lexical type maps. */
if (!inliner->local_types) {
MVMint32 local_types_size = inliner->num_locals * sizeof(MVMuint16);
inliner->local_types = MVM_malloc(local_types_size);
memcpy(inliner->local_types, inliner->sf->body.local_types, local_types_size);
}
inliner->local_types = MVM_realloc(inliner->local_types,
(inliner->num_locals + inlinee->num_locals) * sizeof(MVMuint16));
memcpy(inliner->local_types + inliner->num_locals,
inlinee->local_types ? inlinee->local_types : inlinee->sf->body.local_types,
inlinee->num_locals * sizeof(MVMuint16));
if (!inliner->lexical_types) {
MVMint32 lexical_types_size = inliner->num_lexicals * sizeof(MVMuint16);
inliner->lexical_types = MVM_malloc(lexical_types_size);
memcpy(inliner->lexical_types, inliner->sf->body.lexical_types, lexical_types_size);
}
inliner->lexical_types = MVM_realloc(inliner->lexical_types,
(inliner->num_lexicals + inlinee->num_lexicals) * sizeof(MVMuint16));
memcpy(inliner->lexical_types + inliner->num_lexicals,
inlinee->lexical_types ? inlinee->lexical_types : inlinee->sf->body.lexical_types,
inlinee->num_lexicals * sizeof(MVMuint16));
/* Merge handlers from inlinee. */
if (inlinee->num_handlers) {
MVMuint32 total_handlers = inliner->num_handlers + inlinee->num_handlers;
resize_handlers_table(tc, inliner, total_handlers);
memcpy(inliner->handlers + inliner->num_handlers, inlinee->handlers,
inlinee->num_handlers * sizeof(MVMFrameHandler));
for (i = inliner->num_handlers; i < total_handlers; i++) {
inliner->handlers[i].block_reg += inliner->num_locals;
inliner->handlers[i].label_reg += inliner->num_locals;
}
}
/* If the inliner has handlers in effect at the point of the call that we
* are inlining, then we duplicate those and place them surrounding the
* inlinee, but with the goto still pointing to the original location.
* This means that we can still do a linear scan when searching for an
* exception handler, and don't have to try the (costly and fiddly) matter
* of trying to traverse the post-inlined call chain. */
if (inliner->sf->body.num_handlers) {
/* Walk inliner looking for handlers in effect at the point we hit the
* invoke instruction we're currently inlining; also record all of the
* instructions where the handler "goto" annotation lives. */
MVMuint32 orig_handlers = inliner->sf->body.num_handlers;
MVMuint8 *active = MVM_spesh_alloc(tc, inliner, orig_handlers);
MVMSpeshIns **handler_goto_ins = MVM_spesh_alloc(tc, inliner,
orig_handlers * sizeof(MVMSpeshIns *));
MVMint32 found_invoke = 0;
bb = inliner->entry;
while (bb && !bb->inlined) {
MVMSpeshIns *ins = bb->first_ins;
while (ins) {
MVMSpeshAnn *ann = ins->annotations;
while (ann) {
if (ann->type == MVM_SPESH_ANN_FH_GOTO) {
if (ann->data.frame_handler_index < orig_handlers)
handler_goto_ins[ann->data.frame_handler_index] = ins;
}
else if (!found_invoke) {
/* Only update these to the point we found the invoke
* being inlined, so it serves as a snapshot of what
* is active. */
if (ann->type == MVM_SPESH_ANN_FH_START)
active[ann->data.frame_handler_index] = 1;
else if (ann->type == MVM_SPESH_ANN_FH_END)
active[ann->data.frame_handler_index] = 0;
}
ann = ann->next;
}
if (ins == invoke_ins) {
/* Found it; see if we have any handlers active. If so, we
* will continue walking to collect goto annotations. */
found_invoke = 1;
for (i = 0; i < orig_handlers; i++)
active_handlers_at_invoke += active[i];
if (!active_handlers_at_invoke)
break;
}
ins = ins->next;
}
if (found_invoke && !active_handlers_at_invoke)
break;
bb = bb->linear_next;
}
/* If we found handlers active at the point of invoke, duplicate them
* in the handlers table and add annotations. */
if (active_handlers_at_invoke) {
MVMuint32 insert_pos = inliner->num_handlers + inlinee->num_handlers;
resize_handlers_table(tc, inliner, insert_pos + active_handlers_at_invoke);
for (i = orig_handlers - 1; i >= 0; i--) {
if (active[i]) {
/* Add handler start annotation to first inlinee instruction. */
MVMSpeshAnn *new_ann = MVM_spesh_alloc(tc, inliner, sizeof(MVMSpeshAnn));
new_ann->type = MVM_SPESH_ANN_FH_START;
new_ann->data.frame_handler_index = insert_pos;
new_ann->next = inlinee_first_bb->first_ins->annotations;
inlinee_first_bb->first_ins->annotations = new_ann;
/* Add handler end annotation to last inlinee instruction. */
new_ann = MVM_spesh_alloc(tc, inliner, sizeof(MVMSpeshAnn));
new_ann->type = MVM_SPESH_ANN_FH_END;
new_ann->data.frame_handler_index = insert_pos;
new_ann->next = inlinee_last_bb->last_ins->annotations;
inlinee_last_bb->last_ins->annotations = new_ann;
/* Add handler goto annotation to original target in inliner. */
new_ann = MVM_spesh_alloc(tc, inliner, sizeof(MVMSpeshAnn));
new_ann->type = MVM_SPESH_ANN_FH_GOTO;
new_ann->data.frame_handler_index = insert_pos;
new_ann->next = handler_goto_ins[i]->annotations;
handler_goto_ins[i]->annotations = new_ann;
/* Copy handler entry to new slot. */
memcpy(inliner->handlers + insert_pos, inliner->handlers + i,
sizeof(MVMFrameHandler));
insert_pos++;
}
}
}
}
/* Update total locals, lexicals, basic blocks, and handlers of the
* inliner. */
inliner->num_bbs += inlinee->num_bbs - 1;
inliner->num_locals += inlinee->num_locals;
inliner->num_lexicals += inlinee->num_lexicals;
inliner->num_handlers += inlinee->num_handlers + active_handlers_at_invoke;
}
/* Merges the inlinee's spesh graph into the inliner. */
static void merge_graph(MVMThreadContext *tc, MVMSpeshGraph *inliner,
MVMSpeshGraph *inlinee, MVMCode *inlinee_code,
MVMSpeshIns *invoke_ins) {
MVMSpeshFacts **merged_facts;
MVMuint16 *merged_fact_counts;
MVMint32 i, total_inlines, orig_deopt_addrs;
/* If the inliner and inlinee are from different compilation units, we
* potentially have to fix up extra things. */
MVMint32 same_comp_unit = inliner->sf->body.cu == inlinee->sf->body.cu;
/* Renumber the locals, lexicals, and basic blocks of the inlinee; also
* re-write any indexes in annotations that need it. */
MVMSpeshBB *bb = inlinee->entry;
while (bb) {
MVMSpeshIns *ins = bb->first_ins;
while (ins) {
MVMuint16 opcode = ins->info->opcode;
MVMSpeshAnn *ann = ins->annotations;
while (ann) {
switch (ann->type) {
case MVM_SPESH_ANN_FH_START:
case MVM_SPESH_ANN_FH_END:
case MVM_SPESH_ANN_FH_GOTO:
ann->data.frame_handler_index += inliner->num_handlers;
break;
case MVM_SPESH_ANN_DEOPT_INLINE:
ann->data.deopt_idx += inliner->num_deopt_addrs;
break;
case MVM_SPESH_ANN_INLINE_START:
case MVM_SPESH_ANN_INLINE_END:
ann->data.inline_idx += inliner->num_inlines;
break;
}
ann = ann->next;
}
if (opcode == MVM_SSA_PHI) {
for (i = 0; i < ins->info->num_operands; i++)
ins->operands[i].reg.orig += inliner->num_locals;
}
else {
for (i = 0; i < ins->info->num_operands; i++) {
MVMuint8 flags = ins->info->operands[i];
switch (flags & MVM_operand_rw_mask) {
case MVM_operand_read_reg:
case MVM_operand_write_reg:
ins->operands[i].reg.orig += inliner->num_locals;
break;
case MVM_operand_read_lex:
case MVM_operand_write_lex:
ins->operands[i].lex.idx += inliner->num_lexicals;
break;
default: {
MVMuint32 type = flags & MVM_operand_type_mask;
if (type == MVM_operand_spesh_slot) {
ins->operands[i].lit_i16 += inliner->num_spesh_slots;
}
else if (type == MVM_operand_callsite) {
if (!same_comp_unit)
fix_callsite(tc, inliner, inlinee, &(ins->operands[i]));
}
else if (type == MVM_operand_coderef) {
if (!same_comp_unit)
fix_coderef(tc, inliner, inlinee, &(ins->operands[i]));
}
else if (type == MVM_operand_str) {
if (!same_comp_unit)
fix_str(tc, inliner, inlinee, &(ins->operands[i]));
}
break;
}
}
}
}
ins = ins->next;
}
bb->idx += inliner->num_bbs - 1; /* -1 as we won't include entry */
bb->inlined = 1;
bb = bb->linear_next;
}
/* Incorporate the basic blocks by concatening them onto the end of the
* linear_next chain of the inliner; skip the inlinee's fake entry BB. */
bb = inliner->entry;
while (bb) {
if (!bb->linear_next) {
/* Found the end; insert and we're done. */
bb->linear_next = inlinee->entry->linear_next;
bb = NULL;
}
else {
bb = bb->linear_next;
}
}
/* Merge facts. */
merged_facts = MVM_spesh_alloc(tc, inliner,
(inliner->num_locals + inlinee->num_locals) * sizeof(MVMSpeshFacts *));
memcpy(merged_facts, inliner->facts,
inliner->num_locals * sizeof(MVMSpeshFacts *));
memcpy(merged_facts + inliner->num_locals, inlinee->facts,
inlinee->num_locals * sizeof(MVMSpeshFacts *));
inliner->facts = merged_facts;
merged_fact_counts = MVM_spesh_alloc(tc, inliner,
(inliner->num_locals + inlinee->num_locals) * sizeof(MVMuint16));
memcpy(merged_fact_counts, inliner->fact_counts,
inliner->num_locals * sizeof(MVMuint16));
memcpy(merged_fact_counts + inliner->num_locals, inlinee->fact_counts,
inlinee->num_locals * sizeof(MVMuint16));
inliner->fact_counts = merged_fact_counts;
/* Copy over spesh slots. */
for (i = 0; i < inlinee->num_spesh_slots; i++)
MVM_spesh_add_spesh_slot(tc, inliner, inlinee->spesh_slots[i]);
/* If they are from separate compilation units, make another pass through
* to fix up on wvals. Note we can't do this in the first pass as we must
* not modify the spesh slots once we've got started with the rewrites.
* Now we've resolved all that, we're good to map wvals elsewhere into
* some extra spesh slots. */
if (!same_comp_unit) {
bb = inlinee->entry;
while (bb) {
MVMSpeshIns *ins = bb->first_ins;
while (ins) {
MVMuint16 opcode = ins->info->opcode;
if (opcode == MVM_OP_wval || opcode == MVM_OP_wval_wide)
fix_wval(tc, inliner, inlinee, ins);
ins = ins->next;
}
bb = bb->linear_next;
}
}
/* Merge de-opt tables, if needed. */
orig_deopt_addrs = inliner->num_deopt_addrs;
if (inlinee->num_deopt_addrs) {
assert(inlinee->deopt_addrs != inliner->deopt_addrs);
inliner->alloc_deopt_addrs += inlinee->alloc_deopt_addrs;
if (inliner->deopt_addrs)
inliner->deopt_addrs = MVM_realloc(inliner->deopt_addrs,
inliner->alloc_deopt_addrs * sizeof(MVMint32) * 2);
else
inliner->deopt_addrs = MVM_malloc(inliner->alloc_deopt_addrs * sizeof(MVMint32) * 2);
memcpy(inliner->deopt_addrs + inliner->num_deopt_addrs * 2,
inlinee->deopt_addrs, inlinee->alloc_deopt_addrs * sizeof(MVMint32) * 2);
inliner->num_deopt_addrs += inlinee->num_deopt_addrs;
}
/* Merge inlines table, and add us an entry too. */
total_inlines = inliner->num_inlines + inlinee->num_inlines + 1;
inliner->inlines = inliner->num_inlines
? MVM_realloc(inliner->inlines, total_inlines * sizeof(MVMSpeshInline))
: MVM_malloc(total_inlines * sizeof(MVMSpeshInline));
memcpy(inliner->inlines + inliner->num_inlines, inlinee->inlines,
inlinee->num_inlines * sizeof(MVMSpeshInline));
for (i = inliner->num_inlines; i < total_inlines - 1; i++) {
inliner->inlines[i].locals_start += inliner->num_locals;
inliner->inlines[i].lexicals_start += inliner->num_lexicals;
inliner->inlines[i].return_deopt_idx += orig_deopt_addrs;
}
inliner->inlines[total_inlines - 1].code = inlinee_code;
inliner->inlines[total_inlines - 1].g = inlinee;
inliner->inlines[total_inlines - 1].locals_start = inliner->num_locals;
inliner->inlines[total_inlines - 1].lexicals_start = inliner->num_lexicals;
switch (invoke_ins->info->opcode) {
case MVM_OP_invoke_v:
inliner->inlines[total_inlines - 1].res_type = MVM_RETURN_VOID;
break;
case MVM_OP_invoke_o:
inliner->inlines[total_inlines - 1].res_reg = invoke_ins->operands[0].reg.orig;
inliner->inlines[total_inlines - 1].res_type = MVM_RETURN_OBJ;
break;
case MVM_OP_invoke_i:
inliner->inlines[total_inlines - 1].res_reg = invoke_ins->operands[0].reg.orig;
inliner->inlines[total_inlines - 1].res_type = MVM_RETURN_INT;
break;
case MVM_OP_invoke_n:
inliner->inlines[total_inlines - 1].res_reg = invoke_ins->operands[0].reg.orig;
inliner->inlines[total_inlines - 1].res_type = MVM_RETURN_NUM;
break;
case MVM_OP_invoke_s:
inliner->inlines[total_inlines - 1].res_reg = invoke_ins->operands[0].reg.orig;
inliner->inlines[total_inlines - 1].res_type = MVM_RETURN_STR;
break;
default:
MVM_exception_throw_adhoc(tc, "Spesh inline: unknown invoke instruction");
}
inliner->inlines[total_inlines - 1].return_deopt_idx = return_deopt_idx(tc, invoke_ins);
inliner->num_inlines = total_inlines;
/* Create/update per-specialization local and lexical type maps. */
if (!inliner->local_types) {
MVMint32 local_types_size = inliner->num_locals * sizeof(MVMuint16);
inliner->local_types = MVM_malloc(local_types_size);
memcpy(inliner->local_types, inliner->sf->body.local_types, local_types_size);
}
inliner->local_types = MVM_realloc(inliner->local_types,
(inliner->num_locals + inlinee->num_locals) * sizeof(MVMuint16));
memcpy(inliner->local_types + inliner->num_locals,
inlinee->local_types ? inlinee->local_types : inlinee->sf->body.local_types,
inlinee->num_locals * sizeof(MVMuint16));
if (!inliner->lexical_types) {
MVMint32 lexical_types_size = inliner->num_lexicals * sizeof(MVMuint16);
inliner->lexical_types = MVM_malloc(lexical_types_size);
memcpy(inliner->lexical_types, inliner->sf->body.lexical_types, lexical_types_size);
}
inliner->lexical_types = MVM_realloc(inliner->lexical_types,
(inliner->num_lexicals + inlinee->num_lexicals) * sizeof(MVMuint16));
memcpy(inliner->lexical_types + inliner->num_lexicals,
inlinee->lexical_types ? inlinee->lexical_types : inlinee->sf->body.lexical_types,
inlinee->num_lexicals * sizeof(MVMuint16));
/* Merge handlers. */
if (inlinee->num_handlers) {
MVMuint32 total_handlers = inliner->num_handlers + inlinee->num_handlers;
if (inliner->handlers == inliner->sf->body.handlers) {
/* Original handlers table; need a copy. */
MVMFrameHandler *new_handlers = MVM_malloc(total_handlers * sizeof(MVMFrameHandler));
memcpy(new_handlers, inliner->handlers,
inliner->num_handlers * sizeof(MVMFrameHandler));
inliner->handlers = new_handlers;
}
else {
/* Probably already did some inlines into this frame; resize. */
inliner->handlers = MVM_realloc(inliner->handlers,
total_handlers * sizeof(MVMFrameHandler));
}
memcpy(inliner->handlers + inliner->num_handlers, inlinee->handlers,
inlinee->num_handlers * sizeof(MVMFrameHandler));
for (i = inliner->num_handlers; i < total_handlers; i++) {
inliner->handlers[i].block_reg += inliner->num_locals;
inliner->handlers[i].label_reg += inliner->num_locals;
}
}
/* Update total locals, lexicals, basic blocks, and handlers of the
* inliner. */
inliner->num_bbs += inlinee->num_bbs - 1;
inliner->num_locals += inlinee->num_locals;
inliner->num_lexicals += inlinee->num_lexicals;
inliner->num_handlers += inlinee->num_handlers;
}
/* Check for stability of what was logged, and if it looks sane then add facts
* and turn the log instruction into a */
static void log_facts(MVMThreadContext *tc, MVMSpeshGraph *g, MVMSpeshBB *bb, MVMSpeshIns *ins) {
MVMObject *stable_value = NULL;
MVMObject *stable_cont = NULL;
MVMSpeshFacts *facts;
/* See if all the recorded facts match up; a NULL means there was a code
* path that never reached making a log entry. */
MVMuint16 log_start = ins->operands[1].lit_i16 * MVM_SPESH_LOG_RUNS;
MVMuint16 i;
for (i = log_start; i < log_start + MVM_SPESH_LOG_RUNS; i++) {
MVMObject *consider = (MVMObject *)g->log_slots[i];
if (consider) {
if (!stable_value) {
stable_value = consider;
}
else if (STABLE(stable_value) != STABLE(consider)
|| IS_CONCRETE(stable_value) != IS_CONCRETE(consider)) {
stable_value = NULL;
break;
}
}
}
if (!stable_value)
return;
/* If the value is a container type, need to look inside of it. */
if (STABLE(stable_value)->container_spec && IS_CONCRETE(stable_value)) {
MVMContainerSpec const *contspec = STABLE(stable_value)->container_spec;
if (!contspec->fetch_never_invokes)
return;
stable_cont = stable_value;
stable_value = NULL;
for (i = log_start; i < log_start + MVM_SPESH_LOG_RUNS; i++) {
MVMRegister r;
contspec->fetch(tc, stable_cont, &r);
if (r.o) {
if (!stable_value) {
stable_value = r.o;
}
else if (STABLE(stable_value) != STABLE(r.o)
|| IS_CONCRETE(stable_value) != IS_CONCRETE(r.o)) {
stable_value = NULL;
break;
}
}
}
if (!stable_value)
return;
}
/* Produce a guard op and set facts. */
if (stable_cont) {
MVMSpeshOperand reg = ins->operands[0];
MVMContainerSpec *cs = (MVMContainerSpec *) STABLE(stable_cont)->container_spec;
facts = &g->facts[reg.reg.orig][reg.reg.i];
facts->type = STABLE(stable_cont)->WHAT;
facts->flags |= (MVM_SPESH_FACT_KNOWN_TYPE | MVM_SPESH_FACT_CONCRETE |
MVM_SPESH_FACT_KNOWN_DECONT_TYPE);
facts->decont_type = STABLE(stable_value)->WHAT;
/* If this is a native container, we get away with testing
* against the STABLE only, as the NativeRef REPR has all
* interesting values in its REPRData. */
if (cs->can_store(tc, stable_cont) &&
(MVM_6model_container_iscont_i(tc, stable_cont) ||
MVM_6model_container_iscont_n(tc, stable_cont) ||
MVM_6model_container_iscont_s(tc, stable_cont))) {
facts = &g->facts[ins->operands[0].reg.orig][ins->operands[0].reg.i];
/*facts->type = STABLE(stable_value)->WHAT;*/
facts->flags |= MVM_SPESH_FACT_RW_CONT;
ins->info = MVM_op_get_op(MVM_OP_sp_guardconc);
ins->operands = MVM_spesh_alloc(tc, g, 2 * sizeof(MVMSpeshOperand));
ins->operands[0] = reg;
ins->operands[1].lit_i16 = MVM_spesh_add_spesh_slot(tc, g, (MVMCollectable *)STABLE(stable_cont));
} else {
if (cs->can_store(tc, stable_cont)) {
/* We could do stability testing on rw-ness too, but it's quite
* unlikely we'll have codepaths with a mix of readable and
* writable containers. */
facts->flags |= MVM_SPESH_FACT_RW_CONT;
if (IS_CONCRETE(stable_value)) {
facts->flags |= MVM_SPESH_FACT_DECONT_CONCRETE;
ins->info = MVM_op_get_op(MVM_OP_sp_guardrwconc);
}
else {
facts->flags |= MVM_SPESH_FACT_DECONT_TYPEOBJ;
ins->info = MVM_op_get_op(MVM_OP_sp_guardrwtype);
}
}
else {
if (IS_CONCRETE(stable_value)) {
facts->flags |= MVM_SPESH_FACT_DECONT_CONCRETE;
ins->info = MVM_op_get_op(MVM_OP_sp_guardcontconc);
}
else {
facts->flags |= MVM_SPESH_FACT_DECONT_TYPEOBJ;
ins->info = MVM_op_get_op(MVM_OP_sp_guardconttype);
}
}
ins->operands = MVM_spesh_alloc(tc, g, 3 * sizeof(MVMSpeshOperand));
ins->operands[0] = reg;
ins->operands[1].lit_i16 = MVM_spesh_add_spesh_slot(tc, g, (MVMCollectable *)STABLE(stable_cont));
ins->operands[2].lit_i16 = MVM_spesh_add_spesh_slot(tc, g, (MVMCollectable *)STABLE(stable_value));
}
}
else {
facts = &g->facts[ins->operands[0].reg.orig][ins->operands[0].reg.i];
facts->type = STABLE(stable_value)->WHAT;
facts->flags |= (MVM_SPESH_FACT_KNOWN_TYPE | MVM_SPESH_FACT_DECONTED);
if (IS_CONCRETE(stable_value)) {
facts->flags |= MVM_SPESH_FACT_CONCRETE;
ins->info = MVM_op_get_op(MVM_OP_sp_guardconc);
}
else {
facts->flags |= MVM_SPESH_FACT_TYPEOBJ;
ins->info = MVM_op_get_op(MVM_OP_sp_guardtype);
}
ins->operands[1].lit_i16 = MVM_spesh_add_spesh_slot(tc, g, (MVMCollectable *)STABLE(stable_value));
}
/* Add entry in log guards table, and mark facts as depending on it. */
g->log_guards[g->num_log_guards].ins = ins;
g->log_guards[g->num_log_guards].bb = bb;
facts->flags |= MVM_SPESH_FACT_FROM_LOG_GUARD;
facts->log_guard = g->num_log_guards;
g->num_log_guards++;
}
/* Takes information about the incoming callsite and arguments, and performs
* various optimizations based on that information. */
void MVM_spesh_args(MVMThreadContext *tc, MVMSpeshGraph *g, MVMCallsite *cs, MVMRegister *args) {
/* We need to identify the various arg-related instructions in the graph,
* then manipulate them as a whole. */
MVMSpeshIns *checkarity_ins = NULL;
MVMSpeshBB *checkarity_bb = NULL;
MVMSpeshIns *paramnamesused_ins = NULL;
MVMSpeshBB *paramnamesused_bb = NULL;
MVMSpeshIns *param_sp_ins = NULL;
MVMSpeshBB *param_sp_bb = NULL;
MVMSpeshIns *param_sn_ins = NULL;
MVMSpeshBB *param_sn_bb = NULL;
MVMSpeshIns **pos_ins = MVM_calloc(MAX_POS_ARGS, sizeof(MVMSpeshIns *));
MVMSpeshBB **pos_bb = MVM_calloc(MAX_POS_ARGS, sizeof(MVMSpeshBB *));
MVMuint8 *pos_added = MVM_calloc(MAX_POS_ARGS, sizeof(MVMuint8));
MVMSpeshIns **named_ins = MVM_calloc(MAX_NAMED_ARGS, sizeof(MVMSpeshIns *));
MVMSpeshBB **named_bb = MVM_calloc(MAX_NAMED_ARGS, sizeof(MVMSpeshBB *));
MVMSpeshIns **used_ins = MVM_calloc(MAX_NAMED_ARGS, sizeof(MVMSpeshIns *));
MVMint32 req_max = -1;
MVMint32 opt_min = -1;
MVMint32 opt_max = -1;
MVMint32 num_named = 0;
MVMint32 named_used = 0;
MVMint32 got_named = cs->num_pos != cs->arg_count;
/* Walk through the graph, looking for arg related instructions. */
MVMSpeshBB *bb = g->entry;
while (bb) {
MVMSpeshIns *ins = bb->first_ins;
while (ins) {
switch (ins->info->opcode) {
case MVM_OP_checkarity:
if (checkarity_ins)
goto cleanup; /* Dupe; weird; bail out! */
checkarity_ins = ins;
checkarity_bb = bb;
break;
case MVM_OP_param_rp_i:
case MVM_OP_param_rp_n:
case MVM_OP_param_rp_s:
case MVM_OP_param_rp_o: {
/* Required positional. */
MVMint16 idx = ins->operands[1].lit_i16;
if (idx < 0 || idx >= MAX_POS_ARGS)
goto cleanup;
if (pos_ins[idx]) /* Dupe; weird. */
goto cleanup;
pos_ins[idx] = ins;
pos_bb[idx] = bb;
if (idx > req_max)
req_max = idx;
break;
}
case MVM_OP_param_op_i:
case MVM_OP_param_op_n:
case MVM_OP_param_op_s:
case MVM_OP_param_op_o: {
/* Optional Positional int/num/string/object */
MVMint16 idx = ins->operands[1].lit_i16;
if (idx < 0 || idx >= MAX_POS_ARGS)
goto cleanup;
if (pos_ins[idx]) /* Dupe; weird. */
goto cleanup;
pos_ins[idx] = ins;
pos_bb[idx] = bb;
if (idx > opt_max)
opt_max = idx;
if (opt_min == -1 || idx < opt_min)
opt_min = idx;
break;
}
case MVM_OP_param_on_i:
case MVM_OP_param_on_n:
case MVM_OP_param_on_s:
case MVM_OP_param_on_o:
case MVM_OP_param_rn_i:
case MVM_OP_param_rn_n:
case MVM_OP_param_rn_s:
case MVM_OP_param_rn_o:
/* Named (optional or required). */
if (num_named == MAX_NAMED_ARGS)
goto cleanup;
named_ins[num_named] = ins;
named_bb[num_named] = bb;
num_named++;
break;
case MVM_OP_param_sp:
param_sp_ins = ins;
param_sp_bb = bb;
break;
case MVM_OP_param_sn:
param_sn_ins = ins;
param_sn_bb = bb;
break;
case MVM_OP_usecapture:
case MVM_OP_savecapture:
/* Require full args processing context for now; bail. */
goto cleanup;
case MVM_OP_paramnamesused:
if (paramnamesused_ins)
goto cleanup; /* Dupe; weird; bail out! */
paramnamesused_ins = ins;
paramnamesused_bb = bb;
break;
}
ins = ins->next;
}
bb = bb->linear_next;
}
/* If we didn't find a checkarity instruction, bail. */
if (!checkarity_ins)
goto cleanup;
/* If required and optional aren't contiguous, bail. */
if (opt_min >= 0 && req_max + 1 != opt_min)
goto cleanup;
/* If the number of passed args is in range... */
if (cs->num_pos >= req_max + 1 && (opt_max < 0 || cs->num_pos <= opt_max + 1)) {
/* Ensure we've got all the arg fetch instructions we need, and that
* types match or it's a box/unbox. */
MVMint32 i;
for (i = 0; i < cs->num_pos; i++) {
MVMCallsiteEntry arg_flag = cs->arg_flags[i];
if (!pos_ins[i])
goto cleanup;
switch (pos_ins[i]->info->opcode) {
case MVM_OP_param_rp_i:
case MVM_OP_param_op_i:
if (arg_flag != MVM_CALLSITE_ARG_INT)
if (arg_flag != MVM_CALLSITE_ARG_OBJ ||
prim_spec(tc, args[i].o) != MVM_STORAGE_SPEC_BP_INT)
goto cleanup;
break;
case MVM_OP_param_rp_n:
case MVM_OP_param_op_n:
if (arg_flag != MVM_CALLSITE_ARG_NUM)
if (arg_flag != MVM_CALLSITE_ARG_OBJ ||
prim_spec(tc, args[i].o) != MVM_STORAGE_SPEC_BP_NUM)
goto cleanup;
break;
case MVM_OP_param_rp_s:
case MVM_OP_param_op_s:
if (arg_flag != MVM_CALLSITE_ARG_STR)
if (arg_flag != MVM_CALLSITE_ARG_OBJ ||
prim_spec(tc, args[i].o) != MVM_STORAGE_SPEC_BP_STR)
goto cleanup;
break;
case MVM_OP_param_rp_o:
case MVM_OP_param_op_o:
if (arg_flag != MVM_CALLSITE_ARG_OBJ && arg_flag != MVM_CALLSITE_ARG_INT &&
arg_flag != MVM_CALLSITE_ARG_NUM && arg_flag != MVM_CALLSITE_ARG_STR)
goto cleanup;
break;
}
}
/* If we know there's no incoming nameds we can always turn param_sn into a
* simple hash creation. This will typically be further lowered in optimize. */
if (param_sn_ins && !got_named) {
MVMObject *hash_type = g->sf->body.cu->body.hll_config->slurpy_hash_type;
if (REPR(hash_type)->ID == MVM_REPR_ID_MVMHash) {
MVMSpeshOperand target = param_sn_ins->operands[0];
param_sn_ins->info = MVM_op_get_op(MVM_OP_sp_fastcreate);
param_sn_ins->operands = MVM_spesh_alloc(tc, g, 3 * sizeof(MVMSpeshOperand));
param_sn_ins->operands[0] = target;
param_sn_ins->operands[1].lit_i16 = sizeof(MVMHash);
param_sn_ins->operands[2].lit_i16 = MVM_spesh_add_spesh_slot(tc, g,
(MVMCollectable *)STABLE(hash_type));
}
else {
goto cleanup;
}
}
/* We can optimize. Toss checkarity. */
MVM_spesh_manipulate_delete_ins(tc, g, checkarity_bb, checkarity_ins);
/* Re-write the passed required positionals to spesh ops, and store
* any gurads. */
if (cs->arg_count)
g->arg_guards = MVM_malloc(2 * cs->arg_count * sizeof(MVMSpeshGuard));
for (i = 0; i < cs->num_pos; i++) {
MVMCallsiteEntry arg_flag = cs->arg_flags[i];
switch (pos_ins[i]->info->opcode) {
case MVM_OP_param_rp_i:
case MVM_OP_param_op_i:
if (arg_flag == MVM_CALLSITE_ARG_INT) {
pos_ins[i]->info = MVM_op_get_op(MVM_OP_sp_getarg_i);
}
else {
pos_unbox(tc, g, pos_bb[i], pos_ins[i], MVM_op_get_op(MVM_OP_unbox_i));
pos_added[i]++;
if (args[i].o)
add_guards_and_facts(tc, g, i, args[i].o, pos_ins[i]);
}
break;
case MVM_OP_param_rp_n:
case MVM_OP_param_op_n:
if (arg_flag == MVM_CALLSITE_ARG_NUM) {
pos_ins[i]->info = MVM_op_get_op(MVM_OP_sp_getarg_n);
}
else {
pos_unbox(tc, g, pos_bb[i], pos_ins[i], MVM_op_get_op(MVM_OP_unbox_n));
pos_added[i]++;
if (args[i].o)
add_guards_and_facts(tc, g, i, args[i].o, pos_ins[i]);
}
break;
case MVM_OP_param_rp_s:
case MVM_OP_param_op_s:
if (arg_flag == MVM_CALLSITE_ARG_STR) {
pos_ins[i]->info = MVM_op_get_op(MVM_OP_sp_getarg_s);
}
else {
pos_unbox(tc, g, pos_bb[i], pos_ins[i], MVM_op_get_op(MVM_OP_unbox_s));
pos_added[i]++;
if (args[i].o)
add_guards_and_facts(tc, g, i, args[i].o, pos_ins[i]);
}
break;
case MVM_OP_param_rp_o:
case MVM_OP_param_op_o:
if (arg_flag == MVM_CALLSITE_ARG_OBJ) {
pos_ins[i]->info = MVM_op_get_op(MVM_OP_sp_getarg_o);
if (args[i].o)
add_guards_and_facts(tc, g, i, args[i].o, pos_ins[i]);
}
else if (arg_flag == MVM_CALLSITE_ARG_INT) {
pos_box(tc, g, pos_bb[i], pos_ins[i],
MVM_op_get_op(MVM_OP_hllboxtype_i), MVM_op_get_op(MVM_OP_box_i),
MVM_op_get_op(MVM_OP_sp_getarg_i), MVM_reg_int64);
pos_added[i] += 2;
}
else if (arg_flag == MVM_CALLSITE_ARG_NUM) {
pos_box(tc, g, pos_bb[i], pos_ins[i],
MVM_op_get_op(MVM_OP_hllboxtype_n), MVM_op_get_op(MVM_OP_box_n),
MVM_op_get_op(MVM_OP_sp_getarg_n), MVM_reg_num64);
pos_added[i] += 2;
}
else if (arg_flag == MVM_CALLSITE_ARG_STR) {
pos_box(tc, g, pos_bb[i], pos_ins[i],
MVM_op_get_op(MVM_OP_hllboxtype_s), MVM_op_get_op(MVM_OP_box_s),
MVM_op_get_op(MVM_OP_sp_getarg_s), MVM_reg_str);
pos_added[i] += 2;
}
break;
}
pos_ins[i]->operands[1].lit_i16 = (MVMint16)i;
}
/* Now consider any optionals. */
if (opt_min >= 0) {
for (i = opt_min; i <= opt_max; i++) {
MVMuint8 passed = i < cs->num_pos;
if (passed) {
/* If we know the argument has been passed, then add a goto
* to the "passed" code. */
MVMSpeshIns *after = pos_ins[i];
while (pos_added[i]--)
after = after->next;
MVM_spesh_manipulate_insert_goto(tc, g, pos_bb[i], after,
pos_ins[i]->operands[2].ins_bb);
/* Inserting an unconditional goto makes the linear_next BB
* unreachable, so we remove it from the succ list. */
MVM_spesh_manipulate_remove_successor(tc, pos_bb[i],
pos_bb[i]->linear_next);
} else {
/* If we didn't pass this, just fall through the original
* operation and we'll get the default value set. */
MVM_spesh_manipulate_delete_ins(tc, g, pos_bb[i], pos_ins[i]);
MVM_spesh_manipulate_remove_successor(tc, pos_bb[i],
pos_ins[i]->operands[2].ins_bb);
}
}
}
/* Now consider any nameds. */
for (i = 0; i < num_named; i++) {
/* See if the arg was passed. */
MVMString *arg_name = MVM_spesh_get_string(tc, g, named_ins[i]->operands[1]);
MVMint32 passed_nameds = (cs->arg_count - cs->num_pos) / 2;
MVMint32 cs_flags = cs->num_pos + passed_nameds;
MVMint32 cur_idx = 0;
MVMint32 cur_named = 0;
MVMuint8 found_flag = 0;
MVMint32 found_idx = -1;
MVMint32 j;
for (j = 0; j < cs_flags; j++) {
if (cs->arg_flags[j] & MVM_CALLSITE_ARG_NAMED) {
if (MVM_string_equal(tc, arg_name, cs->arg_names[cur_named])) {
/* Found it. */
found_flag = cs->arg_flags[j];
found_idx = cur_idx;
break;
}
cur_idx += 2;
cur_named++;
}
else {
cur_idx++;
}
}
/* Now go by instruction. */
switch (named_ins[i]->info->opcode) {
case MVM_OP_param_rn_i:
if (found_idx == -1)
goto cleanup;
if (found_flag & MVM_CALLSITE_ARG_INT) {
named_ins[i]->info = MVM_op_get_op(MVM_OP_sp_getarg_i);
named_ins[i]->operands[1].lit_i16 = found_idx + 1;
used_ins[i] = add_named_used_ins(tc, g, named_bb[i], named_ins[i], cur_named);
}
else if (found_flag & MVM_CALLSITE_ARG_OBJ
&& prim_spec(tc, args[found_idx].o) == MVM_STORAGE_SPEC_BP_INT) {
named_ins[i]->operands[1].lit_i16 = found_idx + 1;
pos_unbox(tc, g, named_bb[i], named_ins[i], MVM_op_get_op(MVM_OP_unbox_i));
used_ins[i] = add_named_used_ins(tc, g, named_bb[i], named_ins[i]->next, cur_named);
}
named_used++;
break;
case MVM_OP_param_rn_n:
if (found_idx == -1)
goto cleanup;
if (found_flag & MVM_CALLSITE_ARG_NUM) {
named_ins[i]->info = MVM_op_get_op(MVM_OP_sp_getarg_n);
named_ins[i]->operands[1].lit_i16 = found_idx + 1;
used_ins[i] = add_named_used_ins(tc, g, named_bb[i], named_ins[i], cur_named);
}
else if (found_flag & MVM_CALLSITE_ARG_OBJ
&& prim_spec(tc, args[found_idx].o) == MVM_STORAGE_SPEC_BP_NUM) {
named_ins[i]->operands[1].lit_i16 = found_idx + 1;
pos_unbox(tc, g, named_bb[i], named_ins[i], MVM_op_get_op(MVM_OP_unbox_n));
used_ins[i] = add_named_used_ins(tc, g, named_bb[i], named_ins[i]->next, cur_named);
}
named_used++;
break;
case MVM_OP_param_rn_s:
if (found_idx == -1)
goto cleanup;
if (found_flag & MVM_CALLSITE_ARG_STR) {
named_ins[i]->info = MVM_op_get_op(MVM_OP_sp_getarg_s);
named_ins[i]->operands[1].lit_i16 = found_idx + 1;
used_ins[i] = add_named_used_ins(tc, g, named_bb[i], named_ins[i], cur_named);
}
else if (found_flag & MVM_CALLSITE_ARG_OBJ
&& prim_spec(tc, args[found_idx].o) == MVM_STORAGE_SPEC_BP_STR) {
named_ins[i]->operands[1].lit_i16 = found_idx + 1;
pos_unbox(tc, g, named_bb[i], named_ins[i], MVM_op_get_op(MVM_OP_unbox_s));
used_ins[i] = add_named_used_ins(tc, g, named_bb[i], named_ins[i]->next, cur_named);
}
named_used++;
break;
case MVM_OP_param_rn_o:
if (found_idx == -1)
goto cleanup;
if (found_flag & MVM_CALLSITE_ARG_OBJ) {
MVMuint16 arg_idx = found_idx + 1;
named_ins[i]->info = MVM_op_get_op(MVM_OP_sp_getarg_o);
named_ins[i]->operands[1].lit_i16 = arg_idx;
used_ins[i] = add_named_used_ins(tc, g, named_bb[i], named_ins[i], cur_named);
if (args[arg_idx].o)
add_guards_and_facts(tc, g, arg_idx, args[arg_idx].o, named_ins[i]);
}
else if (found_flag & (MVM_CALLSITE_ARG_INT | MVM_CALLSITE_ARG_NUM | MVM_CALLSITE_ARG_STR)) {
MVMuint16 arg_idx = found_idx + 1;
named_ins[i]->operands[1].lit_i16 = arg_idx;
if (found_flag & MVM_CALLSITE_ARG_INT)
pos_box(tc, g, named_bb[i], named_ins[i],
MVM_op_get_op(MVM_OP_hllboxtype_i), MVM_op_get_op(MVM_OP_box_i),
MVM_op_get_op(MVM_OP_sp_getarg_i), MVM_reg_int64);
else if (found_flag & MVM_CALLSITE_ARG_NUM)
pos_box(tc, g, named_bb[i], named_ins[i],
MVM_op_get_op(MVM_OP_hllboxtype_n), MVM_op_get_op(MVM_OP_box_n),
MVM_op_get_op(MVM_OP_sp_getarg_n), MVM_reg_num64);
else if (found_flag & MVM_CALLSITE_ARG_STR)
pos_box(tc, g, named_bb[i], named_ins[i],
MVM_op_get_op(MVM_OP_hllboxtype_s), MVM_op_get_op(MVM_OP_box_s),
MVM_op_get_op(MVM_OP_sp_getarg_s), MVM_reg_str);
used_ins[i] = add_named_used_ins(tc, g, named_bb[i], named_ins[i]->next->next, cur_named);
}
named_used++;
break;
case MVM_OP_param_on_i:
if (found_idx == -1) {
MVM_spesh_manipulate_delete_ins(tc, g, named_bb[i], named_ins[i]);
}
else if (found_flag & MVM_CALLSITE_ARG_INT) {
named_ins[i]->info = MVM_op_get_op(MVM_OP_sp_getarg_i);
named_ins[i]->operands[1].lit_i16 = found_idx + 1;
MVM_spesh_manipulate_insert_goto(tc, g, named_bb[i], named_ins[i],
named_ins[i]->operands[2].ins_bb);
used_ins[i] = add_named_used_ins(tc, g, named_bb[i], named_ins[i], cur_named);
named_used++;
}
else if (found_flag & MVM_CALLSITE_ARG_OBJ
&& prim_spec(tc, args[found_idx].o) == MVM_STORAGE_SPEC_BP_INT) {
named_ins[i]->operands[1].lit_i16 = found_idx + 1;
pos_unbox(tc, g, named_bb[i], named_ins[i], MVM_op_get_op(MVM_OP_unbox_i));
MVM_spesh_manipulate_insert_goto(tc, g, named_bb[i], named_ins[i]->next,
named_ins[i]->operands[2].ins_bb);
used_ins[i] = add_named_used_ins(tc, g, named_bb[i], named_ins[i]->next, cur_named);
named_used++;
}
break;
case MVM_OP_param_on_n:
if (found_idx == -1) {
MVM_spesh_manipulate_delete_ins(tc, g, named_bb[i], named_ins[i]);
}
else if (found_flag & MVM_CALLSITE_ARG_NUM) {
named_ins[i]->info = MVM_op_get_op(MVM_OP_sp_getarg_n);
named_ins[i]->operands[1].lit_i16 = found_idx + 1;
MVM_spesh_manipulate_insert_goto(tc, g, named_bb[i], named_ins[i],
named_ins[i]->operands[2].ins_bb);
used_ins[i] = add_named_used_ins(tc, g, named_bb[i], named_ins[i], cur_named);
named_used++;
}
else if (found_flag & MVM_CALLSITE_ARG_OBJ
&& prim_spec(tc, args[found_idx].o) == MVM_STORAGE_SPEC_BP_NUM) {
named_ins[i]->operands[1].lit_i16 = found_idx + 1;
pos_unbox(tc, g, named_bb[i], named_ins[i], MVM_op_get_op(MVM_OP_unbox_n));
MVM_spesh_manipulate_insert_goto(tc, g, named_bb[i], named_ins[i]->next,
named_ins[i]->operands[2].ins_bb);
used_ins[i] = add_named_used_ins(tc, g, named_bb[i], named_ins[i]->next, cur_named);
named_used++;
}
break;
case MVM_OP_param_on_s:
if (found_idx == -1) {
MVM_spesh_manipulate_delete_ins(tc, g, named_bb[i], named_ins[i]);
}
else if (found_flag & MVM_CALLSITE_ARG_STR) {
named_ins[i]->info = MVM_op_get_op(MVM_OP_sp_getarg_s);
named_ins[i]->operands[1].lit_i16 = found_idx + 1;
MVM_spesh_manipulate_insert_goto(tc, g, named_bb[i], named_ins[i],
named_ins[i]->operands[2].ins_bb);
used_ins[i] = add_named_used_ins(tc, g, named_bb[i], named_ins[i], cur_named);
named_used++;
}
else if (found_flag & MVM_CALLSITE_ARG_OBJ
&& prim_spec(tc, args[found_idx].o) == MVM_STORAGE_SPEC_BP_STR) {
named_ins[i]->operands[1].lit_i16 = found_idx + 1;
pos_unbox(tc, g, named_bb[i], named_ins[i], MVM_op_get_op(MVM_OP_unbox_s));
MVM_spesh_manipulate_insert_goto(tc, g, named_bb[i], named_ins[i]->next,
named_ins[i]->operands[2].ins_bb);
used_ins[i] = add_named_used_ins(tc, g, named_bb[i], named_ins[i]->next, cur_named);
named_used++;
}
break;
case MVM_OP_param_on_o:
if (found_idx == -1) {
MVM_spesh_manipulate_delete_ins(tc, g, named_bb[i], named_ins[i]);
}
else if (found_flag & MVM_CALLSITE_ARG_OBJ) {
MVMuint16 arg_idx = found_idx + 1;
named_ins[i]->info = MVM_op_get_op(MVM_OP_sp_getarg_o);
named_ins[i]->operands[1].lit_i16 = arg_idx;
MVM_spesh_manipulate_insert_goto(tc, g, named_bb[i], named_ins[i],
named_ins[i]->operands[2].ins_bb);
used_ins[i] = add_named_used_ins(tc, g, named_bb[i], named_ins[i], cur_named);
if (args[arg_idx].o)
add_guards_and_facts(tc, g, arg_idx, args[arg_idx].o, named_ins[i]);
named_used++;
}
else if (found_flag & (MVM_CALLSITE_ARG_INT | MVM_CALLSITE_ARG_NUM | MVM_CALLSITE_ARG_STR)) {
MVMuint16 arg_idx = found_idx + 1;
named_ins[i]->operands[1].lit_i16 = arg_idx;
if (found_flag & MVM_CALLSITE_ARG_INT)
pos_box(tc, g, named_bb[i], named_ins[i],
MVM_op_get_op(MVM_OP_hllboxtype_i), MVM_op_get_op(MVM_OP_box_i),
MVM_op_get_op(MVM_OP_sp_getarg_i), MVM_reg_int64);
else if (found_flag & MVM_CALLSITE_ARG_NUM)
pos_box(tc, g, named_bb[i], named_ins[i],
MVM_op_get_op(MVM_OP_hllboxtype_n), MVM_op_get_op(MVM_OP_box_n),
MVM_op_get_op(MVM_OP_sp_getarg_n), MVM_reg_num64);
else if (found_flag & MVM_CALLSITE_ARG_STR)
pos_box(tc, g, named_bb[i], named_ins[i],
MVM_op_get_op(MVM_OP_hllboxtype_s), MVM_op_get_op(MVM_OP_box_s),
MVM_op_get_op(MVM_OP_sp_getarg_s), MVM_reg_str);
MVM_spesh_manipulate_insert_goto(tc, g, named_bb[i], named_ins[i]->next->next,
named_ins[i]->operands[2].ins_bb);
used_ins[i] = add_named_used_ins(tc, g, named_bb[i], named_ins[i]->next->next, cur_named);
named_used++;
}
break;
}
}
/* If we had no nameds or we used them all, can toss namesused, and we
* don't need to mark used after all. */
if (paramnamesused_ins && num_named == named_used) {
MVM_spesh_manipulate_delete_ins(tc, g, paramnamesused_bb, paramnamesused_ins);
for (i = 0; i < num_named; i++)
if (used_ins[i])
MVM_spesh_manipulate_delete_ins(tc, g, named_bb[i], used_ins[i]);
}
}
cleanup:
MVM_free(pos_ins);
MVM_free(pos_bb);
MVM_free(pos_added);
MVM_free(named_ins);
MVM_free(named_bb);
}
/* Check for stability of what was logged, and if it looks sane then add facts
* and turn the log instruction into a */
static void log_facts(MVMThreadContext *tc, MVMSpeshGraph *g, MVMSpeshBB *bb, MVMSpeshIns *ins) {
MVMObject *stable_value = NULL;
MVMObject *stable_cont = NULL;
MVMSpeshFacts *facts;
/* See if all the recorded facts match up; a NULL means there was a code
* path that never reached making a log entry. */
MVMuint16 log_start = ins->operands[1].lit_i16 * MVM_SPESH_LOG_RUNS;
MVMuint16 i;
for (i = log_start; i < log_start + MVM_SPESH_LOG_RUNS; i++) {
MVMObject *consider = (MVMObject *)g->log_slots[i];
if (consider) {
if (!stable_value) {
stable_value = consider;
}
else if (STABLE(stable_value) != STABLE(consider)
|| IS_CONCRETE(stable_value) != IS_CONCRETE(consider)) {
stable_value = NULL;
break;
}
}
}
if (!stable_value)
return;
/* If the value is a container type, need to look inside of it. */
if (STABLE(stable_value)->container_spec && IS_CONCRETE(stable_value)) {
MVMContainerSpec const *contspec = STABLE(stable_value)->container_spec;
if (!contspec->fetch_never_invokes)
return;
stable_cont = stable_value;
stable_value = NULL;
for (i = log_start; i < log_start + MVM_SPESH_LOG_RUNS; i++) {
MVMRegister r;
contspec->fetch(tc, stable_cont, &r);
if (r.o) {
if (!stable_value) {
stable_value = r.o;
}
else if (STABLE(stable_value) != STABLE(r.o)
|| IS_CONCRETE(stable_value) != IS_CONCRETE(r.o)) {
stable_value = NULL;
break;
}
}
}
if (!stable_value)
return;
}
/* Produce a guard op and set facts. */
if (stable_cont) {
MVMSpeshOperand reg = ins->operands[0];
facts = &g->facts[reg.reg.orig][reg.reg.i];
facts->type = STABLE(stable_cont)->WHAT;
facts->flags |= (MVM_SPESH_FACT_KNOWN_TYPE | MVM_SPESH_FACT_CONCRETE |
MVM_SPESH_FACT_KNOWN_DECONT_TYPE);
facts->decont_type = STABLE(stable_value)->WHAT;
if (IS_CONCRETE(stable_value)) {
facts->flags |= MVM_SPESH_FACT_DECONT_CONCRETE;
ins->info = MVM_op_get_op(MVM_OP_sp_guardcontconc);
}
else {
facts->flags |= MVM_SPESH_FACT_DECONT_TYPEOBJ;
ins->info = MVM_op_get_op(MVM_OP_sp_guardconttype);
}
ins->operands = MVM_spesh_alloc(tc, g, 3 * sizeof(MVMSpeshOperand));
ins->operands[0] = reg;
ins->operands[1].lit_i16 = MVM_spesh_add_spesh_slot(tc, g, (MVMCollectable *)STABLE(stable_cont));
ins->operands[2].lit_i16 = MVM_spesh_add_spesh_slot(tc, g, (MVMCollectable *)STABLE(stable_value));
}
else {
facts = &g->facts[ins->operands[0].reg.orig][ins->operands[0].reg.i];
facts->type = STABLE(stable_value)->WHAT;
facts->flags |= (MVM_SPESH_FACT_KNOWN_TYPE | MVM_SPESH_FACT_DECONTED);
if (IS_CONCRETE(stable_value)) {
facts->flags |= MVM_SPESH_FACT_CONCRETE;
ins->info = MVM_op_get_op(MVM_OP_sp_guardconc);
}
else {
facts->flags |= MVM_SPESH_FACT_TYPEOBJ;
ins->info = MVM_op_get_op(MVM_OP_sp_guardtype);
}
ins->operands[1].lit_i16 = MVM_spesh_add_spesh_slot(tc, g, (MVMCollectable *)STABLE(stable_value));
}
/* Add entry in log guards table, and mark facts as depending on it. */
g->log_guards[g->num_log_guards].ins = ins;
g->log_guards[g->num_log_guards].bb = bb;
facts->flags |= MVM_SPESH_FACT_FROM_LOG_GUARD;
facts->log_guard = g->num_log_guards;
g->num_log_guards++;
}
