/* Add template to nodes, filling in operands and linking tree nodes. Return template root */
static MVMint32 apply_template(MVMThreadContext *tc, MVMJitExprTree *tree, MVMint32 len, char *info,
MVMint32 *code, MVMint32 *operands) {
MVMint32 i, j, root = 0, base = tree->nodes_num;
MVM_VECTOR_ENSURE_SPACE(tree->nodes, len);
/* Loop over string until the end */
for (i = 0, j = base; info[i]; i++, j++) {
switch (info[i]) {
case 'l':
/* template contained a node */
assert(info[code[i]] == 'n');
/* link template-relative to nodes-relative */
tree->nodes[j] = code[i] + base;
break;
case 'i':
/* insert input operand node */
tree->nodes[j] = operands[code[i]];
break;
case 'c':
tree->nodes[j] = MVM_jit_expr_add_const_ptr(tc, tree, MVM_jit_expr_template_constants[code[i]]);
break;
case 'n':
/* next node should contain size */
tree->nodes[j] = code[i];
assert(i + 1 < len && info[i+1] == 's');
root = j;
break;
case 's':
/* Install operator info and read size argument for variadic nodes */
assert(i > 0 && info[i-1] == 'n');
{
const struct OpInfo *op_info = get_op_info(code[i-1]);
MVMJitExprInfo *expr_info = MVM_JIT_EXPR_INFO(tree, j-1);
expr_info->num_links = op_info->nchild < 0 ? code[i] : op_info->nchild;
expr_info->num_args = op_info->nargs;
break;
}
case '.':
default:
/* copy constant from template */
tree->nodes[j] = code[i];
break;
}
}
assert(i == len);
tree->nodes_num = base + len;
return root;
}
t_list *process_routine(t_list *list, t_list *cur, void *param)
{
char *byte;
t_arena *arena;
t_process *proc;
arena = param;
proc = cur->data;
if (proc->cycles_left > 0)
proc->cycles_left--;
else
{
if (proc->cycles_left == 0)
call_asm_func(arena, proc);
byte = get_memory_data(arena, proc->pc, 1);
proc->cycles_left = get_op_info(*byte).nbr_cycles;
free(byte);
}
return (list);
}
static void build_cfg(MVMThreadContext *tc, MVMSpeshGraph *g, MVMStaticFrame *sf,
MVMint32 *existing_deopts, MVMint32 num_existing_deopts) {
MVMSpeshBB *cur_bb, *prev_bb;
MVMSpeshIns *last_ins;
MVMint64 i;
MVMint32 bb_idx;
/* Temporary array of all MVMSpeshIns we create (one per instruction).
* Overestimate at size. Has the flat view, matching the bytecode. */
MVMSpeshIns **ins_flat = MVM_calloc(g->bytecode_size / 2, sizeof(MVMSpeshIns *));
/* Temporary array where each byte in the input bytecode gets a 32-bit
* integer. This is used for two things:
* A) When we make the MVMSpeshIns for an instruction starting at the
* byte, we put the instruction index (into ins_flat) in the slot,
* shifting it by 2 bits to the left. We will use this to do fixups.
* B) The first bit is "I have an incoming branch" - that is, start of
* a basic block. The second bit is "I can branch" - that is, end of
* a basic block. It's possible to have both bits set.
* Anything that's just a zero has no instruction starting there. */
MVMuint32 *byte_to_ins_flags = MVM_calloc(g->bytecode_size, sizeof(MVMuint32));
/* Instruction to basic block mapping. Initialized later. */
MVMSpeshBB **ins_to_bb = NULL;
/* Make first pass through the bytecode. In this pass, we make MVMSpeshIns
* nodes for each instruction and set the start/end of block bits. Also
* set handler targets as basic block starters. */
MVMCompUnit *cu = sf->body.cu;
MVMuint8 *pc = g->bytecode;
MVMuint8 *end = g->bytecode + g->bytecode_size;
MVMuint32 ins_idx = 0;
MVMuint8 next_bbs = 1; /* Next iteration (here, first) starts a BB. */
for (i = 0; i < g->num_handlers; i++)
byte_to_ins_flags[g->handlers[i].goto_offset] |= MVM_CFG_BB_START;
while (pc < end) {
/* Look up op info. */
MVMuint16 opcode = *(MVMuint16 *)pc;
MVMuint8 *args = pc + 2;
MVMuint8 arg_size = 0;
const MVMOpInfo *info = get_op_info(tc, cu, opcode);
/* Create an instruction node, add it, and record its position. */
MVMSpeshIns *ins_node = MVM_spesh_alloc(tc, g, sizeof(MVMSpeshIns));
ins_flat[ins_idx] = ins_node;
byte_to_ins_flags[pc - g->bytecode] |= ins_idx << 2;
/* Did previous instruction end a basic block? */
if (next_bbs) {
byte_to_ins_flags[pc - g->bytecode] |= MVM_CFG_BB_START;
next_bbs = 0;
}
/* Also check we're not already a BB start due to being a branch
* target, in which case we should ensure our prior is marked as
* a BB end. */
else {
if (byte_to_ins_flags[pc - g->bytecode] & MVM_CFG_BB_START) {
MVMuint32 hunt = pc - g->bytecode;
while (!byte_to_ins_flags[--hunt]);
byte_to_ins_flags[hunt] |= MVM_CFG_BB_END;
}
}
/* Store opcode */
ins_node->info = info;
/* Go over operands. */
ins_node->operands = MVM_spesh_alloc(tc, g, info->num_operands * sizeof(MVMSpeshOperand));
for (i = 0; i < info->num_operands; i++) {
MVMuint8 flags = info->operands[i];
MVMuint8 rw = flags & MVM_operand_rw_mask;
switch (rw) {
case MVM_operand_read_reg:
case MVM_operand_write_reg:
ins_node->operands[i].reg.orig = GET_UI16(args, arg_size);
arg_size += 2;
break;
case MVM_operand_read_lex:
case MVM_operand_write_lex:
ins_node->operands[i].lex.idx = GET_UI16(args, arg_size);
ins_node->operands[i].lex.outers = GET_UI16(args, arg_size + 2);
arg_size += 4;
break;
case MVM_operand_literal: {
MVMuint32 type = flags & MVM_operand_type_mask;
switch (type) {
case MVM_operand_int8:
ins_node->operands[i].lit_i8 = GET_I8(args, arg_size);
arg_size += 1;
break;
case MVM_operand_int16:
ins_node->operands[i].lit_i16 = GET_I16(args, arg_size);
arg_size += 2;
break;
case MVM_operand_int32:
ins_node->operands[i].lit_i32 = GET_I32(args, arg_size);
arg_size += 4;
break;
case MVM_operand_int64:
ins_node->operands[i].lit_i64 = MVM_BC_get_I64(args, arg_size);
arg_size += 8;
break;
case MVM_operand_num32:
ins_node->operands[i].lit_n32 = GET_N32(args, arg_size);
arg_size += 4;
break;
case MVM_operand_num64:
ins_node->operands[i].lit_n64 = MVM_BC_get_N64(args, arg_size);
arg_size += 8;
break;
case MVM_operand_callsite:
ins_node->operands[i].callsite_idx = GET_UI16(args, arg_size);
arg_size += 2;
break;
case MVM_operand_coderef:
ins_node->operands[i].coderef_idx = GET_UI16(args, arg_size);
arg_size += 2;
break;
case MVM_operand_str:
ins_node->operands[i].lit_str_idx = GET_UI32(args, arg_size);
arg_size += 4;
break;
case MVM_operand_ins: {
/* Stash instruction offset. */
MVMuint32 target = GET_UI32(args, arg_size);
ins_node->operands[i].ins_offset = target;
/* This is a branching instruction, so it's a BB end. */
byte_to_ins_flags[pc - g->bytecode] |= MVM_CFG_BB_END;
/* Its target is a BB start, and any previous instruction
* we already passed needs marking as a BB end. */
byte_to_ins_flags[target] |= MVM_CFG_BB_START;
if (target > 0 && target < pc - g->bytecode) {
while (!byte_to_ins_flags[--target]);
byte_to_ins_flags[target] |= MVM_CFG_BB_END;
}
/* Next instruction is also a BB start. */
next_bbs = 1;
arg_size += 4;
break;
}
case MVM_operand_spesh_slot:
ins_node->operands[i].lit_i16 = GET_I16(args, arg_size);
arg_size += 2;
break;
default:
MVM_exception_throw_adhoc(tc,
"Spesh: unknown operand type %d in graph building (op %s)",
(int)type, ins_node->info->name);
}
}
break;
}
}
/* We specially handle the jumplist case, which needs to mark all of
* the possible places we could jump to in the following instructions
* as starts of basic blocks. It is, in itself, the end of one. Note
* we jump to the instruction after the n jump points if none match,
* so that is marked too. */
if (opcode == MVM_OP_jumplist) {
MVMint64 n = MVM_BC_get_I64(args, 0);
for (i = 0; i <= n; i++)
byte_to_ins_flags[(pc - g->bytecode) + 12 + i * 6] |= MVM_CFG_BB_START;
byte_to_ins_flags[pc - g->bytecode] |= MVM_CFG_BB_END;
}
/* Invocations, returns, and throws are basic block ends. */
switch (opcode) {
case MVM_OP_invoke_v:
case MVM_OP_invoke_i:
case MVM_OP_invoke_n:
case MVM_OP_invoke_s:
case MVM_OP_invoke_o:
case MVM_OP_return_i:
case MVM_OP_return_n:
case MVM_OP_return_s:
case MVM_OP_return_o:
case MVM_OP_return:
case MVM_OP_throwdyn:
case MVM_OP_throwlex:
case MVM_OP_throwlexotic:
case MVM_OP_throwcatdyn:
case MVM_OP_throwcatlex:
case MVM_OP_throwcatlexotic:
case MVM_OP_die:
case MVM_OP_rethrow:
case MVM_OP_resume:
byte_to_ins_flags[pc - g->bytecode] |= MVM_CFG_BB_END;
next_bbs = 1;
break;
}
/* Final instruction is basic block end. */
if (pc + 2 + arg_size == end)
byte_to_ins_flags[pc - g->bytecode] |= MVM_CFG_BB_END;
/* Caculate next instruction's PC. */
pc += 2 + arg_size;
/* If this is a deopt point opcode... */
if (!existing_deopts && (info->deopt_point & MVM_DEOPT_MARK_ONE))
add_deopt_annotation(tc, g, ins_node, pc, MVM_SPESH_ANN_DEOPT_ONE_INS);
if (!existing_deopts && (info->deopt_point & MVM_DEOPT_MARK_ALL))
add_deopt_annotation(tc, g, ins_node, pc, MVM_SPESH_ANN_DEOPT_ALL_INS);
if (!existing_deopts && (info->deopt_point & MVM_DEOPT_MARK_OSR))
add_deopt_annotation(tc, g, ins_node, pc, MVM_SPESH_ANN_DEOPT_OSR);
/* Go to next instruction. */
ins_idx++;
}
/* Annotate instructions that are handler-significant. */
for (i = 0; i < g->num_handlers; i++) {
MVMSpeshIns *start_ins = ins_flat[byte_to_ins_flags[g->handlers[i].start_offset] >> 2];
MVMSpeshIns *end_ins = ins_flat[byte_to_ins_flags[g->handlers[i].end_offset] >> 2];
MVMSpeshIns *goto_ins = ins_flat[byte_to_ins_flags[g->handlers[i].goto_offset] >> 2];
MVMSpeshAnn *start_ann = MVM_spesh_alloc(tc, g, sizeof(MVMSpeshAnn));
MVMSpeshAnn *end_ann = MVM_spesh_alloc(tc, g, sizeof(MVMSpeshAnn));
MVMSpeshAnn *goto_ann = MVM_spesh_alloc(tc, g, sizeof(MVMSpeshAnn));
start_ann->next = start_ins->annotations;
start_ann->type = MVM_SPESH_ANN_FH_START;
start_ann->data.frame_handler_index = i;
start_ins->annotations = start_ann;
end_ann->next = end_ins->annotations;
end_ann->type = MVM_SPESH_ANN_FH_END;
end_ann->data.frame_handler_index = i;
end_ins->annotations = end_ann;
goto_ann->next = goto_ins->annotations;
goto_ann->type = MVM_SPESH_ANN_FH_GOTO;
goto_ann->data.frame_handler_index = i;
goto_ins->annotations = goto_ann;
}
/* Annotate instructions that are inline start/end points. */
for (i = 0; i < g->num_inlines; i++) {
MVMSpeshIns *start_ins = ins_flat[byte_to_ins_flags[g->inlines[i].start] >> 2];
MVMSpeshIns *end_ins = ins_flat[byte_to_ins_flags[g->inlines[i].end] >> 2];
MVMSpeshAnn *start_ann = MVM_spesh_alloc(tc, g, sizeof(MVMSpeshAnn));
MVMSpeshAnn *end_ann = MVM_spesh_alloc(tc, g, sizeof(MVMSpeshAnn));
start_ann->next = start_ins->annotations;
start_ann->type = MVM_SPESH_ANN_INLINE_START;
start_ann->data.inline_idx = i;
start_ins->annotations = start_ann;
end_ann->next = end_ins->annotations;
end_ann->type = MVM_SPESH_ANN_INLINE_END;
end_ann->data.inline_idx = i;
end_ins->annotations = end_ann;
}
/* Now for the second pass, where we assemble the basic blocks. Also we
* build a lookup table of instructions that start a basic block to that
* basic block, for the final CFG construction. We make the entry block a
* special one, containing a noop; it will have any exception handler
* targets linked from it, so they show up in the graph. */
g->entry = MVM_spesh_alloc(tc, g, sizeof(MVMSpeshBB));
g->entry->first_ins = MVM_spesh_alloc(tc, g, sizeof(MVMSpeshIns));
g->entry->first_ins->info = get_op_info(tc, cu, 0);
g->entry->last_ins = g->entry->first_ins;
g->entry->idx = 0;
cur_bb = NULL;
prev_bb = g->entry;
last_ins = NULL;
ins_to_bb = MVM_calloc(ins_idx, sizeof(MVMSpeshBB *));
ins_idx = 0;
bb_idx = 1;
for (i = 0; i < g->bytecode_size; i++) {
MVMSpeshIns *cur_ins;
/* Skip zeros; no instruction here. */
if (!byte_to_ins_flags[i])
continue;
/* Get current instruction. */
cur_ins = ins_flat[byte_to_ins_flags[i] >> 2];
/* Start of a basic block? */
if (byte_to_ins_flags[i] & MVM_CFG_BB_START) {
/* Should not already be in a basic block. */
if (cur_bb) {
MVM_spesh_graph_destroy(tc, g);
MVM_exception_throw_adhoc(tc, "Spesh: confused during basic block analysis (in block)");
}
/* Create it, and set first instruction and index. */
cur_bb = MVM_spesh_alloc(tc, g, sizeof(MVMSpeshBB));
cur_bb->first_ins = cur_ins;
cur_bb->idx = bb_idx;
bb_idx++;
/* Record instruction -> BB start mapping. */
ins_to_bb[ins_idx] = cur_bb;
/* Link it to the previous one. */
prev_bb->linear_next = cur_bb;
}
/* Should always be in a BB at this point. */
if (!cur_bb) {
MVM_spesh_graph_destroy(tc, g);
MVM_exception_throw_adhoc(tc, "Spesh: confused during basic block analysis (no block)");
}
/* Add instruction into double-linked per-block instruction list. */
if (last_ins) {
last_ins->next = cur_ins;
cur_ins->prev = last_ins;
}
last_ins = cur_ins;
/* End of a basic block? */
if (byte_to_ins_flags[i] & MVM_CFG_BB_END) {
cur_bb->last_ins = cur_ins;
prev_bb = cur_bb;
cur_bb = NULL;
last_ins = NULL;
}
ins_idx++;
}
g->num_bbs = bb_idx;
/* Finally, link the basic blocks up to form a CFG. Along the way, any of
* the instruction operands get the target BB stored. */
cur_bb = g->entry;
while (cur_bb) {
/* If it's the first block, it's a special case; successors are the
* real successor and all exception handlers. */
if (cur_bb == g->entry) {
cur_bb->num_succ = 1 + g->num_handlers;
cur_bb->succ = MVM_spesh_alloc(tc, g, cur_bb->num_succ * sizeof(MVMSpeshBB *));
cur_bb->succ[0] = cur_bb->linear_next;
for (i = 0; i < g->num_handlers; i++) {
MVMuint32 offset = g->handlers[i].goto_offset;
cur_bb->succ[i + 1] = ins_to_bb[byte_to_ins_flags[offset] >> 2];
}
}
/* Otherwise, consider the last instruction, to see how we leave the BB. */
else {
switch (cur_bb->last_ins->info->opcode) {
const char * MVM_jit_expr_operator_name(MVMThreadContext *tc, enum MVMJitExprOperator operator) {
return get_op_info(operator)->name;
}
