static int
sequence_uses_addressof (rtx seq)
{
rtx insn;
for (insn = seq; insn; insn = NEXT_INSN (insn))
if (INSN_P (insn))
{
/* If this is a CALL_PLACEHOLDER, then recursively call ourselves
with each nonempty sequence attached to the CALL_PLACEHOLDER. */
if (GET_CODE (insn) == CALL_INSN
&& GET_CODE (PATTERN (insn)) == CALL_PLACEHOLDER)
{
if (XEXP (PATTERN (insn), 0) != NULL_RTX
&& sequence_uses_addressof (XEXP (PATTERN (insn), 0)))
return 1;
if (XEXP (PATTERN (insn), 1) != NULL_RTX
&& sequence_uses_addressof (XEXP (PATTERN (insn), 1)))
return 1;
if (XEXP (PATTERN (insn), 2) != NULL_RTX
&& sequence_uses_addressof (XEXP (PATTERN (insn), 2)))
return 1;
}
else if (uses_addressof (PATTERN (insn))
|| (REG_NOTES (insn) && uses_addressof (REG_NOTES (insn))))
return 1;
}
return 0;
}
static void
add_test (rtx cond, basic_block bb, basic_block dest)
{
rtx seq, jump, label;
enum machine_mode mode;
rtx op0 = XEXP (cond, 0), op1 = XEXP (cond, 1);
enum rtx_code code = GET_CODE (cond);
mode = GET_MODE (XEXP (cond, 0));
if (mode == VOIDmode)
mode = GET_MODE (XEXP (cond, 1));
start_sequence ();
op0 = force_operand (op0, NULL_RTX);
op1 = force_operand (op1, NULL_RTX);
label = block_label (dest);
do_compare_rtx_and_jump (op0, op1, code, 0, mode, NULL_RTX, NULL_RTX, label);
jump = get_last_insn ();
JUMP_LABEL (jump) = label;
/* The jump is supposed to handle an unlikely special case. */
REG_NOTES (jump)
= gen_rtx_EXPR_LIST (REG_BR_PROB,
const0_rtx, REG_NOTES (jump));
LABEL_NUSES (label)++;
seq = get_insns ();
end_sequence ();
emit_insn_after (seq, BB_END (bb));
}
/* Generate code for transformations 3 and 4 (with MODE and OPERATION,
operands OP1 and OP2, result TARGET, at most SUB subtractions, and
probability of taking the optimal path(s) PROB1 and PROB2). */
static rtx
gen_mod_subtract (enum machine_mode mode, enum rtx_code operation,
rtx target, rtx op1, rtx op2, int sub, int prob1, int prob2)
{
rtx tmp, tmp1, jump;
rtx end_label = gen_label_rtx ();
rtx sequence;
int i;
start_sequence ();
if (!REG_P (op2))
{
tmp = gen_reg_rtx (mode);
emit_move_insn (tmp, copy_rtx (op2));
}
else
tmp = op2;
emit_move_insn (target, copy_rtx (op1));
do_compare_rtx_and_jump (target, tmp, LTU, 0, mode, NULL_RTX,
NULL_RTX, end_label);
/* Add branch probability to jump we just created. */
jump = get_last_insn ();
REG_NOTES (jump) = gen_rtx_EXPR_LIST (REG_BR_PROB,
GEN_INT (prob1), REG_NOTES (jump));
for (i = 0; i < sub; i++)
{
tmp1 = expand_simple_binop (mode, MINUS, target, tmp, target,
0, OPTAB_WIDEN);
if (tmp1 != target)
emit_move_insn (target, tmp1);
do_compare_rtx_and_jump (target, tmp, LTU, 0, mode, NULL_RTX,
NULL_RTX, end_label);
/* Add branch probability to jump we just created. */
jump = get_last_insn ();
REG_NOTES (jump) = gen_rtx_EXPR_LIST (REG_BR_PROB,
GEN_INT (prob2), REG_NOTES (jump));
}
tmp1 = simplify_gen_binary (operation, mode, copy_rtx (target), copy_rtx (tmp));
tmp1 = force_operand (tmp1, target);
if (tmp1 != target)
emit_move_insn (target, tmp1);
emit_label (end_label);
sequence = get_insns ();
end_sequence ();
rebuild_jump_labels (sequence);
return sequence;
}
/* Generate code for transformation 2 (with MODE and OPERATION, operands OP1
and OP2, result TARGET and probability of taking the optimal path PROB). */
static rtx
gen_mod_pow2 (enum machine_mode mode, enum rtx_code operation, rtx target,
rtx op1, rtx op2, int prob)
{
rtx tmp, tmp1, tmp2, tmp3, jump;
rtx neq_label = gen_label_rtx ();
rtx end_label = gen_label_rtx ();
rtx sequence;
start_sequence ();
if (!REG_P (op2))
{
tmp = gen_reg_rtx (mode);
emit_move_insn (tmp, copy_rtx (op2));
}
else
tmp = op2;
tmp1 = expand_simple_binop (mode, PLUS, tmp, constm1_rtx, NULL_RTX,
0, OPTAB_WIDEN);
tmp2 = expand_simple_binop (mode, AND, tmp, tmp1, NULL_RTX,
0, OPTAB_WIDEN);
do_compare_rtx_and_jump (tmp2, const0_rtx, NE, 0, mode, NULL_RTX,
NULL_RTX, neq_label);
/* Add branch probability to jump we just created. */
jump = get_last_insn ();
REG_NOTES (jump) = gen_rtx_EXPR_LIST (REG_BR_PROB,
GEN_INT (REG_BR_PROB_BASE - prob),
REG_NOTES (jump));
tmp3 = expand_simple_binop (mode, AND, op1, tmp1, target,
0, OPTAB_WIDEN);
if (tmp3 != target)
emit_move_insn (copy_rtx (target), tmp3);
emit_jump_insn (gen_jump (end_label));
emit_barrier ();
emit_label (neq_label);
tmp1 = simplify_gen_binary (operation, mode, copy_rtx (op1), copy_rtx (tmp));
tmp1 = force_operand (tmp1, target);
if (tmp1 != target)
emit_move_insn (target, tmp1);
emit_label (end_label);
sequence = get_insns ();
end_sequence ();
rebuild_jump_labels (sequence);
return sequence;
}
/* Generate code for transformation 1 (with MODE and OPERATION, operands OP1
and OP2, whose value is expected to be VALUE, result TARGET and
probability of taking the optimal path PROB). */
static rtx
gen_divmod_fixed_value (enum machine_mode mode, enum rtx_code operation,
rtx target, rtx op1, rtx op2, gcov_type value,
int prob)
{
rtx tmp, tmp1, jump;
rtx neq_label = gen_label_rtx ();
rtx end_label = gen_label_rtx ();
rtx sequence;
start_sequence ();
if (!REG_P (op2))
{
tmp = gen_reg_rtx (mode);
emit_move_insn (tmp, copy_rtx (op2));
}
else
tmp = op2;
do_compare_rtx_and_jump (tmp, GEN_INT (value), NE, 0, mode, NULL_RTX,
NULL_RTX, neq_label);
/* Add branch probability to jump we just created. */
jump = get_last_insn ();
REG_NOTES (jump) = gen_rtx_EXPR_LIST (REG_BR_PROB,
GEN_INT (REG_BR_PROB_BASE - prob),
REG_NOTES (jump));
tmp1 = simplify_gen_binary (operation, mode,
copy_rtx (op1), GEN_INT (value));
tmp1 = force_operand (tmp1, target);
if (tmp1 != target)
emit_move_insn (copy_rtx (target), copy_rtx (tmp1));
emit_jump_insn (gen_jump (end_label));
emit_barrier ();
emit_label (neq_label);
tmp1 = simplify_gen_binary (operation, mode,
copy_rtx (op1), copy_rtx (tmp));
tmp1 = force_operand (tmp1, target);
if (tmp1 != target)
emit_move_insn (copy_rtx (target), copy_rtx (tmp1));
emit_label (end_label);
sequence = get_insns ();
end_sequence ();
rebuild_jump_labels (sequence);
return sequence;
}
static void
init_label_info (rtx f)
{
rtx insn;
for (insn = f; insn; insn = NEXT_INSN (insn))
{
if (LABEL_P (insn))
LABEL_NUSES (insn) = (LABEL_PRESERVE_P (insn) != 0);
/* REG_LABEL_TARGET notes (including the JUMP_LABEL field) are
sticky and not reset here; that way we won't lose association
with a label when e.g. the source for a target register
disappears out of reach for targets that may use jump-target
registers. Jump transformations are supposed to transform
any REG_LABEL_TARGET notes. The target label reference in a
branch may disappear from the branch (and from the
instruction before it) for other reasons, like register
allocation. */
if (INSN_P (insn))
{
rtx note, next;
for (note = REG_NOTES (insn); note; note = next)
{
next = XEXP (note, 1);
if (REG_NOTE_KIND (note) == REG_LABEL_OPERAND
&& ! reg_mentioned_p (XEXP (note, 0), PATTERN (insn)))
remove_note (insn, note);
}
}
}
}
/* Emit a slim dump of X (an insn) to the file F, including any register
note attached to the instruction. */
void
dump_insn_slim (FILE *f, const_rtx x)
{
char t[BUF_LEN + 32];
rtx note;
print_insn (t, x, 1);
fputs (t, f);
putc ('\n', f);
if (INSN_P (x) && REG_NOTES (x))
for (note = REG_NOTES (x); note; note = XEXP (note, 1))
{
print_value (t, XEXP (note, 0), 1);
fprintf (f, " %s: %s\n",
GET_REG_NOTE_NAME (REG_NOTE_KIND (note)), t);
}
}
rtx
compare_and_jump_seq (rtx op0, rtx op1, enum rtx_code comp, rtx label, int prob,
rtx cinsn)
{
rtx seq, jump, cond;
enum machine_mode mode;
mode = GET_MODE (op0);
if (mode == VOIDmode)
mode = GET_MODE (op1);
start_sequence ();
if (GET_MODE_CLASS (mode) == MODE_CC)
{
/* A hack -- there seems to be no easy generic way how to make a
conditional jump from a ccmode comparison. */
gcc_assert (cinsn);
cond = XEXP (SET_SRC (pc_set (cinsn)), 0);
gcc_assert (GET_CODE (cond) == comp);
gcc_assert (rtx_equal_p (op0, XEXP (cond, 0)));
gcc_assert (rtx_equal_p (op1, XEXP (cond, 1)));
emit_jump_insn (copy_insn (PATTERN (cinsn)));
jump = get_last_insn ();
JUMP_LABEL (jump) = JUMP_LABEL (cinsn);
LABEL_NUSES (JUMP_LABEL (jump))++;
redirect_jump (jump, label, 0);
}
else
{
gcc_assert (!cinsn);
op0 = force_operand (op0, NULL_RTX);
op1 = force_operand (op1, NULL_RTX);
do_compare_rtx_and_jump (op0, op1, comp, 0,
mode, NULL_RTX, NULL_RTX, label);
jump = get_last_insn ();
JUMP_LABEL (jump) = label;
LABEL_NUSES (label)++;
}
REG_NOTES (jump) = gen_rtx_EXPR_LIST (REG_BR_PROB, GEN_INT (prob),
REG_NOTES (jump));
seq = get_insns ();
end_sequence ();
return seq;
}
static void
print_insn_with_notes (pretty_printer *pp, const_rtx x)
{
pp_string (pp, print_rtx_head);
print_insn (pp, x, 1);
pp_newline (pp);
if (INSN_P (x) && REG_NOTES (x))
for (rtx note = REG_NOTES (x); note; note = XEXP (note, 1))
{
pp_printf (pp, "%s %s ", print_rtx_head,
GET_REG_NOTE_NAME (REG_NOTE_KIND (note)));
if (GET_CODE (note) == INT_LIST)
pp_printf (pp, "%d", XINT (note, 0));
else
print_pattern (pp, XEXP (note, 0), 1);
pp_newline (pp);
}
}
/* Verify that there is exactly single jump instruction since last and attach
REG_BR_PROB note specifying probability.
??? We really ought to pass the probability down to RTL expanders and let it
re-distribute it when the conditional expands into multiple conditionals.
This is however difficult to do. */
static void
add_reg_br_prob_note (FILE *dump_file, rtx last, int probability)
{
if (profile_status == PROFILE_ABSENT)
return;
for (last = NEXT_INSN (last); last && NEXT_INSN (last); last = NEXT_INSN (last))
if (GET_CODE (last) == JUMP_INSN)
{
/* It is common to emit condjump-around-jump sequence when we don't know
how to reverse the conditional. Special case this. */
if (!any_condjump_p (last)
|| GET_CODE (NEXT_INSN (last)) != JUMP_INSN
|| !simplejump_p (NEXT_INSN (last))
|| GET_CODE (NEXT_INSN (NEXT_INSN (last))) != BARRIER
|| GET_CODE (NEXT_INSN (NEXT_INSN (NEXT_INSN (last)))) != CODE_LABEL
|| NEXT_INSN (NEXT_INSN (NEXT_INSN (NEXT_INSN (last)))))
goto failed;
if (find_reg_note (last, REG_BR_PROB, 0))
abort ();
REG_NOTES (last)
= gen_rtx_EXPR_LIST (REG_BR_PROB,
GEN_INT (REG_BR_PROB_BASE - probability),
REG_NOTES (last));
return;
}
if (!last || GET_CODE (last) != JUMP_INSN || !any_condjump_p (last))
goto failed;
if (find_reg_note (last, REG_BR_PROB, 0))
abort ();
REG_NOTES (last)
= gen_rtx_EXPR_LIST (REG_BR_PROB,
GEN_INT (probability), REG_NOTES (last));
return;
failed:
if (dump_file)
fprintf (dump_file, "Failed to add probability note\n");
}
static void
move_dead_notes (rtx to_insn, rtx from_insn, rtx pattern)
{
rtx note;
rtx next_note;
rtx prev_note = NULL;
for (note = REG_NOTES (from_insn); note; note = next_note)
{
next_note = XEXP (note, 1);
if ((REG_NOTE_KIND (note) == REG_DEAD)
&& pattern == XEXP (note, 0))
{
XEXP (note, 1) = REG_NOTES (to_insn);
REG_NOTES (to_insn) = note;
if (prev_note)
XEXP (prev_note, 1) = next_note;
else
REG_NOTES (from_insn) = next_note;
}
else prev_note = note;
}
}
/* Find offset equivalence note for reg WHAT in INSN and return the
found elmination offset. If the note is not found, return NULL.
Remove the found note. */
static rtx
remove_reg_equal_offset_note (rtx_insn *insn, rtx what)
{
rtx link, *link_loc;
for (link_loc = ®_NOTES (insn);
(link = *link_loc) != NULL_RTX;
link_loc = &XEXP (link, 1))
if (REG_NOTE_KIND (link) == REG_EQUAL
&& GET_CODE (XEXP (link, 0)) == PLUS
&& XEXP (XEXP (link, 0), 0) == what
&& CONST_INT_P (XEXP (XEXP (link, 0), 1)))
{
*link_loc = XEXP (link, 1);
return XEXP (XEXP (link, 0), 1);
}
return NULL_RTX;
}
static bool
update_reg_equal_equiv_notes (rtx insn, enum machine_mode new_mode,
enum machine_mode old_mode, enum rtx_code code)
{
rtx *loc = ®_NOTES (insn);
while (*loc)
{
enum reg_note kind = REG_NOTE_KIND (*loc);
if (kind == REG_EQUAL || kind == REG_EQUIV)
{
rtx orig_src = XEXP (*loc, 0);
/* Update equivalency constants. Recall that RTL constants are
sign-extended. */
if (GET_CODE (orig_src) == CONST_INT
&& HOST_BITS_PER_WIDE_INT >= GET_MODE_BITSIZE (new_mode))
{
if (INTVAL (orig_src) >= 0 || code == SIGN_EXTEND)
/* Nothing needed. */;
else
{
/* Zero-extend the negative constant by masking out the
bits outside the source mode. */
rtx new_const_int
= gen_int_mode (INTVAL (orig_src)
& GET_MODE_MASK (old_mode),
new_mode);
if (!validate_change (insn, &XEXP (*loc, 0),
new_const_int, true))
return false;
}
loc = &XEXP (*loc, 1);
}
/* Drop all other notes, they assume a wrong mode. */
else if (!validate_change (insn, loc, XEXP (*loc, 1), true))
return false;
}
else
loc = &XEXP (*loc, 1);
}
return true;
}
static bool
copyprop_hardreg_forward_1 (basic_block bb, struct value_data *vd)
{
bool anything_changed = false;
rtx insn;
for (insn = BB_HEAD (bb); ; insn = NEXT_INSN (insn))
{
int n_ops, i, alt, predicated;
bool is_asm, any_replacements;
rtx set;
rtx link;
bool replaced[MAX_RECOG_OPERANDS];
bool changed = false;
struct kill_set_value_data ksvd;
if (!NONDEBUG_INSN_P (insn))
{
if (DEBUG_INSN_P (insn))
{
rtx loc = INSN_VAR_LOCATION_LOC (insn);
if (!VAR_LOC_UNKNOWN_P (loc))
replace_oldest_value_addr (&INSN_VAR_LOCATION_LOC (insn),
ALL_REGS, GET_MODE (loc),
ADDR_SPACE_GENERIC, insn, vd);
}
if (insn == BB_END (bb))
break;
else
continue;
}
set = single_set (insn);
extract_insn (insn);
if (! constrain_operands (1))
fatal_insn_not_found (insn);
preprocess_constraints ();
alt = which_alternative;
n_ops = recog_data.n_operands;
is_asm = asm_noperands (PATTERN (insn)) >= 0;
/* Simplify the code below by rewriting things to reflect
matching constraints. Also promote OP_OUT to OP_INOUT
in predicated instructions. */
predicated = GET_CODE (PATTERN (insn)) == COND_EXEC;
for (i = 0; i < n_ops; ++i)
{
int matches = recog_op_alt[i][alt].matches;
if (matches >= 0)
recog_op_alt[i][alt].cl = recog_op_alt[matches][alt].cl;
if (matches >= 0 || recog_op_alt[i][alt].matched >= 0
|| (predicated && recog_data.operand_type[i] == OP_OUT))
recog_data.operand_type[i] = OP_INOUT;
}
/* Apply changes to earlier DEBUG_INSNs if possible. */
if (vd->n_debug_insn_changes)
note_uses (&PATTERN (insn), cprop_find_used_regs, vd);
/* For each earlyclobber operand, zap the value data. */
for (i = 0; i < n_ops; i++)
if (recog_op_alt[i][alt].earlyclobber)
kill_value (recog_data.operand[i], vd);
/* Within asms, a clobber cannot overlap inputs or outputs.
I wouldn't think this were true for regular insns, but
scan_rtx treats them like that... */
note_stores (PATTERN (insn), kill_clobbered_value, vd);
/* Kill all auto-incremented values. */
/* ??? REG_INC is useless, since stack pushes aren't done that way. */
for_each_rtx (&PATTERN (insn), kill_autoinc_value, vd);
/* Kill all early-clobbered operands. */
for (i = 0; i < n_ops; i++)
if (recog_op_alt[i][alt].earlyclobber)
kill_value (recog_data.operand[i], vd);
/* If we have dead sets in the insn, then we need to note these as we
would clobbers. */
for (link = REG_NOTES (insn); link; link = XEXP (link, 1))
{
if (REG_NOTE_KIND (link) == REG_UNUSED)
{
kill_value (XEXP (link, 0), vd);
/* Furthermore, if the insn looked like a single-set,
but the dead store kills the source value of that
set, then we can no-longer use the plain move
special case below. */
if (set
&& reg_overlap_mentioned_p (XEXP (link, 0), SET_SRC (set)))
set = NULL;
}
}
/* Special-case plain move instructions, since we may well
be able to do the move from a different register class. */
if (set && REG_P (SET_SRC (set)))
{
rtx src = SET_SRC (set);
unsigned int regno = REGNO (src);
enum machine_mode mode = GET_MODE (src);
unsigned int i;
rtx new_rtx;
/* If we are accessing SRC in some mode other that what we
set it in, make sure that the replacement is valid. */
if (mode != vd->e[regno].mode)
{
if (hard_regno_nregs[regno][mode]
> hard_regno_nregs[regno][vd->e[regno].mode])
goto no_move_special_case;
/* And likewise, if we are narrowing on big endian the transformation
is also invalid. */
if (hard_regno_nregs[regno][mode]
< hard_regno_nregs[regno][vd->e[regno].mode]
&& (GET_MODE_SIZE (vd->e[regno].mode) > UNITS_PER_WORD
? WORDS_BIG_ENDIAN : BYTES_BIG_ENDIAN))
goto no_move_special_case;
}
/* If the destination is also a register, try to find a source
register in the same class. */
if (REG_P (SET_DEST (set)))
{
new_rtx = find_oldest_value_reg (REGNO_REG_CLASS (regno), src, vd);
if (new_rtx && validate_change (insn, &SET_SRC (set), new_rtx, 0))
{
if (dump_file)
fprintf (dump_file,
"insn %u: replaced reg %u with %u\n",
INSN_UID (insn), regno, REGNO (new_rtx));
changed = true;
goto did_replacement;
}
/* We need to re-extract as validate_change clobbers
recog_data. */
extract_insn (insn);
if (! constrain_operands (1))
fatal_insn_not_found (insn);
preprocess_constraints ();
}
/* Otherwise, try all valid registers and see if its valid. */
for (i = vd->e[regno].oldest_regno; i != regno;
i = vd->e[i].next_regno)
{
new_rtx = maybe_mode_change (vd->e[i].mode, vd->e[regno].mode,
mode, i, regno);
if (new_rtx != NULL_RTX)
{
if (validate_change (insn, &SET_SRC (set), new_rtx, 0))
{
ORIGINAL_REGNO (new_rtx) = ORIGINAL_REGNO (src);
REG_ATTRS (new_rtx) = REG_ATTRS (src);
REG_POINTER (new_rtx) = REG_POINTER (src);
if (dump_file)
fprintf (dump_file,
"insn %u: replaced reg %u with %u\n",
INSN_UID (insn), regno, REGNO (new_rtx));
changed = true;
goto did_replacement;
}
/* We need to re-extract as validate_change clobbers
recog_data. */
extract_insn (insn);
if (! constrain_operands (1))
fatal_insn_not_found (insn);
preprocess_constraints ();
}
}
}
no_move_special_case:
any_replacements = false;
/* For each input operand, replace a hard register with the
eldest live copy that's in an appropriate register class. */
for (i = 0; i < n_ops; i++)
{
replaced[i] = false;
/* Don't scan match_operand here, since we've no reg class
information to pass down. Any operands that we could
substitute in will be represented elsewhere. */
if (recog_data.constraints[i][0] == '\0')
continue;
/* Don't replace in asms intentionally referencing hard regs. */
if (is_asm && REG_P (recog_data.operand[i])
&& (REGNO (recog_data.operand[i])
== ORIGINAL_REGNO (recog_data.operand[i])))
continue;
if (recog_data.operand_type[i] == OP_IN)
{
if (recog_op_alt[i][alt].is_address)
replaced[i]
= replace_oldest_value_addr (recog_data.operand_loc[i],
recog_op_alt[i][alt].cl,
VOIDmode, ADDR_SPACE_GENERIC,
insn, vd);
else if (REG_P (recog_data.operand[i]))
replaced[i]
= replace_oldest_value_reg (recog_data.operand_loc[i],
recog_op_alt[i][alt].cl,
insn, vd);
else if (MEM_P (recog_data.operand[i]))
replaced[i] = replace_oldest_value_mem (recog_data.operand[i],
insn, vd);
}
else if (MEM_P (recog_data.operand[i]))
replaced[i] = replace_oldest_value_mem (recog_data.operand[i],
insn, vd);
/* If we performed any replacement, update match_dups. */
if (replaced[i])
{
int j;
rtx new_rtx;
new_rtx = *recog_data.operand_loc[i];
recog_data.operand[i] = new_rtx;
for (j = 0; j < recog_data.n_dups; j++)
if (recog_data.dup_num[j] == i)
validate_unshare_change (insn, recog_data.dup_loc[j], new_rtx, 1);
any_replacements = true;
}
}
if (any_replacements)
{
if (! apply_change_group ())
{
for (i = 0; i < n_ops; i++)
if (replaced[i])
{
rtx old = *recog_data.operand_loc[i];
recog_data.operand[i] = old;
}
if (dump_file)
fprintf (dump_file,
"insn %u: reg replacements not verified\n",
INSN_UID (insn));
}
else
changed = true;
}
did_replacement:
if (changed)
{
anything_changed = true;
/* If something changed, perhaps further changes to earlier
DEBUG_INSNs can be applied. */
if (vd->n_debug_insn_changes)
note_uses (&PATTERN (insn), cprop_find_used_regs, vd);
}
ksvd.vd = vd;
ksvd.ignore_set_reg = NULL_RTX;
/* Clobber call-clobbered registers. */
if (CALL_P (insn))
{
unsigned int set_regno = INVALID_REGNUM;
unsigned int set_nregs = 0;
unsigned int regno;
rtx exp;
hard_reg_set_iterator hrsi;
for (exp = CALL_INSN_FUNCTION_USAGE (insn); exp; exp = XEXP (exp, 1))
{
rtx x = XEXP (exp, 0);
if (GET_CODE (x) == SET)
{
rtx dest = SET_DEST (x);
kill_value (dest, vd);
set_value_regno (REGNO (dest), GET_MODE (dest), vd);
copy_value (dest, SET_SRC (x), vd);
ksvd.ignore_set_reg = dest;
set_regno = REGNO (dest);
set_nregs
= hard_regno_nregs[set_regno][GET_MODE (dest)];
break;
}
}
EXECUTE_IF_SET_IN_HARD_REG_SET (regs_invalidated_by_call, 0, regno, hrsi)
if (regno < set_regno || regno >= set_regno + set_nregs)
kill_value_regno (regno, 1, vd);
/* If SET was seen in CALL_INSN_FUNCTION_USAGE, and SET_SRC
of the SET isn't in regs_invalidated_by_call hard reg set,
but instead among CLOBBERs on the CALL_INSN, we could wrongly
assume the value in it is still live. */
if (ksvd.ignore_set_reg)
note_stores (PATTERN (insn), kill_clobbered_value, vd);
}
/* Notice stores. */
note_stores (PATTERN (insn), kill_set_value, &ksvd);
/* Notice copies. */
if (set && REG_P (SET_DEST (set)) && REG_P (SET_SRC (set)))
copy_value (SET_DEST (set), SET_SRC (set), vd);
if (insn == BB_END (bb))
break;
}
return anything_changed;
}
static bool
speculative_prefetching_transform (rtx insn)
{
rtx histogram, value;
gcov_type val, count, all;
edge e;
rtx mem, address;
int write;
if (!maybe_hot_bb_p (BLOCK_FOR_INSN (insn)))
return false;
if (!find_mem_reference (insn, &mem, &write))
return false;
address = XEXP (mem, 0);
if (side_effects_p (address))
return false;
if (CONSTANT_P (address))
return false;
for (histogram = REG_NOTES (insn);
histogram;
histogram = XEXP (histogram, 1))
if (REG_NOTE_KIND (histogram) == REG_VALUE_PROFILE
&& XEXP (XEXP (histogram, 0), 0) == GEN_INT (HIST_TYPE_CONST_DELTA))
break;
if (!histogram)
return false;
histogram = XEXP (XEXP (histogram, 0), 1);
value = XEXP (histogram, 0);
histogram = XEXP (histogram, 1);
/* Skip last value referenced. */
histogram = XEXP (histogram, 1);
val = INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
count = INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
all = INTVAL (XEXP (histogram, 0));
/* With that few executions we do not really have a reason to optimize the
statement, and more importantly, the data about differences of addresses
are spoiled by the first item that had no previous value to compare
with. */
if (all < 4)
return false;
/* We require that count be at least half of all; this means
that for the transformation to fire the value must be constant
at least 50% of time (and 75% gives the guarantee of usage). */
if (!rtx_equal_p (address, value) || 2 * count < all)
return false;
/* If the difference is too small, it does not make too much sense to
prefetch, as the memory is probably already in cache. */
if (val >= NOPREFETCH_RANGE_MIN && val <= NOPREFETCH_RANGE_MAX)
return false;
if (dump_file)
fprintf (dump_file, "Speculative prefetching for insn %d\n",
INSN_UID (insn));
e = split_block (BLOCK_FOR_INSN (insn), PREV_INSN (insn));
insert_insn_on_edge (gen_speculative_prefetch (address, val, write), e);
return true;
}
static void
doloop_modify (struct loop *loop, struct niter_desc *desc,
rtx doloop_seq, rtx condition, rtx count)
{
rtx counter_reg;
rtx tmp, noloop = NULL_RTX;
rtx sequence;
rtx jump_insn;
rtx jump_label;
int nonneg = 0, irr;
bool increment_count;
basic_block loop_end = desc->out_edge->src;
enum machine_mode mode;
jump_insn = BB_END (loop_end);
if (dump_file)
{
fprintf (dump_file, "Doloop: Inserting doloop pattern (");
if (desc->const_iter)
fprintf (dump_file, HOST_WIDEST_INT_PRINT_DEC, desc->niter);
else
fputs ("runtime", dump_file);
fputs (" iterations).\n", dump_file);
}
/* Discard original jump to continue loop. The original compare
result may still be live, so it cannot be discarded explicitly. */
delete_insn (jump_insn);
counter_reg = XEXP (condition, 0);
if (GET_CODE (counter_reg) == PLUS)
counter_reg = XEXP (counter_reg, 0);
mode = GET_MODE (counter_reg);
increment_count = false;
switch (GET_CODE (condition))
{
case NE:
/* Currently only NE tests against zero and one are supported. */
if (XEXP (condition, 1) == const1_rtx)
{
increment_count = true;
noloop = const1_rtx;
}
else if (XEXP (condition, 1) == const0_rtx)
noloop = const0_rtx;
else
abort ();
break;
case GE:
/* Currently only GE tests against zero are supported. */
if (XEXP (condition, 1) != const0_rtx)
abort ();
noloop = constm1_rtx;
/* The iteration count does not need incrementing for a GE test. */
increment_count = false;
/* Determine if the iteration counter will be non-negative.
Note that the maximum value loaded is iterations_max - 1. */
if (desc->niter_max
<= ((unsigned HOST_WIDEST_INT) 1
<< (GET_MODE_BITSIZE (mode) - 1)))
nonneg = 1;
break;
/* Abort if an invalid doloop pattern has been generated. */
default:
abort ();
}
if (increment_count)
count = simplify_gen_binary (PLUS, mode, count, const1_rtx);
/* Insert initialization of the count register into the loop header. */
start_sequence ();
tmp = force_operand (count, counter_reg);
convert_move (counter_reg, tmp, 1);
sequence = get_insns ();
end_sequence ();
emit_insn_after (sequence, BB_END (loop_preheader_edge (loop)->src));
if (desc->noloop_assumptions)
{
rtx ass = copy_rtx (desc->noloop_assumptions);
basic_block preheader = loop_preheader_edge (loop)->src;
basic_block set_zero
= loop_split_edge_with (loop_preheader_edge (loop), NULL_RTX);
basic_block new_preheader
= loop_split_edge_with (loop_preheader_edge (loop), NULL_RTX);
basic_block bb;
edge te;
gcov_type cnt;
/* Expand the condition testing the assumptions and if it does not pass,
reset the count register to 0. */
add_test (XEXP (ass, 0), preheader, set_zero);
EDGE_SUCC (preheader, 0)->flags &= ~EDGE_FALLTHRU;
cnt = EDGE_SUCC (preheader, 0)->count;
EDGE_SUCC (preheader, 0)->probability = 0;
EDGE_SUCC (preheader, 0)->count = 0;
irr = EDGE_SUCC (preheader, 0)->flags & EDGE_IRREDUCIBLE_LOOP;
te = make_edge (preheader, new_preheader, EDGE_FALLTHRU | irr);
te->probability = REG_BR_PROB_BASE;
te->count = cnt;
set_immediate_dominator (CDI_DOMINATORS, new_preheader, preheader);
set_zero->count = 0;
set_zero->frequency = 0;
for (ass = XEXP (ass, 1); ass; ass = XEXP (ass, 1))
{
bb = loop_split_edge_with (te, NULL_RTX);
te = EDGE_SUCC (bb, 0);
add_test (XEXP (ass, 0), bb, set_zero);
make_edge (bb, set_zero, irr);
}
start_sequence ();
convert_move (counter_reg, noloop, 0);
sequence = get_insns ();
end_sequence ();
emit_insn_after (sequence, BB_END (set_zero));
}
/* Some targets (eg, C4x) need to initialize special looping
registers. */
#ifdef HAVE_doloop_begin
{
rtx init;
unsigned level = get_loop_level (loop) + 1;
init = gen_doloop_begin (counter_reg,
desc->const_iter ? desc->niter_expr : const0_rtx,
desc->niter_max,
GEN_INT (level));
if (init)
{
start_sequence ();
emit_insn (init);
sequence = get_insns ();
end_sequence ();
emit_insn_after (sequence, BB_END (loop_preheader_edge (loop)->src));
}
}
#endif
/* Insert the new low-overhead looping insn. */
emit_jump_insn_after (doloop_seq, BB_END (loop_end));
jump_insn = BB_END (loop_end);
jump_label = block_label (desc->in_edge->dest);
JUMP_LABEL (jump_insn) = jump_label;
LABEL_NUSES (jump_label)++;
/* Ensure the right fallthru edge is marked, for case we have reversed
the condition. */
desc->in_edge->flags &= ~EDGE_FALLTHRU;
desc->out_edge->flags |= EDGE_FALLTHRU;
/* Add a REG_NONNEG note if the actual or estimated maximum number
of iterations is non-negative. */
if (nonneg)
{
REG_NOTES (jump_insn)
= gen_rtx_EXPR_LIST (REG_NONNEG, NULL_RTX, REG_NOTES (jump_insn));
}
}
/* Do transform 1) on INSN if applicable. */
static bool
divmod_fixed_value_transform (rtx insn)
{
rtx set, set_src, set_dest, op1, op2, value, histogram;
enum rtx_code code;
enum machine_mode mode;
gcov_type val, count, all;
edge e;
int prob;
set = single_set (insn);
if (!set)
return false;
set_src = SET_SRC (set);
set_dest = SET_DEST (set);
code = GET_CODE (set_src);
mode = GET_MODE (set_dest);
if (code != DIV && code != MOD && code != UDIV && code != UMOD)
return false;
op1 = XEXP (set_src, false);
op2 = XEXP (set_src, 1);
for (histogram = REG_NOTES (insn);
histogram;
histogram = XEXP (histogram, 1))
if (REG_NOTE_KIND (histogram) == REG_VALUE_PROFILE
&& XEXP (XEXP (histogram, 0), 0) == GEN_INT (HIST_TYPE_SINGLE_VALUE))
break;
if (!histogram)
return false;
histogram = XEXP (XEXP (histogram, 0), 1);
value = XEXP (histogram, 0);
histogram = XEXP (histogram, 1);
val = INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
count = INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
all = INTVAL (XEXP (histogram, 0));
/* We require that count be at least half of all; this means
that for the transformation to fire the value must be constant
at least 50% of time (and 75% gives the guarantee of usage). */
if (!rtx_equal_p (op2, value) || 2 * count < all)
return false;
if (dump_file)
fprintf (dump_file, "Div/mod by constant transformation on insn %d\n",
INSN_UID (insn));
/* Compute probability of taking the optimal path. */
prob = (count * REG_BR_PROB_BASE + all / 2) / all;
e = split_block (BLOCK_FOR_INSN (insn), PREV_INSN (insn));
delete_insn (insn);
insert_insn_on_edge (
gen_divmod_fixed_value (mode, code, set_dest,
op1, op2, val, prob), e);
return true;
}
static void
fixup_reorder_chain (void)
{
basic_block bb, prev_bb;
int index;
rtx insn = NULL;
if (cfg_layout_function_header)
{
set_first_insn (cfg_layout_function_header);
insn = cfg_layout_function_header;
while (NEXT_INSN (insn))
insn = NEXT_INSN (insn);
}
/* First do the bulk reordering -- rechain the blocks without regard to
the needed changes to jumps and labels. */
for (bb = ENTRY_BLOCK_PTR->next_bb, index = NUM_FIXED_BLOCKS;
bb != 0;
bb = bb->aux, index++)
{
if (bb->il.rtl->header)
{
if (insn)
NEXT_INSN (insn) = bb->il.rtl->header;
else
set_first_insn (bb->il.rtl->header);
PREV_INSN (bb->il.rtl->header) = insn;
insn = bb->il.rtl->header;
while (NEXT_INSN (insn))
insn = NEXT_INSN (insn);
}
if (insn)
NEXT_INSN (insn) = BB_HEAD (bb);
else
set_first_insn (BB_HEAD (bb));
PREV_INSN (BB_HEAD (bb)) = insn;
insn = BB_END (bb);
if (bb->il.rtl->footer)
{
NEXT_INSN (insn) = bb->il.rtl->footer;
PREV_INSN (bb->il.rtl->footer) = insn;
while (NEXT_INSN (insn))
insn = NEXT_INSN (insn);
}
}
gcc_assert (index == n_basic_blocks);
NEXT_INSN (insn) = cfg_layout_function_footer;
if (cfg_layout_function_footer)
PREV_INSN (cfg_layout_function_footer) = insn;
while (NEXT_INSN (insn))
insn = NEXT_INSN (insn);
set_last_insn (insn);
#ifdef ENABLE_CHECKING
verify_insn_chain ();
#endif
delete_dead_jumptables ();
/* Now add jumps and labels as needed to match the blocks new
outgoing edges. */
for (bb = ENTRY_BLOCK_PTR->next_bb; bb ; bb = bb->aux)
{
edge e_fall, e_taken, e;
rtx bb_end_insn;
basic_block nb;
edge_iterator ei;
if (EDGE_COUNT (bb->succs) == 0)
continue;
/* Find the old fallthru edge, and another non-EH edge for
a taken jump. */
e_taken = e_fall = NULL;
FOR_EACH_EDGE (e, ei, bb->succs)
if (e->flags & EDGE_FALLTHRU)
e_fall = e;
else if (! (e->flags & EDGE_EH))
e_taken = e;
bb_end_insn = BB_END (bb);
if (JUMP_P (bb_end_insn))
{
if (any_condjump_p (bb_end_insn))
{
/* If the old fallthru is still next, nothing to do. */
if (bb->aux == e_fall->dest
|| e_fall->dest == EXIT_BLOCK_PTR)
continue;
/* The degenerated case of conditional jump jumping to the next
instruction can happen for jumps with side effects. We need
to construct a forwarder block and this will be done just
fine by force_nonfallthru below. */
if (!e_taken)
;
/* There is another special case: if *neither* block is next,
such as happens at the very end of a function, then we'll
need to add a new unconditional jump. Choose the taken
edge based on known or assumed probability. */
else if (bb->aux != e_taken->dest)
{
rtx note = find_reg_note (bb_end_insn, REG_BR_PROB, 0);
if (note
&& INTVAL (XEXP (note, 0)) < REG_BR_PROB_BASE / 2
&& invert_jump (bb_end_insn,
(e_fall->dest == EXIT_BLOCK_PTR
? NULL_RTX
: label_for_bb (e_fall->dest)), 0))
{
e_fall->flags &= ~EDGE_FALLTHRU;
#ifdef ENABLE_CHECKING
gcc_assert (could_fall_through
(e_taken->src, e_taken->dest));
#endif
e_taken->flags |= EDGE_FALLTHRU;
update_br_prob_note (bb);
e = e_fall, e_fall = e_taken, e_taken = e;
}
}
/* If the "jumping" edge is a crossing edge, and the fall
through edge is non-crossing, leave things as they are. */
else if ((e_taken->flags & EDGE_CROSSING)
&& !(e_fall->flags & EDGE_CROSSING))
continue;
/* Otherwise we can try to invert the jump. This will
basically never fail, however, keep up the pretense. */
else if (invert_jump (bb_end_insn,
(e_fall->dest == EXIT_BLOCK_PTR
? NULL_RTX
: label_for_bb (e_fall->dest)), 0))
{
e_fall->flags &= ~EDGE_FALLTHRU;
#ifdef ENABLE_CHECKING
gcc_assert (could_fall_through
(e_taken->src, e_taken->dest));
#endif
e_taken->flags |= EDGE_FALLTHRU;
update_br_prob_note (bb);
continue;
}
}
else
{
/* Otherwise we have some return, switch or computed
jump. In the 99% case, there should not have been a
fallthru edge. */
gcc_assert (returnjump_p (bb_end_insn) || !e_fall);
continue;
}
}
else
{
/* No fallthru implies a noreturn function with EH edges, or
something similarly bizarre. In any case, we don't need to
do anything. */
if (! e_fall)
continue;
/* If the fallthru block is still next, nothing to do. */
if (bb->aux == e_fall->dest)
continue;
/* A fallthru to exit block. */
if (e_fall->dest == EXIT_BLOCK_PTR)
continue;
}
/* We got here if we need to add a new jump insn. */
nb = force_nonfallthru (e_fall);
if (nb)
{
nb->il.rtl->visited = 1;
nb->aux = bb->aux;
bb->aux = nb;
/* Don't process this new block. */
bb = nb;
/* Make sure new bb is tagged for correct section (same as
fall-thru source, since you cannot fall-throu across
section boundaries). */
BB_COPY_PARTITION (e_fall->src, single_pred (bb));
if (flag_reorder_blocks_and_partition
&& targetm.have_named_sections
&& JUMP_P (BB_END (bb))
&& !any_condjump_p (BB_END (bb))
&& (EDGE_SUCC (bb, 0)->flags & EDGE_CROSSING))
REG_NOTES (BB_END (bb)) = gen_rtx_EXPR_LIST
(REG_CROSSING_JUMP, NULL_RTX, REG_NOTES (BB_END (bb)));
}
}
/* Put basic_block_info in the new order. */
if (dump_file)
{
fprintf (dump_file, "Reordered sequence:\n");
for (bb = ENTRY_BLOCK_PTR->next_bb, index = NUM_FIXED_BLOCKS;
bb;
bb = bb->aux, index++)
{
fprintf (dump_file, " %i ", index);
if (get_bb_original (bb))
fprintf (dump_file, "duplicate of %i ",
get_bb_original (bb)->index);
else if (forwarder_block_p (bb)
&& !LABEL_P (BB_HEAD (bb)))
fprintf (dump_file, "compensation ");
else
fprintf (dump_file, "bb %i ", bb->index);
fprintf (dump_file, " [%i]\n", bb->frequency);
}
}
prev_bb = ENTRY_BLOCK_PTR;
bb = ENTRY_BLOCK_PTR->next_bb;
index = NUM_FIXED_BLOCKS;
for (; bb; prev_bb = bb, bb = bb->aux, index ++)
{
bb->index = index;
SET_BASIC_BLOCK (index, bb);
bb->prev_bb = prev_bb;
prev_bb->next_bb = bb;
}
prev_bb->next_bb = EXIT_BLOCK_PTR;
EXIT_BLOCK_PTR->prev_bb = prev_bb;
/* Annoying special case - jump around dead jumptables left in the code. */
FOR_EACH_BB (bb)
{
edge e;
edge_iterator ei;
FOR_EACH_EDGE (e, ei, bb->succs)
if (e->flags & EDGE_FALLTHRU)
break;
if (e && !can_fallthru (e->src, e->dest))
force_nonfallthru (e);
}
}
static void
adjust_frame_related_expr (rtx last_sp_set, rtx insn,
HOST_WIDE_INT this_adjust)
{
rtx note = find_reg_note (last_sp_set, REG_FRAME_RELATED_EXPR, NULL_RTX);
rtx new_expr = NULL_RTX;
if (note == NULL_RTX && RTX_FRAME_RELATED_P (insn))
return;
if (note
&& GET_CODE (XEXP (note, 0)) == SEQUENCE
&& XVECLEN (XEXP (note, 0), 0) >= 2)
{
rtx expr = XEXP (note, 0);
rtx last = XVECEXP (expr, 0, XVECLEN (expr, 0) - 1);
int i;
if (GET_CODE (last) == SET
&& RTX_FRAME_RELATED_P (last) == RTX_FRAME_RELATED_P (insn)
&& SET_DEST (last) == stack_pointer_rtx
&& GET_CODE (SET_SRC (last)) == PLUS
&& XEXP (SET_SRC (last), 0) == stack_pointer_rtx
&& GET_CODE (XEXP (SET_SRC (last), 1)) == CONST_INT)
{
XEXP (SET_SRC (last), 1)
= GEN_INT (INTVAL (XEXP (SET_SRC (last), 1)) + this_adjust);
return;
}
new_expr = gen_rtx_SEQUENCE (VOIDmode,
rtvec_alloc (XVECLEN (expr, 0) + 1));
for (i = 0; i < XVECLEN (expr, 0); i++)
XVECEXP (new_expr, 0, i) = XVECEXP (expr, 0, i);
}
else
{
new_expr = gen_rtx_SEQUENCE (VOIDmode, rtvec_alloc (2));
if (note)
XVECEXP (new_expr, 0, 0) = XEXP (note, 0);
else
{
rtx expr = copy_rtx (single_set_for_csa (last_sp_set));
XEXP (SET_SRC (expr), 1)
= GEN_INT (INTVAL (XEXP (SET_SRC (expr), 1)) - this_adjust);
RTX_FRAME_RELATED_P (expr) = 1;
XVECEXP (new_expr, 0, 0) = expr;
}
}
XVECEXP (new_expr, 0, XVECLEN (new_expr, 0) - 1)
= copy_rtx (single_set_for_csa (insn));
RTX_FRAME_RELATED_P (XVECEXP (new_expr, 0, XVECLEN (new_expr, 0) - 1))
= RTX_FRAME_RELATED_P (insn);
if (note)
XEXP (note, 0) = new_expr;
else
REG_NOTES (last_sp_set)
= gen_rtx_EXPR_LIST (REG_FRAME_RELATED_EXPR, new_expr,
REG_NOTES (last_sp_set));
}
/* Do transform 2) on INSN if applicable. */
static bool
mod_pow2_value_transform (rtx insn)
{
rtx set, set_src, set_dest, op1, op2, value, histogram;
enum rtx_code code;
enum machine_mode mode;
gcov_type wrong_values, count;
edge e;
int i, all, prob;
set = single_set (insn);
if (!set)
return false;
set_src = SET_SRC (set);
set_dest = SET_DEST (set);
code = GET_CODE (set_src);
mode = GET_MODE (set_dest);
if (code != UMOD)
return false;
op1 = XEXP (set_src, 0);
op2 = XEXP (set_src, 1);
for (histogram = REG_NOTES (insn);
histogram;
histogram = XEXP (histogram, 1))
if (REG_NOTE_KIND (histogram) == REG_VALUE_PROFILE
&& XEXP (XEXP (histogram, 0), 0) == GEN_INT (HIST_TYPE_POW2))
break;
if (!histogram)
return false;
histogram = XEXP (XEXP (histogram, 0), 1);
value = XEXP (histogram, 0);
histogram = XEXP (histogram, 1);
wrong_values =INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
count = 0;
for (i = 0; i < GET_MODE_BITSIZE (mode); i++)
{
count += INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
}
if (!rtx_equal_p (op2, value))
return false;
/* We require that we hit a power of two at least half of all evaluations. */
if (count < wrong_values)
return false;
if (dump_file)
fprintf (dump_file, "Mod power of 2 transformation on insn %d\n",
INSN_UID (insn));
/* Compute probability of taking the optimal path. */
all = count + wrong_values;
prob = (count * REG_BR_PROB_BASE + all / 2) / all;
e = split_block (BLOCK_FOR_INSN (insn), PREV_INSN (insn));
delete_insn (insn);
insert_insn_on_edge (
gen_mod_pow2 (mode, code, set_dest, op1, op2, prob), e);
return true;
}
static int
optimize_mode_switching (void)
{
int e;
basic_block bb;
bool need_commit = false;
static const int num_modes[] = NUM_MODES_FOR_MODE_SWITCHING;
#define N_ENTITIES ARRAY_SIZE (num_modes)
int entity_map[N_ENTITIES];
struct bb_info *bb_info[N_ENTITIES];
int i, j;
int n_entities = 0;
int max_num_modes = 0;
bool emitted ATTRIBUTE_UNUSED = false;
basic_block post_entry = 0;
basic_block pre_exit = 0;
struct edge_list *edge_list = 0;
/* These bitmaps are used for the LCM algorithm. */
sbitmap *kill, *del, *insert, *antic, *transp, *comp;
sbitmap *avin, *avout;
for (e = N_ENTITIES - 1; e >= 0; e--)
if (OPTIMIZE_MODE_SWITCHING (e))
{
int entry_exit_extra = 0;
/* Create the list of segments within each basic block.
If NORMAL_MODE is defined, allow for two extra
blocks split from the entry and exit block. */
if (targetm.mode_switching.entry && targetm.mode_switching.exit)
entry_exit_extra = 3;
bb_info[n_entities]
= XCNEWVEC (struct bb_info,
last_basic_block_for_fn (cfun) + entry_exit_extra);
entity_map[n_entities++] = e;
if (num_modes[e] > max_num_modes)
max_num_modes = num_modes[e];
}
if (! n_entities)
return 0;
/* Make sure if MODE_ENTRY is defined MODE_EXIT is defined. */
gcc_assert ((targetm.mode_switching.entry && targetm.mode_switching.exit)
|| (!targetm.mode_switching.entry
&& !targetm.mode_switching.exit));
if (targetm.mode_switching.entry && targetm.mode_switching.exit)
{
/* Split the edge from the entry block, so that we can note that
there NORMAL_MODE is supplied. */
post_entry = split_edge (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
pre_exit = create_pre_exit (n_entities, entity_map, num_modes);
}
df_analyze ();
/* Create the bitmap vectors. */
antic = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
n_entities * max_num_modes);
transp = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
n_entities * max_num_modes);
comp = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
n_entities * max_num_modes);
avin = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
n_entities * max_num_modes);
avout = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
n_entities * max_num_modes);
kill = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
n_entities * max_num_modes);
bitmap_vector_ones (transp, last_basic_block_for_fn (cfun));
bitmap_vector_clear (antic, last_basic_block_for_fn (cfun));
bitmap_vector_clear (comp, last_basic_block_for_fn (cfun));
for (j = n_entities - 1; j >= 0; j--)
{
int e = entity_map[j];
int no_mode = num_modes[e];
struct bb_info *info = bb_info[j];
rtx_insn *insn;
/* Determine what the first use (if any) need for a mode of entity E is.
This will be the mode that is anticipatable for this block.
Also compute the initial transparency settings. */
FOR_EACH_BB_FN (bb, cfun)
{
struct seginfo *ptr;
int last_mode = no_mode;
bool any_set_required = false;
HARD_REG_SET live_now;
info[bb->index].mode_out = info[bb->index].mode_in = no_mode;
REG_SET_TO_HARD_REG_SET (live_now, df_get_live_in (bb));
/* Pretend the mode is clobbered across abnormal edges. */
{
edge_iterator ei;
edge eg;
FOR_EACH_EDGE (eg, ei, bb->preds)
if (eg->flags & EDGE_COMPLEX)
break;
if (eg)
{
rtx_insn *ins_pos = BB_HEAD (bb);
if (LABEL_P (ins_pos))
ins_pos = NEXT_INSN (ins_pos);
gcc_assert (NOTE_INSN_BASIC_BLOCK_P (ins_pos));
if (ins_pos != BB_END (bb))
ins_pos = NEXT_INSN (ins_pos);
ptr = new_seginfo (no_mode, ins_pos, bb->index, live_now);
add_seginfo (info + bb->index, ptr);
for (i = 0; i < no_mode; i++)
clear_mode_bit (transp[bb->index], j, i);
}
}
FOR_BB_INSNS (bb, insn)
{
if (INSN_P (insn))
{
int mode = targetm.mode_switching.needed (e, insn);
rtx link;
if (mode != no_mode && mode != last_mode)
{
any_set_required = true;
last_mode = mode;
ptr = new_seginfo (mode, insn, bb->index, live_now);
add_seginfo (info + bb->index, ptr);
for (i = 0; i < no_mode; i++)
clear_mode_bit (transp[bb->index], j, i);
}
if (targetm.mode_switching.after)
last_mode = targetm.mode_switching.after (e, last_mode,
insn);
/* Update LIVE_NOW. */
for (link = REG_NOTES (insn); link; link = XEXP (link, 1))
if (REG_NOTE_KIND (link) == REG_DEAD)
reg_dies (XEXP (link, 0), &live_now);
note_stores (PATTERN (insn), reg_becomes_live, &live_now);
for (link = REG_NOTES (insn); link; link = XEXP (link, 1))
if (REG_NOTE_KIND (link) == REG_UNUSED)
reg_dies (XEXP (link, 0), &live_now);
}
}
info[bb->index].computing = last_mode;
/* Check for blocks without ANY mode requirements.
N.B. because of MODE_AFTER, last_mode might still
be different from no_mode, in which case we need to
mark the block as nontransparent. */
if (!any_set_required)
{
ptr = new_seginfo (no_mode, BB_END (bb), bb->index, live_now);
add_seginfo (info + bb->index, ptr);
if (last_mode != no_mode)
for (i = 0; i < no_mode; i++)
clear_mode_bit (transp[bb->index], j, i);
}
}
if (targetm.mode_switching.entry && targetm.mode_switching.exit)
{
int mode = targetm.mode_switching.entry (e);
info[post_entry->index].mode_out =
info[post_entry->index].mode_in = no_mode;
if (pre_exit)
{
info[pre_exit->index].mode_out =
info[pre_exit->index].mode_in = no_mode;
}
if (mode != no_mode)
{
bb = post_entry;
/* By always making this nontransparent, we save
an extra check in make_preds_opaque. We also
need this to avoid confusing pre_edge_lcm when
antic is cleared but transp and comp are set. */
for (i = 0; i < no_mode; i++)
clear_mode_bit (transp[bb->index], j, i);
/* Insert a fake computing definition of MODE into entry
blocks which compute no mode. This represents the mode on
entry. */
info[bb->index].computing = mode;
if (pre_exit)
info[pre_exit->index].seginfo->mode =
targetm.mode_switching.exit (e);
}
}
/* Set the anticipatable and computing arrays. */
for (i = 0; i < no_mode; i++)
{
int m = targetm.mode_switching.priority (entity_map[j], i);
FOR_EACH_BB_FN (bb, cfun)
{
if (info[bb->index].seginfo->mode == m)
set_mode_bit (antic[bb->index], j, m);
if (info[bb->index].computing == m)
set_mode_bit (comp[bb->index], j, m);
}
}
}
/* Calculate the optimal locations for the
placement mode switches to modes with priority I. */
FOR_EACH_BB_FN (bb, cfun)
bitmap_not (kill[bb->index], transp[bb->index]);
edge_list = pre_edge_lcm_avs (n_entities * max_num_modes, transp, comp, antic,
kill, avin, avout, &insert, &del);
for (j = n_entities - 1; j >= 0; j--)
{
int no_mode = num_modes[entity_map[j]];
/* Insert all mode sets that have been inserted by lcm. */
for (int ed = NUM_EDGES (edge_list) - 1; ed >= 0; ed--)
{
edge eg = INDEX_EDGE (edge_list, ed);
eg->aux = (void *)(intptr_t)-1;
for (i = 0; i < no_mode; i++)
{
int m = targetm.mode_switching.priority (entity_map[j], i);
if (mode_bit_p (insert[ed], j, m))
{
eg->aux = (void *)(intptr_t)m;
break;
}
}
}
FOR_EACH_BB_FN (bb, cfun)
{
struct bb_info *info = bb_info[j];
int last_mode = no_mode;
/* intialize mode in availability for bb. */
for (i = 0; i < no_mode; i++)
if (mode_bit_p (avout[bb->index], j, i))
{
if (last_mode == no_mode)
last_mode = i;
if (last_mode != i)
{
last_mode = no_mode;
break;
}
}
info[bb->index].mode_out = last_mode;
/* intialize mode out availability for bb. */
last_mode = no_mode;
for (i = 0; i < no_mode; i++)
if (mode_bit_p (avin[bb->index], j, i))
{
if (last_mode == no_mode)
last_mode = i;
if (last_mode != i)
{
last_mode = no_mode;
break;
}
}
info[bb->index].mode_in = last_mode;
for (i = 0; i < no_mode; i++)
if (mode_bit_p (del[bb->index], j, i))
info[bb->index].seginfo->mode = no_mode;
}
/* Now output the remaining mode sets in all the segments. */
/* In case there was no mode inserted. the mode information on the edge
might not be complete.
Update mode info on edges and commit pending mode sets. */
need_commit |= commit_mode_sets (edge_list, entity_map[j], bb_info[j]);
/* Reset modes for next entity. */
clear_aux_for_edges ();
FOR_EACH_BB_FN (bb, cfun)
{
struct seginfo *ptr, *next;
int cur_mode = bb_info[j][bb->index].mode_in;
for (ptr = bb_info[j][bb->index].seginfo; ptr; ptr = next)
{
next = ptr->next;
if (ptr->mode != no_mode)
{
rtx_insn *mode_set;
rtl_profile_for_bb (bb);
start_sequence ();
targetm.mode_switching.emit (entity_map[j], ptr->mode,
cur_mode, ptr->regs_live);
mode_set = get_insns ();
end_sequence ();
/* modes kill each other inside a basic block. */
cur_mode = ptr->mode;
/* Insert MODE_SET only if it is nonempty. */
if (mode_set != NULL_RTX)
{
emitted = true;
if (NOTE_INSN_BASIC_BLOCK_P (ptr->insn_ptr))
/* We need to emit the insns in a FIFO-like manner,
i.e. the first to be emitted at our insertion
point ends up first in the instruction steam.
Because we made sure that NOTE_INSN_BASIC_BLOCK is
only used for initially empty basic blocks, we
can achieve this by appending at the end of
the block. */
emit_insn_after
(mode_set, BB_END (NOTE_BASIC_BLOCK (ptr->insn_ptr)));
else
emit_insn_before (mode_set, ptr->insn_ptr);
}
default_rtl_profile ();
}
free (ptr);
}
}
free (bb_info[j]);
}
free_edge_list (edge_list);
/* Finished. Free up all the things we've allocated. */
sbitmap_vector_free (del);
sbitmap_vector_free (insert);
sbitmap_vector_free (kill);
sbitmap_vector_free (antic);
sbitmap_vector_free (transp);
sbitmap_vector_free (comp);
sbitmap_vector_free (avin);
sbitmap_vector_free (avout);
if (need_commit)
commit_edge_insertions ();
if (targetm.mode_switching.entry && targetm.mode_switching.exit)
cleanup_cfg (CLEANUP_NO_INSN_DEL);
else if (!need_commit && !emitted)
return 0;
return 1;
}
/* Do transforms 3) and 4) on INSN if applicable. */
static bool
mod_subtract_transform (rtx insn)
{
rtx set, set_src, set_dest, op1, op2, value, histogram;
enum rtx_code code;
enum machine_mode mode;
gcov_type wrong_values, counts[2], count, all;
edge e;
int i, prob1, prob2;
set = single_set (insn);
if (!set)
return false;
set_src = SET_SRC (set);
set_dest = SET_DEST (set);
code = GET_CODE (set_src);
mode = GET_MODE (set_dest);
if (code != UMOD)
return false;
op1 = XEXP (set_src, 0);
op2 = XEXP (set_src, 1);
for (histogram = REG_NOTES (insn);
histogram;
histogram = XEXP (histogram, 1))
if (REG_NOTE_KIND (histogram) == REG_VALUE_PROFILE
&& XEXP (XEXP (histogram, 0), 0) == GEN_INT (HIST_TYPE_INTERVAL))
break;
if (!histogram)
return false;
histogram = XEXP (XEXP (histogram, 0), 1);
value = XEXP (histogram, 0);
histogram = XEXP (histogram, 1);
all = 0;
for (i = 0; i < 2; i++)
{
counts[i] = INTVAL (XEXP (histogram, 0));
all += counts[i];
histogram = XEXP (histogram, 1);
}
wrong_values = INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
wrong_values += INTVAL (XEXP (histogram, 0));
all += wrong_values;
/* We require that we use just subtractions in at least 50% of all
evaluations. */
count = 0;
for (i = 0; i < 2; i++)
{
count += counts[i];
if (count * 2 >= all)
break;
}
if (i == 2)
return false;
if (dump_file)
fprintf (dump_file, "Mod subtract transformation on insn %d\n",
INSN_UID (insn));
/* Compute probability of taking the optimal path(s). */
prob1 = (counts[0] * REG_BR_PROB_BASE + all / 2) / all;
prob2 = (counts[1] * REG_BR_PROB_BASE + all / 2) / all;
e = split_block (BLOCK_FOR_INSN (insn), PREV_INSN (insn));
delete_insn (insn);
insert_insn_on_edge (
gen_mod_subtract (mode, code, set_dest,
op1, op2, i, prob1, prob2), e);
return true;
}