static void
init_label_info (rtx f)
{
rtx insn;
for (insn = f; insn; insn = NEXT_INSN (insn))
{
if (LABEL_P (insn))
LABEL_NUSES (insn) = (LABEL_PRESERVE_P (insn) != 0);
/* REG_LABEL_TARGET notes (including the JUMP_LABEL field) are
sticky and not reset here; that way we won't lose association
with a label when e.g. the source for a target register
disappears out of reach for targets that may use jump-target
registers. Jump transformations are supposed to transform
any REG_LABEL_TARGET notes. The target label reference in a
branch may disappear from the branch (and from the
instruction before it) for other reasons, like register
allocation. */
if (INSN_P (insn))
{
rtx note, next;
for (note = REG_NOTES (insn); note; note = next)
{
next = XEXP (note, 1);
if (REG_NOTE_KIND (note) == REG_LABEL_OPERAND
&& ! reg_mentioned_p (XEXP (note, 0), PATTERN (insn)))
remove_note (insn, note);
}
}
}
}
/* Duplicate the INSN_LIST elements of COPY and prepend them to OLD. */
rtx_insn_list *
concat_INSN_LIST (rtx_insn_list *copy, rtx_insn_list *old)
{
rtx_insn_list *new_rtx = old;
for (; copy ; copy = copy->next ())
{
new_rtx = alloc_INSN_LIST (copy->insn (), new_rtx);
PUT_REG_NOTE_KIND (new_rtx, REG_NOTE_KIND (copy));
}
return new_rtx;
}
/* Emit a slim dump of X (an insn) to the file F, including any register
note attached to the instruction. */
void
dump_insn_slim (FILE *f, const_rtx x)
{
char t[BUF_LEN + 32];
rtx note;
print_insn (t, x, 1);
fputs (t, f);
putc ('\n', f);
if (INSN_P (x) && REG_NOTES (x))
for (note = REG_NOTES (x); note; note = XEXP (note, 1))
{
print_value (t, XEXP (note, 0), 1);
fprintf (f, " %s: %s\n",
GET_REG_NOTE_NAME (REG_NOTE_KIND (note)), t);
}
}
static void
print_insn_with_notes (pretty_printer *pp, const_rtx x)
{
pp_string (pp, print_rtx_head);
print_insn (pp, x, 1);
pp_newline (pp);
if (INSN_P (x) && REG_NOTES (x))
for (rtx note = REG_NOTES (x); note; note = XEXP (note, 1))
{
pp_printf (pp, "%s %s ", print_rtx_head,
GET_REG_NOTE_NAME (REG_NOTE_KIND (note)));
if (GET_CODE (note) == INT_LIST)
pp_printf (pp, "%d", XINT (note, 0));
else
print_pattern (pp, XEXP (note, 0), 1);
pp_newline (pp);
}
}
/* Find offset equivalence note for reg WHAT in INSN and return the
found elmination offset. If the note is not found, return NULL.
Remove the found note. */
static rtx
remove_reg_equal_offset_note (rtx_insn *insn, rtx what)
{
rtx link, *link_loc;
for (link_loc = ®_NOTES (insn);
(link = *link_loc) != NULL_RTX;
link_loc = &XEXP (link, 1))
if (REG_NOTE_KIND (link) == REG_EQUAL
&& GET_CODE (XEXP (link, 0)) == PLUS
&& XEXP (XEXP (link, 0), 0) == what
&& CONST_INT_P (XEXP (XEXP (link, 0), 1)))
{
*link_loc = XEXP (link, 1);
return XEXP (XEXP (link, 0), 1);
}
return NULL_RTX;
}
static bool
update_reg_equal_equiv_notes (rtx insn, enum machine_mode new_mode,
enum machine_mode old_mode, enum rtx_code code)
{
rtx *loc = ®_NOTES (insn);
while (*loc)
{
enum reg_note kind = REG_NOTE_KIND (*loc);
if (kind == REG_EQUAL || kind == REG_EQUIV)
{
rtx orig_src = XEXP (*loc, 0);
/* Update equivalency constants. Recall that RTL constants are
sign-extended. */
if (GET_CODE (orig_src) == CONST_INT
&& HOST_BITS_PER_WIDE_INT >= GET_MODE_BITSIZE (new_mode))
{
if (INTVAL (orig_src) >= 0 || code == SIGN_EXTEND)
/* Nothing needed. */;
else
{
/* Zero-extend the negative constant by masking out the
bits outside the source mode. */
rtx new_const_int
= gen_int_mode (INTVAL (orig_src)
& GET_MODE_MASK (old_mode),
new_mode);
if (!validate_change (insn, &XEXP (*loc, 0),
new_const_int, true))
return false;
}
loc = &XEXP (*loc, 1);
}
/* Drop all other notes, they assume a wrong mode. */
else if (!validate_change (insn, loc, XEXP (*loc, 1), true))
return false;
}
else
loc = &XEXP (*loc, 1);
}
return true;
}
static void
move_dead_notes (rtx to_insn, rtx from_insn, rtx pattern)
{
rtx note;
rtx next_note;
rtx prev_note = NULL;
for (note = REG_NOTES (from_insn); note; note = next_note)
{
next_note = XEXP (note, 1);
if ((REG_NOTE_KIND (note) == REG_DEAD)
&& pattern == XEXP (note, 0))
{
XEXP (note, 1) = REG_NOTES (to_insn);
REG_NOTES (to_insn) = note;
if (prev_note)
XEXP (prev_note, 1) = next_note;
else
REG_NOTES (from_insn) = next_note;
}
else prev_note = note;
}
}
static int
optimize_mode_switching (void)
{
int e;
basic_block bb;
bool need_commit = false;
static const int num_modes[] = NUM_MODES_FOR_MODE_SWITCHING;
#define N_ENTITIES ARRAY_SIZE (num_modes)
int entity_map[N_ENTITIES];
struct bb_info *bb_info[N_ENTITIES];
int i, j;
int n_entities = 0;
int max_num_modes = 0;
bool emitted ATTRIBUTE_UNUSED = false;
basic_block post_entry = 0;
basic_block pre_exit = 0;
struct edge_list *edge_list = 0;
/* These bitmaps are used for the LCM algorithm. */
sbitmap *kill, *del, *insert, *antic, *transp, *comp;
sbitmap *avin, *avout;
for (e = N_ENTITIES - 1; e >= 0; e--)
if (OPTIMIZE_MODE_SWITCHING (e))
{
int entry_exit_extra = 0;
/* Create the list of segments within each basic block.
If NORMAL_MODE is defined, allow for two extra
blocks split from the entry and exit block. */
if (targetm.mode_switching.entry && targetm.mode_switching.exit)
entry_exit_extra = 3;
bb_info[n_entities]
= XCNEWVEC (struct bb_info,
last_basic_block_for_fn (cfun) + entry_exit_extra);
entity_map[n_entities++] = e;
if (num_modes[e] > max_num_modes)
max_num_modes = num_modes[e];
}
if (! n_entities)
return 0;
/* Make sure if MODE_ENTRY is defined MODE_EXIT is defined. */
gcc_assert ((targetm.mode_switching.entry && targetm.mode_switching.exit)
|| (!targetm.mode_switching.entry
&& !targetm.mode_switching.exit));
if (targetm.mode_switching.entry && targetm.mode_switching.exit)
{
/* Split the edge from the entry block, so that we can note that
there NORMAL_MODE is supplied. */
post_entry = split_edge (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
pre_exit = create_pre_exit (n_entities, entity_map, num_modes);
}
df_analyze ();
/* Create the bitmap vectors. */
antic = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
n_entities * max_num_modes);
transp = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
n_entities * max_num_modes);
comp = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
n_entities * max_num_modes);
avin = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
n_entities * max_num_modes);
avout = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
n_entities * max_num_modes);
kill = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
n_entities * max_num_modes);
bitmap_vector_ones (transp, last_basic_block_for_fn (cfun));
bitmap_vector_clear (antic, last_basic_block_for_fn (cfun));
bitmap_vector_clear (comp, last_basic_block_for_fn (cfun));
for (j = n_entities - 1; j >= 0; j--)
{
int e = entity_map[j];
int no_mode = num_modes[e];
struct bb_info *info = bb_info[j];
rtx_insn *insn;
/* Determine what the first use (if any) need for a mode of entity E is.
This will be the mode that is anticipatable for this block.
Also compute the initial transparency settings. */
FOR_EACH_BB_FN (bb, cfun)
{
struct seginfo *ptr;
int last_mode = no_mode;
bool any_set_required = false;
HARD_REG_SET live_now;
info[bb->index].mode_out = info[bb->index].mode_in = no_mode;
REG_SET_TO_HARD_REG_SET (live_now, df_get_live_in (bb));
/* Pretend the mode is clobbered across abnormal edges. */
{
edge_iterator ei;
edge eg;
FOR_EACH_EDGE (eg, ei, bb->preds)
if (eg->flags & EDGE_COMPLEX)
break;
if (eg)
{
rtx_insn *ins_pos = BB_HEAD (bb);
if (LABEL_P (ins_pos))
ins_pos = NEXT_INSN (ins_pos);
gcc_assert (NOTE_INSN_BASIC_BLOCK_P (ins_pos));
if (ins_pos != BB_END (bb))
ins_pos = NEXT_INSN (ins_pos);
ptr = new_seginfo (no_mode, ins_pos, bb->index, live_now);
add_seginfo (info + bb->index, ptr);
for (i = 0; i < no_mode; i++)
clear_mode_bit (transp[bb->index], j, i);
}
}
FOR_BB_INSNS (bb, insn)
{
if (INSN_P (insn))
{
int mode = targetm.mode_switching.needed (e, insn);
rtx link;
if (mode != no_mode && mode != last_mode)
{
any_set_required = true;
last_mode = mode;
ptr = new_seginfo (mode, insn, bb->index, live_now);
add_seginfo (info + bb->index, ptr);
for (i = 0; i < no_mode; i++)
clear_mode_bit (transp[bb->index], j, i);
}
if (targetm.mode_switching.after)
last_mode = targetm.mode_switching.after (e, last_mode,
insn);
/* Update LIVE_NOW. */
for (link = REG_NOTES (insn); link; link = XEXP (link, 1))
if (REG_NOTE_KIND (link) == REG_DEAD)
reg_dies (XEXP (link, 0), &live_now);
note_stores (PATTERN (insn), reg_becomes_live, &live_now);
for (link = REG_NOTES (insn); link; link = XEXP (link, 1))
if (REG_NOTE_KIND (link) == REG_UNUSED)
reg_dies (XEXP (link, 0), &live_now);
}
}
info[bb->index].computing = last_mode;
/* Check for blocks without ANY mode requirements.
N.B. because of MODE_AFTER, last_mode might still
be different from no_mode, in which case we need to
mark the block as nontransparent. */
if (!any_set_required)
{
ptr = new_seginfo (no_mode, BB_END (bb), bb->index, live_now);
add_seginfo (info + bb->index, ptr);
if (last_mode != no_mode)
for (i = 0; i < no_mode; i++)
clear_mode_bit (transp[bb->index], j, i);
}
}
if (targetm.mode_switching.entry && targetm.mode_switching.exit)
{
int mode = targetm.mode_switching.entry (e);
info[post_entry->index].mode_out =
info[post_entry->index].mode_in = no_mode;
if (pre_exit)
{
info[pre_exit->index].mode_out =
info[pre_exit->index].mode_in = no_mode;
}
if (mode != no_mode)
{
bb = post_entry;
/* By always making this nontransparent, we save
an extra check in make_preds_opaque. We also
need this to avoid confusing pre_edge_lcm when
antic is cleared but transp and comp are set. */
for (i = 0; i < no_mode; i++)
clear_mode_bit (transp[bb->index], j, i);
/* Insert a fake computing definition of MODE into entry
blocks which compute no mode. This represents the mode on
entry. */
info[bb->index].computing = mode;
if (pre_exit)
info[pre_exit->index].seginfo->mode =
targetm.mode_switching.exit (e);
}
}
/* Set the anticipatable and computing arrays. */
for (i = 0; i < no_mode; i++)
{
int m = targetm.mode_switching.priority (entity_map[j], i);
FOR_EACH_BB_FN (bb, cfun)
{
if (info[bb->index].seginfo->mode == m)
set_mode_bit (antic[bb->index], j, m);
if (info[bb->index].computing == m)
set_mode_bit (comp[bb->index], j, m);
}
}
}
/* Calculate the optimal locations for the
placement mode switches to modes with priority I. */
FOR_EACH_BB_FN (bb, cfun)
bitmap_not (kill[bb->index], transp[bb->index]);
edge_list = pre_edge_lcm_avs (n_entities * max_num_modes, transp, comp, antic,
kill, avin, avout, &insert, &del);
for (j = n_entities - 1; j >= 0; j--)
{
int no_mode = num_modes[entity_map[j]];
/* Insert all mode sets that have been inserted by lcm. */
for (int ed = NUM_EDGES (edge_list) - 1; ed >= 0; ed--)
{
edge eg = INDEX_EDGE (edge_list, ed);
eg->aux = (void *)(intptr_t)-1;
for (i = 0; i < no_mode; i++)
{
int m = targetm.mode_switching.priority (entity_map[j], i);
if (mode_bit_p (insert[ed], j, m))
{
eg->aux = (void *)(intptr_t)m;
break;
}
}
}
FOR_EACH_BB_FN (bb, cfun)
{
struct bb_info *info = bb_info[j];
int last_mode = no_mode;
/* intialize mode in availability for bb. */
for (i = 0; i < no_mode; i++)
if (mode_bit_p (avout[bb->index], j, i))
{
if (last_mode == no_mode)
last_mode = i;
if (last_mode != i)
{
last_mode = no_mode;
break;
}
}
info[bb->index].mode_out = last_mode;
/* intialize mode out availability for bb. */
last_mode = no_mode;
for (i = 0; i < no_mode; i++)
if (mode_bit_p (avin[bb->index], j, i))
{
if (last_mode == no_mode)
last_mode = i;
if (last_mode != i)
{
last_mode = no_mode;
break;
}
}
info[bb->index].mode_in = last_mode;
for (i = 0; i < no_mode; i++)
if (mode_bit_p (del[bb->index], j, i))
info[bb->index].seginfo->mode = no_mode;
}
/* Now output the remaining mode sets in all the segments. */
/* In case there was no mode inserted. the mode information on the edge
might not be complete.
Update mode info on edges and commit pending mode sets. */
need_commit |= commit_mode_sets (edge_list, entity_map[j], bb_info[j]);
/* Reset modes for next entity. */
clear_aux_for_edges ();
FOR_EACH_BB_FN (bb, cfun)
{
struct seginfo *ptr, *next;
int cur_mode = bb_info[j][bb->index].mode_in;
for (ptr = bb_info[j][bb->index].seginfo; ptr; ptr = next)
{
next = ptr->next;
if (ptr->mode != no_mode)
{
rtx_insn *mode_set;
rtl_profile_for_bb (bb);
start_sequence ();
targetm.mode_switching.emit (entity_map[j], ptr->mode,
cur_mode, ptr->regs_live);
mode_set = get_insns ();
end_sequence ();
/* modes kill each other inside a basic block. */
cur_mode = ptr->mode;
/* Insert MODE_SET only if it is nonempty. */
if (mode_set != NULL_RTX)
{
emitted = true;
if (NOTE_INSN_BASIC_BLOCK_P (ptr->insn_ptr))
/* We need to emit the insns in a FIFO-like manner,
i.e. the first to be emitted at our insertion
point ends up first in the instruction steam.
Because we made sure that NOTE_INSN_BASIC_BLOCK is
only used for initially empty basic blocks, we
can achieve this by appending at the end of
the block. */
emit_insn_after
(mode_set, BB_END (NOTE_BASIC_BLOCK (ptr->insn_ptr)));
else
emit_insn_before (mode_set, ptr->insn_ptr);
}
default_rtl_profile ();
}
free (ptr);
}
}
free (bb_info[j]);
}
free_edge_list (edge_list);
/* Finished. Free up all the things we've allocated. */
sbitmap_vector_free (del);
sbitmap_vector_free (insert);
sbitmap_vector_free (kill);
sbitmap_vector_free (antic);
sbitmap_vector_free (transp);
sbitmap_vector_free (comp);
sbitmap_vector_free (avin);
sbitmap_vector_free (avout);
if (need_commit)
commit_edge_insertions ();
if (targetm.mode_switching.entry && targetm.mode_switching.exit)
cleanup_cfg (CLEANUP_NO_INSN_DEL);
else if (!need_commit && !emitted)
return 0;
return 1;
}
static bool
copyprop_hardreg_forward_1 (basic_block bb, struct value_data *vd)
{
bool anything_changed = false;
rtx insn;
for (insn = BB_HEAD (bb); ; insn = NEXT_INSN (insn))
{
int n_ops, i, alt, predicated;
bool is_asm, any_replacements;
rtx set;
rtx link;
bool replaced[MAX_RECOG_OPERANDS];
bool changed = false;
struct kill_set_value_data ksvd;
if (!NONDEBUG_INSN_P (insn))
{
if (DEBUG_INSN_P (insn))
{
rtx loc = INSN_VAR_LOCATION_LOC (insn);
if (!VAR_LOC_UNKNOWN_P (loc))
replace_oldest_value_addr (&INSN_VAR_LOCATION_LOC (insn),
ALL_REGS, GET_MODE (loc),
ADDR_SPACE_GENERIC, insn, vd);
}
if (insn == BB_END (bb))
break;
else
continue;
}
set = single_set (insn);
extract_insn (insn);
if (! constrain_operands (1))
fatal_insn_not_found (insn);
preprocess_constraints ();
alt = which_alternative;
n_ops = recog_data.n_operands;
is_asm = asm_noperands (PATTERN (insn)) >= 0;
/* Simplify the code below by rewriting things to reflect
matching constraints. Also promote OP_OUT to OP_INOUT
in predicated instructions. */
predicated = GET_CODE (PATTERN (insn)) == COND_EXEC;
for (i = 0; i < n_ops; ++i)
{
int matches = recog_op_alt[i][alt].matches;
if (matches >= 0)
recog_op_alt[i][alt].cl = recog_op_alt[matches][alt].cl;
if (matches >= 0 || recog_op_alt[i][alt].matched >= 0
|| (predicated && recog_data.operand_type[i] == OP_OUT))
recog_data.operand_type[i] = OP_INOUT;
}
/* Apply changes to earlier DEBUG_INSNs if possible. */
if (vd->n_debug_insn_changes)
note_uses (&PATTERN (insn), cprop_find_used_regs, vd);
/* For each earlyclobber operand, zap the value data. */
for (i = 0; i < n_ops; i++)
if (recog_op_alt[i][alt].earlyclobber)
kill_value (recog_data.operand[i], vd);
/* Within asms, a clobber cannot overlap inputs or outputs.
I wouldn't think this were true for regular insns, but
scan_rtx treats them like that... */
note_stores (PATTERN (insn), kill_clobbered_value, vd);
/* Kill all auto-incremented values. */
/* ??? REG_INC is useless, since stack pushes aren't done that way. */
for_each_rtx (&PATTERN (insn), kill_autoinc_value, vd);
/* Kill all early-clobbered operands. */
for (i = 0; i < n_ops; i++)
if (recog_op_alt[i][alt].earlyclobber)
kill_value (recog_data.operand[i], vd);
/* If we have dead sets in the insn, then we need to note these as we
would clobbers. */
for (link = REG_NOTES (insn); link; link = XEXP (link, 1))
{
if (REG_NOTE_KIND (link) == REG_UNUSED)
{
kill_value (XEXP (link, 0), vd);
/* Furthermore, if the insn looked like a single-set,
but the dead store kills the source value of that
set, then we can no-longer use the plain move
special case below. */
if (set
&& reg_overlap_mentioned_p (XEXP (link, 0), SET_SRC (set)))
set = NULL;
}
}
/* Special-case plain move instructions, since we may well
be able to do the move from a different register class. */
if (set && REG_P (SET_SRC (set)))
{
rtx src = SET_SRC (set);
unsigned int regno = REGNO (src);
enum machine_mode mode = GET_MODE (src);
unsigned int i;
rtx new_rtx;
/* If we are accessing SRC in some mode other that what we
set it in, make sure that the replacement is valid. */
if (mode != vd->e[regno].mode)
{
if (hard_regno_nregs[regno][mode]
> hard_regno_nregs[regno][vd->e[regno].mode])
goto no_move_special_case;
/* And likewise, if we are narrowing on big endian the transformation
is also invalid. */
if (hard_regno_nregs[regno][mode]
< hard_regno_nregs[regno][vd->e[regno].mode]
&& (GET_MODE_SIZE (vd->e[regno].mode) > UNITS_PER_WORD
? WORDS_BIG_ENDIAN : BYTES_BIG_ENDIAN))
goto no_move_special_case;
}
/* If the destination is also a register, try to find a source
register in the same class. */
if (REG_P (SET_DEST (set)))
{
new_rtx = find_oldest_value_reg (REGNO_REG_CLASS (regno), src, vd);
if (new_rtx && validate_change (insn, &SET_SRC (set), new_rtx, 0))
{
if (dump_file)
fprintf (dump_file,
"insn %u: replaced reg %u with %u\n",
INSN_UID (insn), regno, REGNO (new_rtx));
changed = true;
goto did_replacement;
}
/* We need to re-extract as validate_change clobbers
recog_data. */
extract_insn (insn);
if (! constrain_operands (1))
fatal_insn_not_found (insn);
preprocess_constraints ();
}
/* Otherwise, try all valid registers and see if its valid. */
for (i = vd->e[regno].oldest_regno; i != regno;
i = vd->e[i].next_regno)
{
new_rtx = maybe_mode_change (vd->e[i].mode, vd->e[regno].mode,
mode, i, regno);
if (new_rtx != NULL_RTX)
{
if (validate_change (insn, &SET_SRC (set), new_rtx, 0))
{
ORIGINAL_REGNO (new_rtx) = ORIGINAL_REGNO (src);
REG_ATTRS (new_rtx) = REG_ATTRS (src);
REG_POINTER (new_rtx) = REG_POINTER (src);
if (dump_file)
fprintf (dump_file,
"insn %u: replaced reg %u with %u\n",
INSN_UID (insn), regno, REGNO (new_rtx));
changed = true;
goto did_replacement;
}
/* We need to re-extract as validate_change clobbers
recog_data. */
extract_insn (insn);
if (! constrain_operands (1))
fatal_insn_not_found (insn);
preprocess_constraints ();
}
}
}
no_move_special_case:
any_replacements = false;
/* For each input operand, replace a hard register with the
eldest live copy that's in an appropriate register class. */
for (i = 0; i < n_ops; i++)
{
replaced[i] = false;
/* Don't scan match_operand here, since we've no reg class
information to pass down. Any operands that we could
substitute in will be represented elsewhere. */
if (recog_data.constraints[i][0] == '\0')
continue;
/* Don't replace in asms intentionally referencing hard regs. */
if (is_asm && REG_P (recog_data.operand[i])
&& (REGNO (recog_data.operand[i])
== ORIGINAL_REGNO (recog_data.operand[i])))
continue;
if (recog_data.operand_type[i] == OP_IN)
{
if (recog_op_alt[i][alt].is_address)
replaced[i]
= replace_oldest_value_addr (recog_data.operand_loc[i],
recog_op_alt[i][alt].cl,
VOIDmode, ADDR_SPACE_GENERIC,
insn, vd);
else if (REG_P (recog_data.operand[i]))
replaced[i]
= replace_oldest_value_reg (recog_data.operand_loc[i],
recog_op_alt[i][alt].cl,
insn, vd);
else if (MEM_P (recog_data.operand[i]))
replaced[i] = replace_oldest_value_mem (recog_data.operand[i],
insn, vd);
}
else if (MEM_P (recog_data.operand[i]))
replaced[i] = replace_oldest_value_mem (recog_data.operand[i],
insn, vd);
/* If we performed any replacement, update match_dups. */
if (replaced[i])
{
int j;
rtx new_rtx;
new_rtx = *recog_data.operand_loc[i];
recog_data.operand[i] = new_rtx;
for (j = 0; j < recog_data.n_dups; j++)
if (recog_data.dup_num[j] == i)
validate_unshare_change (insn, recog_data.dup_loc[j], new_rtx, 1);
any_replacements = true;
}
}
if (any_replacements)
{
if (! apply_change_group ())
{
for (i = 0; i < n_ops; i++)
if (replaced[i])
{
rtx old = *recog_data.operand_loc[i];
recog_data.operand[i] = old;
}
if (dump_file)
fprintf (dump_file,
"insn %u: reg replacements not verified\n",
INSN_UID (insn));
}
else
changed = true;
}
did_replacement:
if (changed)
{
anything_changed = true;
/* If something changed, perhaps further changes to earlier
DEBUG_INSNs can be applied. */
if (vd->n_debug_insn_changes)
note_uses (&PATTERN (insn), cprop_find_used_regs, vd);
}
ksvd.vd = vd;
ksvd.ignore_set_reg = NULL_RTX;
/* Clobber call-clobbered registers. */
if (CALL_P (insn))
{
unsigned int set_regno = INVALID_REGNUM;
unsigned int set_nregs = 0;
unsigned int regno;
rtx exp;
hard_reg_set_iterator hrsi;
for (exp = CALL_INSN_FUNCTION_USAGE (insn); exp; exp = XEXP (exp, 1))
{
rtx x = XEXP (exp, 0);
if (GET_CODE (x) == SET)
{
rtx dest = SET_DEST (x);
kill_value (dest, vd);
set_value_regno (REGNO (dest), GET_MODE (dest), vd);
copy_value (dest, SET_SRC (x), vd);
ksvd.ignore_set_reg = dest;
set_regno = REGNO (dest);
set_nregs
= hard_regno_nregs[set_regno][GET_MODE (dest)];
break;
}
}
EXECUTE_IF_SET_IN_HARD_REG_SET (regs_invalidated_by_call, 0, regno, hrsi)
if (regno < set_regno || regno >= set_regno + set_nregs)
kill_value_regno (regno, 1, vd);
/* If SET was seen in CALL_INSN_FUNCTION_USAGE, and SET_SRC
of the SET isn't in regs_invalidated_by_call hard reg set,
but instead among CLOBBERs on the CALL_INSN, we could wrongly
assume the value in it is still live. */
if (ksvd.ignore_set_reg)
note_stores (PATTERN (insn), kill_clobbered_value, vd);
}
/* Notice stores. */
note_stores (PATTERN (insn), kill_set_value, &ksvd);
/* Notice copies. */
if (set && REG_P (SET_DEST (set)) && REG_P (SET_SRC (set)))
copy_value (SET_DEST (set), SET_SRC (set), vd);
if (insn == BB_END (bb))
break;
}
return anything_changed;
}
/* Scan X and replace any eliminable registers (such as fp) with a
replacement (such as sp) if SUBST_P, plus an offset. The offset is
a change in the offset between the eliminable register and its
substitution if UPDATE_P, or the full offset if FULL_P, or
otherwise zero. If FULL_P, we also use the SP offsets for
elimination to SP. If UPDATE_P, use UPDATE_SP_OFFSET for updating
offsets of register elimnable to SP. If UPDATE_SP_OFFSET is
non-zero, don't use difference of the offset and the previous
offset.
MEM_MODE is the mode of an enclosing MEM. We need this to know how
much to adjust a register for, e.g., PRE_DEC. Also, if we are
inside a MEM, we are allowed to replace a sum of a hard register
and the constant zero with the hard register, which we cannot do
outside a MEM. In addition, we need to record the fact that a
hard register is referenced outside a MEM.
If we make full substitution to SP for non-null INSN, add the insn
sp offset. */
rtx
lra_eliminate_regs_1 (rtx_insn *insn, rtx x, machine_mode mem_mode,
bool subst_p, bool update_p,
HOST_WIDE_INT update_sp_offset, bool full_p)
{
enum rtx_code code = GET_CODE (x);
struct lra_elim_table *ep;
rtx new_rtx;
int i, j;
const char *fmt;
int copied = 0;
lra_assert (!update_p || !full_p);
lra_assert (update_sp_offset == 0 || (!subst_p && update_p && !full_p));
if (! current_function_decl)
return x;
switch (code)
{
CASE_CONST_ANY:
case CONST:
case SYMBOL_REF:
case CODE_LABEL:
case PC:
case CC0:
case ASM_INPUT:
case ADDR_VEC:
case ADDR_DIFF_VEC:
case RETURN:
return x;
case REG:
/* First handle the case where we encounter a bare hard register
that is eliminable. Replace it with a PLUS. */
if ((ep = get_elimination (x)) != NULL)
{
rtx to = subst_p ? ep->to_rtx : ep->from_rtx;
if (update_sp_offset != 0)
{
if (ep->to_rtx == stack_pointer_rtx)
return plus_constant (Pmode, to, update_sp_offset);
return to;
}
else if (update_p)
return plus_constant (Pmode, to, ep->offset - ep->previous_offset);
else if (full_p)
return plus_constant (Pmode, to,
ep->offset
- (insn != NULL_RTX
&& ep->to_rtx == stack_pointer_rtx
? lra_get_insn_recog_data (insn)->sp_offset
: 0));
else
return to;
}
return x;
case PLUS:
/* If this is the sum of an eliminable register and a constant, rework
the sum. */
if (REG_P (XEXP (x, 0)) && CONSTANT_P (XEXP (x, 1)))
{
if ((ep = get_elimination (XEXP (x, 0))) != NULL)
{
HOST_WIDE_INT offset;
rtx to = subst_p ? ep->to_rtx : ep->from_rtx;
if (! update_p && ! full_p)
return gen_rtx_PLUS (Pmode, to, XEXP (x, 1));
if (update_sp_offset != 0)
offset = ep->to_rtx == stack_pointer_rtx ? update_sp_offset : 0;
else
offset = (update_p
? ep->offset - ep->previous_offset : ep->offset);
if (full_p && insn != NULL_RTX && ep->to_rtx == stack_pointer_rtx)
offset -= lra_get_insn_recog_data (insn)->sp_offset;
if (CONST_INT_P (XEXP (x, 1)) && INTVAL (XEXP (x, 1)) == -offset)
return to;
else
return gen_rtx_PLUS (Pmode, to,
plus_constant (Pmode,
XEXP (x, 1), offset));
}
/* If the hard register is not eliminable, we are done since
the other operand is a constant. */
return x;
}
/* If this is part of an address, we want to bring any constant
to the outermost PLUS. We will do this by doing hard
register replacement in our operands and seeing if a constant
shows up in one of them.
Note that there is no risk of modifying the structure of the
insn, since we only get called for its operands, thus we are
either modifying the address inside a MEM, or something like
an address operand of a load-address insn. */
{
rtx new0 = lra_eliminate_regs_1 (insn, XEXP (x, 0), mem_mode,
subst_p, update_p,
update_sp_offset, full_p);
rtx new1 = lra_eliminate_regs_1 (insn, XEXP (x, 1), mem_mode,
subst_p, update_p,
update_sp_offset, full_p);
new0 = move_plus_up (new0);
new1 = move_plus_up (new1);
if (new0 != XEXP (x, 0) || new1 != XEXP (x, 1))
return form_sum (new0, new1);
}
return x;
case MULT:
/* If this is the product of an eliminable hard register and a
constant, apply the distribute law and move the constant out
so that we have (plus (mult ..) ..). This is needed in order
to keep load-address insns valid. This case is pathological.
We ignore the possibility of overflow here. */
if (REG_P (XEXP (x, 0)) && CONST_INT_P (XEXP (x, 1))
&& (ep = get_elimination (XEXP (x, 0))) != NULL)
{
rtx to = subst_p ? ep->to_rtx : ep->from_rtx;
if (update_sp_offset != 0)
{
if (ep->to_rtx == stack_pointer_rtx)
return plus_constant (Pmode,
gen_rtx_MULT (Pmode, to, XEXP (x, 1)),
update_sp_offset * INTVAL (XEXP (x, 1)));
return gen_rtx_MULT (Pmode, to, XEXP (x, 1));
}
else if (update_p)
return plus_constant (Pmode,
gen_rtx_MULT (Pmode, to, XEXP (x, 1)),
(ep->offset - ep->previous_offset)
* INTVAL (XEXP (x, 1)));
else if (full_p)
{
HOST_WIDE_INT offset = ep->offset;
if (insn != NULL_RTX && ep->to_rtx == stack_pointer_rtx)
offset -= lra_get_insn_recog_data (insn)->sp_offset;
return
plus_constant (Pmode,
gen_rtx_MULT (Pmode, to, XEXP (x, 1)),
offset * INTVAL (XEXP (x, 1)));
}
else
return gen_rtx_MULT (Pmode, to, XEXP (x, 1));
}
/* fall through */
case CALL:
case COMPARE:
/* See comments before PLUS about handling MINUS. */
case MINUS:
case DIV: case UDIV:
case MOD: case UMOD:
case AND: case IOR: case XOR:
case ROTATERT: case ROTATE:
case ASHIFTRT: case LSHIFTRT: case ASHIFT:
case NE: case EQ:
case GE: case GT: case GEU: case GTU:
case LE: case LT: case LEU: case LTU:
{
rtx new0 = lra_eliminate_regs_1 (insn, XEXP (x, 0), mem_mode,
subst_p, update_p,
update_sp_offset, full_p);
rtx new1 = XEXP (x, 1)
? lra_eliminate_regs_1 (insn, XEXP (x, 1), mem_mode,
subst_p, update_p,
update_sp_offset, full_p) : 0;
if (new0 != XEXP (x, 0) || new1 != XEXP (x, 1))
return gen_rtx_fmt_ee (code, GET_MODE (x), new0, new1);
}
return x;
case EXPR_LIST:
/* If we have something in XEXP (x, 0), the usual case,
eliminate it. */
if (XEXP (x, 0))
{
new_rtx = lra_eliminate_regs_1 (insn, XEXP (x, 0), mem_mode,
subst_p, update_p,
update_sp_offset, full_p);
if (new_rtx != XEXP (x, 0))
{
/* If this is a REG_DEAD note, it is not valid anymore.
Using the eliminated version could result in creating a
REG_DEAD note for the stack or frame pointer. */
if (REG_NOTE_KIND (x) == REG_DEAD)
return (XEXP (x, 1)
? lra_eliminate_regs_1 (insn, XEXP (x, 1), mem_mode,
subst_p, update_p,
update_sp_offset, full_p)
: NULL_RTX);
x = alloc_reg_note (REG_NOTE_KIND (x), new_rtx, XEXP (x, 1));
}
}
/* fall through */
case INSN_LIST:
case INT_LIST:
/* Now do eliminations in the rest of the chain. If this was
an EXPR_LIST, this might result in allocating more memory than is
strictly needed, but it simplifies the code. */
if (XEXP (x, 1))
{
new_rtx = lra_eliminate_regs_1 (insn, XEXP (x, 1), mem_mode,
subst_p, update_p,
update_sp_offset, full_p);
if (new_rtx != XEXP (x, 1))
return
gen_rtx_fmt_ee (GET_CODE (x), GET_MODE (x),
XEXP (x, 0), new_rtx);
}
return x;
case PRE_INC:
case POST_INC:
case PRE_DEC:
case POST_DEC:
/* We do not support elimination of a register that is modified.
elimination_effects has already make sure that this does not
happen. */
return x;
case PRE_MODIFY:
case POST_MODIFY:
/* We do not support elimination of a hard register that is
modified. LRA has already make sure that this does not
happen. The only remaining case we need to consider here is
that the increment value may be an eliminable register. */
if (GET_CODE (XEXP (x, 1)) == PLUS
&& XEXP (XEXP (x, 1), 0) == XEXP (x, 0))
{
rtx new_rtx = lra_eliminate_regs_1 (insn, XEXP (XEXP (x, 1), 1),
mem_mode, subst_p, update_p,
update_sp_offset, full_p);
if (new_rtx != XEXP (XEXP (x, 1), 1))
return gen_rtx_fmt_ee (code, GET_MODE (x), XEXP (x, 0),
gen_rtx_PLUS (GET_MODE (x),
XEXP (x, 0), new_rtx));
}
return x;
case STRICT_LOW_PART:
case NEG: case NOT:
case SIGN_EXTEND: case ZERO_EXTEND:
case TRUNCATE: case FLOAT_EXTEND: case FLOAT_TRUNCATE:
case FLOAT: case FIX:
case UNSIGNED_FIX: case UNSIGNED_FLOAT:
case ABS:
case SQRT:
case FFS:
case CLZ:
case CTZ:
case POPCOUNT:
case PARITY:
case BSWAP:
new_rtx = lra_eliminate_regs_1 (insn, XEXP (x, 0), mem_mode,
subst_p, update_p,
update_sp_offset, full_p);
if (new_rtx != XEXP (x, 0))
return gen_rtx_fmt_e (code, GET_MODE (x), new_rtx);
return x;
case SUBREG:
new_rtx = lra_eliminate_regs_1 (insn, SUBREG_REG (x), mem_mode,
subst_p, update_p,
update_sp_offset, full_p);
if (new_rtx != SUBREG_REG (x))
{
int x_size = GET_MODE_SIZE (GET_MODE (x));
int new_size = GET_MODE_SIZE (GET_MODE (new_rtx));
if (MEM_P (new_rtx) && x_size <= new_size)
{
SUBREG_REG (x) = new_rtx;
alter_subreg (&x, false);
return x;
}
else if (! subst_p)
{
/* LRA can transform subregs itself. So don't call
simplify_gen_subreg until LRA transformations are
finished. Function simplify_gen_subreg can do
non-trivial transformations (like truncation) which
might make LRA work to fail. */
SUBREG_REG (x) = new_rtx;
return x;
}
else
return simplify_gen_subreg (GET_MODE (x), new_rtx,
GET_MODE (new_rtx), SUBREG_BYTE (x));
}
return x;
case MEM:
/* Our only special processing is to pass the mode of the MEM to our
recursive call and copy the flags. While we are here, handle this
case more efficiently. */
return
replace_equiv_address_nv
(x,
lra_eliminate_regs_1 (insn, XEXP (x, 0), GET_MODE (x),
subst_p, update_p, update_sp_offset, full_p));
case USE:
/* Handle insn_list USE that a call to a pure function may generate. */
new_rtx = lra_eliminate_regs_1 (insn, XEXP (x, 0), VOIDmode,
subst_p, update_p, update_sp_offset, full_p);
if (new_rtx != XEXP (x, 0))
return gen_rtx_USE (GET_MODE (x), new_rtx);
return x;
case CLOBBER:
case SET:
gcc_unreachable ();
default:
break;
}
/* Process each of our operands recursively. If any have changed, make a
copy of the rtx. */
fmt = GET_RTX_FORMAT (code);
for (i = 0; i < GET_RTX_LENGTH (code); i++, fmt++)
{
if (*fmt == 'e')
{
new_rtx = lra_eliminate_regs_1 (insn, XEXP (x, i), mem_mode,
subst_p, update_p,
update_sp_offset, full_p);
if (new_rtx != XEXP (x, i) && ! copied)
{
x = shallow_copy_rtx (x);
copied = 1;
}
XEXP (x, i) = new_rtx;
}
else if (*fmt == 'E')
{
int copied_vec = 0;
for (j = 0; j < XVECLEN (x, i); j++)
{
new_rtx = lra_eliminate_regs_1 (insn, XVECEXP (x, i, j), mem_mode,
subst_p, update_p,
update_sp_offset, full_p);
if (new_rtx != XVECEXP (x, i, j) && ! copied_vec)
{
rtvec new_v = gen_rtvec_v (XVECLEN (x, i),
XVEC (x, i)->elem);
if (! copied)
{
x = shallow_copy_rtx (x);
copied = 1;
}
XVEC (x, i) = new_v;
copied_vec = 1;
}
XVECEXP (x, i, j) = new_rtx;
}
}
}
return x;
}
static bool
speculative_prefetching_transform (rtx insn)
{
rtx histogram, value;
gcov_type val, count, all;
edge e;
rtx mem, address;
int write;
if (!maybe_hot_bb_p (BLOCK_FOR_INSN (insn)))
return false;
if (!find_mem_reference (insn, &mem, &write))
return false;
address = XEXP (mem, 0);
if (side_effects_p (address))
return false;
if (CONSTANT_P (address))
return false;
for (histogram = REG_NOTES (insn);
histogram;
histogram = XEXP (histogram, 1))
if (REG_NOTE_KIND (histogram) == REG_VALUE_PROFILE
&& XEXP (XEXP (histogram, 0), 0) == GEN_INT (HIST_TYPE_CONST_DELTA))
break;
if (!histogram)
return false;
histogram = XEXP (XEXP (histogram, 0), 1);
value = XEXP (histogram, 0);
histogram = XEXP (histogram, 1);
/* Skip last value referenced. */
histogram = XEXP (histogram, 1);
val = INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
count = INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
all = INTVAL (XEXP (histogram, 0));
/* With that few executions we do not really have a reason to optimize the
statement, and more importantly, the data about differences of addresses
are spoiled by the first item that had no previous value to compare
with. */
if (all < 4)
return false;
/* We require that count be at least half of all; this means
that for the transformation to fire the value must be constant
at least 50% of time (and 75% gives the guarantee of usage). */
if (!rtx_equal_p (address, value) || 2 * count < all)
return false;
/* If the difference is too small, it does not make too much sense to
prefetch, as the memory is probably already in cache. */
if (val >= NOPREFETCH_RANGE_MIN && val <= NOPREFETCH_RANGE_MAX)
return false;
if (dump_file)
fprintf (dump_file, "Speculative prefetching for insn %d\n",
INSN_UID (insn));
e = split_block (BLOCK_FOR_INSN (insn), PREV_INSN (insn));
insert_insn_on_edge (gen_speculative_prefetch (address, val, write), e);
return true;
}
/* Do transforms 3) and 4) on INSN if applicable. */
static bool
mod_subtract_transform (rtx insn)
{
rtx set, set_src, set_dest, op1, op2, value, histogram;
enum rtx_code code;
enum machine_mode mode;
gcov_type wrong_values, counts[2], count, all;
edge e;
int i, prob1, prob2;
set = single_set (insn);
if (!set)
return false;
set_src = SET_SRC (set);
set_dest = SET_DEST (set);
code = GET_CODE (set_src);
mode = GET_MODE (set_dest);
if (code != UMOD)
return false;
op1 = XEXP (set_src, 0);
op2 = XEXP (set_src, 1);
for (histogram = REG_NOTES (insn);
histogram;
histogram = XEXP (histogram, 1))
if (REG_NOTE_KIND (histogram) == REG_VALUE_PROFILE
&& XEXP (XEXP (histogram, 0), 0) == GEN_INT (HIST_TYPE_INTERVAL))
break;
if (!histogram)
return false;
histogram = XEXP (XEXP (histogram, 0), 1);
value = XEXP (histogram, 0);
histogram = XEXP (histogram, 1);
all = 0;
for (i = 0; i < 2; i++)
{
counts[i] = INTVAL (XEXP (histogram, 0));
all += counts[i];
histogram = XEXP (histogram, 1);
}
wrong_values = INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
wrong_values += INTVAL (XEXP (histogram, 0));
all += wrong_values;
/* We require that we use just subtractions in at least 50% of all
evaluations. */
count = 0;
for (i = 0; i < 2; i++)
{
count += counts[i];
if (count * 2 >= all)
break;
}
if (i == 2)
return false;
if (dump_file)
fprintf (dump_file, "Mod subtract transformation on insn %d\n",
INSN_UID (insn));
/* Compute probability of taking the optimal path(s). */
prob1 = (counts[0] * REG_BR_PROB_BASE + all / 2) / all;
prob2 = (counts[1] * REG_BR_PROB_BASE + all / 2) / all;
e = split_block (BLOCK_FOR_INSN (insn), PREV_INSN (insn));
delete_insn (insn);
insert_insn_on_edge (
gen_mod_subtract (mode, code, set_dest,
op1, op2, i, prob1, prob2), e);
return true;
}
/* Do transform 2) on INSN if applicable. */
static bool
mod_pow2_value_transform (rtx insn)
{
rtx set, set_src, set_dest, op1, op2, value, histogram;
enum rtx_code code;
enum machine_mode mode;
gcov_type wrong_values, count;
edge e;
int i, all, prob;
set = single_set (insn);
if (!set)
return false;
set_src = SET_SRC (set);
set_dest = SET_DEST (set);
code = GET_CODE (set_src);
mode = GET_MODE (set_dest);
if (code != UMOD)
return false;
op1 = XEXP (set_src, 0);
op2 = XEXP (set_src, 1);
for (histogram = REG_NOTES (insn);
histogram;
histogram = XEXP (histogram, 1))
if (REG_NOTE_KIND (histogram) == REG_VALUE_PROFILE
&& XEXP (XEXP (histogram, 0), 0) == GEN_INT (HIST_TYPE_POW2))
break;
if (!histogram)
return false;
histogram = XEXP (XEXP (histogram, 0), 1);
value = XEXP (histogram, 0);
histogram = XEXP (histogram, 1);
wrong_values =INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
count = 0;
for (i = 0; i < GET_MODE_BITSIZE (mode); i++)
{
count += INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
}
if (!rtx_equal_p (op2, value))
return false;
/* We require that we hit a power of two at least half of all evaluations. */
if (count < wrong_values)
return false;
if (dump_file)
fprintf (dump_file, "Mod power of 2 transformation on insn %d\n",
INSN_UID (insn));
/* Compute probability of taking the optimal path. */
all = count + wrong_values;
prob = (count * REG_BR_PROB_BASE + all / 2) / all;
e = split_block (BLOCK_FOR_INSN (insn), PREV_INSN (insn));
delete_insn (insn);
insert_insn_on_edge (
gen_mod_pow2 (mode, code, set_dest, op1, op2, prob), e);
return true;
}
/* Do transform 1) on INSN if applicable. */
static bool
divmod_fixed_value_transform (rtx insn)
{
rtx set, set_src, set_dest, op1, op2, value, histogram;
enum rtx_code code;
enum machine_mode mode;
gcov_type val, count, all;
edge e;
int prob;
set = single_set (insn);
if (!set)
return false;
set_src = SET_SRC (set);
set_dest = SET_DEST (set);
code = GET_CODE (set_src);
mode = GET_MODE (set_dest);
if (code != DIV && code != MOD && code != UDIV && code != UMOD)
return false;
op1 = XEXP (set_src, false);
op2 = XEXP (set_src, 1);
for (histogram = REG_NOTES (insn);
histogram;
histogram = XEXP (histogram, 1))
if (REG_NOTE_KIND (histogram) == REG_VALUE_PROFILE
&& XEXP (XEXP (histogram, 0), 0) == GEN_INT (HIST_TYPE_SINGLE_VALUE))
break;
if (!histogram)
return false;
histogram = XEXP (XEXP (histogram, 0), 1);
value = XEXP (histogram, 0);
histogram = XEXP (histogram, 1);
val = INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
count = INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
all = INTVAL (XEXP (histogram, 0));
/* We require that count be at least half of all; this means
that for the transformation to fire the value must be constant
at least 50% of time (and 75% gives the guarantee of usage). */
if (!rtx_equal_p (op2, value) || 2 * count < all)
return false;
if (dump_file)
fprintf (dump_file, "Div/mod by constant transformation on insn %d\n",
INSN_UID (insn));
/* Compute probability of taking the optimal path. */
prob = (count * REG_BR_PROB_BASE + all / 2) / all;
e = split_block (BLOCK_FOR_INSN (insn), PREV_INSN (insn));
delete_insn (insn);
insert_insn_on_edge (
gen_divmod_fixed_value (mode, code, set_dest,
op1, op2, val, prob), e);
return true;
}
