static void
erase_matching_seqs (void)
{
seq_block sb;
matching_seq mseq;
rtx insn;
basic_block bb;
rtx retlabel, saveinsn, callinsn;
int i;
for (sb = seq_blocks; sb; sb = sb->next_seq_block)
{
for (mseq = sb->matching_seqs; mseq; mseq = mseq->next_matching_seq)
{
insn = mseq->insn;
bb = BLOCK_FOR_INSN (insn);
/* Get the label after the sequence. This will be the return
address. The label will be referenced using a symbol_ref so
protect it from deleting. */
retlabel = block_label_after (insn);
LABEL_PRESERVE_P (retlabel) = 1;
/* Delete the insns of the sequence. */
for (i = 0; i < sb->length; i++)
insn = prev_insn_in_block (insn);
delete_basic_block (split_block_and_df_analyze (bb, insn));
/* Emit an insn saving the return address to the link register
before the deleted sequence. */
saveinsn = emit_insn_after (gen_move_insn (pattern_seqs->link_reg,
gen_symbol_ref_rtx_for_label
(retlabel)),
BB_END (bb));
BLOCK_FOR_INSN (saveinsn) = bb;
/* Emit a jump to the appropriate part of the pattern sequence
after the save insn. Also update the basic block. */
callinsn = emit_jump_insn_after (gen_jump (sb->label), saveinsn);
JUMP_LABEL (callinsn) = sb->label;
LABEL_NUSES (sb->label)++;
BLOCK_FOR_INSN (callinsn) = bb;
BB_END (bb) = callinsn;
/* Maintain control flow and liveness information. */
SET_REGNO_REG_SET (df_get_live_out (bb),
REGNO (pattern_seqs->link_reg));
emit_barrier_after (BB_END (bb));
make_single_succ_edge (bb, BLOCK_FOR_INSN (sb->label), 0);
IOR_REG_SET (df_get_live_out (bb),
df_get_live_in (BLOCK_FOR_INSN (sb->label)));
make_edge (BLOCK_FOR_INSN (seq_blocks->label),
BLOCK_FOR_INSN (retlabel), EDGE_ABNORMAL);
}
}
}
void
ebb_compute_jump_reg_dependencies (rtx insn, regset cond_set, regset used,
regset set)
{
basic_block b = BLOCK_FOR_INSN (insn);
edge e;
edge_iterator ei;
FOR_EACH_EDGE (e, ei, b->succs)
if (e->flags & EDGE_FALLTHRU)
/* The jump may be a by-product of a branch that has been merged
in the main codepath after being conditionalized. Therefore
it may guard the fallthrough block from using a value that has
conditionally overwritten that of the main codepath. So we
consider that it restores the value of the main codepath. */
bitmap_and (set, df_get_live_in (e->dest), cond_set);
else
bitmap_ior_into (used, df_get_live_in (e->dest));
}
/* The confluence function used by the DF equation solver to propagate
live info from successor to predecessor on edge E according to the
following equation:
bb.liveout = 0 for entry block | OR (livein of successors)
*/
static bool
live_con_fun_n (edge e)
{
basic_block bb = e->src;
basic_block dest = e->dest;
bitmap bb_liveout = df_get_live_out (bb);
bitmap dest_livein = df_get_live_in (dest);
return bitmap_ior_and_compl_into (bb_liveout,
dest_livein, &all_hard_regs_bitmap);
}
void
ebb_compute_jump_reg_dependencies (rtx insn, regset used)
{
basic_block b = BLOCK_FOR_INSN (insn);
edge e;
edge_iterator ei;
FOR_EACH_EDGE (e, ei, b->succs)
if ((e->flags & EDGE_FALLTHRU) == 0)
bitmap_ior_into (used, df_get_live_in (e->dest));
}
/* The transfer function used by the DF equation solver to propagate
live info through block with BB_INDEX according to the following
equation:
bb.livein = (bb.liveout - bb.kill) OR bb.gen
*/
static bool
live_trans_fun (int bb_index)
{
basic_block bb = get_bb_data_by_index (bb_index)->bb;
bitmap bb_liveout = df_get_live_out (bb);
bitmap bb_livein = df_get_live_in (bb);
bb_data_t bb_info = get_bb_data (bb);
bitmap_and_compl (&temp_bitmap, bb_liveout, &all_hard_regs_bitmap);
return bitmap_ior_and_compl (bb_livein, &bb_info->gen_pseudos,
&temp_bitmap, &bb_info->killed_pseudos);
}
static int
optimize_mode_switching (void)
{
int e;
basic_block bb;
bool need_commit = false;
static const int num_modes[] = NUM_MODES_FOR_MODE_SWITCHING;
#define N_ENTITIES ARRAY_SIZE (num_modes)
int entity_map[N_ENTITIES];
struct bb_info *bb_info[N_ENTITIES];
int i, j;
int n_entities = 0;
int max_num_modes = 0;
bool emitted ATTRIBUTE_UNUSED = false;
basic_block post_entry = 0;
basic_block pre_exit = 0;
struct edge_list *edge_list = 0;
/* These bitmaps are used for the LCM algorithm. */
sbitmap *kill, *del, *insert, *antic, *transp, *comp;
sbitmap *avin, *avout;
for (e = N_ENTITIES - 1; e >= 0; e--)
if (OPTIMIZE_MODE_SWITCHING (e))
{
int entry_exit_extra = 0;
/* Create the list of segments within each basic block.
If NORMAL_MODE is defined, allow for two extra
blocks split from the entry and exit block. */
if (targetm.mode_switching.entry && targetm.mode_switching.exit)
entry_exit_extra = 3;
bb_info[n_entities]
= XCNEWVEC (struct bb_info,
last_basic_block_for_fn (cfun) + entry_exit_extra);
entity_map[n_entities++] = e;
if (num_modes[e] > max_num_modes)
max_num_modes = num_modes[e];
}
if (! n_entities)
return 0;
/* Make sure if MODE_ENTRY is defined MODE_EXIT is defined. */
gcc_assert ((targetm.mode_switching.entry && targetm.mode_switching.exit)
|| (!targetm.mode_switching.entry
&& !targetm.mode_switching.exit));
if (targetm.mode_switching.entry && targetm.mode_switching.exit)
{
/* Split the edge from the entry block, so that we can note that
there NORMAL_MODE is supplied. */
post_entry = split_edge (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
pre_exit = create_pre_exit (n_entities, entity_map, num_modes);
}
df_analyze ();
/* Create the bitmap vectors. */
antic = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
n_entities * max_num_modes);
transp = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
n_entities * max_num_modes);
comp = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
n_entities * max_num_modes);
avin = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
n_entities * max_num_modes);
avout = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
n_entities * max_num_modes);
kill = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
n_entities * max_num_modes);
bitmap_vector_ones (transp, last_basic_block_for_fn (cfun));
bitmap_vector_clear (antic, last_basic_block_for_fn (cfun));
bitmap_vector_clear (comp, last_basic_block_for_fn (cfun));
for (j = n_entities - 1; j >= 0; j--)
{
int e = entity_map[j];
int no_mode = num_modes[e];
struct bb_info *info = bb_info[j];
rtx_insn *insn;
/* Determine what the first use (if any) need for a mode of entity E is.
This will be the mode that is anticipatable for this block.
Also compute the initial transparency settings. */
FOR_EACH_BB_FN (bb, cfun)
{
struct seginfo *ptr;
int last_mode = no_mode;
bool any_set_required = false;
HARD_REG_SET live_now;
info[bb->index].mode_out = info[bb->index].mode_in = no_mode;
REG_SET_TO_HARD_REG_SET (live_now, df_get_live_in (bb));
/* Pretend the mode is clobbered across abnormal edges. */
{
edge_iterator ei;
edge eg;
FOR_EACH_EDGE (eg, ei, bb->preds)
if (eg->flags & EDGE_COMPLEX)
break;
if (eg)
{
rtx_insn *ins_pos = BB_HEAD (bb);
if (LABEL_P (ins_pos))
ins_pos = NEXT_INSN (ins_pos);
gcc_assert (NOTE_INSN_BASIC_BLOCK_P (ins_pos));
if (ins_pos != BB_END (bb))
ins_pos = NEXT_INSN (ins_pos);
ptr = new_seginfo (no_mode, ins_pos, bb->index, live_now);
add_seginfo (info + bb->index, ptr);
for (i = 0; i < no_mode; i++)
clear_mode_bit (transp[bb->index], j, i);
}
}
FOR_BB_INSNS (bb, insn)
{
if (INSN_P (insn))
{
int mode = targetm.mode_switching.needed (e, insn);
rtx link;
if (mode != no_mode && mode != last_mode)
{
any_set_required = true;
last_mode = mode;
ptr = new_seginfo (mode, insn, bb->index, live_now);
add_seginfo (info + bb->index, ptr);
for (i = 0; i < no_mode; i++)
clear_mode_bit (transp[bb->index], j, i);
}
if (targetm.mode_switching.after)
last_mode = targetm.mode_switching.after (e, last_mode,
insn);
/* Update LIVE_NOW. */
for (link = REG_NOTES (insn); link; link = XEXP (link, 1))
if (REG_NOTE_KIND (link) == REG_DEAD)
reg_dies (XEXP (link, 0), &live_now);
note_stores (PATTERN (insn), reg_becomes_live, &live_now);
for (link = REG_NOTES (insn); link; link = XEXP (link, 1))
if (REG_NOTE_KIND (link) == REG_UNUSED)
reg_dies (XEXP (link, 0), &live_now);
}
}
info[bb->index].computing = last_mode;
/* Check for blocks without ANY mode requirements.
N.B. because of MODE_AFTER, last_mode might still
be different from no_mode, in which case we need to
mark the block as nontransparent. */
if (!any_set_required)
{
ptr = new_seginfo (no_mode, BB_END (bb), bb->index, live_now);
add_seginfo (info + bb->index, ptr);
if (last_mode != no_mode)
for (i = 0; i < no_mode; i++)
clear_mode_bit (transp[bb->index], j, i);
}
}
if (targetm.mode_switching.entry && targetm.mode_switching.exit)
{
int mode = targetm.mode_switching.entry (e);
info[post_entry->index].mode_out =
info[post_entry->index].mode_in = no_mode;
if (pre_exit)
{
info[pre_exit->index].mode_out =
info[pre_exit->index].mode_in = no_mode;
}
if (mode != no_mode)
{
bb = post_entry;
/* By always making this nontransparent, we save
an extra check in make_preds_opaque. We also
need this to avoid confusing pre_edge_lcm when
antic is cleared but transp and comp are set. */
for (i = 0; i < no_mode; i++)
clear_mode_bit (transp[bb->index], j, i);
/* Insert a fake computing definition of MODE into entry
blocks which compute no mode. This represents the mode on
entry. */
info[bb->index].computing = mode;
if (pre_exit)
info[pre_exit->index].seginfo->mode =
targetm.mode_switching.exit (e);
}
}
/* Set the anticipatable and computing arrays. */
for (i = 0; i < no_mode; i++)
{
int m = targetm.mode_switching.priority (entity_map[j], i);
FOR_EACH_BB_FN (bb, cfun)
{
if (info[bb->index].seginfo->mode == m)
set_mode_bit (antic[bb->index], j, m);
if (info[bb->index].computing == m)
set_mode_bit (comp[bb->index], j, m);
}
}
}
/* Calculate the optimal locations for the
placement mode switches to modes with priority I. */
FOR_EACH_BB_FN (bb, cfun)
bitmap_not (kill[bb->index], transp[bb->index]);
edge_list = pre_edge_lcm_avs (n_entities * max_num_modes, transp, comp, antic,
kill, avin, avout, &insert, &del);
for (j = n_entities - 1; j >= 0; j--)
{
int no_mode = num_modes[entity_map[j]];
/* Insert all mode sets that have been inserted by lcm. */
for (int ed = NUM_EDGES (edge_list) - 1; ed >= 0; ed--)
{
edge eg = INDEX_EDGE (edge_list, ed);
eg->aux = (void *)(intptr_t)-1;
for (i = 0; i < no_mode; i++)
{
int m = targetm.mode_switching.priority (entity_map[j], i);
if (mode_bit_p (insert[ed], j, m))
{
eg->aux = (void *)(intptr_t)m;
break;
}
}
}
FOR_EACH_BB_FN (bb, cfun)
{
struct bb_info *info = bb_info[j];
int last_mode = no_mode;
/* intialize mode in availability for bb. */
for (i = 0; i < no_mode; i++)
if (mode_bit_p (avout[bb->index], j, i))
{
if (last_mode == no_mode)
last_mode = i;
if (last_mode != i)
{
last_mode = no_mode;
break;
}
}
info[bb->index].mode_out = last_mode;
/* intialize mode out availability for bb. */
last_mode = no_mode;
for (i = 0; i < no_mode; i++)
if (mode_bit_p (avin[bb->index], j, i))
{
if (last_mode == no_mode)
last_mode = i;
if (last_mode != i)
{
last_mode = no_mode;
break;
}
}
info[bb->index].mode_in = last_mode;
for (i = 0; i < no_mode; i++)
if (mode_bit_p (del[bb->index], j, i))
info[bb->index].seginfo->mode = no_mode;
}
/* Now output the remaining mode sets in all the segments. */
/* In case there was no mode inserted. the mode information on the edge
might not be complete.
Update mode info on edges and commit pending mode sets. */
need_commit |= commit_mode_sets (edge_list, entity_map[j], bb_info[j]);
/* Reset modes for next entity. */
clear_aux_for_edges ();
FOR_EACH_BB_FN (bb, cfun)
{
struct seginfo *ptr, *next;
int cur_mode = bb_info[j][bb->index].mode_in;
for (ptr = bb_info[j][bb->index].seginfo; ptr; ptr = next)
{
next = ptr->next;
if (ptr->mode != no_mode)
{
rtx_insn *mode_set;
rtl_profile_for_bb (bb);
start_sequence ();
targetm.mode_switching.emit (entity_map[j], ptr->mode,
cur_mode, ptr->regs_live);
mode_set = get_insns ();
end_sequence ();
/* modes kill each other inside a basic block. */
cur_mode = ptr->mode;
/* Insert MODE_SET only if it is nonempty. */
if (mode_set != NULL_RTX)
{
emitted = true;
if (NOTE_INSN_BASIC_BLOCK_P (ptr->insn_ptr))
/* We need to emit the insns in a FIFO-like manner,
i.e. the first to be emitted at our insertion
point ends up first in the instruction steam.
Because we made sure that NOTE_INSN_BASIC_BLOCK is
only used for initially empty basic blocks, we
can achieve this by appending at the end of
the block. */
emit_insn_after
(mode_set, BB_END (NOTE_BASIC_BLOCK (ptr->insn_ptr)));
else
emit_insn_before (mode_set, ptr->insn_ptr);
}
default_rtl_profile ();
}
free (ptr);
}
}
free (bb_info[j]);
}
free_edge_list (edge_list);
/* Finished. Free up all the things we've allocated. */
sbitmap_vector_free (del);
sbitmap_vector_free (insert);
sbitmap_vector_free (kill);
sbitmap_vector_free (antic);
sbitmap_vector_free (transp);
sbitmap_vector_free (comp);
sbitmap_vector_free (avin);
sbitmap_vector_free (avout);
if (need_commit)
commit_edge_insertions ();
if (targetm.mode_switching.entry && targetm.mode_switching.exit)
cleanup_cfg (CLEANUP_NO_INSN_DEL);
else if (!need_commit && !emitted)
return 0;
return 1;
}
/* The major function for aggressive pseudo coalescing of moves only
if the both pseudos were spilled and not special reload pseudos. */
bool
lra_coalesce (void)
{
basic_block bb;
rtx mv, set, insn, next, *sorted_moves;
int i, mv_num, sregno, dregno;
int coalesced_moves;
int max_regno = max_reg_num ();
bitmap_head involved_insns_bitmap;
timevar_push (TV_LRA_COALESCE);
if (lra_dump_file != NULL)
fprintf (lra_dump_file,
"\n********** Pseudos coalescing #%d: **********\n\n",
++lra_coalesce_iter);
first_coalesced_pseudo = XNEWVEC (int, max_regno);
next_coalesced_pseudo = XNEWVEC (int, max_regno);
for (i = 0; i < max_regno; i++)
first_coalesced_pseudo[i] = next_coalesced_pseudo[i] = i;
sorted_moves = XNEWVEC (rtx, get_max_uid ());
mv_num = 0;
/* Collect moves. */
coalesced_moves = 0;
FOR_EACH_BB (bb)
{
FOR_BB_INSNS_SAFE (bb, insn, next)
if (INSN_P (insn)
&& (set = single_set (insn)) != NULL_RTX
&& REG_P (SET_DEST (set)) && REG_P (SET_SRC (set))
&& (sregno = REGNO (SET_SRC (set))) >= FIRST_PSEUDO_REGISTER
&& (dregno = REGNO (SET_DEST (set))) >= FIRST_PSEUDO_REGISTER
&& mem_move_p (sregno, dregno)
&& coalescable_pseudo_p (sregno) && coalescable_pseudo_p (dregno)
&& ! side_effects_p (set)
&& !(lra_intersected_live_ranges_p
(lra_reg_info[sregno].live_ranges,
lra_reg_info[dregno].live_ranges)))
sorted_moves[mv_num++] = insn;
}
qsort (sorted_moves, mv_num, sizeof (rtx), move_freq_compare_func);
/* Coalesced copies, most frequently executed first. */
bitmap_initialize (&coalesced_pseudos_bitmap, ®_obstack);
bitmap_initialize (&involved_insns_bitmap, ®_obstack);
for (i = 0; i < mv_num; i++)
{
mv = sorted_moves[i];
set = single_set (mv);
lra_assert (set != NULL && REG_P (SET_SRC (set))
&& REG_P (SET_DEST (set)));
sregno = REGNO (SET_SRC (set));
dregno = REGNO (SET_DEST (set));
if (first_coalesced_pseudo[sregno] == first_coalesced_pseudo[dregno])
{
coalesced_moves++;
if (lra_dump_file != NULL)
fprintf
(lra_dump_file, " Coalescing move %i:r%d-r%d (freq=%d)\n",
INSN_UID (mv), sregno, dregno,
BLOCK_FOR_INSN (mv)->frequency);
/* We updated involved_insns_bitmap when doing the merge. */
}
else if (!(lra_intersected_live_ranges_p
(lra_reg_info[first_coalesced_pseudo[sregno]].live_ranges,
lra_reg_info[first_coalesced_pseudo[dregno]].live_ranges)))
{
coalesced_moves++;
if (lra_dump_file != NULL)
fprintf
(lra_dump_file,
" Coalescing move %i:r%d(%d)-r%d(%d) (freq=%d)\n",
INSN_UID (mv), sregno, ORIGINAL_REGNO (SET_SRC (set)),
dregno, ORIGINAL_REGNO (SET_DEST (set)),
BLOCK_FOR_INSN (mv)->frequency);
bitmap_ior_into (&involved_insns_bitmap,
&lra_reg_info[sregno].insn_bitmap);
bitmap_ior_into (&involved_insns_bitmap,
&lra_reg_info[dregno].insn_bitmap);
merge_pseudos (sregno, dregno);
}
}
bitmap_initialize (&used_pseudos_bitmap, ®_obstack);
FOR_EACH_BB (bb)
{
update_live_info (df_get_live_in (bb));
update_live_info (df_get_live_out (bb));
FOR_BB_INSNS_SAFE (bb, insn, next)
if (INSN_P (insn)
&& bitmap_bit_p (&involved_insns_bitmap, INSN_UID (insn)))
{
if (! substitute (&insn))
continue;
lra_update_insn_regno_info (insn);
if ((set = single_set (insn)) != NULL_RTX && set_noop_p (set))
{
/* Coalesced move. */
if (lra_dump_file != NULL)
fprintf (lra_dump_file, " Removing move %i (freq=%d)\n",
INSN_UID (insn), BLOCK_FOR_INSN (insn)->frequency);
lra_set_insn_deleted (insn);
}
}
}
bitmap_clear (&used_pseudos_bitmap);
bitmap_clear (&involved_insns_bitmap);
bitmap_clear (&coalesced_pseudos_bitmap);
if (lra_dump_file != NULL && coalesced_moves != 0)
fprintf (lra_dump_file, "Coalesced Moves = %d\n", coalesced_moves);
free (sorted_moves);
free (next_coalesced_pseudo);
free (first_coalesced_pseudo);
timevar_pop (TV_LRA_COALESCE);
return coalesced_moves != 0;
}