static void add_test (rtx cond, basic_block bb, basic_block dest) { rtx seq, jump, label; enum machine_mode mode; rtx op0 = XEXP (cond, 0), op1 = XEXP (cond, 1); enum rtx_code code = GET_CODE (cond); mode = GET_MODE (XEXP (cond, 0)); if (mode == VOIDmode) mode = GET_MODE (XEXP (cond, 1)); start_sequence (); op0 = force_operand (op0, NULL_RTX); op1 = force_operand (op1, NULL_RTX); label = block_label (dest); do_compare_rtx_and_jump (op0, op1, code, 0, mode, NULL_RTX, NULL_RTX, label); jump = get_last_insn (); JUMP_LABEL (jump) = label; /* The jump is supposed to handle an unlikely special case. */ REG_NOTES (jump) = gen_rtx_EXPR_LIST (REG_BR_PROB, const0_rtx, REG_NOTES (jump)); LABEL_NUSES (label)++; seq = get_insns (); end_sequence (); emit_insn_after (seq, BB_END (bb)); }
static void erase_matching_seqs (void) { seq_block sb; matching_seq mseq; rtx insn; basic_block bb; rtx retlabel, saveinsn, callinsn; int i; for (sb = seq_blocks; sb; sb = sb->next_seq_block) { for (mseq = sb->matching_seqs; mseq; mseq = mseq->next_matching_seq) { insn = mseq->insn; bb = BLOCK_FOR_INSN (insn); /* Get the label after the sequence. This will be the return address. The label will be referenced using a symbol_ref so protect it from deleting. */ retlabel = block_label_after (insn); LABEL_PRESERVE_P (retlabel) = 1; /* Delete the insns of the sequence. */ for (i = 0; i < sb->length; i++) insn = prev_insn_in_block (insn); delete_basic_block (split_block_and_df_analyze (bb, insn)); /* Emit an insn saving the return address to the link register before the deleted sequence. */ saveinsn = emit_insn_after (gen_move_insn (pattern_seqs->link_reg, gen_symbol_ref_rtx_for_label (retlabel)), BB_END (bb)); BLOCK_FOR_INSN (saveinsn) = bb; /* Emit a jump to the appropriate part of the pattern sequence after the save insn. Also update the basic block. */ callinsn = emit_jump_insn_after (gen_jump (sb->label), saveinsn); JUMP_LABEL (callinsn) = sb->label; LABEL_NUSES (sb->label)++; BLOCK_FOR_INSN (callinsn) = bb; BB_END (bb) = callinsn; /* Maintain control flow and liveness information. */ SET_REGNO_REG_SET (df_get_live_out (bb), REGNO (pattern_seqs->link_reg)); emit_barrier_after (BB_END (bb)); make_single_succ_edge (bb, BLOCK_FOR_INSN (sb->label), 0); IOR_REG_SET (df_get_live_out (bb), df_get_live_in (BLOCK_FOR_INSN (sb->label))); make_edge (BLOCK_FOR_INSN (seq_blocks->label), BLOCK_FOR_INSN (retlabel), EDGE_ABNORMAL); } } }
static void split_pattern_seq (void) { rtx insn; basic_block bb; rtx retlabel, retjmp, saveinsn; int i; seq_block sb; insn = pattern_seqs->insn; bb = BLOCK_FOR_INSN (insn); /* Get the label after the sequence. This will be the return address. The label will be referenced using a symbol_ref so protect it from deleting. */ retlabel = block_label_after (insn); LABEL_PRESERVE_P (retlabel) = 1; /* Emit an indirect jump via the link register after the sequence acting as the return insn. Also emit a barrier and update the basic block. */ if (!find_reg_note (BB_END (bb), REG_NORETURN, NULL)) retjmp = emit_jump_insn_after (gen_indirect_jump (pattern_seqs->link_reg), BB_END (bb)); emit_barrier_after (BB_END (bb)); /* Replace all outgoing edges with a new one to the block of RETLABEL. */ while (EDGE_COUNT (bb->succs) != 0) remove_edge (EDGE_SUCC (bb, 0)); make_edge (bb, BLOCK_FOR_INSN (retlabel), EDGE_ABNORMAL); /* Split the sequence according to SEQ_BLOCKS and cache the label of the resulting basic blocks. */ i = 0; for (sb = seq_blocks; sb; sb = sb->next_seq_block) { for (; i < sb->length; i++) insn = prev_insn_in_block (insn); sb->label = block_label (split_block_and_df_analyze (bb, insn)); } /* Emit an insn saving the return address to the link register before the sequence. */ saveinsn = emit_insn_after (gen_move_insn (pattern_seqs->link_reg, gen_symbol_ref_rtx_for_label (retlabel)), BB_END (bb)); /* Update liveness info. */ SET_REGNO_REG_SET (df_get_live_out (bb), REGNO (pattern_seqs->link_reg)); }
static int optimize_mode_switching (void) { int e; basic_block bb; bool need_commit = false; static const int num_modes[] = NUM_MODES_FOR_MODE_SWITCHING; #define N_ENTITIES ARRAY_SIZE (num_modes) int entity_map[N_ENTITIES]; struct bb_info *bb_info[N_ENTITIES]; int i, j; int n_entities = 0; int max_num_modes = 0; bool emitted ATTRIBUTE_UNUSED = false; basic_block post_entry = 0; basic_block pre_exit = 0; struct edge_list *edge_list = 0; /* These bitmaps are used for the LCM algorithm. */ sbitmap *kill, *del, *insert, *antic, *transp, *comp; sbitmap *avin, *avout; for (e = N_ENTITIES - 1; e >= 0; e--) if (OPTIMIZE_MODE_SWITCHING (e)) { int entry_exit_extra = 0; /* Create the list of segments within each basic block. If NORMAL_MODE is defined, allow for two extra blocks split from the entry and exit block. */ if (targetm.mode_switching.entry && targetm.mode_switching.exit) entry_exit_extra = 3; bb_info[n_entities] = XCNEWVEC (struct bb_info, last_basic_block_for_fn (cfun) + entry_exit_extra); entity_map[n_entities++] = e; if (num_modes[e] > max_num_modes) max_num_modes = num_modes[e]; } if (! n_entities) return 0; /* Make sure if MODE_ENTRY is defined MODE_EXIT is defined. */ gcc_assert ((targetm.mode_switching.entry && targetm.mode_switching.exit) || (!targetm.mode_switching.entry && !targetm.mode_switching.exit)); if (targetm.mode_switching.entry && targetm.mode_switching.exit) { /* Split the edge from the entry block, so that we can note that there NORMAL_MODE is supplied. */ post_entry = split_edge (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun))); pre_exit = create_pre_exit (n_entities, entity_map, num_modes); } df_analyze (); /* Create the bitmap vectors. */ antic = sbitmap_vector_alloc (last_basic_block_for_fn (cfun), n_entities * max_num_modes); transp = sbitmap_vector_alloc (last_basic_block_for_fn (cfun), n_entities * max_num_modes); comp = sbitmap_vector_alloc (last_basic_block_for_fn (cfun), n_entities * max_num_modes); avin = sbitmap_vector_alloc (last_basic_block_for_fn (cfun), n_entities * max_num_modes); avout = sbitmap_vector_alloc (last_basic_block_for_fn (cfun), n_entities * max_num_modes); kill = sbitmap_vector_alloc (last_basic_block_for_fn (cfun), n_entities * max_num_modes); bitmap_vector_ones (transp, last_basic_block_for_fn (cfun)); bitmap_vector_clear (antic, last_basic_block_for_fn (cfun)); bitmap_vector_clear (comp, last_basic_block_for_fn (cfun)); for (j = n_entities - 1; j >= 0; j--) { int e = entity_map[j]; int no_mode = num_modes[e]; struct bb_info *info = bb_info[j]; rtx_insn *insn; /* Determine what the first use (if any) need for a mode of entity E is. This will be the mode that is anticipatable for this block. Also compute the initial transparency settings. */ FOR_EACH_BB_FN (bb, cfun) { struct seginfo *ptr; int last_mode = no_mode; bool any_set_required = false; HARD_REG_SET live_now; info[bb->index].mode_out = info[bb->index].mode_in = no_mode; REG_SET_TO_HARD_REG_SET (live_now, df_get_live_in (bb)); /* Pretend the mode is clobbered across abnormal edges. */ { edge_iterator ei; edge eg; FOR_EACH_EDGE (eg, ei, bb->preds) if (eg->flags & EDGE_COMPLEX) break; if (eg) { rtx_insn *ins_pos = BB_HEAD (bb); if (LABEL_P (ins_pos)) ins_pos = NEXT_INSN (ins_pos); gcc_assert (NOTE_INSN_BASIC_BLOCK_P (ins_pos)); if (ins_pos != BB_END (bb)) ins_pos = NEXT_INSN (ins_pos); ptr = new_seginfo (no_mode, ins_pos, bb->index, live_now); add_seginfo (info + bb->index, ptr); for (i = 0; i < no_mode; i++) clear_mode_bit (transp[bb->index], j, i); } } FOR_BB_INSNS (bb, insn) { if (INSN_P (insn)) { int mode = targetm.mode_switching.needed (e, insn); rtx link; if (mode != no_mode && mode != last_mode) { any_set_required = true; last_mode = mode; ptr = new_seginfo (mode, insn, bb->index, live_now); add_seginfo (info + bb->index, ptr); for (i = 0; i < no_mode; i++) clear_mode_bit (transp[bb->index], j, i); } if (targetm.mode_switching.after) last_mode = targetm.mode_switching.after (e, last_mode, insn); /* Update LIVE_NOW. */ for (link = REG_NOTES (insn); link; link = XEXP (link, 1)) if (REG_NOTE_KIND (link) == REG_DEAD) reg_dies (XEXP (link, 0), &live_now); note_stores (PATTERN (insn), reg_becomes_live, &live_now); for (link = REG_NOTES (insn); link; link = XEXP (link, 1)) if (REG_NOTE_KIND (link) == REG_UNUSED) reg_dies (XEXP (link, 0), &live_now); } } info[bb->index].computing = last_mode; /* Check for blocks without ANY mode requirements. N.B. because of MODE_AFTER, last_mode might still be different from no_mode, in which case we need to mark the block as nontransparent. */ if (!any_set_required) { ptr = new_seginfo (no_mode, BB_END (bb), bb->index, live_now); add_seginfo (info + bb->index, ptr); if (last_mode != no_mode) for (i = 0; i < no_mode; i++) clear_mode_bit (transp[bb->index], j, i); } } if (targetm.mode_switching.entry && targetm.mode_switching.exit) { int mode = targetm.mode_switching.entry (e); info[post_entry->index].mode_out = info[post_entry->index].mode_in = no_mode; if (pre_exit) { info[pre_exit->index].mode_out = info[pre_exit->index].mode_in = no_mode; } if (mode != no_mode) { bb = post_entry; /* By always making this nontransparent, we save an extra check in make_preds_opaque. We also need this to avoid confusing pre_edge_lcm when antic is cleared but transp and comp are set. */ for (i = 0; i < no_mode; i++) clear_mode_bit (transp[bb->index], j, i); /* Insert a fake computing definition of MODE into entry blocks which compute no mode. This represents the mode on entry. */ info[bb->index].computing = mode; if (pre_exit) info[pre_exit->index].seginfo->mode = targetm.mode_switching.exit (e); } } /* Set the anticipatable and computing arrays. */ for (i = 0; i < no_mode; i++) { int m = targetm.mode_switching.priority (entity_map[j], i); FOR_EACH_BB_FN (bb, cfun) { if (info[bb->index].seginfo->mode == m) set_mode_bit (antic[bb->index], j, m); if (info[bb->index].computing == m) set_mode_bit (comp[bb->index], j, m); } } } /* Calculate the optimal locations for the placement mode switches to modes with priority I. */ FOR_EACH_BB_FN (bb, cfun) bitmap_not (kill[bb->index], transp[bb->index]); edge_list = pre_edge_lcm_avs (n_entities * max_num_modes, transp, comp, antic, kill, avin, avout, &insert, &del); for (j = n_entities - 1; j >= 0; j--) { int no_mode = num_modes[entity_map[j]]; /* Insert all mode sets that have been inserted by lcm. */ for (int ed = NUM_EDGES (edge_list) - 1; ed >= 0; ed--) { edge eg = INDEX_EDGE (edge_list, ed); eg->aux = (void *)(intptr_t)-1; for (i = 0; i < no_mode; i++) { int m = targetm.mode_switching.priority (entity_map[j], i); if (mode_bit_p (insert[ed], j, m)) { eg->aux = (void *)(intptr_t)m; break; } } } FOR_EACH_BB_FN (bb, cfun) { struct bb_info *info = bb_info[j]; int last_mode = no_mode; /* intialize mode in availability for bb. */ for (i = 0; i < no_mode; i++) if (mode_bit_p (avout[bb->index], j, i)) { if (last_mode == no_mode) last_mode = i; if (last_mode != i) { last_mode = no_mode; break; } } info[bb->index].mode_out = last_mode; /* intialize mode out availability for bb. */ last_mode = no_mode; for (i = 0; i < no_mode; i++) if (mode_bit_p (avin[bb->index], j, i)) { if (last_mode == no_mode) last_mode = i; if (last_mode != i) { last_mode = no_mode; break; } } info[bb->index].mode_in = last_mode; for (i = 0; i < no_mode; i++) if (mode_bit_p (del[bb->index], j, i)) info[bb->index].seginfo->mode = no_mode; } /* Now output the remaining mode sets in all the segments. */ /* In case there was no mode inserted. the mode information on the edge might not be complete. Update mode info on edges and commit pending mode sets. */ need_commit |= commit_mode_sets (edge_list, entity_map[j], bb_info[j]); /* Reset modes for next entity. */ clear_aux_for_edges (); FOR_EACH_BB_FN (bb, cfun) { struct seginfo *ptr, *next; int cur_mode = bb_info[j][bb->index].mode_in; for (ptr = bb_info[j][bb->index].seginfo; ptr; ptr = next) { next = ptr->next; if (ptr->mode != no_mode) { rtx_insn *mode_set; rtl_profile_for_bb (bb); start_sequence (); targetm.mode_switching.emit (entity_map[j], ptr->mode, cur_mode, ptr->regs_live); mode_set = get_insns (); end_sequence (); /* modes kill each other inside a basic block. */ cur_mode = ptr->mode; /* Insert MODE_SET only if it is nonempty. */ if (mode_set != NULL_RTX) { emitted = true; if (NOTE_INSN_BASIC_BLOCK_P (ptr->insn_ptr)) /* We need to emit the insns in a FIFO-like manner, i.e. the first to be emitted at our insertion point ends up first in the instruction steam. Because we made sure that NOTE_INSN_BASIC_BLOCK is only used for initially empty basic blocks, we can achieve this by appending at the end of the block. */ emit_insn_after (mode_set, BB_END (NOTE_BASIC_BLOCK (ptr->insn_ptr))); else emit_insn_before (mode_set, ptr->insn_ptr); } default_rtl_profile (); } free (ptr); } } free (bb_info[j]); } free_edge_list (edge_list); /* Finished. Free up all the things we've allocated. */ sbitmap_vector_free (del); sbitmap_vector_free (insert); sbitmap_vector_free (kill); sbitmap_vector_free (antic); sbitmap_vector_free (transp); sbitmap_vector_free (comp); sbitmap_vector_free (avin); sbitmap_vector_free (avout); if (need_commit) commit_edge_insertions (); if (targetm.mode_switching.entry && targetm.mode_switching.exit) cleanup_cfg (CLEANUP_NO_INSN_DEL); else if (!need_commit && !emitted) return 0; return 1; }
static struct loop * unswitch_loop (struct loop *loop, basic_block unswitch_on, rtx cond, rtx cinsn) { edge entry, latch_edge, true_edge, false_edge, e; basic_block switch_bb, unswitch_on_alt; struct loop *nloop; int irred_flag, prob; rtx seq; /* Some sanity checking. */ gcc_assert (flow_bb_inside_loop_p (loop, unswitch_on)); gcc_assert (EDGE_COUNT (unswitch_on->succs) == 2); gcc_assert (just_once_each_iteration_p (loop, unswitch_on)); gcc_assert (!loop->inner); gcc_assert (flow_bb_inside_loop_p (loop, EDGE_SUCC (unswitch_on, 0)->dest)); gcc_assert (flow_bb_inside_loop_p (loop, EDGE_SUCC (unswitch_on, 1)->dest)); entry = loop_preheader_edge (loop); /* Make a copy. */ irred_flag = entry->flags & EDGE_IRREDUCIBLE_LOOP; entry->flags &= ~EDGE_IRREDUCIBLE_LOOP; if (!duplicate_loop_to_header_edge (loop, entry, 1, NULL, NULL, NULL, 0)) return NULL; entry->flags |= irred_flag; /* Record the block with condition we unswitch on. */ unswitch_on_alt = get_bb_copy (unswitch_on); true_edge = BRANCH_EDGE (unswitch_on_alt); false_edge = FALLTHRU_EDGE (unswitch_on); latch_edge = single_succ_edge (get_bb_copy (loop->latch)); /* Create a block with the condition. */ prob = true_edge->probability; switch_bb = create_empty_bb (EXIT_BLOCK_PTR->prev_bb); seq = compare_and_jump_seq (XEXP (cond, 0), XEXP (cond, 1), GET_CODE (cond), block_label (true_edge->dest), prob, cinsn); emit_insn_after (seq, BB_END (switch_bb)); e = make_edge (switch_bb, true_edge->dest, 0); e->probability = prob; e->count = latch_edge->count * prob / REG_BR_PROB_BASE; e = make_edge (switch_bb, FALLTHRU_EDGE (unswitch_on)->dest, EDGE_FALLTHRU); e->probability = false_edge->probability; e->count = latch_edge->count * (false_edge->probability) / REG_BR_PROB_BASE; if (irred_flag) { switch_bb->flags |= BB_IRREDUCIBLE_LOOP; EDGE_SUCC (switch_bb, 0)->flags |= EDGE_IRREDUCIBLE_LOOP; EDGE_SUCC (switch_bb, 1)->flags |= EDGE_IRREDUCIBLE_LOOP; } else { switch_bb->flags &= ~BB_IRREDUCIBLE_LOOP; EDGE_SUCC (switch_bb, 0)->flags &= ~EDGE_IRREDUCIBLE_LOOP; EDGE_SUCC (switch_bb, 1)->flags &= ~EDGE_IRREDUCIBLE_LOOP; } /* Loopify from the copy of LOOP body, constructing the new loop. */ nloop = loopify (latch_edge, single_pred_edge (get_bb_copy (loop->header)), switch_bb, BRANCH_EDGE (switch_bb), FALLTHRU_EDGE (switch_bb), true, prob, REG_BR_PROB_BASE - prob); copy_loop_info (loop, nloop); /* Remove branches that are now unreachable in new loops. */ remove_path (true_edge); remove_path (false_edge); /* Preserve the simple loop preheaders. */ split_edge (loop_preheader_edge (loop)); split_edge (loop_preheader_edge (nloop)); return nloop; }
static void doloop_modify (struct loop *loop, struct niter_desc *desc, rtx doloop_seq, rtx condition, rtx count) { rtx counter_reg; rtx tmp, noloop = NULL_RTX; rtx sequence; rtx jump_insn; rtx jump_label; int nonneg = 0, irr; bool increment_count; basic_block loop_end = desc->out_edge->src; enum machine_mode mode; jump_insn = BB_END (loop_end); if (dump_file) { fprintf (dump_file, "Doloop: Inserting doloop pattern ("); if (desc->const_iter) fprintf (dump_file, HOST_WIDEST_INT_PRINT_DEC, desc->niter); else fputs ("runtime", dump_file); fputs (" iterations).\n", dump_file); } /* Discard original jump to continue loop. The original compare result may still be live, so it cannot be discarded explicitly. */ delete_insn (jump_insn); counter_reg = XEXP (condition, 0); if (GET_CODE (counter_reg) == PLUS) counter_reg = XEXP (counter_reg, 0); mode = GET_MODE (counter_reg); increment_count = false; switch (GET_CODE (condition)) { case NE: /* Currently only NE tests against zero and one are supported. */ if (XEXP (condition, 1) == const1_rtx) { increment_count = true; noloop = const1_rtx; } else if (XEXP (condition, 1) == const0_rtx) noloop = const0_rtx; else abort (); break; case GE: /* Currently only GE tests against zero are supported. */ if (XEXP (condition, 1) != const0_rtx) abort (); noloop = constm1_rtx; /* The iteration count does not need incrementing for a GE test. */ increment_count = false; /* Determine if the iteration counter will be non-negative. Note that the maximum value loaded is iterations_max - 1. */ if (desc->niter_max <= ((unsigned HOST_WIDEST_INT) 1 << (GET_MODE_BITSIZE (mode) - 1))) nonneg = 1; break; /* Abort if an invalid doloop pattern has been generated. */ default: abort (); } if (increment_count) count = simplify_gen_binary (PLUS, mode, count, const1_rtx); /* Insert initialization of the count register into the loop header. */ start_sequence (); tmp = force_operand (count, counter_reg); convert_move (counter_reg, tmp, 1); sequence = get_insns (); end_sequence (); emit_insn_after (sequence, BB_END (loop_preheader_edge (loop)->src)); if (desc->noloop_assumptions) { rtx ass = copy_rtx (desc->noloop_assumptions); basic_block preheader = loop_preheader_edge (loop)->src; basic_block set_zero = loop_split_edge_with (loop_preheader_edge (loop), NULL_RTX); basic_block new_preheader = loop_split_edge_with (loop_preheader_edge (loop), NULL_RTX); basic_block bb; edge te; gcov_type cnt; /* Expand the condition testing the assumptions and if it does not pass, reset the count register to 0. */ add_test (XEXP (ass, 0), preheader, set_zero); EDGE_SUCC (preheader, 0)->flags &= ~EDGE_FALLTHRU; cnt = EDGE_SUCC (preheader, 0)->count; EDGE_SUCC (preheader, 0)->probability = 0; EDGE_SUCC (preheader, 0)->count = 0; irr = EDGE_SUCC (preheader, 0)->flags & EDGE_IRREDUCIBLE_LOOP; te = make_edge (preheader, new_preheader, EDGE_FALLTHRU | irr); te->probability = REG_BR_PROB_BASE; te->count = cnt; set_immediate_dominator (CDI_DOMINATORS, new_preheader, preheader); set_zero->count = 0; set_zero->frequency = 0; for (ass = XEXP (ass, 1); ass; ass = XEXP (ass, 1)) { bb = loop_split_edge_with (te, NULL_RTX); te = EDGE_SUCC (bb, 0); add_test (XEXP (ass, 0), bb, set_zero); make_edge (bb, set_zero, irr); } start_sequence (); convert_move (counter_reg, noloop, 0); sequence = get_insns (); end_sequence (); emit_insn_after (sequence, BB_END (set_zero)); } /* Some targets (eg, C4x) need to initialize special looping registers. */ #ifdef HAVE_doloop_begin { rtx init; unsigned level = get_loop_level (loop) + 1; init = gen_doloop_begin (counter_reg, desc->const_iter ? desc->niter_expr : const0_rtx, desc->niter_max, GEN_INT (level)); if (init) { start_sequence (); emit_insn (init); sequence = get_insns (); end_sequence (); emit_insn_after (sequence, BB_END (loop_preheader_edge (loop)->src)); } } #endif /* Insert the new low-overhead looping insn. */ emit_jump_insn_after (doloop_seq, BB_END (loop_end)); jump_insn = BB_END (loop_end); jump_label = block_label (desc->in_edge->dest); JUMP_LABEL (jump_insn) = jump_label; LABEL_NUSES (jump_label)++; /* Ensure the right fallthru edge is marked, for case we have reversed the condition. */ desc->in_edge->flags &= ~EDGE_FALLTHRU; desc->out_edge->flags |= EDGE_FALLTHRU; /* Add a REG_NONNEG note if the actual or estimated maximum number of iterations is non-negative. */ if (nonneg) { REG_NOTES (jump_insn) = gen_rtx_EXPR_LIST (REG_NONNEG, NULL_RTX, REG_NOTES (jump_insn)); } }