static rtx block_label_after (rtx insn) { basic_block bb = BLOCK_FOR_INSN (insn); if ((insn == BB_END (bb)) && (bb->next_bb != EXIT_BLOCK_PTR)) return block_label (bb->next_bb); else return block_label (split_block_and_df_analyze (bb, insn)); }
static void add_test (rtx cond, basic_block bb, basic_block dest) { rtx seq, jump, label; enum machine_mode mode; rtx op0 = XEXP (cond, 0), op1 = XEXP (cond, 1); enum rtx_code code = GET_CODE (cond); mode = GET_MODE (XEXP (cond, 0)); if (mode == VOIDmode) mode = GET_MODE (XEXP (cond, 1)); start_sequence (); op0 = force_operand (op0, NULL_RTX); op1 = force_operand (op1, NULL_RTX); label = block_label (dest); do_compare_rtx_and_jump (op0, op1, code, 0, mode, NULL_RTX, NULL_RTX, label); jump = get_last_insn (); JUMP_LABEL (jump) = label; /* The jump is supposed to handle an unlikely special case. */ REG_NOTES (jump) = gen_rtx_EXPR_LIST (REG_BR_PROB, const0_rtx, REG_NOTES (jump)); LABEL_NUSES (label)++; seq = get_insns (); end_sequence (); emit_insn_after (seq, BB_END (bb)); }
static void compute_init_costs (void) { rtx rtx_jump, rtx_store, rtx_return, reg, label; basic_block bb; FOR_EACH_BB (bb) if (BB_HEAD (bb)) break; label = block_label (bb); reg = gen_rtx_REG (Pmode, 0); /* Pattern for indirect jump. */ rtx_jump = gen_indirect_jump (reg); /* Pattern for storing address. */ rtx_store = gen_rtx_SET (VOIDmode, reg, gen_symbol_ref_rtx_for_label (label)); /* Pattern for return insn. */ rtx_return = gen_jump (label); /* The cost of jump. */ seq_jump_cost = compute_rtx_cost (make_jump_insn_raw (rtx_jump)); /* The cost of calling sequence. */ seq_call_cost = seq_jump_cost + compute_rtx_cost (make_insn_raw (rtx_store)); /* The cost of return. */ seq_return_cost = compute_rtx_cost (make_jump_insn_raw (rtx_return)); /* Simple heuristic for minimal sequence cost. */ seq_call_cost = (int)(seq_call_cost * (double)SEQ_CALL_COST_MULTIPLIER); }
static void split_pattern_seq (void) { rtx insn; basic_block bb; rtx retlabel, retjmp, saveinsn; int i; seq_block sb; insn = pattern_seqs->insn; bb = BLOCK_FOR_INSN (insn); /* Get the label after the sequence. This will be the return address. The label will be referenced using a symbol_ref so protect it from deleting. */ retlabel = block_label_after (insn); LABEL_PRESERVE_P (retlabel) = 1; /* Emit an indirect jump via the link register after the sequence acting as the return insn. Also emit a barrier and update the basic block. */ if (!find_reg_note (BB_END (bb), REG_NORETURN, NULL)) retjmp = emit_jump_insn_after (gen_indirect_jump (pattern_seqs->link_reg), BB_END (bb)); emit_barrier_after (BB_END (bb)); /* Replace all outgoing edges with a new one to the block of RETLABEL. */ while (EDGE_COUNT (bb->succs) != 0) remove_edge (EDGE_SUCC (bb, 0)); make_edge (bb, BLOCK_FOR_INSN (retlabel), EDGE_ABNORMAL); /* Split the sequence according to SEQ_BLOCKS and cache the label of the resulting basic blocks. */ i = 0; for (sb = seq_blocks; sb; sb = sb->next_seq_block) { for (; i < sb->length; i++) insn = prev_insn_in_block (insn); sb->label = block_label (split_block_and_df_analyze (bb, insn)); } /* Emit an insn saving the return address to the link register before the sequence. */ saveinsn = emit_insn_after (gen_move_insn (pattern_seqs->link_reg, gen_symbol_ref_rtx_for_label (retlabel)), BB_END (bb)); /* Update liveness info. */ SET_REGNO_REG_SET (df_get_live_out (bb), REGNO (pattern_seqs->link_reg)); }
static rtx label_for_bb (basic_block bb) { rtx label = BB_HEAD (bb); if (GET_CODE (label) != CODE_LABEL) { if (rtl_dump_file) fprintf (rtl_dump_file, "Emitting label for block %d\n", bb->index); label = block_label (bb); } return label; }
static rtx label_for_bb (basic_block bb) { rtx label = BB_HEAD (bb); if (!LABEL_P (label)) { if (dump_file) fprintf (dump_file, "Emitting label for block %d\n", bb->index); label = block_label (bb); } return label; }
static struct loop * unswitch_loop (struct loop *loop, basic_block unswitch_on, rtx cond, rtx cinsn) { edge entry, latch_edge, true_edge, false_edge, e; basic_block switch_bb, unswitch_on_alt; struct loop *nloop; int irred_flag, prob; rtx seq; /* Some sanity checking. */ gcc_assert (flow_bb_inside_loop_p (loop, unswitch_on)); gcc_assert (EDGE_COUNT (unswitch_on->succs) == 2); gcc_assert (just_once_each_iteration_p (loop, unswitch_on)); gcc_assert (!loop->inner); gcc_assert (flow_bb_inside_loop_p (loop, EDGE_SUCC (unswitch_on, 0)->dest)); gcc_assert (flow_bb_inside_loop_p (loop, EDGE_SUCC (unswitch_on, 1)->dest)); entry = loop_preheader_edge (loop); /* Make a copy. */ irred_flag = entry->flags & EDGE_IRREDUCIBLE_LOOP; entry->flags &= ~EDGE_IRREDUCIBLE_LOOP; if (!duplicate_loop_to_header_edge (loop, entry, 1, NULL, NULL, NULL, 0)) return NULL; entry->flags |= irred_flag; /* Record the block with condition we unswitch on. */ unswitch_on_alt = get_bb_copy (unswitch_on); true_edge = BRANCH_EDGE (unswitch_on_alt); false_edge = FALLTHRU_EDGE (unswitch_on); latch_edge = single_succ_edge (get_bb_copy (loop->latch)); /* Create a block with the condition. */ prob = true_edge->probability; switch_bb = create_empty_bb (EXIT_BLOCK_PTR->prev_bb); seq = compare_and_jump_seq (XEXP (cond, 0), XEXP (cond, 1), GET_CODE (cond), block_label (true_edge->dest), prob, cinsn); emit_insn_after (seq, BB_END (switch_bb)); e = make_edge (switch_bb, true_edge->dest, 0); e->probability = prob; e->count = latch_edge->count * prob / REG_BR_PROB_BASE; e = make_edge (switch_bb, FALLTHRU_EDGE (unswitch_on)->dest, EDGE_FALLTHRU); e->probability = false_edge->probability; e->count = latch_edge->count * (false_edge->probability) / REG_BR_PROB_BASE; if (irred_flag) { switch_bb->flags |= BB_IRREDUCIBLE_LOOP; EDGE_SUCC (switch_bb, 0)->flags |= EDGE_IRREDUCIBLE_LOOP; EDGE_SUCC (switch_bb, 1)->flags |= EDGE_IRREDUCIBLE_LOOP; } else { switch_bb->flags &= ~BB_IRREDUCIBLE_LOOP; EDGE_SUCC (switch_bb, 0)->flags &= ~EDGE_IRREDUCIBLE_LOOP; EDGE_SUCC (switch_bb, 1)->flags &= ~EDGE_IRREDUCIBLE_LOOP; } /* Loopify from the copy of LOOP body, constructing the new loop. */ nloop = loopify (latch_edge, single_pred_edge (get_bb_copy (loop->header)), switch_bb, BRANCH_EDGE (switch_bb), FALLTHRU_EDGE (switch_bb), true, prob, REG_BR_PROB_BASE - prob); copy_loop_info (loop, nloop); /* Remove branches that are now unreachable in new loops. */ remove_path (true_edge); remove_path (false_edge); /* Preserve the simple loop preheaders. */ split_edge (loop_preheader_edge (loop)); split_edge (loop_preheader_edge (nloop)); return nloop; }
static void fixup_reorder_chain (void) { basic_block bb, prev_bb; int index; rtx insn = NULL; if (cfg_layout_function_header) { set_first_insn (cfg_layout_function_header); insn = cfg_layout_function_header; while (NEXT_INSN (insn)) insn = NEXT_INSN (insn); } /* First do the bulk reordering -- rechain the blocks without regard to the needed changes to jumps and labels. */ for (bb = ENTRY_BLOCK_PTR->next_bb, index = 0; bb != 0; bb = bb->rbi->next, index++) { if (bb->rbi->header) { if (insn) NEXT_INSN (insn) = bb->rbi->header; else set_first_insn (bb->rbi->header); PREV_INSN (bb->rbi->header) = insn; insn = bb->rbi->header; while (NEXT_INSN (insn)) insn = NEXT_INSN (insn); } if (insn) NEXT_INSN (insn) = BB_HEAD (bb); else set_first_insn (BB_HEAD (bb)); PREV_INSN (BB_HEAD (bb)) = insn; insn = BB_END (bb); if (bb->rbi->footer) { NEXT_INSN (insn) = bb->rbi->footer; PREV_INSN (bb->rbi->footer) = insn; while (NEXT_INSN (insn)) insn = NEXT_INSN (insn); } } if (index != n_basic_blocks) abort (); NEXT_INSN (insn) = cfg_layout_function_footer; if (cfg_layout_function_footer) PREV_INSN (cfg_layout_function_footer) = insn; while (NEXT_INSN (insn)) insn = NEXT_INSN (insn); set_last_insn (insn); #ifdef ENABLE_CHECKING verify_insn_chain (); #endif delete_dead_jumptables (); /* Now add jumps and labels as needed to match the blocks new outgoing edges. */ for (bb = ENTRY_BLOCK_PTR->next_bb; bb ; bb = bb->rbi->next) { edge e_fall, e_taken, e; rtx bb_end_insn; basic_block nb; if (bb->succ == NULL) continue; /* Find the old fallthru edge, and another non-EH edge for a taken jump. */ e_taken = e_fall = NULL; for (e = bb->succ; e ; e = e->succ_next) if (e->flags & EDGE_FALLTHRU) e_fall = e; else if (! (e->flags & EDGE_EH)) e_taken = e; bb_end_insn = BB_END (bb); if (GET_CODE (bb_end_insn) == JUMP_INSN) { if (any_condjump_p (bb_end_insn)) { /* If the old fallthru is still next, nothing to do. */ if (bb->rbi->next == e_fall->dest || (!bb->rbi->next && e_fall->dest == EXIT_BLOCK_PTR)) continue; /* The degenerated case of conditional jump jumping to the next instruction can happen on target having jumps with side effects. Create temporarily the duplicated edge representing branch. It will get unidentified by force_nonfallthru_and_redirect that would otherwise get confused by fallthru edge not pointing to the next basic block. */ if (!e_taken) { rtx note; edge e_fake; e_fake = unchecked_make_edge (bb, e_fall->dest, 0); if (!redirect_jump (BB_END (bb), block_label (bb), 0)) abort (); note = find_reg_note (BB_END (bb), REG_BR_PROB, NULL_RTX); if (note) { int prob = INTVAL (XEXP (note, 0)); e_fake->probability = prob; e_fake->count = e_fall->count * prob / REG_BR_PROB_BASE; e_fall->probability -= e_fall->probability; e_fall->count -= e_fake->count; if (e_fall->probability < 0) e_fall->probability = 0; if (e_fall->count < 0) e_fall->count = 0; } } /* There is one special case: if *neither* block is next, such as happens at the very end of a function, then we'll need to add a new unconditional jump. Choose the taken edge based on known or assumed probability. */ else if (bb->rbi->next != e_taken->dest) { rtx note = find_reg_note (bb_end_insn, REG_BR_PROB, 0); if (note && INTVAL (XEXP (note, 0)) < REG_BR_PROB_BASE / 2 && invert_jump (bb_end_insn, label_for_bb (e_fall->dest), 0)) { e_fall->flags &= ~EDGE_FALLTHRU; e_taken->flags |= EDGE_FALLTHRU; update_br_prob_note (bb); e = e_fall, e_fall = e_taken, e_taken = e; } } /* Otherwise we can try to invert the jump. This will basically never fail, however, keep up the pretense. */ else if (invert_jump (bb_end_insn, label_for_bb (e_fall->dest), 0)) { e_fall->flags &= ~EDGE_FALLTHRU; e_taken->flags |= EDGE_FALLTHRU; update_br_prob_note (bb); continue; } } else if (returnjump_p (bb_end_insn)) continue; else { /* Otherwise we have some switch or computed jump. In the 99% case, there should not have been a fallthru edge. */ if (! e_fall) continue; #ifdef CASE_DROPS_THROUGH /* Except for VAX. Since we didn't have predication for the tablejump, the fallthru block should not have moved. */ if (bb->rbi->next == e_fall->dest) continue; bb_end_insn = skip_insns_after_block (bb); #else abort (); #endif } } else { /* No fallthru implies a noreturn function with EH edges, or something similarly bizarre. In any case, we don't need to do anything. */ if (! e_fall) continue; /* If the fallthru block is still next, nothing to do. */ if (bb->rbi->next == e_fall->dest) continue; /* A fallthru to exit block. */ if (!bb->rbi->next && e_fall->dest == EXIT_BLOCK_PTR) continue; } /* We got here if we need to add a new jump insn. */ nb = force_nonfallthru (e_fall); if (nb) { cfg_layout_initialize_rbi (nb); nb->rbi->visited = 1; nb->rbi->next = bb->rbi->next; bb->rbi->next = nb; /* Don't process this new block. */ bb = nb; } } /* Put basic_block_info in the new order. */ if (rtl_dump_file) { fprintf (rtl_dump_file, "Reordered sequence:\n"); for (bb = ENTRY_BLOCK_PTR->next_bb, index = 0; bb; bb = bb->rbi->next, index ++) { fprintf (rtl_dump_file, " %i ", index); if (bb->rbi->original) fprintf (rtl_dump_file, "duplicate of %i ", bb->rbi->original->index); else if (forwarder_block_p (bb) && GET_CODE (BB_HEAD (bb)) != CODE_LABEL) fprintf (rtl_dump_file, "compensation "); else fprintf (rtl_dump_file, "bb %i ", bb->index); fprintf (rtl_dump_file, " [%i]\n", bb->frequency); } } prev_bb = ENTRY_BLOCK_PTR; bb = ENTRY_BLOCK_PTR->next_bb; index = 0; for (; bb; prev_bb = bb, bb = bb->rbi->next, index ++) { bb->index = index; BASIC_BLOCK (index) = bb; bb->prev_bb = prev_bb; prev_bb->next_bb = bb; } prev_bb->next_bb = EXIT_BLOCK_PTR; EXIT_BLOCK_PTR->prev_bb = prev_bb; /* Annoying special case - jump around dead jumptables left in the code. */ FOR_EACH_BB (bb) { edge e; for (e = bb->succ; e && !(e->flags & EDGE_FALLTHRU); e = e->succ_next) continue; if (e && !can_fallthru (e->src, e->dest)) force_nonfallthru (e); } }
static bool doloop_optimize (struct loop *loop) { enum machine_mode mode; rtx doloop_seq, doloop_pat, doloop_reg; rtx iterations, count; rtx iterations_max; rtx start_label; rtx condition; unsigned level, est_niter; struct niter_desc *desc; unsigned word_mode_size; unsigned HOST_WIDE_INT word_mode_max; if (dump_file) fprintf (dump_file, "Doloop: Processing loop %d.\n", loop->num); /* APPLE LOCAL begin lno */ /* Ignore large loops. */ if (loop->ninsns > (unsigned) PARAM_VALUE (PARAM_MAX_DOLOOP_INSNS)) { if (dump_file) fprintf (dump_file, "Doloop: The loop is too large.\n"); return false; } /* APPLE LOCAL end lno */ iv_analysis_loop_init (loop); /* Find the simple exit of a LOOP. */ desc = get_simple_loop_desc (loop); /* Check that loop is a candidate for a low-overhead looping insn. */ if (!doloop_valid_p (loop, desc)) { if (dump_file) fprintf (dump_file, "Doloop: The loop is not suitable.\n"); return false; } mode = desc->mode; est_niter = 3; if (desc->const_iter) est_niter = desc->niter; /* If the estimate on number of iterations is reliable (comes from profile feedback), use it. Do not use it normally, since the expected number of iterations of an unrolled loop is 2. */ if (loop->header->count) est_niter = expected_loop_iterations (loop); if (est_niter < 3) { if (dump_file) fprintf (dump_file, "Doloop: Too few iterations (%u) to be profitable.\n", est_niter); return false; } count = copy_rtx (desc->niter_expr); iterations = desc->const_iter ? desc->niter_expr : const0_rtx; iterations_max = GEN_INT (desc->niter_max); level = get_loop_level (loop) + 1; /* Generate looping insn. If the pattern FAILs then give up trying to modify the loop since there is some aspect the back-end does not like. */ start_label = block_label (desc->in_edge->dest); doloop_reg = gen_reg_rtx (mode); doloop_seq = gen_doloop_end (doloop_reg, iterations, iterations_max, GEN_INT (level), start_label); word_mode_size = GET_MODE_BITSIZE (word_mode); word_mode_max = ((unsigned HOST_WIDE_INT) 1 << (word_mode_size - 1) << 1) - 1; if (! doloop_seq && mode != word_mode /* Before trying mode different from the one in that # of iterations is computed, we must be sure that the number of iterations fits into the new mode. */ && (word_mode_size >= GET_MODE_BITSIZE (mode) || desc->niter_max <= word_mode_max)) { if (word_mode_size > GET_MODE_BITSIZE (mode)) { count = simplify_gen_unary (ZERO_EXTEND, word_mode, count, mode); iterations = simplify_gen_unary (ZERO_EXTEND, word_mode, iterations, mode); iterations_max = simplify_gen_unary (ZERO_EXTEND, word_mode, iterations_max, mode); } else { count = lowpart_subreg (word_mode, count, mode); iterations = lowpart_subreg (word_mode, iterations, mode); iterations_max = lowpart_subreg (word_mode, iterations_max, mode); } PUT_MODE (doloop_reg, word_mode); doloop_seq = gen_doloop_end (doloop_reg, iterations, iterations_max, GEN_INT (level), start_label); } if (! doloop_seq) { if (dump_file) fprintf (dump_file, "Doloop: Target unwilling to use doloop pattern!\n"); return false; } /* If multiple instructions were created, the last must be the jump instruction. Also, a raw define_insn may yield a plain pattern. */ doloop_pat = doloop_seq; if (INSN_P (doloop_pat)) { while (NEXT_INSN (doloop_pat) != NULL_RTX) doloop_pat = NEXT_INSN (doloop_pat); if (JUMP_P (doloop_pat)) doloop_pat = PATTERN (doloop_pat); else doloop_pat = NULL_RTX; } if (! doloop_pat || ! (condition = doloop_condition_get (doloop_pat))) { if (dump_file) fprintf (dump_file, "Doloop: Unrecognizable doloop pattern!\n"); return false; } doloop_modify (loop, desc, doloop_seq, condition, count); return true; }
static void doloop_modify (struct loop *loop, struct niter_desc *desc, rtx doloop_seq, rtx condition, rtx count) { rtx counter_reg; rtx tmp, noloop = NULL_RTX; rtx sequence; rtx jump_insn; rtx jump_label; int nonneg = 0, irr; bool increment_count; basic_block loop_end = desc->out_edge->src; enum machine_mode mode; jump_insn = BB_END (loop_end); if (dump_file) { fprintf (dump_file, "Doloop: Inserting doloop pattern ("); if (desc->const_iter) fprintf (dump_file, HOST_WIDEST_INT_PRINT_DEC, desc->niter); else fputs ("runtime", dump_file); fputs (" iterations).\n", dump_file); } /* Discard original jump to continue loop. The original compare result may still be live, so it cannot be discarded explicitly. */ delete_insn (jump_insn); counter_reg = XEXP (condition, 0); if (GET_CODE (counter_reg) == PLUS) counter_reg = XEXP (counter_reg, 0); mode = GET_MODE (counter_reg); increment_count = false; switch (GET_CODE (condition)) { case NE: /* Currently only NE tests against zero and one are supported. */ if (XEXP (condition, 1) == const1_rtx) { increment_count = true; noloop = const1_rtx; } else if (XEXP (condition, 1) == const0_rtx) noloop = const0_rtx; else abort (); break; case GE: /* Currently only GE tests against zero are supported. */ if (XEXP (condition, 1) != const0_rtx) abort (); noloop = constm1_rtx; /* The iteration count does not need incrementing for a GE test. */ increment_count = false; /* Determine if the iteration counter will be non-negative. Note that the maximum value loaded is iterations_max - 1. */ if (desc->niter_max <= ((unsigned HOST_WIDEST_INT) 1 << (GET_MODE_BITSIZE (mode) - 1))) nonneg = 1; break; /* Abort if an invalid doloop pattern has been generated. */ default: abort (); } if (increment_count) count = simplify_gen_binary (PLUS, mode, count, const1_rtx); /* Insert initialization of the count register into the loop header. */ start_sequence (); tmp = force_operand (count, counter_reg); convert_move (counter_reg, tmp, 1); sequence = get_insns (); end_sequence (); emit_insn_after (sequence, BB_END (loop_preheader_edge (loop)->src)); if (desc->noloop_assumptions) { rtx ass = copy_rtx (desc->noloop_assumptions); basic_block preheader = loop_preheader_edge (loop)->src; basic_block set_zero = loop_split_edge_with (loop_preheader_edge (loop), NULL_RTX); basic_block new_preheader = loop_split_edge_with (loop_preheader_edge (loop), NULL_RTX); basic_block bb; edge te; gcov_type cnt; /* Expand the condition testing the assumptions and if it does not pass, reset the count register to 0. */ add_test (XEXP (ass, 0), preheader, set_zero); EDGE_SUCC (preheader, 0)->flags &= ~EDGE_FALLTHRU; cnt = EDGE_SUCC (preheader, 0)->count; EDGE_SUCC (preheader, 0)->probability = 0; EDGE_SUCC (preheader, 0)->count = 0; irr = EDGE_SUCC (preheader, 0)->flags & EDGE_IRREDUCIBLE_LOOP; te = make_edge (preheader, new_preheader, EDGE_FALLTHRU | irr); te->probability = REG_BR_PROB_BASE; te->count = cnt; set_immediate_dominator (CDI_DOMINATORS, new_preheader, preheader); set_zero->count = 0; set_zero->frequency = 0; for (ass = XEXP (ass, 1); ass; ass = XEXP (ass, 1)) { bb = loop_split_edge_with (te, NULL_RTX); te = EDGE_SUCC (bb, 0); add_test (XEXP (ass, 0), bb, set_zero); make_edge (bb, set_zero, irr); } start_sequence (); convert_move (counter_reg, noloop, 0); sequence = get_insns (); end_sequence (); emit_insn_after (sequence, BB_END (set_zero)); } /* Some targets (eg, C4x) need to initialize special looping registers. */ #ifdef HAVE_doloop_begin { rtx init; unsigned level = get_loop_level (loop) + 1; init = gen_doloop_begin (counter_reg, desc->const_iter ? desc->niter_expr : const0_rtx, desc->niter_max, GEN_INT (level)); if (init) { start_sequence (); emit_insn (init); sequence = get_insns (); end_sequence (); emit_insn_after (sequence, BB_END (loop_preheader_edge (loop)->src)); } } #endif /* Insert the new low-overhead looping insn. */ emit_jump_insn_after (doloop_seq, BB_END (loop_end)); jump_insn = BB_END (loop_end); jump_label = block_label (desc->in_edge->dest); JUMP_LABEL (jump_insn) = jump_label; LABEL_NUSES (jump_label)++; /* Ensure the right fallthru edge is marked, for case we have reversed the condition. */ desc->in_edge->flags &= ~EDGE_FALLTHRU; desc->out_edge->flags |= EDGE_FALLTHRU; /* Add a REG_NONNEG note if the actual or estimated maximum number of iterations is non-negative. */ if (nonneg) { REG_NOTES (jump_insn) = gen_rtx_EXPR_LIST (REG_NONNEG, NULL_RTX, REG_NOTES (jump_insn)); } }