static void test_uncond_jump () { rtx_insn *label = gen_label_rtx (); rtx jump_pat = gen_rtx_SET (pc_rtx, gen_rtx_LABEL_REF (VOIDmode, label)); ASSERT_EQ (SET, jump_pat->code); ASSERT_EQ (LABEL_REF, SET_SRC (jump_pat)->code); ASSERT_EQ (label, label_ref_label (SET_SRC (jump_pat))); ASSERT_EQ (PC, SET_DEST (jump_pat)->code); verify_print_pattern ("pc=L0", jump_pat); ASSERT_RTL_DUMP_EQ ("(set (pc)\n" " (label_ref 0))", jump_pat); rtx_insn *jump_insn = emit_jump_insn (jump_pat); ASSERT_FALSE (any_condjump_p (jump_insn)); ASSERT_TRUE (any_uncondjump_p (jump_insn)); ASSERT_TRUE (pc_set (jump_insn)); ASSERT_TRUE (simplejump_p (jump_insn)); ASSERT_TRUE (onlyjump_p (jump_insn)); ASSERT_TRUE (control_flow_insn_p (jump_insn)); ASSERT_RTL_DUMP_EQ ("(cjump_insn 1 (set (pc)\n" " (label_ref 0)))\n", jump_insn); }
/* Verify that there is exactly single jump instruction since last and attach REG_BR_PROB note specifying probability. ??? We really ought to pass the probability down to RTL expanders and let it re-distribute it when the conditional expands into multiple conditionals. This is however difficult to do. */ static void add_reg_br_prob_note (FILE *dump_file, rtx last, int probability) { if (profile_status == PROFILE_ABSENT) return; for (last = NEXT_INSN (last); last && NEXT_INSN (last); last = NEXT_INSN (last)) if (GET_CODE (last) == JUMP_INSN) { /* It is common to emit condjump-around-jump sequence when we don't know how to reverse the conditional. Special case this. */ if (!any_condjump_p (last) || GET_CODE (NEXT_INSN (last)) != JUMP_INSN || !simplejump_p (NEXT_INSN (last)) || GET_CODE (NEXT_INSN (NEXT_INSN (last))) != BARRIER || GET_CODE (NEXT_INSN (NEXT_INSN (NEXT_INSN (last)))) != CODE_LABEL || NEXT_INSN (NEXT_INSN (NEXT_INSN (NEXT_INSN (last))))) goto failed; if (find_reg_note (last, REG_BR_PROB, 0)) abort (); REG_NOTES (last) = gen_rtx_EXPR_LIST (REG_BR_PROB, GEN_INT (REG_BR_PROB_BASE - probability), REG_NOTES (last)); return; } if (!last || GET_CODE (last) != JUMP_INSN || !any_condjump_p (last)) goto failed; if (find_reg_note (last, REG_BR_PROB, 0)) abort (); REG_NOTES (last) = gen_rtx_EXPR_LIST (REG_BR_PROB, GEN_INT (probability), REG_NOTES (last)); return; failed: if (dump_file) fprintf (dump_file, "Failed to add probability note\n"); }
void ubsan_expand_si_overflow_mul_check (gimple stmt) { rtx res, op0, op1; tree lhs, fn, arg0, arg1; rtx_code_label *done_label, *do_error; rtx target = NULL_RTX; lhs = gimple_call_lhs (stmt); arg0 = gimple_call_arg (stmt, 0); arg1 = gimple_call_arg (stmt, 1); done_label = gen_label_rtx (); do_error = gen_label_rtx (); do_pending_stack_adjust (); op0 = expand_normal (arg0); op1 = expand_normal (arg1); machine_mode mode = TYPE_MODE (TREE_TYPE (arg0)); if (lhs) target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); enum insn_code icode = optab_handler (mulv4_optab, mode); if (icode != CODE_FOR_nothing) { struct expand_operand ops[4]; rtx_insn *last = get_last_insn (); res = gen_reg_rtx (mode); create_output_operand (&ops[0], res, mode); create_input_operand (&ops[1], op0, mode); create_input_operand (&ops[2], op1, mode); create_fixed_operand (&ops[3], do_error); if (maybe_expand_insn (icode, 4, ops)) { last = get_last_insn (); if (profile_status_for_fn (cfun) != PROFILE_ABSENT && JUMP_P (last) && any_condjump_p (last) && !find_reg_note (last, REG_BR_PROB, 0)) add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY); emit_jump (done_label); } else { delete_insns_since (last); icode = CODE_FOR_nothing; } } if (icode == CODE_FOR_nothing) { struct separate_ops ops; machine_mode hmode = mode_for_size (GET_MODE_PRECISION (mode) / 2, MODE_INT, 1); ops.op0 = arg0; ops.op1 = arg1; ops.op2 = NULL_TREE; ops.location = gimple_location (stmt); if (GET_MODE_2XWIDER_MODE (mode) != VOIDmode && targetm.scalar_mode_supported_p (GET_MODE_2XWIDER_MODE (mode))) { machine_mode wmode = GET_MODE_2XWIDER_MODE (mode); ops.code = WIDEN_MULT_EXPR; ops.type = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode), 0); res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL); rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res, GET_MODE_PRECISION (mode), NULL_RTX, 0); hipart = gen_lowpart (mode, hipart); res = gen_lowpart (mode, res); rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, GET_MODE_PRECISION (mode) - 1, NULL_RTX, 0); /* RES is low half of the double width result, HIPART the high half. There was overflow if HIPART is different from RES < 0 ? -1 : 0. */ emit_cmp_and_jump_insns (signbit, hipart, EQ, NULL_RTX, mode, false, done_label, PROB_VERY_LIKELY); } else if (hmode != BLKmode && 2 * GET_MODE_PRECISION (hmode) == GET_MODE_PRECISION (mode)) { rtx_code_label *large_op0 = gen_label_rtx (); rtx_code_label *small_op0_large_op1 = gen_label_rtx (); rtx_code_label *one_small_one_large = gen_label_rtx (); rtx_code_label *both_ops_large = gen_label_rtx (); rtx_code_label *after_hipart_neg = gen_label_rtx (); rtx_code_label *after_lopart_neg = gen_label_rtx (); rtx_code_label *do_overflow = gen_label_rtx (); rtx_code_label *hipart_different = gen_label_rtx (); unsigned int hprec = GET_MODE_PRECISION (hmode); rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec, NULL_RTX, 0); hipart0 = gen_lowpart (hmode, hipart0); rtx lopart0 = gen_lowpart (hmode, op0); rtx signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1, NULL_RTX, 0); rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec, NULL_RTX, 0); hipart1 = gen_lowpart (hmode, hipart1); rtx lopart1 = gen_lowpart (hmode, op1); rtx signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1, NULL_RTX, 0); res = gen_reg_rtx (mode); /* True if op0 resp. op1 are known to be in the range of halfstype. */ bool op0_small_p = false; bool op1_small_p = false; /* True if op0 resp. op1 are known to have all zeros or all ones in the upper half of bits, but are not known to be op{0,1}_small_p. */ bool op0_medium_p = false; bool op1_medium_p = false; /* -1 if op{0,1} is known to be negative, 0 if it is known to be nonnegative, 1 if unknown. */ int op0_sign = 1; int op1_sign = 1; if (TREE_CODE (arg0) == SSA_NAME) { wide_int arg0_min, arg0_max; if (get_range_info (arg0, &arg0_min, &arg0_max) == VR_RANGE) { unsigned int mprec0 = wi::min_precision (arg0_min, SIGNED); unsigned int mprec1 = wi::min_precision (arg0_max, SIGNED); if (mprec0 <= hprec && mprec1 <= hprec) op0_small_p = true; else if (mprec0 <= hprec + 1 && mprec1 <= hprec + 1) op0_medium_p = true; if (!wi::neg_p (arg0_min, TYPE_SIGN (TREE_TYPE (arg0)))) op0_sign = 0; else if (wi::neg_p (arg0_max, TYPE_SIGN (TREE_TYPE (arg0)))) op0_sign = -1; } } if (TREE_CODE (arg1) == SSA_NAME) { wide_int arg1_min, arg1_max; if (get_range_info (arg1, &arg1_min, &arg1_max) == VR_RANGE) { unsigned int mprec0 = wi::min_precision (arg1_min, SIGNED); unsigned int mprec1 = wi::min_precision (arg1_max, SIGNED); if (mprec0 <= hprec && mprec1 <= hprec) op1_small_p = true; else if (mprec0 <= hprec + 1 && mprec1 <= hprec + 1) op1_medium_p = true; if (!wi::neg_p (arg1_min, TYPE_SIGN (TREE_TYPE (arg1)))) op1_sign = 0; else if (wi::neg_p (arg1_max, TYPE_SIGN (TREE_TYPE (arg1)))) op1_sign = -1; } } int smaller_sign = 1; int larger_sign = 1; if (op0_small_p) { smaller_sign = op0_sign; larger_sign = op1_sign; } else if (op1_small_p) { smaller_sign = op1_sign; larger_sign = op0_sign; } else if (op0_sign == op1_sign) { smaller_sign = op0_sign; larger_sign = op0_sign; } if (!op0_small_p) emit_cmp_and_jump_insns (signbit0, hipart0, NE, NULL_RTX, hmode, false, large_op0, PROB_UNLIKELY); if (!op1_small_p) emit_cmp_and_jump_insns (signbit1, hipart1, NE, NULL_RTX, hmode, false, small_op0_large_op1, PROB_UNLIKELY); /* If both op0 and op1 are sign extended from hmode to mode, the multiplication will never overflow. We can do just one hmode x hmode => mode widening multiplication. */ if (GET_CODE (lopart0) == SUBREG) { SUBREG_PROMOTED_VAR_P (lopart0) = 1; SUBREG_PROMOTED_SET (lopart0, 0); } if (GET_CODE (lopart1) == SUBREG) { SUBREG_PROMOTED_VAR_P (lopart1) = 1; SUBREG_PROMOTED_SET (lopart1, 0); } tree halfstype = build_nonstandard_integer_type (hprec, 0); ops.op0 = make_tree (halfstype, lopart0); ops.op1 = make_tree (halfstype, lopart1); ops.code = WIDEN_MULT_EXPR; ops.type = TREE_TYPE (arg0); rtx thisres = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); emit_move_insn (res, thisres); emit_jump (done_label); emit_label (small_op0_large_op1); /* If op0 is sign extended from hmode to mode, but op1 is not, just swap the arguments and handle it as op1 sign extended, op0 not. */ rtx larger = gen_reg_rtx (mode); rtx hipart = gen_reg_rtx (hmode); rtx lopart = gen_reg_rtx (hmode); emit_move_insn (larger, op1); emit_move_insn (hipart, hipart1); emit_move_insn (lopart, lopart0); emit_jump (one_small_one_large); emit_label (large_op0); if (!op1_small_p) emit_cmp_and_jump_insns (signbit1, hipart1, NE, NULL_RTX, hmode, false, both_ops_large, PROB_UNLIKELY); /* If op1 is sign extended from hmode to mode, but op0 is not, prepare larger, hipart and lopart pseudos and handle it together with small_op0_large_op1. */ emit_move_insn (larger, op0); emit_move_insn (hipart, hipart0); emit_move_insn (lopart, lopart1); emit_label (one_small_one_large); /* lopart is the low part of the operand that is sign extended to mode, larger is the the other operand, hipart is the high part of larger and lopart0 and lopart1 are the low parts of both operands. We perform lopart0 * lopart1 and lopart * hipart widening multiplications. */ tree halfutype = build_nonstandard_integer_type (hprec, 1); ops.op0 = make_tree (halfutype, lopart0); ops.op1 = make_tree (halfutype, lopart1); rtx lo0xlo1 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); ops.op0 = make_tree (halfutype, lopart); ops.op1 = make_tree (halfutype, hipart); rtx loxhi = gen_reg_rtx (mode); rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); emit_move_insn (loxhi, tem); /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */ if (larger_sign == 0) emit_jump (after_hipart_neg); else if (larger_sign != -1) emit_cmp_and_jump_insns (hipart, const0_rtx, GE, NULL_RTX, hmode, false, after_hipart_neg, PROB_EVEN); tem = convert_modes (mode, hmode, lopart, 1); tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1); tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX, 1, OPTAB_DIRECT); emit_move_insn (loxhi, tem); emit_label (after_hipart_neg); /* if (lopart < 0) loxhi -= larger; */ if (smaller_sign == 0) emit_jump (after_lopart_neg); else if (smaller_sign != -1) emit_cmp_and_jump_insns (lopart, const0_rtx, GE, NULL_RTX, hmode, false, after_lopart_neg, PROB_EVEN); tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX, 1, OPTAB_DIRECT); emit_move_insn (loxhi, tem); emit_label (after_lopart_neg); /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */ tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1); tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX, 1, OPTAB_DIRECT); emit_move_insn (loxhi, tem); /* if (loxhi >> (bitsize / 2) == (hmode) loxhi >> (bitsize / 2 - 1)) */ rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec, NULL_RTX, 0); hipartloxhi = gen_lowpart (hmode, hipartloxhi); rtx lopartloxhi = gen_lowpart (hmode, loxhi); rtx signbitloxhi = expand_shift (RSHIFT_EXPR, hmode, lopartloxhi, hprec - 1, NULL_RTX, 0); emit_cmp_and_jump_insns (signbitloxhi, hipartloxhi, NE, NULL_RTX, hmode, false, do_overflow, PROB_VERY_UNLIKELY); /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */ rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec, NULL_RTX, 1); tem = convert_modes (mode, hmode, gen_lowpart (hmode, lo0xlo1), 1); tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res, 1, OPTAB_DIRECT); if (tem != res) emit_move_insn (res, tem); emit_jump (done_label); emit_label (both_ops_large); /* If both operands are large (not sign extended from hmode), then perform the full multiplication which will be the result of the operation. The only cases which don't overflow are some cases where both hipart0 and highpart1 are 0 or -1. */ ops.code = MULT_EXPR; ops.op0 = make_tree (TREE_TYPE (arg0), op0); ops.op1 = make_tree (TREE_TYPE (arg0), op1); tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); emit_move_insn (res, tem); if (!op0_medium_p) { tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx, NULL_RTX, 1, OPTAB_DIRECT); emit_cmp_and_jump_insns (tem, const1_rtx, GTU, NULL_RTX, hmode, true, do_error, PROB_VERY_UNLIKELY); } if (!op1_medium_p) { tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx, NULL_RTX, 1, OPTAB_DIRECT); emit_cmp_and_jump_insns (tem, const1_rtx, GTU, NULL_RTX, hmode, true, do_error, PROB_VERY_UNLIKELY); } /* At this point hipart{0,1} are both in [-1, 0]. If they are the same, overflow happened if res is negative, if they are different, overflow happened if res is positive. */ if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign) emit_jump (hipart_different); else if (op0_sign == 1 || op1_sign == 1) emit_cmp_and_jump_insns (hipart0, hipart1, NE, NULL_RTX, hmode, true, hipart_different, PROB_EVEN); emit_cmp_and_jump_insns (res, const0_rtx, LT, NULL_RTX, mode, false, do_error, PROB_VERY_UNLIKELY); emit_jump (done_label); emit_label (hipart_different); emit_cmp_and_jump_insns (res, const0_rtx, GE, NULL_RTX, mode, false, do_error, PROB_VERY_UNLIKELY); emit_jump (done_label); emit_label (do_overflow); /* Overflow, do full multiplication and fallthru into do_error. */ ops.op0 = make_tree (TREE_TYPE (arg0), op0); ops.op1 = make_tree (TREE_TYPE (arg0), op1); tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); emit_move_insn (res, tem); } else { ops.code = MULT_EXPR; ops.type = TREE_TYPE (arg0); res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); emit_jump (done_label); } } emit_label (do_error); /* Expand the ubsan builtin call. */ push_temp_slots (); fn = ubsan_build_overflow_builtin (MULT_EXPR, gimple_location (stmt), TREE_TYPE (arg0), arg0, arg1); expand_normal (fn); pop_temp_slots (); do_pending_stack_adjust (); /* We're done. */ emit_label (done_label); if (lhs) emit_move_insn (target, res); }
void ubsan_expand_si_overflow_neg_check (gimple stmt) { rtx res, op1; tree lhs, fn, arg1; rtx_code_label *done_label, *do_error; rtx target = NULL_RTX; lhs = gimple_call_lhs (stmt); arg1 = gimple_call_arg (stmt, 1); done_label = gen_label_rtx (); do_error = gen_label_rtx (); do_pending_stack_adjust (); op1 = expand_normal (arg1); machine_mode mode = TYPE_MODE (TREE_TYPE (arg1)); if (lhs) target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); enum insn_code icode = optab_handler (negv3_optab, mode); if (icode != CODE_FOR_nothing) { struct expand_operand ops[3]; rtx_insn *last = get_last_insn (); res = gen_reg_rtx (mode); create_output_operand (&ops[0], res, mode); create_input_operand (&ops[1], op1, mode); create_fixed_operand (&ops[2], do_error); if (maybe_expand_insn (icode, 3, ops)) { last = get_last_insn (); if (profile_status_for_fn (cfun) != PROFILE_ABSENT && JUMP_P (last) && any_condjump_p (last) && !find_reg_note (last, REG_BR_PROB, 0)) add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY); emit_jump (done_label); } else { delete_insns_since (last); icode = CODE_FOR_nothing; } } if (icode == CODE_FOR_nothing) { /* Compute the operation. On RTL level, the addition is always unsigned. */ res = expand_unop (mode, neg_optab, op1, NULL_RTX, false); /* Compare the operand with the most negative value. */ rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1))); emit_cmp_and_jump_insns (op1, minv, NE, NULL_RTX, mode, false, done_label, PROB_VERY_LIKELY); } emit_label (do_error); /* Expand the ubsan builtin call. */ push_temp_slots (); fn = ubsan_build_overflow_builtin (NEGATE_EXPR, gimple_location (stmt), TREE_TYPE (arg1), arg1, NULL_TREE); expand_normal (fn); pop_temp_slots (); do_pending_stack_adjust (); /* We're done. */ emit_label (done_label); if (lhs) emit_move_insn (target, res); }
void ubsan_expand_si_overflow_addsub_check (tree_code code, gimple stmt) { rtx res, op0, op1; tree lhs, fn, arg0, arg1; rtx_code_label *done_label, *do_error; rtx target = NULL_RTX; lhs = gimple_call_lhs (stmt); arg0 = gimple_call_arg (stmt, 0); arg1 = gimple_call_arg (stmt, 1); done_label = gen_label_rtx (); do_error = gen_label_rtx (); do_pending_stack_adjust (); op0 = expand_normal (arg0); op1 = expand_normal (arg1); machine_mode mode = TYPE_MODE (TREE_TYPE (arg0)); if (lhs) target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); enum insn_code icode = optab_handler (code == PLUS_EXPR ? addv4_optab : subv4_optab, mode); if (icode != CODE_FOR_nothing) { struct expand_operand ops[4]; rtx_insn *last = get_last_insn (); res = gen_reg_rtx (mode); create_output_operand (&ops[0], res, mode); create_input_operand (&ops[1], op0, mode); create_input_operand (&ops[2], op1, mode); create_fixed_operand (&ops[3], do_error); if (maybe_expand_insn (icode, 4, ops)) { last = get_last_insn (); if (profile_status_for_fn (cfun) != PROFILE_ABSENT && JUMP_P (last) && any_condjump_p (last) && !find_reg_note (last, REG_BR_PROB, 0)) add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY); emit_jump (done_label); } else { delete_insns_since (last); icode = CODE_FOR_nothing; } } if (icode == CODE_FOR_nothing) { rtx_code_label *sub_check = gen_label_rtx (); int pos_neg = 3; /* Compute the operation. On RTL level, the addition is always unsigned. */ res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab, op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN); /* If we can prove one of the arguments (for MINUS_EXPR only the second operand, as subtraction is not commutative) is always non-negative or always negative, we can do just one comparison and conditional jump instead of 2 at runtime, 3 present in the emitted code. If one of the arguments is CONST_INT, all we need is to make sure it is op1, then the first emit_cmp_and_jump_insns will be just folded. Otherwise try to use range info if available. */ if (code == PLUS_EXPR && CONST_INT_P (op0)) { rtx tem = op0; op0 = op1; op1 = tem; } else if (CONST_INT_P (op1)) ; else if (code == PLUS_EXPR && TREE_CODE (arg0) == SSA_NAME) { wide_int arg0_min, arg0_max; if (get_range_info (arg0, &arg0_min, &arg0_max) == VR_RANGE) { if (!wi::neg_p (arg0_min, TYPE_SIGN (TREE_TYPE (arg0)))) pos_neg = 1; else if (wi::neg_p (arg0_max, TYPE_SIGN (TREE_TYPE (arg0)))) pos_neg = 2; } if (pos_neg != 3) { rtx tem = op0; op0 = op1; op1 = tem; } } if (pos_neg == 3 && !CONST_INT_P (op1) && TREE_CODE (arg1) == SSA_NAME) { wide_int arg1_min, arg1_max; if (get_range_info (arg1, &arg1_min, &arg1_max) == VR_RANGE) { if (!wi::neg_p (arg1_min, TYPE_SIGN (TREE_TYPE (arg1)))) pos_neg = 1; else if (wi::neg_p (arg1_max, TYPE_SIGN (TREE_TYPE (arg1)))) pos_neg = 2; } } /* If the op1 is negative, we have to use a different check. */ if (pos_neg == 3) emit_cmp_and_jump_insns (op1, const0_rtx, LT, NULL_RTX, mode, false, sub_check, PROB_EVEN); /* Compare the result of the operation with one of the operands. */ if (pos_neg & 1) emit_cmp_and_jump_insns (res, op0, code == PLUS_EXPR ? GE : LE, NULL_RTX, mode, false, done_label, PROB_VERY_LIKELY); /* If we get here, we have to print the error. */ if (pos_neg == 3) { emit_jump (do_error); emit_label (sub_check); } /* We have k = a + b for b < 0 here. k <= a must hold. */ if (pos_neg & 2) emit_cmp_and_jump_insns (res, op0, code == PLUS_EXPR ? LE : GE, NULL_RTX, mode, false, done_label, PROB_VERY_LIKELY); } emit_label (do_error); /* Expand the ubsan builtin call. */ push_temp_slots (); fn = ubsan_build_overflow_builtin (code, gimple_location (stmt), TREE_TYPE (arg0), arg0, arg1); expand_normal (fn); pop_temp_slots (); do_pending_stack_adjust (); /* We're done. */ emit_label (done_label); if (lhs) emit_move_insn (target, res); }
void do_compare_rtx_and_jump (rtx op0, rtx op1, enum rtx_code code, int unsignedp, enum machine_mode mode, rtx size, rtx if_false_label, rtx if_true_label, int prob) { rtx tem; rtx dummy_label = NULL_RTX; rtx last; /* Reverse the comparison if that is safe and we want to jump if it is false. Also convert to the reverse comparison if the target can implement it. */ if ((! if_true_label || ! can_compare_p (code, mode, ccp_jump)) && (! FLOAT_MODE_P (mode) || code == ORDERED || code == UNORDERED || (! HONOR_NANS (mode) && (code == LTGT || code == UNEQ)) || (! HONOR_SNANS (mode) && (code == EQ || code == NE)))) { enum rtx_code rcode; if (FLOAT_MODE_P (mode)) rcode = reverse_condition_maybe_unordered (code); else rcode = reverse_condition (code); /* Canonicalize to UNORDERED for the libcall. */ if (can_compare_p (rcode, mode, ccp_jump) || (code == ORDERED && ! can_compare_p (ORDERED, mode, ccp_jump))) { tem = if_true_label; if_true_label = if_false_label; if_false_label = tem; code = rcode; prob = inv (prob); } } /* If one operand is constant, make it the second one. Only do this if the other operand is not constant as well. */ if (swap_commutative_operands_p (op0, op1)) { tem = op0; op0 = op1; op1 = tem; code = swap_condition (code); } do_pending_stack_adjust (); code = unsignedp ? unsigned_condition (code) : code; if (0 != (tem = simplify_relational_operation (code, mode, VOIDmode, op0, op1))) { if (CONSTANT_P (tem)) { rtx label = (tem == const0_rtx || tem == CONST0_RTX (mode)) ? if_false_label : if_true_label; if (label) emit_jump (label); return; } code = GET_CODE (tem); mode = GET_MODE (tem); op0 = XEXP (tem, 0); op1 = XEXP (tem, 1); unsignedp = (code == GTU || code == LTU || code == GEU || code == LEU); } if (! if_true_label) dummy_label = if_true_label = gen_label_rtx (); if (GET_MODE_CLASS (mode) == MODE_INT && ! can_compare_p (code, mode, ccp_jump)) { switch (code) { case LTU: do_jump_by_parts_greater_rtx (mode, 1, op1, op0, if_false_label, if_true_label, prob); break; case LEU: do_jump_by_parts_greater_rtx (mode, 1, op0, op1, if_true_label, if_false_label, inv (prob)); break; case GTU: do_jump_by_parts_greater_rtx (mode, 1, op0, op1, if_false_label, if_true_label, prob); break; case GEU: do_jump_by_parts_greater_rtx (mode, 1, op1, op0, if_true_label, if_false_label, inv (prob)); break; case LT: do_jump_by_parts_greater_rtx (mode, 0, op1, op0, if_false_label, if_true_label, prob); break; case LE: do_jump_by_parts_greater_rtx (mode, 0, op0, op1, if_true_label, if_false_label, inv (prob)); break; case GT: do_jump_by_parts_greater_rtx (mode, 0, op0, op1, if_false_label, if_true_label, prob); break; case GE: do_jump_by_parts_greater_rtx (mode, 0, op1, op0, if_true_label, if_false_label, inv (prob)); break; case EQ: do_jump_by_parts_equality_rtx (mode, op0, op1, if_false_label, if_true_label, prob); break; case NE: do_jump_by_parts_equality_rtx (mode, op0, op1, if_true_label, if_false_label, inv (prob)); break; default: gcc_unreachable (); } } else { if (GET_MODE_CLASS (mode) == MODE_FLOAT && ! can_compare_p (code, mode, ccp_jump) && can_compare_p (swap_condition (code), mode, ccp_jump)) { rtx tmp; code = swap_condition (code); tmp = op0; op0 = op1; op1 = tmp; } else if (GET_MODE_CLASS (mode) == MODE_FLOAT && ! can_compare_p (code, mode, ccp_jump) /* Never split ORDERED and UNORDERED. These must be implemented. */ && (code != ORDERED && code != UNORDERED) /* Split a floating-point comparison if we can jump on other conditions... */ && (have_insn_for (COMPARE, mode) /* ... or if there is no libcall for it. */ || code_to_optab[code] == NULL)) { enum rtx_code first_code; bool and_them = split_comparison (code, mode, &first_code, &code); /* If there are no NaNs, the first comparison should always fall through. */ if (!HONOR_NANS (mode)) gcc_assert (first_code == (and_them ? ORDERED : UNORDERED)); else { if (and_them) { rtx dest_label; /* If we only jump if true, just bypass the second jump. */ if (! if_false_label) { if (! dummy_label) dummy_label = gen_label_rtx (); dest_label = dummy_label; } else dest_label = if_false_label; do_compare_rtx_and_jump (op0, op1, first_code, unsignedp, mode, size, dest_label, NULL_RTX, prob); } else do_compare_rtx_and_jump (op0, op1, first_code, unsignedp, mode, size, NULL_RTX, if_true_label, prob); } } last = get_last_insn (); emit_cmp_and_jump_insns (op0, op1, code, size, mode, unsignedp, if_true_label); if (prob != -1 && profile_status != PROFILE_ABSENT) { for (last = NEXT_INSN (last); last && NEXT_INSN (last); last = NEXT_INSN (last)) if (JUMP_P (last)) break; if (!last || !JUMP_P (last) || NEXT_INSN (last) || !any_condjump_p (last)) { if (dump_file) fprintf (dump_file, "Failed to add probability note\n"); } else { gcc_assert (!find_reg_note (last, REG_BR_PROB, 0)); add_reg_note (last, REG_BR_PROB, GEN_INT (prob)); } } } if (if_false_label) emit_jump (if_false_label); if (dummy_label) emit_label (dummy_label); }
static basic_block rotate_loop (edge back_edge, struct trace *trace, int trace_n) { basic_block bb; /* Information about the best end (end after rotation) of the loop. */ basic_block best_bb = NULL; edge best_edge = NULL; int best_freq = -1; gcov_type best_count = -1; /* The best edge is preferred when its destination is not visited yet or is a start block of some trace. */ bool is_preferred = false; /* Find the most frequent edge that goes out from current trace. */ bb = back_edge->dest; do { edge e; for (e = bb->succ; e; e = e->succ_next) if (e->dest != EXIT_BLOCK_PTR && e->dest->rbi->visited != trace_n && (e->flags & EDGE_CAN_FALLTHRU) && !(e->flags & EDGE_COMPLEX)) { if (is_preferred) { /* The best edge is preferred. */ if (!e->dest->rbi->visited || bbd[e->dest->index].start_of_trace >= 0) { /* The current edge E is also preferred. */ int freq = EDGE_FREQUENCY (e); if (freq > best_freq || e->count > best_count) { best_freq = freq; best_count = e->count; best_edge = e; best_bb = bb; } } } else { if (!e->dest->rbi->visited || bbd[e->dest->index].start_of_trace >= 0) { /* The current edge E is preferred. */ is_preferred = true; best_freq = EDGE_FREQUENCY (e); best_count = e->count; best_edge = e; best_bb = bb; } else { int freq = EDGE_FREQUENCY (e); if (!best_edge || freq > best_freq || e->count > best_count) { best_freq = freq; best_count = e->count; best_edge = e; best_bb = bb; } } } } bb = bb->rbi->next; } while (bb != back_edge->dest); if (best_bb) { /* Rotate the loop so that the BEST_EDGE goes out from the last block of the trace. */ if (back_edge->dest == trace->first) { trace->first = best_bb->rbi->next; } else { basic_block prev_bb; for (prev_bb = trace->first; prev_bb->rbi->next != back_edge->dest; prev_bb = prev_bb->rbi->next) ; prev_bb->rbi->next = best_bb->rbi->next; /* Try to get rid of uncond jump to cond jump. */ if (prev_bb->succ && !prev_bb->succ->succ_next) { basic_block header = prev_bb->succ->dest; /* Duplicate HEADER if it is a small block containing cond jump in the end. */ if (any_condjump_p (BB_END (header)) && copy_bb_p (header, 0)) { copy_bb (header, prev_bb->succ, prev_bb, trace_n); } } } } else { /* We have not found suitable loop tail so do no rotation. */ best_bb = back_edge->src; } best_bb->rbi->next = NULL; return best_bb; }
static rtx may_unswitch_on (basic_block bb, struct loop *loop, rtx *cinsn) { rtx test, at, op[2], stest; struct rtx_iv iv; unsigned i; enum machine_mode mode; /* BB must end in a simple conditional jump. */ if (EDGE_COUNT (bb->succs) != 2) return NULL_RTX; if (!any_condjump_p (BB_END (bb))) return NULL_RTX; /* With branches inside loop. */ if (!flow_bb_inside_loop_p (loop, EDGE_SUCC (bb, 0)->dest) || !flow_bb_inside_loop_p (loop, EDGE_SUCC (bb, 1)->dest)) return NULL_RTX; /* It must be executed just once each iteration (because otherwise we are unable to update dominator/irreducible loop information correctly). */ if (!just_once_each_iteration_p (loop, bb)) return NULL_RTX; /* Condition must be invariant. */ test = get_condition (BB_END (bb), &at, true, false); if (!test) return NULL_RTX; for (i = 0; i < 2; i++) { op[i] = XEXP (test, i); if (CONSTANT_P (op[i])) continue; if (!iv_analyze (at, op[i], &iv)) return NULL_RTX; if (iv.step != const0_rtx || iv.first_special) return NULL_RTX; op[i] = get_iv_value (&iv, const0_rtx); } mode = GET_MODE (op[0]); if (mode == VOIDmode) mode = GET_MODE (op[1]); if (GET_MODE_CLASS (mode) == MODE_CC) { if (at != BB_END (bb)) return NULL_RTX; if (!rtx_equal_p (op[0], XEXP (test, 0)) || !rtx_equal_p (op[1], XEXP (test, 1))) return NULL_RTX; *cinsn = BB_END (bb); return test; } stest = simplify_gen_relational (GET_CODE (test), SImode, mode, op[0], op[1]); if (stest == const0_rtx || stest == const_true_rtx) return stest; return canon_condition (gen_rtx_fmt_ee (GET_CODE (test), SImode, op[0], op[1])); }
static void fixup_reorder_chain (void) { basic_block bb, prev_bb; int index; rtx insn = NULL; if (cfg_layout_function_header) { set_first_insn (cfg_layout_function_header); insn = cfg_layout_function_header; while (NEXT_INSN (insn)) insn = NEXT_INSN (insn); } /* First do the bulk reordering -- rechain the blocks without regard to the needed changes to jumps and labels. */ for (bb = ENTRY_BLOCK_PTR->next_bb, index = 0; bb != 0; bb = bb->rbi->next, index++) { if (bb->rbi->header) { if (insn) NEXT_INSN (insn) = bb->rbi->header; else set_first_insn (bb->rbi->header); PREV_INSN (bb->rbi->header) = insn; insn = bb->rbi->header; while (NEXT_INSN (insn)) insn = NEXT_INSN (insn); } if (insn) NEXT_INSN (insn) = BB_HEAD (bb); else set_first_insn (BB_HEAD (bb)); PREV_INSN (BB_HEAD (bb)) = insn; insn = BB_END (bb); if (bb->rbi->footer) { NEXT_INSN (insn) = bb->rbi->footer; PREV_INSN (bb->rbi->footer) = insn; while (NEXT_INSN (insn)) insn = NEXT_INSN (insn); } } if (index != n_basic_blocks) abort (); NEXT_INSN (insn) = cfg_layout_function_footer; if (cfg_layout_function_footer) PREV_INSN (cfg_layout_function_footer) = insn; while (NEXT_INSN (insn)) insn = NEXT_INSN (insn); set_last_insn (insn); #ifdef ENABLE_CHECKING verify_insn_chain (); #endif delete_dead_jumptables (); /* Now add jumps and labels as needed to match the blocks new outgoing edges. */ for (bb = ENTRY_BLOCK_PTR->next_bb; bb ; bb = bb->rbi->next) { edge e_fall, e_taken, e; rtx bb_end_insn; basic_block nb; if (bb->succ == NULL) continue; /* Find the old fallthru edge, and another non-EH edge for a taken jump. */ e_taken = e_fall = NULL; for (e = bb->succ; e ; e = e->succ_next) if (e->flags & EDGE_FALLTHRU) e_fall = e; else if (! (e->flags & EDGE_EH)) e_taken = e; bb_end_insn = BB_END (bb); if (GET_CODE (bb_end_insn) == JUMP_INSN) { if (any_condjump_p (bb_end_insn)) { /* If the old fallthru is still next, nothing to do. */ if (bb->rbi->next == e_fall->dest || (!bb->rbi->next && e_fall->dest == EXIT_BLOCK_PTR)) continue; /* The degenerated case of conditional jump jumping to the next instruction can happen on target having jumps with side effects. Create temporarily the duplicated edge representing branch. It will get unidentified by force_nonfallthru_and_redirect that would otherwise get confused by fallthru edge not pointing to the next basic block. */ if (!e_taken) { rtx note; edge e_fake; e_fake = unchecked_make_edge (bb, e_fall->dest, 0); if (!redirect_jump (BB_END (bb), block_label (bb), 0)) abort (); note = find_reg_note (BB_END (bb), REG_BR_PROB, NULL_RTX); if (note) { int prob = INTVAL (XEXP (note, 0)); e_fake->probability = prob; e_fake->count = e_fall->count * prob / REG_BR_PROB_BASE; e_fall->probability -= e_fall->probability; e_fall->count -= e_fake->count; if (e_fall->probability < 0) e_fall->probability = 0; if (e_fall->count < 0) e_fall->count = 0; } } /* There is one special case: if *neither* block is next, such as happens at the very end of a function, then we'll need to add a new unconditional jump. Choose the taken edge based on known or assumed probability. */ else if (bb->rbi->next != e_taken->dest) { rtx note = find_reg_note (bb_end_insn, REG_BR_PROB, 0); if (note && INTVAL (XEXP (note, 0)) < REG_BR_PROB_BASE / 2 && invert_jump (bb_end_insn, label_for_bb (e_fall->dest), 0)) { e_fall->flags &= ~EDGE_FALLTHRU; e_taken->flags |= EDGE_FALLTHRU; update_br_prob_note (bb); e = e_fall, e_fall = e_taken, e_taken = e; } } /* Otherwise we can try to invert the jump. This will basically never fail, however, keep up the pretense. */ else if (invert_jump (bb_end_insn, label_for_bb (e_fall->dest), 0)) { e_fall->flags &= ~EDGE_FALLTHRU; e_taken->flags |= EDGE_FALLTHRU; update_br_prob_note (bb); continue; } } else if (returnjump_p (bb_end_insn)) continue; else { /* Otherwise we have some switch or computed jump. In the 99% case, there should not have been a fallthru edge. */ if (! e_fall) continue; #ifdef CASE_DROPS_THROUGH /* Except for VAX. Since we didn't have predication for the tablejump, the fallthru block should not have moved. */ if (bb->rbi->next == e_fall->dest) continue; bb_end_insn = skip_insns_after_block (bb); #else abort (); #endif } } else { /* No fallthru implies a noreturn function with EH edges, or something similarly bizarre. In any case, we don't need to do anything. */ if (! e_fall) continue; /* If the fallthru block is still next, nothing to do. */ if (bb->rbi->next == e_fall->dest) continue; /* A fallthru to exit block. */ if (!bb->rbi->next && e_fall->dest == EXIT_BLOCK_PTR) continue; } /* We got here if we need to add a new jump insn. */ nb = force_nonfallthru (e_fall); if (nb) { cfg_layout_initialize_rbi (nb); nb->rbi->visited = 1; nb->rbi->next = bb->rbi->next; bb->rbi->next = nb; /* Don't process this new block. */ bb = nb; } } /* Put basic_block_info in the new order. */ if (rtl_dump_file) { fprintf (rtl_dump_file, "Reordered sequence:\n"); for (bb = ENTRY_BLOCK_PTR->next_bb, index = 0; bb; bb = bb->rbi->next, index ++) { fprintf (rtl_dump_file, " %i ", index); if (bb->rbi->original) fprintf (rtl_dump_file, "duplicate of %i ", bb->rbi->original->index); else if (forwarder_block_p (bb) && GET_CODE (BB_HEAD (bb)) != CODE_LABEL) fprintf (rtl_dump_file, "compensation "); else fprintf (rtl_dump_file, "bb %i ", bb->index); fprintf (rtl_dump_file, " [%i]\n", bb->frequency); } } prev_bb = ENTRY_BLOCK_PTR; bb = ENTRY_BLOCK_PTR->next_bb; index = 0; for (; bb; prev_bb = bb, bb = bb->rbi->next, index ++) { bb->index = index; BASIC_BLOCK (index) = bb; bb->prev_bb = prev_bb; prev_bb->next_bb = bb; } prev_bb->next_bb = EXIT_BLOCK_PTR; EXIT_BLOCK_PTR->prev_bb = prev_bb; /* Annoying special case - jump around dead jumptables left in the code. */ FOR_EACH_BB (bb) { edge e; for (e = bb->succ; e && !(e->flags & EDGE_FALLTHRU); e = e->succ_next) continue; if (e && !can_fallthru (e->src, e->dest)) force_nonfallthru (e); } }
static void fixup_reorder_chain (void) { basic_block bb, prev_bb; int index; rtx insn = NULL; if (cfg_layout_function_header) { set_first_insn (cfg_layout_function_header); insn = cfg_layout_function_header; while (NEXT_INSN (insn)) insn = NEXT_INSN (insn); } /* First do the bulk reordering -- rechain the blocks without regard to the needed changes to jumps and labels. */ for (bb = ENTRY_BLOCK_PTR->next_bb, index = NUM_FIXED_BLOCKS; bb != 0; bb = bb->aux, index++) { if (bb->il.rtl->header) { if (insn) NEXT_INSN (insn) = bb->il.rtl->header; else set_first_insn (bb->il.rtl->header); PREV_INSN (bb->il.rtl->header) = insn; insn = bb->il.rtl->header; while (NEXT_INSN (insn)) insn = NEXT_INSN (insn); } if (insn) NEXT_INSN (insn) = BB_HEAD (bb); else set_first_insn (BB_HEAD (bb)); PREV_INSN (BB_HEAD (bb)) = insn; insn = BB_END (bb); if (bb->il.rtl->footer) { NEXT_INSN (insn) = bb->il.rtl->footer; PREV_INSN (bb->il.rtl->footer) = insn; while (NEXT_INSN (insn)) insn = NEXT_INSN (insn); } } gcc_assert (index == n_basic_blocks); NEXT_INSN (insn) = cfg_layout_function_footer; if (cfg_layout_function_footer) PREV_INSN (cfg_layout_function_footer) = insn; while (NEXT_INSN (insn)) insn = NEXT_INSN (insn); set_last_insn (insn); #ifdef ENABLE_CHECKING verify_insn_chain (); #endif delete_dead_jumptables (); /* Now add jumps and labels as needed to match the blocks new outgoing edges. */ for (bb = ENTRY_BLOCK_PTR->next_bb; bb ; bb = bb->aux) { edge e_fall, e_taken, e; rtx bb_end_insn; basic_block nb; edge_iterator ei; if (EDGE_COUNT (bb->succs) == 0) continue; /* Find the old fallthru edge, and another non-EH edge for a taken jump. */ e_taken = e_fall = NULL; FOR_EACH_EDGE (e, ei, bb->succs) if (e->flags & EDGE_FALLTHRU) e_fall = e; else if (! (e->flags & EDGE_EH)) e_taken = e; bb_end_insn = BB_END (bb); if (JUMP_P (bb_end_insn)) { if (any_condjump_p (bb_end_insn)) { /* If the old fallthru is still next, nothing to do. */ if (bb->aux == e_fall->dest || e_fall->dest == EXIT_BLOCK_PTR) continue; /* The degenerated case of conditional jump jumping to the next instruction can happen for jumps with side effects. We need to construct a forwarder block and this will be done just fine by force_nonfallthru below. */ if (!e_taken) ; /* There is another special case: if *neither* block is next, such as happens at the very end of a function, then we'll need to add a new unconditional jump. Choose the taken edge based on known or assumed probability. */ else if (bb->aux != e_taken->dest) { rtx note = find_reg_note (bb_end_insn, REG_BR_PROB, 0); if (note && INTVAL (XEXP (note, 0)) < REG_BR_PROB_BASE / 2 && invert_jump (bb_end_insn, (e_fall->dest == EXIT_BLOCK_PTR ? NULL_RTX : label_for_bb (e_fall->dest)), 0)) { e_fall->flags &= ~EDGE_FALLTHRU; #ifdef ENABLE_CHECKING gcc_assert (could_fall_through (e_taken->src, e_taken->dest)); #endif e_taken->flags |= EDGE_FALLTHRU; update_br_prob_note (bb); e = e_fall, e_fall = e_taken, e_taken = e; } } /* If the "jumping" edge is a crossing edge, and the fall through edge is non-crossing, leave things as they are. */ else if ((e_taken->flags & EDGE_CROSSING) && !(e_fall->flags & EDGE_CROSSING)) continue; /* Otherwise we can try to invert the jump. This will basically never fail, however, keep up the pretense. */ else if (invert_jump (bb_end_insn, (e_fall->dest == EXIT_BLOCK_PTR ? NULL_RTX : label_for_bb (e_fall->dest)), 0)) { e_fall->flags &= ~EDGE_FALLTHRU; #ifdef ENABLE_CHECKING gcc_assert (could_fall_through (e_taken->src, e_taken->dest)); #endif e_taken->flags |= EDGE_FALLTHRU; update_br_prob_note (bb); continue; } } else { /* Otherwise we have some return, switch or computed jump. In the 99% case, there should not have been a fallthru edge. */ gcc_assert (returnjump_p (bb_end_insn) || !e_fall); continue; } } else { /* No fallthru implies a noreturn function with EH edges, or something similarly bizarre. In any case, we don't need to do anything. */ if (! e_fall) continue; /* If the fallthru block is still next, nothing to do. */ if (bb->aux == e_fall->dest) continue; /* A fallthru to exit block. */ if (e_fall->dest == EXIT_BLOCK_PTR) continue; } /* We got here if we need to add a new jump insn. */ nb = force_nonfallthru (e_fall); if (nb) { nb->il.rtl->visited = 1; nb->aux = bb->aux; bb->aux = nb; /* Don't process this new block. */ bb = nb; /* Make sure new bb is tagged for correct section (same as fall-thru source, since you cannot fall-throu across section boundaries). */ BB_COPY_PARTITION (e_fall->src, single_pred (bb)); if (flag_reorder_blocks_and_partition && targetm.have_named_sections && JUMP_P (BB_END (bb)) && !any_condjump_p (BB_END (bb)) && (EDGE_SUCC (bb, 0)->flags & EDGE_CROSSING)) REG_NOTES (BB_END (bb)) = gen_rtx_EXPR_LIST (REG_CROSSING_JUMP, NULL_RTX, REG_NOTES (BB_END (bb))); } } /* Put basic_block_info in the new order. */ if (dump_file) { fprintf (dump_file, "Reordered sequence:\n"); for (bb = ENTRY_BLOCK_PTR->next_bb, index = NUM_FIXED_BLOCKS; bb; bb = bb->aux, index++) { fprintf (dump_file, " %i ", index); if (get_bb_original (bb)) fprintf (dump_file, "duplicate of %i ", get_bb_original (bb)->index); else if (forwarder_block_p (bb) && !LABEL_P (BB_HEAD (bb))) fprintf (dump_file, "compensation "); else fprintf (dump_file, "bb %i ", bb->index); fprintf (dump_file, " [%i]\n", bb->frequency); } } prev_bb = ENTRY_BLOCK_PTR; bb = ENTRY_BLOCK_PTR->next_bb; index = NUM_FIXED_BLOCKS; for (; bb; prev_bb = bb, bb = bb->aux, index ++) { bb->index = index; SET_BASIC_BLOCK (index, bb); bb->prev_bb = prev_bb; prev_bb->next_bb = bb; } prev_bb->next_bb = EXIT_BLOCK_PTR; EXIT_BLOCK_PTR->prev_bb = prev_bb; /* Annoying special case - jump around dead jumptables left in the code. */ FOR_EACH_BB (bb) { edge e; edge_iterator ei; FOR_EACH_EDGE (e, ei, bb->succs) if (e->flags & EDGE_FALLTHRU) break; if (e && !can_fallthru (e->src, e->dest)) force_nonfallthru (e); } }