void
ubsan_expand_si_overflow_mul_check (gimple stmt)
{
rtx res, op0, op1;
tree lhs, fn, arg0, arg1;
rtx_code_label *done_label, *do_error;
rtx target = NULL_RTX;
lhs = gimple_call_lhs (stmt);
arg0 = gimple_call_arg (stmt, 0);
arg1 = gimple_call_arg (stmt, 1);
done_label = gen_label_rtx ();
do_error = gen_label_rtx ();
do_pending_stack_adjust ();
op0 = expand_normal (arg0);
op1 = expand_normal (arg1);
machine_mode mode = TYPE_MODE (TREE_TYPE (arg0));
if (lhs)
target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
enum insn_code icode = optab_handler (mulv4_optab, mode);
if (icode != CODE_FOR_nothing)
{
struct expand_operand ops[4];
rtx_insn *last = get_last_insn ();
res = gen_reg_rtx (mode);
create_output_operand (&ops[0], res, mode);
create_input_operand (&ops[1], op0, mode);
create_input_operand (&ops[2], op1, mode);
create_fixed_operand (&ops[3], do_error);
if (maybe_expand_insn (icode, 4, ops))
{
last = get_last_insn ();
if (profile_status_for_fn (cfun) != PROFILE_ABSENT
&& JUMP_P (last)
&& any_condjump_p (last)
&& !find_reg_note (last, REG_BR_PROB, 0))
add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
emit_jump (done_label);
}
else
{
delete_insns_since (last);
icode = CODE_FOR_nothing;
}
}
if (icode == CODE_FOR_nothing)
{
struct separate_ops ops;
machine_mode hmode
= mode_for_size (GET_MODE_PRECISION (mode) / 2, MODE_INT, 1);
ops.op0 = arg0;
ops.op1 = arg1;
ops.op2 = NULL_TREE;
ops.location = gimple_location (stmt);
if (GET_MODE_2XWIDER_MODE (mode) != VOIDmode
&& targetm.scalar_mode_supported_p (GET_MODE_2XWIDER_MODE (mode)))
{
machine_mode wmode = GET_MODE_2XWIDER_MODE (mode);
ops.code = WIDEN_MULT_EXPR;
ops.type
= build_nonstandard_integer_type (GET_MODE_PRECISION (wmode), 0);
res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL);
rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res,
GET_MODE_PRECISION (mode), NULL_RTX, 0);
hipart = gen_lowpart (mode, hipart);
res = gen_lowpart (mode, res);
rtx signbit = expand_shift (RSHIFT_EXPR, mode, res,
GET_MODE_PRECISION (mode) - 1,
NULL_RTX, 0);
/* RES is low half of the double width result, HIPART
the high half. There was overflow if
HIPART is different from RES < 0 ? -1 : 0. */
emit_cmp_and_jump_insns (signbit, hipart, EQ, NULL_RTX, mode,
false, done_label, PROB_VERY_LIKELY);
}
else if (hmode != BLKmode
&& 2 * GET_MODE_PRECISION (hmode) == GET_MODE_PRECISION (mode))
{
rtx_code_label *large_op0 = gen_label_rtx ();
rtx_code_label *small_op0_large_op1 = gen_label_rtx ();
rtx_code_label *one_small_one_large = gen_label_rtx ();
rtx_code_label *both_ops_large = gen_label_rtx ();
rtx_code_label *after_hipart_neg = gen_label_rtx ();
rtx_code_label *after_lopart_neg = gen_label_rtx ();
rtx_code_label *do_overflow = gen_label_rtx ();
rtx_code_label *hipart_different = gen_label_rtx ();
unsigned int hprec = GET_MODE_PRECISION (hmode);
rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec,
NULL_RTX, 0);
hipart0 = gen_lowpart (hmode, hipart0);
rtx lopart0 = gen_lowpart (hmode, op0);
rtx signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1,
NULL_RTX, 0);
rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec,
NULL_RTX, 0);
hipart1 = gen_lowpart (hmode, hipart1);
rtx lopart1 = gen_lowpart (hmode, op1);
rtx signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1,
NULL_RTX, 0);
res = gen_reg_rtx (mode);
/* True if op0 resp. op1 are known to be in the range of
halfstype. */
bool op0_small_p = false;
bool op1_small_p = false;
/* True if op0 resp. op1 are known to have all zeros or all ones
in the upper half of bits, but are not known to be
op{0,1}_small_p. */
bool op0_medium_p = false;
bool op1_medium_p = false;
/* -1 if op{0,1} is known to be negative, 0 if it is known to be
nonnegative, 1 if unknown. */
int op0_sign = 1;
int op1_sign = 1;
if (TREE_CODE (arg0) == SSA_NAME)
{
wide_int arg0_min, arg0_max;
if (get_range_info (arg0, &arg0_min, &arg0_max) == VR_RANGE)
{
unsigned int mprec0 = wi::min_precision (arg0_min, SIGNED);
unsigned int mprec1 = wi::min_precision (arg0_max, SIGNED);
if (mprec0 <= hprec && mprec1 <= hprec)
op0_small_p = true;
else if (mprec0 <= hprec + 1 && mprec1 <= hprec + 1)
op0_medium_p = true;
if (!wi::neg_p (arg0_min, TYPE_SIGN (TREE_TYPE (arg0))))
op0_sign = 0;
else if (wi::neg_p (arg0_max, TYPE_SIGN (TREE_TYPE (arg0))))
op0_sign = -1;
}
}
if (TREE_CODE (arg1) == SSA_NAME)
{
wide_int arg1_min, arg1_max;
if (get_range_info (arg1, &arg1_min, &arg1_max) == VR_RANGE)
{
unsigned int mprec0 = wi::min_precision (arg1_min, SIGNED);
unsigned int mprec1 = wi::min_precision (arg1_max, SIGNED);
if (mprec0 <= hprec && mprec1 <= hprec)
op1_small_p = true;
else if (mprec0 <= hprec + 1 && mprec1 <= hprec + 1)
op1_medium_p = true;
if (!wi::neg_p (arg1_min, TYPE_SIGN (TREE_TYPE (arg1))))
op1_sign = 0;
else if (wi::neg_p (arg1_max, TYPE_SIGN (TREE_TYPE (arg1))))
op1_sign = -1;
}
}
int smaller_sign = 1;
int larger_sign = 1;
if (op0_small_p)
{
smaller_sign = op0_sign;
larger_sign = op1_sign;
}
else if (op1_small_p)
{
smaller_sign = op1_sign;
larger_sign = op0_sign;
}
else if (op0_sign == op1_sign)
{
smaller_sign = op0_sign;
larger_sign = op0_sign;
}
if (!op0_small_p)
emit_cmp_and_jump_insns (signbit0, hipart0, NE, NULL_RTX, hmode,
false, large_op0, PROB_UNLIKELY);
if (!op1_small_p)
emit_cmp_and_jump_insns (signbit1, hipart1, NE, NULL_RTX, hmode,
false, small_op0_large_op1,
PROB_UNLIKELY);
/* If both op0 and op1 are sign extended from hmode to mode,
the multiplication will never overflow. We can do just one
hmode x hmode => mode widening multiplication. */
if (GET_CODE (lopart0) == SUBREG)
{
SUBREG_PROMOTED_VAR_P (lopart0) = 1;
SUBREG_PROMOTED_SET (lopart0, 0);
}
if (GET_CODE (lopart1) == SUBREG)
{
SUBREG_PROMOTED_VAR_P (lopart1) = 1;
SUBREG_PROMOTED_SET (lopart1, 0);
}
tree halfstype = build_nonstandard_integer_type (hprec, 0);
ops.op0 = make_tree (halfstype, lopart0);
ops.op1 = make_tree (halfstype, lopart1);
ops.code = WIDEN_MULT_EXPR;
ops.type = TREE_TYPE (arg0);
rtx thisres
= expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
emit_move_insn (res, thisres);
emit_jump (done_label);
emit_label (small_op0_large_op1);
/* If op0 is sign extended from hmode to mode, but op1 is not,
just swap the arguments and handle it as op1 sign extended,
op0 not. */
rtx larger = gen_reg_rtx (mode);
rtx hipart = gen_reg_rtx (hmode);
rtx lopart = gen_reg_rtx (hmode);
emit_move_insn (larger, op1);
emit_move_insn (hipart, hipart1);
emit_move_insn (lopart, lopart0);
emit_jump (one_small_one_large);
emit_label (large_op0);
if (!op1_small_p)
emit_cmp_and_jump_insns (signbit1, hipart1, NE, NULL_RTX, hmode,
false, both_ops_large, PROB_UNLIKELY);
/* If op1 is sign extended from hmode to mode, but op0 is not,
prepare larger, hipart and lopart pseudos and handle it together
with small_op0_large_op1. */
emit_move_insn (larger, op0);
emit_move_insn (hipart, hipart0);
emit_move_insn (lopart, lopart1);
emit_label (one_small_one_large);
/* lopart is the low part of the operand that is sign extended
to mode, larger is the the other operand, hipart is the
high part of larger and lopart0 and lopart1 are the low parts
of both operands.
We perform lopart0 * lopart1 and lopart * hipart widening
multiplications. */
tree halfutype = build_nonstandard_integer_type (hprec, 1);
ops.op0 = make_tree (halfutype, lopart0);
ops.op1 = make_tree (halfutype, lopart1);
rtx lo0xlo1
= expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
ops.op0 = make_tree (halfutype, lopart);
ops.op1 = make_tree (halfutype, hipart);
rtx loxhi = gen_reg_rtx (mode);
rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
emit_move_insn (loxhi, tem);
/* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
if (larger_sign == 0)
emit_jump (after_hipart_neg);
else if (larger_sign != -1)
emit_cmp_and_jump_insns (hipart, const0_rtx, GE, NULL_RTX, hmode,
false, after_hipart_neg, PROB_EVEN);
tem = convert_modes (mode, hmode, lopart, 1);
tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1);
tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX,
1, OPTAB_DIRECT);
emit_move_insn (loxhi, tem);
emit_label (after_hipart_neg);
/* if (lopart < 0) loxhi -= larger; */
if (smaller_sign == 0)
emit_jump (after_lopart_neg);
else if (smaller_sign != -1)
emit_cmp_and_jump_insns (lopart, const0_rtx, GE, NULL_RTX, hmode,
false, after_lopart_neg, PROB_EVEN);
tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX,
1, OPTAB_DIRECT);
emit_move_insn (loxhi, tem);
emit_label (after_lopart_neg);
/* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1);
tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX,
1, OPTAB_DIRECT);
emit_move_insn (loxhi, tem);
/* if (loxhi >> (bitsize / 2)
== (hmode) loxhi >> (bitsize / 2 - 1)) */
rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec,
NULL_RTX, 0);
hipartloxhi = gen_lowpart (hmode, hipartloxhi);
rtx lopartloxhi = gen_lowpart (hmode, loxhi);
rtx signbitloxhi = expand_shift (RSHIFT_EXPR, hmode, lopartloxhi,
hprec - 1, NULL_RTX, 0);
emit_cmp_and_jump_insns (signbitloxhi, hipartloxhi, NE, NULL_RTX,
hmode, false, do_overflow,
PROB_VERY_UNLIKELY);
/* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec,
NULL_RTX, 1);
tem = convert_modes (mode, hmode, gen_lowpart (hmode, lo0xlo1), 1);
tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res,
1, OPTAB_DIRECT);
if (tem != res)
emit_move_insn (res, tem);
emit_jump (done_label);
emit_label (both_ops_large);
/* If both operands are large (not sign extended from hmode),
then perform the full multiplication which will be the result
of the operation. The only cases which don't overflow are
some cases where both hipart0 and highpart1 are 0 or -1. */
ops.code = MULT_EXPR;
ops.op0 = make_tree (TREE_TYPE (arg0), op0);
ops.op1 = make_tree (TREE_TYPE (arg0), op1);
tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
emit_move_insn (res, tem);
if (!op0_medium_p)
{
tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx,
NULL_RTX, 1, OPTAB_DIRECT);
emit_cmp_and_jump_insns (tem, const1_rtx, GTU, NULL_RTX, hmode,
true, do_error, PROB_VERY_UNLIKELY);
}
if (!op1_medium_p)
{
tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx,
NULL_RTX, 1, OPTAB_DIRECT);
emit_cmp_and_jump_insns (tem, const1_rtx, GTU, NULL_RTX, hmode,
true, do_error, PROB_VERY_UNLIKELY);
}
/* At this point hipart{0,1} are both in [-1, 0]. If they are the
same, overflow happened if res is negative, if they are different,
overflow happened if res is positive. */
if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign)
emit_jump (hipart_different);
else if (op0_sign == 1 || op1_sign == 1)
emit_cmp_and_jump_insns (hipart0, hipart1, NE, NULL_RTX, hmode,
true, hipart_different, PROB_EVEN);
emit_cmp_and_jump_insns (res, const0_rtx, LT, NULL_RTX, mode, false,
do_error, PROB_VERY_UNLIKELY);
emit_jump (done_label);
emit_label (hipart_different);
emit_cmp_and_jump_insns (res, const0_rtx, GE, NULL_RTX, mode, false,
do_error, PROB_VERY_UNLIKELY);
emit_jump (done_label);
emit_label (do_overflow);
/* Overflow, do full multiplication and fallthru into do_error. */
ops.op0 = make_tree (TREE_TYPE (arg0), op0);
ops.op1 = make_tree (TREE_TYPE (arg0), op1);
tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
emit_move_insn (res, tem);
}
else
{
ops.code = MULT_EXPR;
ops.type = TREE_TYPE (arg0);
res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
emit_jump (done_label);
}
}
emit_label (do_error);
/* Expand the ubsan builtin call. */
push_temp_slots ();
fn = ubsan_build_overflow_builtin (MULT_EXPR, gimple_location (stmt),
TREE_TYPE (arg0), arg0, arg1);
expand_normal (fn);
pop_temp_slots ();
do_pending_stack_adjust ();
/* We're done. */
emit_label (done_label);
if (lhs)
emit_move_insn (target, res);
}
void
eliminate_regs_in_insn (rtx_insn *insn, bool replace_p, bool first_p,
HOST_WIDE_INT update_sp_offset)
{
int icode = recog_memoized (insn);
rtx old_set = single_set (insn);
bool validate_p;
int i;
rtx substed_operand[MAX_RECOG_OPERANDS];
rtx orig_operand[MAX_RECOG_OPERANDS];
struct lra_elim_table *ep;
rtx plus_src, plus_cst_src;
lra_insn_recog_data_t id;
struct lra_static_insn_data *static_id;
if (icode < 0 && asm_noperands (PATTERN (insn)) < 0 && ! DEBUG_INSN_P (insn))
{
lra_assert (GET_CODE (PATTERN (insn)) == USE
|| GET_CODE (PATTERN (insn)) == CLOBBER
|| GET_CODE (PATTERN (insn)) == ASM_INPUT);
return;
}
/* Check for setting an eliminable register. */
if (old_set != 0 && REG_P (SET_DEST (old_set))
&& (ep = get_elimination (SET_DEST (old_set))) != NULL)
{
for (ep = reg_eliminate; ep < ®_eliminate[NUM_ELIMINABLE_REGS]; ep++)
if (ep->from_rtx == SET_DEST (old_set) && ep->can_eliminate)
{
bool delete_p = replace_p;
#ifdef HARD_FRAME_POINTER_REGNUM
if (ep->from == FRAME_POINTER_REGNUM
&& ep->to == HARD_FRAME_POINTER_REGNUM)
/* If this is setting the frame pointer register to the
hardware frame pointer register and this is an
elimination that will be done (tested above), this
insn is really adjusting the frame pointer downward
to compensate for the adjustment done before a
nonlocal goto. */
{
rtx src = SET_SRC (old_set);
rtx off = remove_reg_equal_offset_note (insn, ep->to_rtx);
/* We should never process such insn with non-zero
UPDATE_SP_OFFSET. */
lra_assert (update_sp_offset == 0);
if (off != NULL_RTX
|| src == ep->to_rtx
|| (GET_CODE (src) == PLUS
&& XEXP (src, 0) == ep->to_rtx
&& CONST_INT_P (XEXP (src, 1))))
{
HOST_WIDE_INT offset;
if (replace_p)
{
SET_DEST (old_set) = ep->to_rtx;
lra_update_insn_recog_data (insn);
return;
}
offset = (off != NULL_RTX ? INTVAL (off)
: src == ep->to_rtx ? 0 : INTVAL (XEXP (src, 1)));
offset -= (ep->offset - ep->previous_offset);
src = plus_constant (Pmode, ep->to_rtx, offset);
/* First see if this insn remains valid when we
make the change. If not, keep the INSN_CODE
the same and let the constraint pass fit it
up. */
validate_change (insn, &SET_SRC (old_set), src, 1);
validate_change (insn, &SET_DEST (old_set),
ep->from_rtx, 1);
if (! apply_change_group ())
{
SET_SRC (old_set) = src;
SET_DEST (old_set) = ep->from_rtx;
}
lra_update_insn_recog_data (insn);
/* Add offset note for future updates. */
add_reg_note (insn, REG_EQUAL, src);
return;
}
}
#endif
/* This insn isn't serving a useful purpose. We delete it
when REPLACE is set. */
if (delete_p)
lra_delete_dead_insn (insn);
return;
}
}
/* We allow one special case which happens to work on all machines we
currently support: a single set with the source or a REG_EQUAL
note being a PLUS of an eliminable register and a constant. */
plus_src = plus_cst_src = 0;
if (old_set && REG_P (SET_DEST (old_set)))
{
if (GET_CODE (SET_SRC (old_set)) == PLUS)
plus_src = SET_SRC (old_set);
/* First see if the source is of the form (plus (...) CST). */
if (plus_src
&& CONST_INT_P (XEXP (plus_src, 1)))
plus_cst_src = plus_src;
/* Check that the first operand of the PLUS is a hard reg or
the lowpart subreg of one. */
if (plus_cst_src)
{
rtx reg = XEXP (plus_cst_src, 0);
if (GET_CODE (reg) == SUBREG && subreg_lowpart_p (reg))
reg = SUBREG_REG (reg);
if (!REG_P (reg) || REGNO (reg) >= FIRST_PSEUDO_REGISTER)
plus_cst_src = 0;
}
}
if (plus_cst_src)
{
rtx reg = XEXP (plus_cst_src, 0);
HOST_WIDE_INT offset = INTVAL (XEXP (plus_cst_src, 1));
if (GET_CODE (reg) == SUBREG)
reg = SUBREG_REG (reg);
if (REG_P (reg) && (ep = get_elimination (reg)) != NULL)
{
rtx to_rtx = replace_p ? ep->to_rtx : ep->from_rtx;
if (! replace_p)
{
if (update_sp_offset == 0)
offset += (ep->offset - ep->previous_offset);
if (ep->to_rtx == stack_pointer_rtx)
{
if (first_p)
offset -= lra_get_insn_recog_data (insn)->sp_offset;
else
offset += update_sp_offset;
}
offset = trunc_int_for_mode (offset, GET_MODE (plus_cst_src));
}
if (GET_CODE (XEXP (plus_cst_src, 0)) == SUBREG)
to_rtx = gen_lowpart (GET_MODE (XEXP (plus_cst_src, 0)), to_rtx);
/* If we have a nonzero offset, and the source is already a
simple REG, the following transformation would increase
the cost of the insn by replacing a simple REG with (plus
(reg sp) CST). So try only when we already had a PLUS
before. */
if (offset == 0 || plus_src)
{
rtx new_src = plus_constant (GET_MODE (to_rtx), to_rtx, offset);
old_set = single_set (insn);
/* First see if this insn remains valid when we make the
change. If not, try to replace the whole pattern
with a simple set (this may help if the original insn
was a PARALLEL that was only recognized as single_set
due to REG_UNUSED notes). If this isn't valid
either, keep the INSN_CODE the same and let the
constraint pass fix it up. */
if (! validate_change (insn, &SET_SRC (old_set), new_src, 0))
{
rtx new_pat = gen_rtx_SET (SET_DEST (old_set), new_src);
if (! validate_change (insn, &PATTERN (insn), new_pat, 0))
SET_SRC (old_set) = new_src;
}
lra_update_insn_recog_data (insn);
/* This can't have an effect on elimination offsets, so skip
right to the end. */
return;
}
}
}
/* Eliminate all eliminable registers occurring in operands that
can be handled by the constraint pass. */
id = lra_get_insn_recog_data (insn);
static_id = id->insn_static_data;
validate_p = false;
for (i = 0; i < static_id->n_operands; i++)
{
orig_operand[i] = *id->operand_loc[i];
substed_operand[i] = *id->operand_loc[i];
/* For an asm statement, every operand is eliminable. */
if (icode < 0 || insn_data[icode].operand[i].eliminable)
{
/* Check for setting a hard register that we know about. */
if (static_id->operand[i].type != OP_IN
&& REG_P (orig_operand[i]))
{
/* If we are assigning to a hard register that can be
eliminated, it must be as part of a PARALLEL, since
the code above handles single SETs. This reg can not
be longer eliminated -- it is forced by
mark_not_eliminable. */
for (ep = reg_eliminate;
ep < ®_eliminate[NUM_ELIMINABLE_REGS];
ep++)
lra_assert (ep->from_rtx != orig_operand[i]
|| ! ep->can_eliminate);
}
/* Companion to the above plus substitution, we can allow
invariants as the source of a plain move. */
substed_operand[i]
= lra_eliminate_regs_1 (insn, *id->operand_loc[i], VOIDmode,
replace_p, ! replace_p && ! first_p,
update_sp_offset, first_p);
if (substed_operand[i] != orig_operand[i])
validate_p = true;
}
}
if (! validate_p)
return;
/* Substitute the operands; the new values are in the substed_operand
array. */
for (i = 0; i < static_id->n_operands; i++)
*id->operand_loc[i] = substed_operand[i];
for (i = 0; i < static_id->n_dups; i++)
*id->dup_loc[i] = substed_operand[(int) static_id->dup_num[i]];
/* If we had a move insn but now we don't, re-recognize it.
This will cause spurious re-recognition if the old move had a
PARALLEL since the new one still will, but we can't call
single_set without having put new body into the insn and the
re-recognition won't hurt in this rare case. */
id = lra_update_insn_recog_data (insn);
static_id = id->insn_static_data;
}
