void ConstantOffsetExtractor::ComputeKnownBits(Value *V, APInt &KnownOne,
APInt &KnownZero) const {
IntegerType *IT = cast<IntegerType>(V->getType());
KnownOne = APInt(IT->getBitWidth(), 0);
KnownZero = APInt(IT->getBitWidth(), 0);
llvm::computeKnownBits(V, KnownZero, KnownOne, DL, 0);
}
FastDivInsertionTask::FastDivInsertionTask(Instruction *I,
const BypassWidthsTy &BypassWidths) {
switch (I->getOpcode()) {
case Instruction::UDiv:
case Instruction::SDiv:
case Instruction::URem:
case Instruction::SRem:
SlowDivOrRem = I;
break;
default:
// I is not a div/rem operation.
return;
}
// Skip division on vector types. Only optimize integer instructions.
IntegerType *SlowType = dyn_cast<IntegerType>(SlowDivOrRem->getType());
if (!SlowType)
return;
// Skip if this bitwidth is not bypassed.
auto BI = BypassWidths.find(SlowType->getBitWidth());
if (BI == BypassWidths.end())
return;
// Get type for div/rem instruction with bypass bitwidth.
IntegerType *BT = IntegerType::get(I->getContext(), BI->second);
BypassType = BT;
// The original basic block.
MainBB = I->getParent();
// The instruction is indeed a slow div or rem operation.
IsValidTask = true;
}
ObjectSizeOffsetVisitor::ObjectSizeOffsetVisitor(const TargetData *TD,
LLVMContext &Context,
bool RoundToAlign)
: TD(TD), RoundToAlign(RoundToAlign) {
IntegerType *IntTy = TD->getIntPtrType(Context);
IntTyBits = IntTy->getBitWidth();
Zero = APInt::getNullValue(IntTyBits);
}
ObjectSizeOffsetVisitor::ObjectSizeOffsetVisitor(const DataLayout *DL,
const TargetLibraryInfo *TLI,
LLVMContext &Context,
bool RoundToAlign)
: DL(DL), TLI(TLI), RoundToAlign(RoundToAlign) {
IntegerType *IntTy = DL->getIntPtrType(Context);
IntTyBits = IntTy->getBitWidth();
Zero = APInt::getNullValue(IntTyBits);
}
void RuntimeDebugBuilder::createIntPrinter(Value *V) {
IntegerType *Ty = dyn_cast<IntegerType>(V->getType());
assert(Ty && Ty->getBitWidth() == 64 &&
"Cannot insert printer for this type.");
Function *F = getPrintF();
Value *String = Builder.CreateGlobalStringPtr("%ld");
Builder.CreateCall2(F, String, V);
createFlush();
}
Value *IslExprBuilder::createInt(__isl_take isl_ast_expr *Expr) {
assert(isl_ast_expr_get_type(Expr) == isl_ast_expr_int &&
"Expression not of type isl_ast_expr_int");
isl_int Int;
Value *V;
APInt APValue;
IntegerType *T;
isl_int_init(Int);
isl_ast_expr_get_int(Expr, &Int);
APValue = APInt_from_MPZ(Int);
T = getType(Expr);
APValue = APValue.sextOrSelf(T->getBitWidth());
V = ConstantInt::get(T, APValue);
isl_ast_expr_free(Expr);
isl_int_clear(Int);
return V;
}
// bypassSlowDivision - This optimization identifies DIV instructions that can
// be profitably bypassed and carried out with a shorter, faster divide.
bool llvm::bypassSlowDivision(Function &F,
Function::iterator &I,
const DenseMap<unsigned int, unsigned int> &BypassWidths) {
DivCacheTy DivCache;
bool MadeChange = false;
for (BasicBlock::iterator J = I->begin(); J != I->end(); J++) {
// Get instruction details
unsigned Opcode = J->getOpcode();
bool UseDivOp = Opcode == Instruction::SDiv || Opcode == Instruction::UDiv;
bool UseRemOp = Opcode == Instruction::SRem || Opcode == Instruction::URem;
bool UseSignedOp = Opcode == Instruction::SDiv ||
Opcode == Instruction::SRem;
// Only optimize div or rem ops
if (!UseDivOp && !UseRemOp)
continue;
// Skip division on vector types, only optimize integer instructions
if (!J->getType()->isIntegerTy())
continue;
// Get bitwidth of div/rem instruction
IntegerType *T = cast<IntegerType>(J->getType());
unsigned int bitwidth = T->getBitWidth();
// Continue if bitwidth is not bypassed
DenseMap<unsigned int, unsigned int>::const_iterator BI = BypassWidths.find(bitwidth);
if (BI == BypassWidths.end())
continue;
// Get type for div/rem instruction with bypass bitwidth
IntegerType *BT = IntegerType::get(J->getContext(), BI->second);
MadeChange |= reuseOrInsertFastDiv(F, I, J, BT, UseDivOp,
UseSignedOp, DivCache);
}
return MadeChange;
}
unsigned TypeUtils::getPrimitiveTypeWidth(const Type* type) {
if (type->isVoidTy()) {
return 0;
} else if (type->isFloatTy()) {
return 32 / 8;
} else if (type->isDoubleTy()) {
return 64 / 8;
} else if (type->isX86_FP80Ty()) {
return 80 / 8;
} else if (type->isFP128Ty()) {
return 128 / 8;
} else if (type->isPPC_FP128Ty()) {
return 128 / 8;
} else if (type->isX86_MMXTy()) {
return 64 / 8;
} else if (type->isIntegerTy()) {
IntegerType* integerTy = (IntegerType*) type;
return integerTy->getBitWidth() / 8;
} else {
assert(0 && "must be a primitive type");
return -1;
}
}
Instruction *InstCombiner::visitAdd(BinaryOperator &I) {
bool Changed = SimplifyAssociativeOrCommutative(I);
Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
if (Value *V = SimplifyAddInst(LHS, RHS, I.hasNoSignedWrap(),
I.hasNoUnsignedWrap(), TD))
return ReplaceInstUsesWith(I, V);
// (A*B)+(A*C) -> A*(B+C) etc
if (Value *V = SimplifyUsingDistributiveLaws(I))
return ReplaceInstUsesWith(I, V);
if (ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) {
// X + (signbit) --> X ^ signbit
const APInt &Val = CI->getValue();
if (Val.isSignBit())
return BinaryOperator::CreateXor(LHS, RHS);
// See if SimplifyDemandedBits can simplify this. This handles stuff like
// (X & 254)+1 -> (X&254)|1
if (SimplifyDemandedInstructionBits(I))
return &I;
// zext(bool) + C -> bool ? C + 1 : C
if (ZExtInst *ZI = dyn_cast<ZExtInst>(LHS))
if (ZI->getSrcTy()->isIntegerTy(1))
return SelectInst::Create(ZI->getOperand(0), AddOne(CI), CI);
Value *XorLHS = 0; ConstantInt *XorRHS = 0;
if (match(LHS, m_Xor(m_Value(XorLHS), m_ConstantInt(XorRHS)))) {
uint32_t TySizeBits = I.getType()->getScalarSizeInBits();
const APInt &RHSVal = CI->getValue();
unsigned ExtendAmt = 0;
// If we have ADD(XOR(AND(X, 0xFF), 0x80), 0xF..F80), it's a sext.
// If we have ADD(XOR(AND(X, 0xFF), 0xF..F80), 0x80), it's a sext.
if (XorRHS->getValue() == -RHSVal) {
if (RHSVal.isPowerOf2())
ExtendAmt = TySizeBits - RHSVal.logBase2() - 1;
else if (XorRHS->getValue().isPowerOf2())
ExtendAmt = TySizeBits - XorRHS->getValue().logBase2() - 1;
}
if (ExtendAmt) {
APInt Mask = APInt::getHighBitsSet(TySizeBits, ExtendAmt);
if (!MaskedValueIsZero(XorLHS, Mask))
ExtendAmt = 0;
}
if (ExtendAmt) {
Constant *ShAmt = ConstantInt::get(I.getType(), ExtendAmt);
Value *NewShl = Builder->CreateShl(XorLHS, ShAmt, "sext");
return BinaryOperator::CreateAShr(NewShl, ShAmt);
}
// If this is a xor that was canonicalized from a sub, turn it back into
// a sub and fuse this add with it.
if (LHS->hasOneUse() && (XorRHS->getValue()+1).isPowerOf2()) {
IntegerType *IT = cast<IntegerType>(I.getType());
APInt Mask = APInt::getAllOnesValue(IT->getBitWidth());
APInt LHSKnownOne(IT->getBitWidth(), 0);
APInt LHSKnownZero(IT->getBitWidth(), 0);
ComputeMaskedBits(XorLHS, Mask, LHSKnownZero, LHSKnownOne);
if ((XorRHS->getValue() | LHSKnownZero).isAllOnesValue())
return BinaryOperator::CreateSub(ConstantExpr::getAdd(XorRHS, CI),
XorLHS);
}
}
}
if (isa<Constant>(RHS) && isa<PHINode>(LHS))
if (Instruction *NV = FoldOpIntoPhi(I))
return NV;
if (I.getType()->isIntegerTy(1))
return BinaryOperator::CreateXor(LHS, RHS);
// X + X --> X << 1
if (LHS == RHS) {
BinaryOperator *New =
BinaryOperator::CreateShl(LHS, ConstantInt::get(I.getType(), 1));
New->setHasNoSignedWrap(I.hasNoSignedWrap());
New->setHasNoUnsignedWrap(I.hasNoUnsignedWrap());
return New;
}
// -A + B --> B - A
// -A + -B --> -(A + B)
if (Value *LHSV = dyn_castNegVal(LHS)) {
if (Value *RHSV = dyn_castNegVal(RHS)) {
Value *NewAdd = Builder->CreateAdd(LHSV, RHSV, "sum");
return BinaryOperator::CreateNeg(NewAdd);
}
return BinaryOperator::CreateSub(RHS, LHSV);
}
// A + -B --> A - B
if (!isa<Constant>(RHS))
if (Value *V = dyn_castNegVal(RHS))
return BinaryOperator::CreateSub(LHS, V);
ConstantInt *C2;
if (Value *X = dyn_castFoldableMul(LHS, C2)) {
if (X == RHS) // X*C + X --> X * (C+1)
return BinaryOperator::CreateMul(RHS, AddOne(C2));
// X*C1 + X*C2 --> X * (C1+C2)
ConstantInt *C1;
if (X == dyn_castFoldableMul(RHS, C1))
return BinaryOperator::CreateMul(X, ConstantExpr::getAdd(C1, C2));
}
// X + X*C --> X * (C+1)
if (dyn_castFoldableMul(RHS, C2) == LHS)
return BinaryOperator::CreateMul(LHS, AddOne(C2));
// A+B --> A|B iff A and B have no bits set in common.
if (IntegerType *IT = dyn_cast<IntegerType>(I.getType())) {
APInt Mask = APInt::getAllOnesValue(IT->getBitWidth());
APInt LHSKnownOne(IT->getBitWidth(), 0);
APInt LHSKnownZero(IT->getBitWidth(), 0);
ComputeMaskedBits(LHS, Mask, LHSKnownZero, LHSKnownOne);
if (LHSKnownZero != 0) {
APInt RHSKnownOne(IT->getBitWidth(), 0);
APInt RHSKnownZero(IT->getBitWidth(), 0);
ComputeMaskedBits(RHS, Mask, RHSKnownZero, RHSKnownOne);
// No bits in common -> bitwise or.
if ((LHSKnownZero|RHSKnownZero).isAllOnesValue())
return BinaryOperator::CreateOr(LHS, RHS);
}
}
// W*X + Y*Z --> W * (X+Z) iff W == Y
{
Value *W, *X, *Y, *Z;
if (match(LHS, m_Mul(m_Value(W), m_Value(X))) &&
match(RHS, m_Mul(m_Value(Y), m_Value(Z)))) {
if (W != Y) {
if (W == Z) {
std::swap(Y, Z);
} else if (Y == X) {
std::swap(W, X);
} else if (X == Z) {
std::swap(Y, Z);
std::swap(W, X);
}
}
if (W == Y) {
Value *NewAdd = Builder->CreateAdd(X, Z, LHS->getName());
return BinaryOperator::CreateMul(W, NewAdd);
}
}
}
if (ConstantInt *CRHS = dyn_cast<ConstantInt>(RHS)) {
Value *X = 0;
if (match(LHS, m_Not(m_Value(X)))) // ~X + C --> (C-1) - X
return BinaryOperator::CreateSub(SubOne(CRHS), X);
// (X & FF00) + xx00 -> (X+xx00) & FF00
if (LHS->hasOneUse() &&
match(LHS, m_And(m_Value(X), m_ConstantInt(C2))) &&
CRHS->getValue() == (CRHS->getValue() & C2->getValue())) {
// See if all bits from the first bit set in the Add RHS up are included
// in the mask. First, get the rightmost bit.
const APInt &AddRHSV = CRHS->getValue();
// Form a mask of all bits from the lowest bit added through the top.
APInt AddRHSHighBits(~((AddRHSV & -AddRHSV)-1));
// See if the and mask includes all of these bits.
APInt AddRHSHighBitsAnd(AddRHSHighBits & C2->getValue());
if (AddRHSHighBits == AddRHSHighBitsAnd) {
// Okay, the xform is safe. Insert the new add pronto.
Value *NewAdd = Builder->CreateAdd(X, CRHS, LHS->getName());
return BinaryOperator::CreateAnd(NewAdd, C2);
}
}
// Try to fold constant add into select arguments.
if (SelectInst *SI = dyn_cast<SelectInst>(LHS))
if (Instruction *R = FoldOpIntoSelect(I, SI))
return R;
}
// add (select X 0 (sub n A)) A --> select X A n
{
SelectInst *SI = dyn_cast<SelectInst>(LHS);
Value *A = RHS;
if (!SI) {
SI = dyn_cast<SelectInst>(RHS);
A = LHS;
}
if (SI && SI->hasOneUse()) {
Value *TV = SI->getTrueValue();
Value *FV = SI->getFalseValue();
Value *N;
// Can we fold the add into the argument of the select?
// We check both true and false select arguments for a matching subtract.
if (match(FV, m_Zero()) && match(TV, m_Sub(m_Value(N), m_Specific(A))))
// Fold the add into the true select value.
return SelectInst::Create(SI->getCondition(), N, A);
if (match(TV, m_Zero()) && match(FV, m_Sub(m_Value(N), m_Specific(A))))
// Fold the add into the false select value.
return SelectInst::Create(SI->getCondition(), A, N);
}
}
// Check for (add (sext x), y), see if we can merge this into an
// integer add followed by a sext.
if (SExtInst *LHSConv = dyn_cast<SExtInst>(LHS)) {
// (add (sext x), cst) --> (sext (add x, cst'))
if (ConstantInt *RHSC = dyn_cast<ConstantInt>(RHS)) {
Constant *CI =
ConstantExpr::getTrunc(RHSC, LHSConv->getOperand(0)->getType());
if (LHSConv->hasOneUse() &&
ConstantExpr::getSExt(CI, I.getType()) == RHSC &&
WillNotOverflowSignedAdd(LHSConv->getOperand(0), CI)) {
// Insert the new, smaller add.
Value *NewAdd = Builder->CreateNSWAdd(LHSConv->getOperand(0),
CI, "addconv");
return new SExtInst(NewAdd, I.getType());
}
}
// (add (sext x), (sext y)) --> (sext (add int x, y))
if (SExtInst *RHSConv = dyn_cast<SExtInst>(RHS)) {
// Only do this if x/y have the same type, if at last one of them has a
// single use (so we don't increase the number of sexts), and if the
// integer add will not overflow.
if (LHSConv->getOperand(0)->getType()==RHSConv->getOperand(0)->getType()&&
(LHSConv->hasOneUse() || RHSConv->hasOneUse()) &&
WillNotOverflowSignedAdd(LHSConv->getOperand(0),
RHSConv->getOperand(0))) {
// Insert the new integer add.
Value *NewAdd = Builder->CreateNSWAdd(LHSConv->getOperand(0),
RHSConv->getOperand(0), "addconv");
return new SExtInst(NewAdd, I.getType());
}
}
}
return Changed ? &I : 0;
}
/// visitSelectInstWithICmp - Visit a SelectInst that has an
/// ICmpInst as its first operand.
///
Instruction *InstCombiner::visitSelectInstWithICmp(SelectInst &SI,
ICmpInst *ICI) {
bool Changed = false;
ICmpInst::Predicate Pred = ICI->getPredicate();
Value *CmpLHS = ICI->getOperand(0);
Value *CmpRHS = ICI->getOperand(1);
Value *TrueVal = SI.getTrueValue();
Value *FalseVal = SI.getFalseValue();
// Check cases where the comparison is with a constant that
// can be adjusted to fit the min/max idiom. We may move or edit ICI
// here, so make sure the select is the only user.
if (ICI->hasOneUse())
if (ConstantInt *CI = dyn_cast<ConstantInt>(CmpRHS)) {
// X < MIN ? T : F --> F
if ((Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_ULT)
&& CI->isMinValue(Pred == ICmpInst::ICMP_SLT))
return ReplaceInstUsesWith(SI, FalseVal);
// X > MAX ? T : F --> F
else if ((Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_UGT)
&& CI->isMaxValue(Pred == ICmpInst::ICMP_SGT))
return ReplaceInstUsesWith(SI, FalseVal);
switch (Pred) {
default: break;
case ICmpInst::ICMP_ULT:
case ICmpInst::ICMP_SLT:
case ICmpInst::ICMP_UGT:
case ICmpInst::ICMP_SGT: {
// These transformations only work for selects over integers.
IntegerType *SelectTy = dyn_cast<IntegerType>(SI.getType());
if (!SelectTy)
break;
Constant *AdjustedRHS;
if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_SGT)
AdjustedRHS = ConstantInt::get(CI->getContext(), CI->getValue() + 1);
else // (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_SLT)
AdjustedRHS = ConstantInt::get(CI->getContext(), CI->getValue() - 1);
// X > C ? X : C+1 --> X < C+1 ? C+1 : X
// X < C ? X : C-1 --> X > C-1 ? C-1 : X
if ((CmpLHS == TrueVal && AdjustedRHS == FalseVal) ||
(CmpLHS == FalseVal && AdjustedRHS == TrueVal))
; // Nothing to do here. Values match without any sign/zero extension.
// Types do not match. Instead of calculating this with mixed types
// promote all to the larger type. This enables scalar evolution to
// analyze this expression.
else if (CmpRHS->getType()->getScalarSizeInBits()
< SelectTy->getBitWidth()) {
Constant *sextRHS = ConstantExpr::getSExt(AdjustedRHS, SelectTy);
// X = sext x; x >s c ? X : C+1 --> X = sext x; X <s C+1 ? C+1 : X
// X = sext x; x <s c ? X : C-1 --> X = sext x; X >s C-1 ? C-1 : X
// X = sext x; x >u c ? X : C+1 --> X = sext x; X <u C+1 ? C+1 : X
// X = sext x; x <u c ? X : C-1 --> X = sext x; X >u C-1 ? C-1 : X
if (match(TrueVal, m_SExt(m_Specific(CmpLHS))) &&
sextRHS == FalseVal) {
CmpLHS = TrueVal;
AdjustedRHS = sextRHS;
} else if (match(FalseVal, m_SExt(m_Specific(CmpLHS))) &&
sextRHS == TrueVal) {
CmpLHS = FalseVal;
AdjustedRHS = sextRHS;
} else if (ICI->isUnsigned()) {
Constant *zextRHS = ConstantExpr::getZExt(AdjustedRHS, SelectTy);
// X = zext x; x >u c ? X : C+1 --> X = zext x; X <u C+1 ? C+1 : X
// X = zext x; x <u c ? X : C-1 --> X = zext x; X >u C-1 ? C-1 : X
// zext + signed compare cannot be changed:
// 0xff <s 0x00, but 0x00ff >s 0x0000
if (match(TrueVal, m_ZExt(m_Specific(CmpLHS))) &&
zextRHS == FalseVal) {
CmpLHS = TrueVal;
AdjustedRHS = zextRHS;
} else if (match(FalseVal, m_ZExt(m_Specific(CmpLHS))) &&
zextRHS == TrueVal) {
CmpLHS = FalseVal;
AdjustedRHS = zextRHS;
} else
break;
} else
break;
} else
break;
Pred = ICmpInst::getSwappedPredicate(Pred);
CmpRHS = AdjustedRHS;
std::swap(FalseVal, TrueVal);
ICI->setPredicate(Pred);
ICI->setOperand(0, CmpLHS);
ICI->setOperand(1, CmpRHS);
SI.setOperand(1, TrueVal);
SI.setOperand(2, FalseVal);
// Move ICI instruction right before the select instruction. Otherwise
// the sext/zext value may be defined after the ICI instruction uses it.
ICI->moveBefore(&SI);
Changed = true;
break;
}
}
}
// Transform (X >s -1) ? C1 : C2 --> ((X >>s 31) & (C2 - C1)) + C1
// and (X <s 0) ? C2 : C1 --> ((X >>s 31) & (C2 - C1)) + C1
// FIXME: Type and constness constraints could be lifted, but we have to
// watch code size carefully. We should consider xor instead of
// sub/add when we decide to do that.
if (IntegerType *Ty = dyn_cast<IntegerType>(CmpLHS->getType())) {
if (TrueVal->getType() == Ty) {
if (ConstantInt *Cmp = dyn_cast<ConstantInt>(CmpRHS)) {
ConstantInt *C1 = nullptr, *C2 = nullptr;
if (Pred == ICmpInst::ICMP_SGT && Cmp->isAllOnesValue()) {
C1 = dyn_cast<ConstantInt>(TrueVal);
C2 = dyn_cast<ConstantInt>(FalseVal);
} else if (Pred == ICmpInst::ICMP_SLT && Cmp->isNullValue()) {
C1 = dyn_cast<ConstantInt>(FalseVal);
C2 = dyn_cast<ConstantInt>(TrueVal);
}
if (C1 && C2) {
// This shift results in either -1 or 0.
Value *AShr = Builder->CreateAShr(CmpLHS, Ty->getBitWidth()-1);
// Check if we can express the operation with a single or.
if (C2->isAllOnesValue())
return ReplaceInstUsesWith(SI, Builder->CreateOr(AShr, C1));
Value *And = Builder->CreateAnd(AShr, C2->getValue()-C1->getValue());
return ReplaceInstUsesWith(SI, Builder->CreateAdd(And, C1));
}
}
}
}
// If we have an equality comparison then we know the value in one of the
// arms of the select. See if substituting this value into the arm and
// simplifying the result yields the same value as the other arm.
if (Pred == ICmpInst::ICMP_EQ) {
if (SimplifyWithOpReplaced(FalseVal, CmpLHS, CmpRHS, DL, TLI, DT, AC) ==
TrueVal ||
SimplifyWithOpReplaced(FalseVal, CmpRHS, CmpLHS, DL, TLI, DT, AC) ==
TrueVal)
return ReplaceInstUsesWith(SI, FalseVal);
if (SimplifyWithOpReplaced(TrueVal, CmpLHS, CmpRHS, DL, TLI, DT, AC) ==
FalseVal ||
SimplifyWithOpReplaced(TrueVal, CmpRHS, CmpLHS, DL, TLI, DT, AC) ==
FalseVal)
return ReplaceInstUsesWith(SI, FalseVal);
} else if (Pred == ICmpInst::ICMP_NE) {
if (SimplifyWithOpReplaced(TrueVal, CmpLHS, CmpRHS, DL, TLI, DT, AC) ==
FalseVal ||
SimplifyWithOpReplaced(TrueVal, CmpRHS, CmpLHS, DL, TLI, DT, AC) ==
FalseVal)
return ReplaceInstUsesWith(SI, TrueVal);
if (SimplifyWithOpReplaced(FalseVal, CmpLHS, CmpRHS, DL, TLI, DT, AC) ==
TrueVal ||
SimplifyWithOpReplaced(FalseVal, CmpRHS, CmpLHS, DL, TLI, DT, AC) ==
TrueVal)
return ReplaceInstUsesWith(SI, TrueVal);
}
// NOTE: if we wanted to, this is where to detect integer MIN/MAX
if (CmpRHS != CmpLHS && isa<Constant>(CmpRHS)) {
if (CmpLHS == TrueVal && Pred == ICmpInst::ICMP_EQ) {
// Transform (X == C) ? X : Y -> (X == C) ? C : Y
SI.setOperand(1, CmpRHS);
Changed = true;
} else if (CmpLHS == FalseVal && Pred == ICmpInst::ICMP_NE) {
// Transform (X != C) ? Y : X -> (X != C) ? Y : C
SI.setOperand(2, CmpRHS);
Changed = true;
}
}
if (unsigned BitWidth = TrueVal->getType()->getScalarSizeInBits()) {
APInt MinSignedValue = APInt::getSignBit(BitWidth);
Value *X;
const APInt *Y, *C;
bool TrueWhenUnset;
bool IsBitTest = false;
if (ICmpInst::isEquality(Pred) &&
match(CmpLHS, m_And(m_Value(X), m_Power2(Y))) &&
match(CmpRHS, m_Zero())) {
IsBitTest = true;
TrueWhenUnset = Pred == ICmpInst::ICMP_EQ;
} else if (Pred == ICmpInst::ICMP_SLT && match(CmpRHS, m_Zero())) {
X = CmpLHS;
Y = &MinSignedValue;
IsBitTest = true;
TrueWhenUnset = false;
} else if (Pred == ICmpInst::ICMP_SGT && match(CmpRHS, m_AllOnes())) {
X = CmpLHS;
Y = &MinSignedValue;
IsBitTest = true;
TrueWhenUnset = true;
}
if (IsBitTest) {
Value *V = nullptr;
// (X & Y) == 0 ? X : X ^ Y --> X & ~Y
if (TrueWhenUnset && TrueVal == X &&
match(FalseVal, m_Xor(m_Specific(X), m_APInt(C))) && *Y == *C)
V = Builder->CreateAnd(X, ~(*Y));
// (X & Y) != 0 ? X ^ Y : X --> X & ~Y
else if (!TrueWhenUnset && FalseVal == X &&
match(TrueVal, m_Xor(m_Specific(X), m_APInt(C))) && *Y == *C)
V = Builder->CreateAnd(X, ~(*Y));
// (X & Y) == 0 ? X ^ Y : X --> X | Y
else if (TrueWhenUnset && FalseVal == X &&
match(TrueVal, m_Xor(m_Specific(X), m_APInt(C))) && *Y == *C)
V = Builder->CreateOr(X, *Y);
// (X & Y) != 0 ? X : X ^ Y --> X | Y
else if (!TrueWhenUnset && TrueVal == X &&
match(FalseVal, m_Xor(m_Specific(X), m_APInt(C))) && *Y == *C)
V = Builder->CreateOr(X, *Y);
if (V)
return ReplaceInstUsesWith(SI, V);
}
}
if (Value *V = foldSelectICmpAndOr(SI, TrueVal, FalseVal, Builder))
return ReplaceInstUsesWith(SI, V);
return Changed ? &SI : nullptr;
}
bool LazyValueInfoCache::solveBlockValueConstantRange(LVILatticeVal &BBLV,
Instruction *BBI,
BasicBlock *BB) {
// Figure out the range of the LHS. If that fails, bail.
if (!hasBlockValue(BBI->getOperand(0), BB)) {
BlockValueStack.push(std::make_pair(BB, BBI->getOperand(0)));
return false;
}
LVILatticeVal LHSVal = getBlockValue(BBI->getOperand(0), BB);
if (!LHSVal.isConstantRange()) {
BBLV.markOverdefined();
return true;
}
ConstantRange LHSRange = LHSVal.getConstantRange();
ConstantRange RHSRange(1);
IntegerType *ResultTy = cast<IntegerType>(BBI->getType());
if (isa<BinaryOperator>(BBI)) {
if (ConstantInt *RHS = dyn_cast<ConstantInt>(BBI->getOperand(1))) {
RHSRange = ConstantRange(RHS->getValue());
} else {
BBLV.markOverdefined();
return true;
}
}
// NOTE: We're currently limited by the set of operations that ConstantRange
// can evaluate symbolically. Enhancing that set will allows us to analyze
// more definitions.
LVILatticeVal Result;
switch (BBI->getOpcode()) {
case Instruction::Add:
Result.markConstantRange(LHSRange.add(RHSRange));
break;
case Instruction::Sub:
Result.markConstantRange(LHSRange.sub(RHSRange));
break;
case Instruction::Mul:
Result.markConstantRange(LHSRange.multiply(RHSRange));
break;
case Instruction::UDiv:
Result.markConstantRange(LHSRange.udiv(RHSRange));
break;
case Instruction::Shl:
Result.markConstantRange(LHSRange.shl(RHSRange));
break;
case Instruction::LShr:
Result.markConstantRange(LHSRange.lshr(RHSRange));
break;
case Instruction::Trunc:
Result.markConstantRange(LHSRange.truncate(ResultTy->getBitWidth()));
break;
case Instruction::SExt:
Result.markConstantRange(LHSRange.signExtend(ResultTy->getBitWidth()));
break;
case Instruction::ZExt:
Result.markConstantRange(LHSRange.zeroExtend(ResultTy->getBitWidth()));
break;
case Instruction::BitCast:
Result.markConstantRange(LHSRange);
break;
case Instruction::And:
Result.markConstantRange(LHSRange.binaryAnd(RHSRange));
break;
case Instruction::Or:
Result.markConstantRange(LHSRange.binaryOr(RHSRange));
break;
// Unhandled instructions are overdefined.
default:
DEBUG(dbgs() << " compute BB '" << BB->getName()
<< "' - overdefined because inst def found.\n");
Result.markOverdefined();
break;
}
BBLV = Result;
return true;
}
/// Generates code to divide two unsigned scalar 32-bit or 64-bit integers.
/// Returns the quotient, rounded towards 0. Builder's insert point should
/// point where the caller wants code generated, e.g. at the udiv instruction.
static Value *generateUnsignedDivisionCode(Value *Dividend, Value *Divisor,
IRBuilder<> &Builder) {
// The basic algorithm can be found in the compiler-rt project's
// implementation of __udivsi3.c. Here, we do a lower-level IR based approach
// that's been hand-tuned to lessen the amount of control flow involved.
// Some helper values
IntegerType *DivTy = cast<IntegerType>(Dividend->getType());
unsigned BitWidth = DivTy->getBitWidth();
ConstantInt *Zero;
ConstantInt *One;
ConstantInt *NegOne;
ConstantInt *MSB;
if (BitWidth == 64) {
Zero = Builder.getInt64(0);
One = Builder.getInt64(1);
NegOne = ConstantInt::getSigned(DivTy, -1);
MSB = Builder.getInt64(63);
} else {
assert(BitWidth == 32 && "Unexpected bit width");
Zero = Builder.getInt32(0);
One = Builder.getInt32(1);
NegOne = ConstantInt::getSigned(DivTy, -1);
MSB = Builder.getInt32(31);
}
ConstantInt *True = Builder.getTrue();
BasicBlock *IBB = Builder.GetInsertBlock();
Function *F = IBB->getParent();
Function *CTLZ = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
DivTy);
// Our CFG is going to look like:
// +---------------------+
// | special-cases |
// | ... |
// +---------------------+
// | |
// | +----------+
// | | bb1 |
// | | ... |
// | +----------+
// | | |
// | | +------------+
// | | | preheader |
// | | | ... |
// | | +------------+
// | | |
// | | | +---+
// | | | | |
// | | +------------+ |
// | | | do-while | |
// | | | ... | |
// | | +------------+ |
// | | | | |
// | +-----------+ +---+
// | | loop-exit |
// | | ... |
// | +-----------+
// | |
// +-------+
// | ... |
// | end |
// +-------+
BasicBlock *SpecialCases = Builder.GetInsertBlock();
SpecialCases->setName(Twine(SpecialCases->getName(), "_udiv-special-cases"));
BasicBlock *End = SpecialCases->splitBasicBlock(Builder.GetInsertPoint(),
"udiv-end");
BasicBlock *LoopExit = BasicBlock::Create(Builder.getContext(),
"udiv-loop-exit", F, End);
BasicBlock *DoWhile = BasicBlock::Create(Builder.getContext(),
"udiv-do-while", F, End);
BasicBlock *Preheader = BasicBlock::Create(Builder.getContext(),
"udiv-preheader", F, End);
BasicBlock *BB1 = BasicBlock::Create(Builder.getContext(),
"udiv-bb1", F, End);
// We'll be overwriting the terminator to insert our extra blocks
SpecialCases->getTerminator()->eraseFromParent();
// Same instructions are generated for both i32 (msb 31) and i64 (msb 63).
// First off, check for special cases: dividend or divisor is zero, divisor
// is greater than dividend, and divisor is 1.
// ; special-cases:
// ; %ret0_1 = icmp eq i32 %divisor, 0
// ; %ret0_2 = icmp eq i32 %dividend, 0
// ; %ret0_3 = or i1 %ret0_1, %ret0_2
// ; %tmp0 = tail call i32 @llvm.ctlz.i32(i32 %divisor, i1 true)
// ; %tmp1 = tail call i32 @llvm.ctlz.i32(i32 %dividend, i1 true)
// ; %sr = sub nsw i32 %tmp0, %tmp1
// ; %ret0_4 = icmp ugt i32 %sr, 31
// ; %ret0 = or i1 %ret0_3, %ret0_4
// ; %retDividend = icmp eq i32 %sr, 31
// ; %retVal = select i1 %ret0, i32 0, i32 %dividend
// ; %earlyRet = or i1 %ret0, %retDividend
// ; br i1 %earlyRet, label %end, label %bb1
Builder.SetInsertPoint(SpecialCases);
Value *Ret0_1 = Builder.CreateICmpEQ(Divisor, Zero);
Value *Ret0_2 = Builder.CreateICmpEQ(Dividend, Zero);
Value *Ret0_3 = Builder.CreateOr(Ret0_1, Ret0_2);
Value *Tmp0 = Builder.CreateCall(CTLZ, {Divisor, True});
Value *Tmp1 = Builder.CreateCall(CTLZ, {Dividend, True});
Value *SR = Builder.CreateSub(Tmp0, Tmp1);
Value *Ret0_4 = Builder.CreateICmpUGT(SR, MSB);
Value *Ret0 = Builder.CreateOr(Ret0_3, Ret0_4);
Value *RetDividend = Builder.CreateICmpEQ(SR, MSB);
Value *RetVal = Builder.CreateSelect(Ret0, Zero, Dividend);
Value *EarlyRet = Builder.CreateOr(Ret0, RetDividend);
Builder.CreateCondBr(EarlyRet, End, BB1);
// ; bb1: ; preds = %special-cases
// ; %sr_1 = add i32 %sr, 1
// ; %tmp2 = sub i32 31, %sr
// ; %q = shl i32 %dividend, %tmp2
// ; %skipLoop = icmp eq i32 %sr_1, 0
// ; br i1 %skipLoop, label %loop-exit, label %preheader
Builder.SetInsertPoint(BB1);
Value *SR_1 = Builder.CreateAdd(SR, One);
Value *Tmp2 = Builder.CreateSub(MSB, SR);
Value *Q = Builder.CreateShl(Dividend, Tmp2);
Value *SkipLoop = Builder.CreateICmpEQ(SR_1, Zero);
Builder.CreateCondBr(SkipLoop, LoopExit, Preheader);
// ; preheader: ; preds = %bb1
// ; %tmp3 = lshr i32 %dividend, %sr_1
// ; %tmp4 = add i32 %divisor, -1
// ; br label %do-while
Builder.SetInsertPoint(Preheader);
Value *Tmp3 = Builder.CreateLShr(Dividend, SR_1);
Value *Tmp4 = Builder.CreateAdd(Divisor, NegOne);
Builder.CreateBr(DoWhile);
// ; do-while: ; preds = %do-while, %preheader
// ; %carry_1 = phi i32 [ 0, %preheader ], [ %carry, %do-while ]
// ; %sr_3 = phi i32 [ %sr_1, %preheader ], [ %sr_2, %do-while ]
// ; %r_1 = phi i32 [ %tmp3, %preheader ], [ %r, %do-while ]
// ; %q_2 = phi i32 [ %q, %preheader ], [ %q_1, %do-while ]
// ; %tmp5 = shl i32 %r_1, 1
// ; %tmp6 = lshr i32 %q_2, 31
// ; %tmp7 = or i32 %tmp5, %tmp6
// ; %tmp8 = shl i32 %q_2, 1
// ; %q_1 = or i32 %carry_1, %tmp8
// ; %tmp9 = sub i32 %tmp4, %tmp7
// ; %tmp10 = ashr i32 %tmp9, 31
// ; %carry = and i32 %tmp10, 1
// ; %tmp11 = and i32 %tmp10, %divisor
// ; %r = sub i32 %tmp7, %tmp11
// ; %sr_2 = add i32 %sr_3, -1
// ; %tmp12 = icmp eq i32 %sr_2, 0
// ; br i1 %tmp12, label %loop-exit, label %do-while
Builder.SetInsertPoint(DoWhile);
PHINode *Carry_1 = Builder.CreatePHI(DivTy, 2);
PHINode *SR_3 = Builder.CreatePHI(DivTy, 2);
PHINode *R_1 = Builder.CreatePHI(DivTy, 2);
PHINode *Q_2 = Builder.CreatePHI(DivTy, 2);
Value *Tmp5 = Builder.CreateShl(R_1, One);
Value *Tmp6 = Builder.CreateLShr(Q_2, MSB);
Value *Tmp7 = Builder.CreateOr(Tmp5, Tmp6);
Value *Tmp8 = Builder.CreateShl(Q_2, One);
Value *Q_1 = Builder.CreateOr(Carry_1, Tmp8);
Value *Tmp9 = Builder.CreateSub(Tmp4, Tmp7);
Value *Tmp10 = Builder.CreateAShr(Tmp9, MSB);
Value *Carry = Builder.CreateAnd(Tmp10, One);
Value *Tmp11 = Builder.CreateAnd(Tmp10, Divisor);
Value *R = Builder.CreateSub(Tmp7, Tmp11);
Value *SR_2 = Builder.CreateAdd(SR_3, NegOne);
Value *Tmp12 = Builder.CreateICmpEQ(SR_2, Zero);
Builder.CreateCondBr(Tmp12, LoopExit, DoWhile);
// ; loop-exit: ; preds = %do-while, %bb1
// ; %carry_2 = phi i32 [ 0, %bb1 ], [ %carry, %do-while ]
// ; %q_3 = phi i32 [ %q, %bb1 ], [ %q_1, %do-while ]
// ; %tmp13 = shl i32 %q_3, 1
// ; %q_4 = or i32 %carry_2, %tmp13
// ; br label %end
Builder.SetInsertPoint(LoopExit);
PHINode *Carry_2 = Builder.CreatePHI(DivTy, 2);
PHINode *Q_3 = Builder.CreatePHI(DivTy, 2);
Value *Tmp13 = Builder.CreateShl(Q_3, One);
Value *Q_4 = Builder.CreateOr(Carry_2, Tmp13);
Builder.CreateBr(End);
// ; end: ; preds = %loop-exit, %special-cases
// ; %q_5 = phi i32 [ %q_4, %loop-exit ], [ %retVal, %special-cases ]
// ; ret i32 %q_5
Builder.SetInsertPoint(End, End->begin());
PHINode *Q_5 = Builder.CreatePHI(DivTy, 2);
// Populate the Phis, since all values have now been created. Our Phis were:
// ; %carry_1 = phi i32 [ 0, %preheader ], [ %carry, %do-while ]
Carry_1->addIncoming(Zero, Preheader);
Carry_1->addIncoming(Carry, DoWhile);
// ; %sr_3 = phi i32 [ %sr_1, %preheader ], [ %sr_2, %do-while ]
SR_3->addIncoming(SR_1, Preheader);
SR_3->addIncoming(SR_2, DoWhile);
// ; %r_1 = phi i32 [ %tmp3, %preheader ], [ %r, %do-while ]
R_1->addIncoming(Tmp3, Preheader);
R_1->addIncoming(R, DoWhile);
// ; %q_2 = phi i32 [ %q, %preheader ], [ %q_1, %do-while ]
Q_2->addIncoming(Q, Preheader);
Q_2->addIncoming(Q_1, DoWhile);
// ; %carry_2 = phi i32 [ 0, %bb1 ], [ %carry, %do-while ]
Carry_2->addIncoming(Zero, BB1);
Carry_2->addIncoming(Carry, DoWhile);
// ; %q_3 = phi i32 [ %q, %bb1 ], [ %q_1, %do-while ]
Q_3->addIncoming(Q, BB1);
Q_3->addIncoming(Q_1, DoWhile);
// ; %q_5 = phi i32 [ %q_4, %loop-exit ], [ %retVal, %special-cases ]
Q_5->addIncoming(Q_4, LoopExit);
Q_5->addIncoming(RetVal, SpecialCases);
return Q_5;
}