bool LowerExpectIntrinsic::HandleSwitchExpect(SwitchInst *SI) {
CallInst *CI = dyn_cast<CallInst>(SI->getCondition());
if (!CI)
return false;
Function *Fn = CI->getCalledFunction();
if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect)
return false;
Value *ArgValue = CI->getArgOperand(0);
ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1));
if (!ExpectedValue)
return false;
LLVMContext &Context = CI->getContext();
const Type *Int32Ty = Type::getInt32Ty(Context);
unsigned caseNo = SI->findCaseValue(ExpectedValue);
std::vector<Value *> Vec;
unsigned n = SI->getNumCases();
Vec.resize(n + 1); // +1 for MDString
Vec[0] = MDString::get(Context, "branch_weights");
for (unsigned i = 0; i < n; ++i) {
Vec[i + 1] = ConstantInt::get(Int32Ty, i == caseNo ? LikelyBranchWeight : UnlikelyBranchWeight);
}
MDNode *WeightsNode = llvm::MDNode::get(Context, Vec);
SI->setMetadata(LLVMContext::MD_prof, WeightsNode);
SI->setCondition(ArgValue);
return true;
}
bool LowerExpectIntrinsic::HandleSwitchExpect(SwitchInst *SI) {
CallInst *CI = dyn_cast<CallInst>(SI->getCondition());
if (!CI)
return false;
Function *Fn = CI->getCalledFunction();
if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect)
return false;
Value *ArgValue = CI->getArgOperand(0);
ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1));
if (!ExpectedValue)
return false;
SwitchInst::CaseIt Case = SI->findCaseValue(ExpectedValue);
unsigned n = SI->getNumCases(); // +1 for default case.
std::vector<uint32_t> Weights(n + 1);
Weights[0] = Case == SI->case_default() ? LikelyBranchWeight
: UnlikelyBranchWeight;
for (unsigned i = 0; i != n; ++i)
Weights[i + 1] = i == Case.getCaseIndex() ? LikelyBranchWeight
: UnlikelyBranchWeight;
SI->setMetadata(LLVMContext::MD_prof,
MDBuilder(CI->getContext()).createBranchWeights(Weights));
SI->setCondition(ArgValue);
return true;
}
bool LowerExpectIntrinsic::HandleIfExpect(BranchInst *BI) {
if (BI->isUnconditional())
return false;
// Handle non-optimized IR code like:
// %expval = call i64 @llvm.expect.i64(i64 %conv1, i64 1)
// %tobool = icmp ne i64 %expval, 0
// br i1 %tobool, label %if.then, label %if.end
//
// Or the following simpler case:
// %expval = call i1 @llvm.expect.i1(i1 %cmp, i1 1)
// br i1 %expval, label %if.then, label %if.end
CallInst *CI;
ICmpInst *CmpI = dyn_cast<ICmpInst>(BI->getCondition());
if (!CmpI) {
CI = dyn_cast<CallInst>(BI->getCondition());
} else {
if (CmpI->getPredicate() != CmpInst::ICMP_NE)
return false;
CI = dyn_cast<CallInst>(CmpI->getOperand(0));
}
if (!CI)
return false;
Function *Fn = CI->getCalledFunction();
if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect)
return false;
Value *ArgValue = CI->getArgOperand(0);
ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1));
if (!ExpectedValue)
return false;
MDBuilder MDB(CI->getContext());
MDNode *Node;
// If expect value is equal to 1 it means that we are more likely to take
// branch 0, in other case more likely is branch 1.
if (ExpectedValue->isOne())
Node = MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight);
else
Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight);
BI->setMetadata(LLVMContext::MD_prof, Node);
if (CmpI)
CmpI->setOperand(0, ArgValue);
else
BI->setCondition(ArgValue);
return true;
}
bool LowerExpectIntrinsic::HandleIfExpect(BranchInst *BI) {
if (BI->isUnconditional())
return false;
// Handle non-optimized IR code like:
// %expval = call i64 @llvm.expect.i64.i64(i64 %conv1, i64 1)
// %tobool = icmp ne i64 %expval, 0
// br i1 %tobool, label %if.then, label %if.end
ICmpInst *CmpI = dyn_cast<ICmpInst>(BI->getCondition());
if (!CmpI || CmpI->getPredicate() != CmpInst::ICMP_NE)
return false;
CallInst *CI = dyn_cast<CallInst>(CmpI->getOperand(0));
if (!CI)
return false;
Function *Fn = CI->getCalledFunction();
if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect)
return false;
Value *ArgValue = CI->getArgOperand(0);
ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1));
if (!ExpectedValue)
return false;
LLVMContext &Context = CI->getContext();
const Type *Int32Ty = Type::getInt32Ty(Context);
bool Likely = ExpectedValue->isOne();
// If expect value is equal to 1 it means that we are more likely to take
// branch 0, in other case more likely is branch 1.
Value *Ops[] = {
MDString::get(Context, "branch_weights"),
ConstantInt::get(Int32Ty, Likely ? LikelyBranchWeight : UnlikelyBranchWeight),
ConstantInt::get(Int32Ty, Likely ? UnlikelyBranchWeight : LikelyBranchWeight)
};
MDNode *WeightsNode = MDNode::get(Context, Ops);
BI->setMetadata(LLVMContext::MD_prof, WeightsNode);
CmpI->setOperand(0, ArgValue);
return true;
}
CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
MDNode *ScopeTag, MDNode *NoAliasTag) {
assert(DstAlign >= ElementSize &&
"Pointer alignment must be at least element size");
assert(SrcAlign >= ElementSize &&
"Pointer alignment must be at least element size");
Dst = getCastedInt8PtrValue(Dst);
Src = getCastedInt8PtrValue(Src);
Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
Module *M = BB->getParent()->getParent();
Value *TheFn = Intrinsic::getDeclaration(
M, Intrinsic::memcpy_element_unordered_atomic, Tys);
CallInst *CI = createCallHelper(TheFn, Ops, this);
// Set the alignment of the pointer args.
CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
// Set the TBAA info if present.
if (TBAATag)
CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
// Set the TBAA Struct info if present.
if (TBAAStructTag)
CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
if (ScopeTag)
CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
if (NoAliasTag)
CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
return CI;
}
