void setUnrollID(Loop *L, bool enable) {
if (!enable && disabledLoops.find(L) != disabledLoops.end())
return;
LLVMContext &Context = L->getHeader()->getContext();
#if LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 6
SmallVector<Metadata *, 2> forceUnroll;
forceUnroll.push_back(MDString::get(Context, "llvm.loop.unroll.enable"));
forceUnroll.push_back(ConstantAsMetadata::get(ConstantInt::get(Type::getInt1Ty(Context), enable)));
MDNode *forceUnrollNode = MDNode::get(Context, forceUnroll);
SmallVector<Metadata *, 4> Vals;
Vals.push_back(NULL);
Vals.push_back(forceUnrollNode);
#else
SmallVector<Value *, 2> forceUnroll;
forceUnroll.push_back(MDString::get(Context, "llvm.loop.unroll.enable"));
forceUnroll.push_back(ConstantInt::get(Type::getInt1Ty(Context), enable));
MDNode *forceUnrollNode = MDNode::get(Context, forceUnroll);
SmallVector<Value *, 4> Vals;
Vals.push_back(NULL);
Vals.push_back(forceUnrollNode);
#endif
MDNode *NewLoopID = MDNode::get(Context, Vals);
// Set operand 0 to refer to the loop id itself.
NewLoopID->replaceOperandWith(0, NewLoopID);
L->setLoopID(NewLoopID);
if (!enable)
disabledLoops.insert(L);
}
/// Remap the operands of an MDNode.
///
/// If \c Node is temporary, uniquing cycles are ignored. If \c Node is
/// distinct, uniquing cycles are resolved as they're found.
///
/// \pre \c Node.isDistinct() or \c Node.isTemporary().
static bool remapOperands(MDNode &Node,
SmallVectorImpl<MDNode *> &DistinctWorklist,
ValueToValueMapTy &VM, RemapFlags Flags,
ValueMapTypeRemapper *TypeMapper,
ValueMaterializer *Materializer) {
assert(!Node.isUniqued() && "Expected temporary or distinct node");
const bool IsDistinct = Node.isDistinct();
bool AnyChanged = false;
for (unsigned I = 0, E = Node.getNumOperands(); I != E; ++I) {
Metadata *Old = Node.getOperand(I);
Metadata *New = mapMetadataOp(Old, DistinctWorklist, VM, Flags, TypeMapper,
Materializer);
if (Old != New) {
AnyChanged = true;
Node.replaceOperandWith(I, New);
// Resolve uniquing cycles underneath distinct nodes on the fly so they
// don't infect later operands.
if (IsDistinct)
resolveCycles(New);
}
}
return AnyChanged;
}
static llvm::MDNode *stripDebugLocFromLoopID(llvm::MDNode *N) {
assert(N->op_begin() != N->op_end() && "Missing self reference?");
// if there is no debug location, we do not have to rewrite this MDNode.
if (std::none_of(N->op_begin() + 1, N->op_end(), [](const MDOperand &Op) {
return isa<DILocation>(Op.get());
}))
return N;
// If there is only the debug location without any actual loop metadata, we
// can remove the metadata.
if (std::none_of(N->op_begin() + 1, N->op_end(), [](const MDOperand &Op) {
return !isa<DILocation>(Op.get());
}))
return nullptr;
SmallVector<Metadata *, 4> Args;
// Reserve operand 0 for loop id self reference.
auto TempNode = MDNode::getTemporary(N->getContext(), None);
Args.push_back(TempNode.get());
// Add all non-debug location operands back.
for (auto Op = N->op_begin() + 1; Op != N->op_end(); Op++) {
if (!isa<DILocation>(*Op))
Args.push_back(*Op);
}
// Set the first operand to itself.
MDNode *LoopID = MDNode::get(N->getContext(), Args);
LoopID->replaceOperandWith(0, LoopID);
return LoopID;
}
// Remove existing unroll metadata and add unroll disable metadata to
// indicate the loop has already been unrolled. This prevents a loop
// from being unrolled more than is directed by a pragma if the loop
// unrolling pass is run more than once (which it generally is).
static void SetLoopAlreadyUnrolled(Loop *L) {
MDNode *LoopID = L->getLoopID();
if (!LoopID) return;
// First remove any existing loop unrolling metadata.
SmallVector<Metadata *, 4> MDs;
// Reserve first location for self reference to the LoopID metadata node.
MDs.push_back(nullptr);
for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {
bool IsUnrollMetadata = false;
MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i));
if (MD) {
const MDString *S = dyn_cast<MDString>(MD->getOperand(0));
IsUnrollMetadata = S && S->getString().startswith("llvm.loop.unroll.");
}
if (!IsUnrollMetadata)
MDs.push_back(LoopID->getOperand(i));
}
// Add unroll(disable) metadata to disable future unrolling.
LLVMContext &Context = L->getHeader()->getContext();
SmallVector<Metadata *, 1> DisableOperands;
DisableOperands.push_back(MDString::get(Context, "llvm.loop.unroll.disable"));
MDNode *DisableNode = MDNode::get(Context, DisableOperands);
MDs.push_back(DisableNode);
MDNode *NewLoopID = MDNode::get(Context, MDs);
// Set operand 0 to refer to the loop id itself.
NewLoopID->replaceOperandWith(0, NewLoopID);
L->setLoopID(NewLoopID);
}
void DIDescriptor::replaceFunctionField(unsigned Elt, Function *F) {
if (!DbgNode)
return;
if (Elt < DbgNode->getNumOperands()) {
MDNode *Node = const_cast<MDNode *>(DbgNode);
Node->replaceOperandWith(Elt, F ? ConstantAsMetadata::get(F) : nullptr);
}
}
void DIDescriptor::replaceFunctionField(unsigned Elt, Function *F) {
if (DbgNode == 0)
return;
if (Elt < DbgNode->getNumOperands()) {
MDNode *Node = const_cast<MDNode*>(DbgNode);
Node->replaceOperandWith(Elt, F);
}
}
void MDNodeMapper::remapOperands(MDNode &N, OperandMapper mapOperand) {
assert(!N.isUniqued() && "Expected distinct or temporary nodes");
for (unsigned I = 0, E = N.getNumOperands(); I != E; ++I) {
Metadata *Old = N.getOperand(I);
Metadata *New = mapOperand(Old);
if (Old != New)
N.replaceOperandWith(I, New);
}
}
static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs) {
if (!Attrs.IsParallel && Attrs.VectorizerWidth == 0 &&
Attrs.VectorizerUnroll == 0 &&
Attrs.VectorizerEnable == LoopAttributes::VecUnspecified)
return nullptr;
SmallVector<Value *, 4> Args;
// Reserve operand 0 for loop id self reference.
MDNode *TempNode = MDNode::getTemporary(Ctx, None);
Args.push_back(TempNode);
// Setting vectorizer.width
if (Attrs.VectorizerWidth > 0) {
Value *Vals[] = { MDString::get(Ctx, "llvm.loop.vectorize.width"),
ConstantInt::get(Type::getInt32Ty(Ctx),
Attrs.VectorizerWidth) };
Args.push_back(MDNode::get(Ctx, Vals));
}
// Setting vectorizer.unroll
if (Attrs.VectorizerUnroll > 0) {
Value *Vals[] = { MDString::get(Ctx, "llvm.loop.interleave.count"),
ConstantInt::get(Type::getInt32Ty(Ctx),
Attrs.VectorizerUnroll) };
Args.push_back(MDNode::get(Ctx, Vals));
}
// Setting vectorizer.enable
if (Attrs.VectorizerEnable != LoopAttributes::VecUnspecified) {
Value *Vals[] = { MDString::get(Ctx, "llvm.loop.vectorize.enable"),
ConstantInt::get(Type::getInt1Ty(Ctx),
(Attrs.VectorizerEnable ==
LoopAttributes::VecEnable)) };
Args.push_back(MDNode::get(Ctx, Vals));
}
MDNode *LoopID = MDNode::get(Ctx, Args);
assert(LoopID->use_empty() && "LoopID should not be used");
// Set the first operand to itself.
LoopID->replaceOperandWith(0, LoopID);
MDNode::deleteTemporary(TempNode);
return LoopID;
}
void MDNodeMapper::remapOperands(const Data &D, MDNode &N) {
for (unsigned I = 0, E = N.getNumOperands(); I != E; ++I) {
Metadata *Old = N.getOperand(I);
Metadata *New;
if (Optional<Metadata *> MappedOp = getMappedOp(Old)){
New = *MappedOp;
} else {
assert(!N.isDistinct() &&
"Expected all nodes to be pre-mapped for distinct operands");
MDNode &OldN = *cast<MDNode>(Old);
assert(!OldN.isDistinct() && "Expected distinct nodes to be pre-mapped");
New = &getFwdReference(D, OldN);
}
if (Old != New)
N.replaceOperandWith(I, New);
}
}
/// @brief Get a self referencing id metadata node.
///
/// The MDNode looks like this (if arg0/arg1 are not null):
///
/// '!n = metadata !{metadata !n, arg0, arg1}'
///
/// @return The self referencing id metadata node.
static MDNode *getID(LLVMContext &Ctx, Metadata *arg0 = nullptr,
Metadata *arg1 = nullptr) {
MDNode *ID;
SmallVector<Metadata *, 3> Args;
// Use a temporary node to safely create a unique pointer for the first arg.
auto TempNode = MDNode::getTemporary(Ctx, None);
// Reserve operand 0 for loop id self reference.
Args.push_back(TempNode.get());
if (arg0)
Args.push_back(arg0);
if (arg1)
Args.push_back(arg1);
ID = MDNode::get(Ctx, Args);
ID->replaceOperandWith(0, ID);
return ID;
}
static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs) {
if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
Attrs.InterleaveCount == 0 &&
Attrs.VectorizeEnable == LoopAttributes::Unspecified)
return nullptr;
SmallVector<Metadata *, 4> Args;
// Reserve operand 0 for loop id self reference.
auto TempNode = MDNode::getTemporary(Ctx, None);
Args.push_back(TempNode.get());
// Setting vectorize.width
if (Attrs.VectorizeWidth > 0) {
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"),
ConstantAsMetadata::get(ConstantInt::get(
Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))};
Args.push_back(MDNode::get(Ctx, Vals));
}
// Setting interleave.count
if (Attrs.InterleaveCount > 0) {
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"),
ConstantAsMetadata::get(ConstantInt::get(
Type::getInt32Ty(Ctx), Attrs.InterleaveCount))};
Args.push_back(MDNode::get(Ctx, Vals));
}
// Setting vectorize.enable
if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
ConstantAsMetadata::get(ConstantInt::get(
Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable ==
LoopAttributes::Enable)))};
Args.push_back(MDNode::get(Ctx, Vals));
}
// Set the first operand to itself.
MDNode *LoopID = MDNode::get(Ctx, Args);
LoopID->replaceOperandWith(0, LoopID);
return LoopID;
}
/// \brief Set input string into loop metadata by keeping other values intact.
void llvm::addStringMetadataToLoop(Loop *TheLoop, const char *MDString,
unsigned V) {
SmallVector<Metadata *, 4> MDs(1);
// If the loop already has metadata, retain it.
MDNode *LoopID = TheLoop->getLoopID();
if (LoopID) {
for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {
MDNode *Node = cast<MDNode>(LoopID->getOperand(i));
MDs.push_back(Node);
}
}
// Add new metadata.
MDs.push_back(createStringMetadata(TheLoop, MDString, V));
// Replace current metadata node with new one.
LLVMContext &Context = TheLoop->getHeader()->getContext();
MDNode *NewLoopID = MDNode::get(Context, MDs);
// Set operand 0 to refer to the loop id itself.
NewLoopID->replaceOperandWith(0, NewLoopID);
TheLoop->setLoopID(NewLoopID);
}
/// Remap the operands of an MDNode.
static bool remapOperands(MDNode &Node,
SmallVectorImpl<MDNode *> &DistinctWorklist,
ValueToValueMapTy &VM, RemapFlags Flags,
ValueMapTypeRemapper *TypeMapper,
ValueMaterializer *Materializer) {
assert(!Node.isUniqued() && "Expected temporary or distinct node");
bool AnyChanged = false;
for (unsigned I = 0, E = Node.getNumOperands(); I != E; ++I) {
Metadata *Old = Node.getOperand(I);
Metadata *New = mapMetadataOp(Old, DistinctWorklist, VM, Flags, TypeMapper,
Materializer);
if (Old != New) {
AnyChanged = true;
Node.replaceOperandWith(I, New);
}
}
return AnyChanged;
}
MDNode *MDBuilder::createAnonymousAARoot(StringRef Name, MDNode *Extra) {
// To ensure uniqueness the root node is self-referential.
auto Dummy = MDNode::getTemporary(Context, None);
SmallVector<Metadata *, 3> Args(1, Dummy.get());
if (Extra)
Args.push_back(Extra);
if (!Name.empty())
Args.push_back(createString(Name));
MDNode *Root = MDNode::get(Context, Args);
// At this point we have
// !0 = metadata !{} <- dummy
// !1 = metadata !{metadata !0} <- root
// Replace the dummy operand with the root node itself and delete the dummy.
Root->replaceOperandWith(0, Root);
// We now have
// !1 = metadata !{metadata !1} <- self-referential root
return Root;
}
MDNode *llvm::makePostTransformationMetadata(LLVMContext &Context,
MDNode *OrigLoopID,
ArrayRef<StringRef> RemovePrefixes,
ArrayRef<MDNode *> AddAttrs) {
// First remove any existing loop metadata related to this transformation.
SmallVector<Metadata *, 4> MDs;
// Reserve first location for self reference to the LoopID metadata node.
TempMDTuple TempNode = MDNode::getTemporary(Context, None);
MDs.push_back(TempNode.get());
// Remove metadata for the transformation that has been applied or that became
// outdated.
if (OrigLoopID) {
for (unsigned i = 1, ie = OrigLoopID->getNumOperands(); i < ie; ++i) {
bool IsVectorMetadata = false;
Metadata *Op = OrigLoopID->getOperand(i);
if (MDNode *MD = dyn_cast<MDNode>(Op)) {
const MDString *S = dyn_cast<MDString>(MD->getOperand(0));
if (S)
IsVectorMetadata =
llvm::any_of(RemovePrefixes, [S](StringRef Prefix) -> bool {
return S->getString().startswith(Prefix);
});
}
if (!IsVectorMetadata)
MDs.push_back(Op);
}
}
// Add metadata to avoid reapplying a transformation, such as
// llvm.loop.unroll.disable and llvm.loop.isvectorized.
MDs.append(AddAttrs.begin(), AddAttrs.end());
MDNode *NewLoopID = MDNode::getDistinct(Context, MDs);
// Replace the temporary node with a self-reference.
NewLoopID->replaceOperandWith(0, NewLoopID);
return NewLoopID;
}
/// Create a clone of the blocks in a loop and connect them together.
/// If UnrollProlog is true, loop structure will not be cloned, otherwise a new
/// loop will be created including all cloned blocks, and the iterator of it
/// switches to count NewIter down to 0.
///
static void CloneLoopBlocks(Loop *L, Value *NewIter, const bool UnrollProlog,
BasicBlock *InsertTop, BasicBlock *InsertBot,
std::vector<BasicBlock *> &NewBlocks,
LoopBlocksDFS &LoopBlocks, ValueToValueMapTy &VMap,
LoopInfo *LI) {
BasicBlock *Preheader = L->getLoopPreheader();
BasicBlock *Header = L->getHeader();
BasicBlock *Latch = L->getLoopLatch();
Function *F = Header->getParent();
LoopBlocksDFS::RPOIterator BlockBegin = LoopBlocks.beginRPO();
LoopBlocksDFS::RPOIterator BlockEnd = LoopBlocks.endRPO();
Loop *NewLoop = 0;
Loop *ParentLoop = L->getParentLoop();
if (!UnrollProlog) {
NewLoop = new Loop();
if (ParentLoop)
ParentLoop->addChildLoop(NewLoop);
else
LI->addTopLevelLoop(NewLoop);
}
// For each block in the original loop, create a new copy,
// and update the value map with the newly created values.
for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) {
BasicBlock *NewBB = CloneBasicBlock(*BB, VMap, ".prol", F);
NewBlocks.push_back(NewBB);
if (NewLoop)
NewLoop->addBasicBlockToLoop(NewBB, *LI);
else if (ParentLoop)
ParentLoop->addBasicBlockToLoop(NewBB, *LI);
VMap[*BB] = NewBB;
if (Header == *BB) {
// For the first block, add a CFG connection to this newly
// created block.
InsertTop->getTerminator()->setSuccessor(0, NewBB);
}
if (Latch == *BB) {
// For the last block, if UnrollProlog is true, create a direct jump to
// InsertBot. If not, create a loop back to cloned head.
VMap.erase((*BB)->getTerminator());
BasicBlock *FirstLoopBB = cast<BasicBlock>(VMap[Header]);
BranchInst *LatchBR = cast<BranchInst>(NewBB->getTerminator());
IRBuilder<> Builder(LatchBR);
if (UnrollProlog) {
Builder.CreateBr(InsertBot);
} else {
PHINode *NewIdx = PHINode::Create(NewIter->getType(), 2, "prol.iter",
FirstLoopBB->getFirstNonPHI());
Value *IdxSub =
Builder.CreateSub(NewIdx, ConstantInt::get(NewIdx->getType(), 1),
NewIdx->getName() + ".sub");
Value *IdxCmp =
Builder.CreateIsNotNull(IdxSub, NewIdx->getName() + ".cmp");
Builder.CreateCondBr(IdxCmp, FirstLoopBB, InsertBot);
NewIdx->addIncoming(NewIter, InsertTop);
NewIdx->addIncoming(IdxSub, NewBB);
}
LatchBR->eraseFromParent();
}
}
// Change the incoming values to the ones defined in the preheader or
// cloned loop.
for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) {
PHINode *NewPHI = cast<PHINode>(VMap[I]);
if (UnrollProlog) {
VMap[I] = NewPHI->getIncomingValueForBlock(Preheader);
cast<BasicBlock>(VMap[Header])->getInstList().erase(NewPHI);
} else {
unsigned idx = NewPHI->getBasicBlockIndex(Preheader);
NewPHI->setIncomingBlock(idx, InsertTop);
BasicBlock *NewLatch = cast<BasicBlock>(VMap[Latch]);
idx = NewPHI->getBasicBlockIndex(Latch);
Value *InVal = NewPHI->getIncomingValue(idx);
NewPHI->setIncomingBlock(idx, NewLatch);
if (VMap[InVal])
NewPHI->setIncomingValue(idx, VMap[InVal]);
}
}
if (NewLoop) {
// Add unroll disable metadata to disable future unrolling for this loop.
SmallVector<Metadata *, 4> MDs;
// Reserve first location for self reference to the LoopID metadata node.
MDs.push_back(nullptr);
MDNode *LoopID = NewLoop->getLoopID();
if (LoopID) {
// First remove any existing loop unrolling metadata.
for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {
bool IsUnrollMetadata = false;
MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i));
if (MD) {
const MDString *S = dyn_cast<MDString>(MD->getOperand(0));
IsUnrollMetadata = S && S->getString().startswith("llvm.loop.unroll.");
}
if (!IsUnrollMetadata)
MDs.push_back(LoopID->getOperand(i));
}
}
LLVMContext &Context = NewLoop->getHeader()->getContext();
SmallVector<Metadata *, 1> DisableOperands;
DisableOperands.push_back(MDString::get(Context, "llvm.loop.unroll.disable"));
MDNode *DisableNode = MDNode::get(Context, DisableOperands);
MDs.push_back(DisableNode);
MDNode *NewLoopID = MDNode::get(Context, MDs);
// Set operand 0 to refer to the loop id itself.
NewLoopID->replaceOperandWith(0, NewLoopID);
NewLoop->setLoopID(NewLoopID);
}
}
void LowerSIMDLoop::enableUnsafeAlgebraIfReduction(PHINode* Phi, Loop* L) const
{
typedef SmallVector<Instruction*, 8> chainVector;
chainVector chain;
Instruction *J;
unsigned opcode = 0;
for (Instruction *I = Phi; ; I=J) {
J = NULL;
// Find the user of instruction I that is within loop L.
#ifdef LLVM35
for (User *UI : I->users()) { /*}*/
Instruction *U = cast<Instruction>(UI);
#else
for (Value::use_iterator UI = I->use_begin(), UE = I->use_end(); UI != UE; ++UI) {
Instruction *U = cast<Instruction>(*UI);
#endif
if (L->contains(U)) {
if (J) {
DEBUG(dbgs() << "LSL: not a reduction var because op has two internal uses: " << *I << "\n");
return;
}
J = U;
}
}
if (!J) {
DEBUG(dbgs() << "LSL: chain prematurely terminated at " << *I << "\n");
return;
}
if (J==Phi) {
// Found the entire chain.
break;
}
if (opcode) {
// Check that arithmetic op matches prior arithmetic ops in the chain.
if (J->getOpcode()!=opcode) {
DEBUG(dbgs() << "LSL: chain broke at " << *J << " because of wrong opcode\n");
return;
}
}
else {
// First arithmetic op in the chain.
opcode = J->getOpcode();
if (opcode!=Instruction::FAdd && opcode!=Instruction::FMul) {
DEBUG(dbgs() << "LSL: first arithmetic op in chain is uninteresting" << *J << "\n");
return;
}
}
chain.push_back(J);
}
for (chainVector::const_iterator K=chain.begin(); K!=chain.end(); ++K) {
DEBUG(dbgs() << "LSL: marking " << **K << "\n");
(*K)->setHasUnsafeAlgebra(true);
}
}
bool LowerSIMDLoop::runOnLoop(Loop *L, LPPassManager &LPM)
{
if (!simd_loop_mdkind)
return false; // Fast rejection test.
if (!hasSIMDLoopMetadata(L))
return false;
DEBUG(dbgs() << "LSL: simd_loop found\n");
#ifdef LLVM34
MDNode* n = L->getLoopID();
if (!n) {
// Loop does not have a LoopID yet, so give it one.
#ifdef LLVM36
n = MDNode::get(getGlobalContext(), ArrayRef<Metadata*>(NULL));
#else
n = MDNode::get(getGlobalContext(), ArrayRef<Value*>(NULL));
#endif
n->replaceOperandWith(0,n);
L->setLoopID(n);
}
#else
MDNode* n = MDNode::get(getGlobalContext(), ArrayRef<Value*>());
L->getLoopLatch()->getTerminator()->setMetadata("llvm.loop.parallel", n);
#endif
#ifdef LLVM36
MDNode* m = MDNode::get(getGlobalContext(), ArrayRef<Metadata*>(n));
#else
MDNode* m = MDNode::get(getGlobalContext(), ArrayRef<Value*>(n));
#endif
// Mark memory references so that Loop::isAnnotatedParallel will return true for this loop.
for(Loop::block_iterator BBI = L->block_begin(), E=L->block_end(); BBI!=E; ++BBI)
for (BasicBlock::iterator I = (*BBI)->begin(), EE = (*BBI)->end(); I!=EE; ++I)
if (I->mayReadOrWriteMemory())
I->setMetadata("llvm.mem.parallel_loop_access", m);
assert(L->isAnnotatedParallel());
// Mark floating-point reductions as okay to reassociate/commute.
BasicBlock* Lh = L->getHeader();
DEBUG(dbgs() << "LSL: loop header: " << *Lh << "\n");
for (BasicBlock::iterator I = Lh->begin(), E = Lh->end(); I!=E; ++I)
if (PHINode *Phi = dyn_cast<PHINode>(I))
enableUnsafeAlgebraIfReduction(Phi,L);
return true;
}
char LowerSIMDLoop::ID = 0;
static RegisterPass<LowerSIMDLoop> X("LowerSIMDLoop", "LowerSIMDLoop Pass",
false /* Only looks at CFG */,
false /* Analysis Pass */);
JL_DLLEXPORT Pass* createLowerSimdLoopPass() {
return new LowerSIMDLoop();
}
} // namespace llvm
static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs,
llvm::DebugLoc Location) {
if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
Attrs.UnrollEnable == LoopAttributes::Unspecified &&
Attrs.DistributeEnable == LoopAttributes::Unspecified &&
!Location)
return nullptr;
SmallVector<Metadata *, 4> Args;
// Reserve operand 0 for loop id self reference.
auto TempNode = MDNode::getTemporary(Ctx, None);
Args.push_back(TempNode.get());
// If we have a valid debug location for the loop, add it.
if (Location)
Args.push_back(Location.getAsMDNode());
// Setting vectorize.width
if (Attrs.VectorizeWidth > 0) {
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"),
ConstantAsMetadata::get(ConstantInt::get(
Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))};
Args.push_back(MDNode::get(Ctx, Vals));
}
// Setting interleave.count
if (Attrs.InterleaveCount > 0) {
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"),
ConstantAsMetadata::get(ConstantInt::get(
Type::getInt32Ty(Ctx), Attrs.InterleaveCount))};
Args.push_back(MDNode::get(Ctx, Vals));
}
// Setting interleave.count
if (Attrs.UnrollCount > 0) {
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
ConstantAsMetadata::get(ConstantInt::get(
Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
Args.push_back(MDNode::get(Ctx, Vals));
}
// Setting vectorize.enable
if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
ConstantAsMetadata::get(ConstantInt::get(
Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable ==
LoopAttributes::Enable)))};
Args.push_back(MDNode::get(Ctx, Vals));
}
// Setting unroll.full or unroll.disable
if (Attrs.UnrollEnable != LoopAttributes::Unspecified) {
std::string Name;
if (Attrs.UnrollEnable == LoopAttributes::Enable)
Name = "llvm.loop.unroll.enable";
else if (Attrs.UnrollEnable == LoopAttributes::Full)
Name = "llvm.loop.unroll.full";
else
Name = "llvm.loop.unroll.disable";
Metadata *Vals[] = {MDString::get(Ctx, Name)};
Args.push_back(MDNode::get(Ctx, Vals));
}
if (Attrs.DistributeEnable != LoopAttributes::Unspecified) {
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
ConstantAsMetadata::get(ConstantInt::get(
Type::getInt1Ty(Ctx), (Attrs.DistributeEnable ==
LoopAttributes::Enable)))};
Args.push_back(MDNode::get(Ctx, Vals));
}
// Set the first operand to itself.
MDNode *LoopID = MDNode::get(Ctx, Args);
LoopID->replaceOperandWith(0, LoopID);
return LoopID;
}
/// Create a clone of the blocks in a loop and connect them together.
/// If CreateRemainderLoop is false, loop structure will not be cloned,
/// otherwise a new loop will be created including all cloned blocks, and the
/// iterator of it switches to count NewIter down to 0.
/// The cloned blocks should be inserted between InsertTop and InsertBot.
/// If loop structure is cloned InsertTop should be new preheader, InsertBot
/// new loop exit.
/// Return the new cloned loop that is created when CreateRemainderLoop is true.
static Loop *
CloneLoopBlocks(Loop *L, Value *NewIter, const bool CreateRemainderLoop,
const bool UseEpilogRemainder, const bool UnrollRemainder,
BasicBlock *InsertTop,
BasicBlock *InsertBot, BasicBlock *Preheader,
std::vector<BasicBlock *> &NewBlocks, LoopBlocksDFS &LoopBlocks,
ValueToValueMapTy &VMap, DominatorTree *DT, LoopInfo *LI) {
StringRef suffix = UseEpilogRemainder ? "epil" : "prol";
BasicBlock *Header = L->getHeader();
BasicBlock *Latch = L->getLoopLatch();
Function *F = Header->getParent();
LoopBlocksDFS::RPOIterator BlockBegin = LoopBlocks.beginRPO();
LoopBlocksDFS::RPOIterator BlockEnd = LoopBlocks.endRPO();
Loop *ParentLoop = L->getParentLoop();
NewLoopsMap NewLoops;
NewLoops[ParentLoop] = ParentLoop;
if (!CreateRemainderLoop)
NewLoops[L] = ParentLoop;
// For each block in the original loop, create a new copy,
// and update the value map with the newly created values.
for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) {
BasicBlock *NewBB = CloneBasicBlock(*BB, VMap, "." + suffix, F);
NewBlocks.push_back(NewBB);
// If we're unrolling the outermost loop, there's no remainder loop,
// and this block isn't in a nested loop, then the new block is not
// in any loop. Otherwise, add it to loopinfo.
if (CreateRemainderLoop || LI->getLoopFor(*BB) != L || ParentLoop)
addClonedBlockToLoopInfo(*BB, NewBB, LI, NewLoops);
VMap[*BB] = NewBB;
if (Header == *BB) {
// For the first block, add a CFG connection to this newly
// created block.
InsertTop->getTerminator()->setSuccessor(0, NewBB);
}
if (DT) {
if (Header == *BB) {
// The header is dominated by the preheader.
DT->addNewBlock(NewBB, InsertTop);
} else {
// Copy information from original loop to unrolled loop.
BasicBlock *IDomBB = DT->getNode(*BB)->getIDom()->getBlock();
DT->addNewBlock(NewBB, cast<BasicBlock>(VMap[IDomBB]));
}
}
if (Latch == *BB) {
// For the last block, if CreateRemainderLoop is false, create a direct
// jump to InsertBot. If not, create a loop back to cloned head.
VMap.erase((*BB)->getTerminator());
BasicBlock *FirstLoopBB = cast<BasicBlock>(VMap[Header]);
BranchInst *LatchBR = cast<BranchInst>(NewBB->getTerminator());
IRBuilder<> Builder(LatchBR);
if (!CreateRemainderLoop) {
Builder.CreateBr(InsertBot);
} else {
PHINode *NewIdx = PHINode::Create(NewIter->getType(), 2,
suffix + ".iter",
FirstLoopBB->getFirstNonPHI());
Value *IdxSub =
Builder.CreateSub(NewIdx, ConstantInt::get(NewIdx->getType(), 1),
NewIdx->getName() + ".sub");
Value *IdxCmp =
Builder.CreateIsNotNull(IdxSub, NewIdx->getName() + ".cmp");
Builder.CreateCondBr(IdxCmp, FirstLoopBB, InsertBot);
NewIdx->addIncoming(NewIter, InsertTop);
NewIdx->addIncoming(IdxSub, NewBB);
}
LatchBR->eraseFromParent();
}
}
// Change the incoming values to the ones defined in the preheader or
// cloned loop.
for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) {
PHINode *NewPHI = cast<PHINode>(VMap[&*I]);
if (!CreateRemainderLoop) {
if (UseEpilogRemainder) {
unsigned idx = NewPHI->getBasicBlockIndex(Preheader);
NewPHI->setIncomingBlock(idx, InsertTop);
NewPHI->removeIncomingValue(Latch, false);
} else {
VMap[&*I] = NewPHI->getIncomingValueForBlock(Preheader);
cast<BasicBlock>(VMap[Header])->getInstList().erase(NewPHI);
}
} else {
unsigned idx = NewPHI->getBasicBlockIndex(Preheader);
NewPHI->setIncomingBlock(idx, InsertTop);
BasicBlock *NewLatch = cast<BasicBlock>(VMap[Latch]);
idx = NewPHI->getBasicBlockIndex(Latch);
Value *InVal = NewPHI->getIncomingValue(idx);
NewPHI->setIncomingBlock(idx, NewLatch);
if (Value *V = VMap.lookup(InVal))
NewPHI->setIncomingValue(idx, V);
}
}
if (CreateRemainderLoop) {
Loop *NewLoop = NewLoops[L];
assert(NewLoop && "L should have been cloned");
// Only add loop metadata if the loop is not going to be completely
// unrolled.
if (UnrollRemainder)
return NewLoop;
// Add unroll disable metadata to disable future unrolling for this loop.
SmallVector<Metadata *, 4> MDs;
// Reserve first location for self reference to the LoopID metadata node.
MDs.push_back(nullptr);
MDNode *LoopID = NewLoop->getLoopID();
if (LoopID) {
// First remove any existing loop unrolling metadata.
for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {
bool IsUnrollMetadata = false;
MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i));
if (MD) {
const MDString *S = dyn_cast<MDString>(MD->getOperand(0));
IsUnrollMetadata = S && S->getString().startswith("llvm.loop.unroll.");
}
if (!IsUnrollMetadata)
MDs.push_back(LoopID->getOperand(i));
}
}
LLVMContext &Context = NewLoop->getHeader()->getContext();
SmallVector<Metadata *, 1> DisableOperands;
DisableOperands.push_back(MDString::get(Context,
"llvm.loop.unroll.disable"));
MDNode *DisableNode = MDNode::get(Context, DisableOperands);
MDs.push_back(DisableNode);
MDNode *NewLoopID = MDNode::get(Context, MDs);
// Set operand 0 to refer to the loop id itself.
NewLoopID->replaceOperandWith(0, NewLoopID);
NewLoop->setLoopID(NewLoopID);
return NewLoop;
}
else
return nullptr;
}
