void setUnrollID(Loop *L, bool enable) { if (!enable && disabledLoops.find(L) != disabledLoops.end()) return; LLVMContext &Context = L->getHeader()->getContext(); #if LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 6 SmallVector<Metadata *, 2> forceUnroll; forceUnroll.push_back(MDString::get(Context, "llvm.loop.unroll.enable")); forceUnroll.push_back(ConstantAsMetadata::get(ConstantInt::get(Type::getInt1Ty(Context), enable))); MDNode *forceUnrollNode = MDNode::get(Context, forceUnroll); SmallVector<Metadata *, 4> Vals; Vals.push_back(NULL); Vals.push_back(forceUnrollNode); #else SmallVector<Value *, 2> forceUnroll; forceUnroll.push_back(MDString::get(Context, "llvm.loop.unroll.enable")); forceUnroll.push_back(ConstantInt::get(Type::getInt1Ty(Context), enable)); MDNode *forceUnrollNode = MDNode::get(Context, forceUnroll); SmallVector<Value *, 4> Vals; Vals.push_back(NULL); Vals.push_back(forceUnrollNode); #endif MDNode *NewLoopID = MDNode::get(Context, Vals); // Set operand 0 to refer to the loop id itself. NewLoopID->replaceOperandWith(0, NewLoopID); L->setLoopID(NewLoopID); if (!enable) disabledLoops.insert(L); }
/// Remap the operands of an MDNode. /// /// If \c Node is temporary, uniquing cycles are ignored. If \c Node is /// distinct, uniquing cycles are resolved as they're found. /// /// \pre \c Node.isDistinct() or \c Node.isTemporary(). static bool remapOperands(MDNode &Node, SmallVectorImpl<MDNode *> &DistinctWorklist, ValueToValueMapTy &VM, RemapFlags Flags, ValueMapTypeRemapper *TypeMapper, ValueMaterializer *Materializer) { assert(!Node.isUniqued() && "Expected temporary or distinct node"); const bool IsDistinct = Node.isDistinct(); bool AnyChanged = false; for (unsigned I = 0, E = Node.getNumOperands(); I != E; ++I) { Metadata *Old = Node.getOperand(I); Metadata *New = mapMetadataOp(Old, DistinctWorklist, VM, Flags, TypeMapper, Materializer); if (Old != New) { AnyChanged = true; Node.replaceOperandWith(I, New); // Resolve uniquing cycles underneath distinct nodes on the fly so they // don't infect later operands. if (IsDistinct) resolveCycles(New); } } return AnyChanged; }
static llvm::MDNode *stripDebugLocFromLoopID(llvm::MDNode *N) { assert(N->op_begin() != N->op_end() && "Missing self reference?"); // if there is no debug location, we do not have to rewrite this MDNode. if (std::none_of(N->op_begin() + 1, N->op_end(), [](const MDOperand &Op) { return isa<DILocation>(Op.get()); })) return N; // If there is only the debug location without any actual loop metadata, we // can remove the metadata. if (std::none_of(N->op_begin() + 1, N->op_end(), [](const MDOperand &Op) { return !isa<DILocation>(Op.get()); })) return nullptr; SmallVector<Metadata *, 4> Args; // Reserve operand 0 for loop id self reference. auto TempNode = MDNode::getTemporary(N->getContext(), None); Args.push_back(TempNode.get()); // Add all non-debug location operands back. for (auto Op = N->op_begin() + 1; Op != N->op_end(); Op++) { if (!isa<DILocation>(*Op)) Args.push_back(*Op); } // Set the first operand to itself. MDNode *LoopID = MDNode::get(N->getContext(), Args); LoopID->replaceOperandWith(0, LoopID); return LoopID; }
// Remove existing unroll metadata and add unroll disable metadata to // indicate the loop has already been unrolled. This prevents a loop // from being unrolled more than is directed by a pragma if the loop // unrolling pass is run more than once (which it generally is). static void SetLoopAlreadyUnrolled(Loop *L) { MDNode *LoopID = L->getLoopID(); if (!LoopID) return; // First remove any existing loop unrolling metadata. SmallVector<Metadata *, 4> MDs; // Reserve first location for self reference to the LoopID metadata node. MDs.push_back(nullptr); for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) { bool IsUnrollMetadata = false; MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i)); if (MD) { const MDString *S = dyn_cast<MDString>(MD->getOperand(0)); IsUnrollMetadata = S && S->getString().startswith("llvm.loop.unroll."); } if (!IsUnrollMetadata) MDs.push_back(LoopID->getOperand(i)); } // Add unroll(disable) metadata to disable future unrolling. LLVMContext &Context = L->getHeader()->getContext(); SmallVector<Metadata *, 1> DisableOperands; DisableOperands.push_back(MDString::get(Context, "llvm.loop.unroll.disable")); MDNode *DisableNode = MDNode::get(Context, DisableOperands); MDs.push_back(DisableNode); MDNode *NewLoopID = MDNode::get(Context, MDs); // Set operand 0 to refer to the loop id itself. NewLoopID->replaceOperandWith(0, NewLoopID); L->setLoopID(NewLoopID); }
void DIDescriptor::replaceFunctionField(unsigned Elt, Function *F) { if (!DbgNode) return; if (Elt < DbgNode->getNumOperands()) { MDNode *Node = const_cast<MDNode *>(DbgNode); Node->replaceOperandWith(Elt, F ? ConstantAsMetadata::get(F) : nullptr); } }
void DIDescriptor::replaceFunctionField(unsigned Elt, Function *F) { if (DbgNode == 0) return; if (Elt < DbgNode->getNumOperands()) { MDNode *Node = const_cast<MDNode*>(DbgNode); Node->replaceOperandWith(Elt, F); } }
void MDNodeMapper::remapOperands(MDNode &N, OperandMapper mapOperand) { assert(!N.isUniqued() && "Expected distinct or temporary nodes"); for (unsigned I = 0, E = N.getNumOperands(); I != E; ++I) { Metadata *Old = N.getOperand(I); Metadata *New = mapOperand(Old); if (Old != New) N.replaceOperandWith(I, New); } }
static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs) { if (!Attrs.IsParallel && Attrs.VectorizerWidth == 0 && Attrs.VectorizerUnroll == 0 && Attrs.VectorizerEnable == LoopAttributes::VecUnspecified) return nullptr; SmallVector<Value *, 4> Args; // Reserve operand 0 for loop id self reference. MDNode *TempNode = MDNode::getTemporary(Ctx, None); Args.push_back(TempNode); // Setting vectorizer.width if (Attrs.VectorizerWidth > 0) { Value *Vals[] = { MDString::get(Ctx, "llvm.loop.vectorize.width"), ConstantInt::get(Type::getInt32Ty(Ctx), Attrs.VectorizerWidth) }; Args.push_back(MDNode::get(Ctx, Vals)); } // Setting vectorizer.unroll if (Attrs.VectorizerUnroll > 0) { Value *Vals[] = { MDString::get(Ctx, "llvm.loop.interleave.count"), ConstantInt::get(Type::getInt32Ty(Ctx), Attrs.VectorizerUnroll) }; Args.push_back(MDNode::get(Ctx, Vals)); } // Setting vectorizer.enable if (Attrs.VectorizerEnable != LoopAttributes::VecUnspecified) { Value *Vals[] = { MDString::get(Ctx, "llvm.loop.vectorize.enable"), ConstantInt::get(Type::getInt1Ty(Ctx), (Attrs.VectorizerEnable == LoopAttributes::VecEnable)) }; Args.push_back(MDNode::get(Ctx, Vals)); } MDNode *LoopID = MDNode::get(Ctx, Args); assert(LoopID->use_empty() && "LoopID should not be used"); // Set the first operand to itself. LoopID->replaceOperandWith(0, LoopID); MDNode::deleteTemporary(TempNode); return LoopID; }
void MDNodeMapper::remapOperands(const Data &D, MDNode &N) { for (unsigned I = 0, E = N.getNumOperands(); I != E; ++I) { Metadata *Old = N.getOperand(I); Metadata *New; if (Optional<Metadata *> MappedOp = getMappedOp(Old)){ New = *MappedOp; } else { assert(!N.isDistinct() && "Expected all nodes to be pre-mapped for distinct operands"); MDNode &OldN = *cast<MDNode>(Old); assert(!OldN.isDistinct() && "Expected distinct nodes to be pre-mapped"); New = &getFwdReference(D, OldN); } if (Old != New) N.replaceOperandWith(I, New); } }
/// @brief Get a self referencing id metadata node. /// /// The MDNode looks like this (if arg0/arg1 are not null): /// /// '!n = metadata !{metadata !n, arg0, arg1}' /// /// @return The self referencing id metadata node. static MDNode *getID(LLVMContext &Ctx, Metadata *arg0 = nullptr, Metadata *arg1 = nullptr) { MDNode *ID; SmallVector<Metadata *, 3> Args; // Use a temporary node to safely create a unique pointer for the first arg. auto TempNode = MDNode::getTemporary(Ctx, None); // Reserve operand 0 for loop id self reference. Args.push_back(TempNode.get()); if (arg0) Args.push_back(arg0); if (arg1) Args.push_back(arg1); ID = MDNode::get(Ctx, Args); ID->replaceOperandWith(0, ID); return ID; }
static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs) { if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 && Attrs.InterleaveCount == 0 && Attrs.VectorizeEnable == LoopAttributes::Unspecified) return nullptr; SmallVector<Metadata *, 4> Args; // Reserve operand 0 for loop id self reference. auto TempNode = MDNode::getTemporary(Ctx, None); Args.push_back(TempNode.get()); // Setting vectorize.width if (Attrs.VectorizeWidth > 0) { Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"), ConstantAsMetadata::get(ConstantInt::get( Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))}; Args.push_back(MDNode::get(Ctx, Vals)); } // Setting interleave.count if (Attrs.InterleaveCount > 0) { Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"), ConstantAsMetadata::get(ConstantInt::get( Type::getInt32Ty(Ctx), Attrs.InterleaveCount))}; Args.push_back(MDNode::get(Ctx, Vals)); } // Setting vectorize.enable if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) { Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"), ConstantAsMetadata::get(ConstantInt::get( Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable == LoopAttributes::Enable)))}; Args.push_back(MDNode::get(Ctx, Vals)); } // Set the first operand to itself. MDNode *LoopID = MDNode::get(Ctx, Args); LoopID->replaceOperandWith(0, LoopID); return LoopID; }
/// \brief Set input string into loop metadata by keeping other values intact. void llvm::addStringMetadataToLoop(Loop *TheLoop, const char *MDString, unsigned V) { SmallVector<Metadata *, 4> MDs(1); // If the loop already has metadata, retain it. MDNode *LoopID = TheLoop->getLoopID(); if (LoopID) { for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) { MDNode *Node = cast<MDNode>(LoopID->getOperand(i)); MDs.push_back(Node); } } // Add new metadata. MDs.push_back(createStringMetadata(TheLoop, MDString, V)); // Replace current metadata node with new one. LLVMContext &Context = TheLoop->getHeader()->getContext(); MDNode *NewLoopID = MDNode::get(Context, MDs); // Set operand 0 to refer to the loop id itself. NewLoopID->replaceOperandWith(0, NewLoopID); TheLoop->setLoopID(NewLoopID); }
/// Remap the operands of an MDNode. static bool remapOperands(MDNode &Node, SmallVectorImpl<MDNode *> &DistinctWorklist, ValueToValueMapTy &VM, RemapFlags Flags, ValueMapTypeRemapper *TypeMapper, ValueMaterializer *Materializer) { assert(!Node.isUniqued() && "Expected temporary or distinct node"); bool AnyChanged = false; for (unsigned I = 0, E = Node.getNumOperands(); I != E; ++I) { Metadata *Old = Node.getOperand(I); Metadata *New = mapMetadataOp(Old, DistinctWorklist, VM, Flags, TypeMapper, Materializer); if (Old != New) { AnyChanged = true; Node.replaceOperandWith(I, New); } } return AnyChanged; }
MDNode *MDBuilder::createAnonymousAARoot(StringRef Name, MDNode *Extra) { // To ensure uniqueness the root node is self-referential. auto Dummy = MDNode::getTemporary(Context, None); SmallVector<Metadata *, 3> Args(1, Dummy.get()); if (Extra) Args.push_back(Extra); if (!Name.empty()) Args.push_back(createString(Name)); MDNode *Root = MDNode::get(Context, Args); // At this point we have // !0 = metadata !{} <- dummy // !1 = metadata !{metadata !0} <- root // Replace the dummy operand with the root node itself and delete the dummy. Root->replaceOperandWith(0, Root); // We now have // !1 = metadata !{metadata !1} <- self-referential root return Root; }
MDNode *llvm::makePostTransformationMetadata(LLVMContext &Context, MDNode *OrigLoopID, ArrayRef<StringRef> RemovePrefixes, ArrayRef<MDNode *> AddAttrs) { // First remove any existing loop metadata related to this transformation. SmallVector<Metadata *, 4> MDs; // Reserve first location for self reference to the LoopID metadata node. TempMDTuple TempNode = MDNode::getTemporary(Context, None); MDs.push_back(TempNode.get()); // Remove metadata for the transformation that has been applied or that became // outdated. if (OrigLoopID) { for (unsigned i = 1, ie = OrigLoopID->getNumOperands(); i < ie; ++i) { bool IsVectorMetadata = false; Metadata *Op = OrigLoopID->getOperand(i); if (MDNode *MD = dyn_cast<MDNode>(Op)) { const MDString *S = dyn_cast<MDString>(MD->getOperand(0)); if (S) IsVectorMetadata = llvm::any_of(RemovePrefixes, [S](StringRef Prefix) -> bool { return S->getString().startswith(Prefix); }); } if (!IsVectorMetadata) MDs.push_back(Op); } } // Add metadata to avoid reapplying a transformation, such as // llvm.loop.unroll.disable and llvm.loop.isvectorized. MDs.append(AddAttrs.begin(), AddAttrs.end()); MDNode *NewLoopID = MDNode::getDistinct(Context, MDs); // Replace the temporary node with a self-reference. NewLoopID->replaceOperandWith(0, NewLoopID); return NewLoopID; }
/// Create a clone of the blocks in a loop and connect them together. /// If UnrollProlog is true, loop structure will not be cloned, otherwise a new /// loop will be created including all cloned blocks, and the iterator of it /// switches to count NewIter down to 0. /// static void CloneLoopBlocks(Loop *L, Value *NewIter, const bool UnrollProlog, BasicBlock *InsertTop, BasicBlock *InsertBot, std::vector<BasicBlock *> &NewBlocks, LoopBlocksDFS &LoopBlocks, ValueToValueMapTy &VMap, LoopInfo *LI) { BasicBlock *Preheader = L->getLoopPreheader(); BasicBlock *Header = L->getHeader(); BasicBlock *Latch = L->getLoopLatch(); Function *F = Header->getParent(); LoopBlocksDFS::RPOIterator BlockBegin = LoopBlocks.beginRPO(); LoopBlocksDFS::RPOIterator BlockEnd = LoopBlocks.endRPO(); Loop *NewLoop = 0; Loop *ParentLoop = L->getParentLoop(); if (!UnrollProlog) { NewLoop = new Loop(); if (ParentLoop) ParentLoop->addChildLoop(NewLoop); else LI->addTopLevelLoop(NewLoop); } // For each block in the original loop, create a new copy, // and update the value map with the newly created values. for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) { BasicBlock *NewBB = CloneBasicBlock(*BB, VMap, ".prol", F); NewBlocks.push_back(NewBB); if (NewLoop) NewLoop->addBasicBlockToLoop(NewBB, *LI); else if (ParentLoop) ParentLoop->addBasicBlockToLoop(NewBB, *LI); VMap[*BB] = NewBB; if (Header == *BB) { // For the first block, add a CFG connection to this newly // created block. InsertTop->getTerminator()->setSuccessor(0, NewBB); } if (Latch == *BB) { // For the last block, if UnrollProlog is true, create a direct jump to // InsertBot. If not, create a loop back to cloned head. VMap.erase((*BB)->getTerminator()); BasicBlock *FirstLoopBB = cast<BasicBlock>(VMap[Header]); BranchInst *LatchBR = cast<BranchInst>(NewBB->getTerminator()); IRBuilder<> Builder(LatchBR); if (UnrollProlog) { Builder.CreateBr(InsertBot); } else { PHINode *NewIdx = PHINode::Create(NewIter->getType(), 2, "prol.iter", FirstLoopBB->getFirstNonPHI()); Value *IdxSub = Builder.CreateSub(NewIdx, ConstantInt::get(NewIdx->getType(), 1), NewIdx->getName() + ".sub"); Value *IdxCmp = Builder.CreateIsNotNull(IdxSub, NewIdx->getName() + ".cmp"); Builder.CreateCondBr(IdxCmp, FirstLoopBB, InsertBot); NewIdx->addIncoming(NewIter, InsertTop); NewIdx->addIncoming(IdxSub, NewBB); } LatchBR->eraseFromParent(); } } // Change the incoming values to the ones defined in the preheader or // cloned loop. for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) { PHINode *NewPHI = cast<PHINode>(VMap[I]); if (UnrollProlog) { VMap[I] = NewPHI->getIncomingValueForBlock(Preheader); cast<BasicBlock>(VMap[Header])->getInstList().erase(NewPHI); } else { unsigned idx = NewPHI->getBasicBlockIndex(Preheader); NewPHI->setIncomingBlock(idx, InsertTop); BasicBlock *NewLatch = cast<BasicBlock>(VMap[Latch]); idx = NewPHI->getBasicBlockIndex(Latch); Value *InVal = NewPHI->getIncomingValue(idx); NewPHI->setIncomingBlock(idx, NewLatch); if (VMap[InVal]) NewPHI->setIncomingValue(idx, VMap[InVal]); } } if (NewLoop) { // Add unroll disable metadata to disable future unrolling for this loop. SmallVector<Metadata *, 4> MDs; // Reserve first location for self reference to the LoopID metadata node. MDs.push_back(nullptr); MDNode *LoopID = NewLoop->getLoopID(); if (LoopID) { // First remove any existing loop unrolling metadata. for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) { bool IsUnrollMetadata = false; MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i)); if (MD) { const MDString *S = dyn_cast<MDString>(MD->getOperand(0)); IsUnrollMetadata = S && S->getString().startswith("llvm.loop.unroll."); } if (!IsUnrollMetadata) MDs.push_back(LoopID->getOperand(i)); } } LLVMContext &Context = NewLoop->getHeader()->getContext(); SmallVector<Metadata *, 1> DisableOperands; DisableOperands.push_back(MDString::get(Context, "llvm.loop.unroll.disable")); MDNode *DisableNode = MDNode::get(Context, DisableOperands); MDs.push_back(DisableNode); MDNode *NewLoopID = MDNode::get(Context, MDs); // Set operand 0 to refer to the loop id itself. NewLoopID->replaceOperandWith(0, NewLoopID); NewLoop->setLoopID(NewLoopID); } }
void LowerSIMDLoop::enableUnsafeAlgebraIfReduction(PHINode* Phi, Loop* L) const { typedef SmallVector<Instruction*, 8> chainVector; chainVector chain; Instruction *J; unsigned opcode = 0; for (Instruction *I = Phi; ; I=J) { J = NULL; // Find the user of instruction I that is within loop L. #ifdef LLVM35 for (User *UI : I->users()) { /*}*/ Instruction *U = cast<Instruction>(UI); #else for (Value::use_iterator UI = I->use_begin(), UE = I->use_end(); UI != UE; ++UI) { Instruction *U = cast<Instruction>(*UI); #endif if (L->contains(U)) { if (J) { DEBUG(dbgs() << "LSL: not a reduction var because op has two internal uses: " << *I << "\n"); return; } J = U; } } if (!J) { DEBUG(dbgs() << "LSL: chain prematurely terminated at " << *I << "\n"); return; } if (J==Phi) { // Found the entire chain. break; } if (opcode) { // Check that arithmetic op matches prior arithmetic ops in the chain. if (J->getOpcode()!=opcode) { DEBUG(dbgs() << "LSL: chain broke at " << *J << " because of wrong opcode\n"); return; } } else { // First arithmetic op in the chain. opcode = J->getOpcode(); if (opcode!=Instruction::FAdd && opcode!=Instruction::FMul) { DEBUG(dbgs() << "LSL: first arithmetic op in chain is uninteresting" << *J << "\n"); return; } } chain.push_back(J); } for (chainVector::const_iterator K=chain.begin(); K!=chain.end(); ++K) { DEBUG(dbgs() << "LSL: marking " << **K << "\n"); (*K)->setHasUnsafeAlgebra(true); } } bool LowerSIMDLoop::runOnLoop(Loop *L, LPPassManager &LPM) { if (!simd_loop_mdkind) return false; // Fast rejection test. if (!hasSIMDLoopMetadata(L)) return false; DEBUG(dbgs() << "LSL: simd_loop found\n"); #ifdef LLVM34 MDNode* n = L->getLoopID(); if (!n) { // Loop does not have a LoopID yet, so give it one. #ifdef LLVM36 n = MDNode::get(getGlobalContext(), ArrayRef<Metadata*>(NULL)); #else n = MDNode::get(getGlobalContext(), ArrayRef<Value*>(NULL)); #endif n->replaceOperandWith(0,n); L->setLoopID(n); } #else MDNode* n = MDNode::get(getGlobalContext(), ArrayRef<Value*>()); L->getLoopLatch()->getTerminator()->setMetadata("llvm.loop.parallel", n); #endif #ifdef LLVM36 MDNode* m = MDNode::get(getGlobalContext(), ArrayRef<Metadata*>(n)); #else MDNode* m = MDNode::get(getGlobalContext(), ArrayRef<Value*>(n)); #endif // Mark memory references so that Loop::isAnnotatedParallel will return true for this loop. for(Loop::block_iterator BBI = L->block_begin(), E=L->block_end(); BBI!=E; ++BBI) for (BasicBlock::iterator I = (*BBI)->begin(), EE = (*BBI)->end(); I!=EE; ++I) if (I->mayReadOrWriteMemory()) I->setMetadata("llvm.mem.parallel_loop_access", m); assert(L->isAnnotatedParallel()); // Mark floating-point reductions as okay to reassociate/commute. BasicBlock* Lh = L->getHeader(); DEBUG(dbgs() << "LSL: loop header: " << *Lh << "\n"); for (BasicBlock::iterator I = Lh->begin(), E = Lh->end(); I!=E; ++I) if (PHINode *Phi = dyn_cast<PHINode>(I)) enableUnsafeAlgebraIfReduction(Phi,L); return true; } char LowerSIMDLoop::ID = 0; static RegisterPass<LowerSIMDLoop> X("LowerSIMDLoop", "LowerSIMDLoop Pass", false /* Only looks at CFG */, false /* Analysis Pass */); JL_DLLEXPORT Pass* createLowerSimdLoopPass() { return new LowerSIMDLoop(); } } // namespace llvm
static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs, llvm::DebugLoc Location) { if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 && Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 && Attrs.VectorizeEnable == LoopAttributes::Unspecified && Attrs.UnrollEnable == LoopAttributes::Unspecified && Attrs.DistributeEnable == LoopAttributes::Unspecified && !Location) return nullptr; SmallVector<Metadata *, 4> Args; // Reserve operand 0 for loop id self reference. auto TempNode = MDNode::getTemporary(Ctx, None); Args.push_back(TempNode.get()); // If we have a valid debug location for the loop, add it. if (Location) Args.push_back(Location.getAsMDNode()); // Setting vectorize.width if (Attrs.VectorizeWidth > 0) { Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"), ConstantAsMetadata::get(ConstantInt::get( Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))}; Args.push_back(MDNode::get(Ctx, Vals)); } // Setting interleave.count if (Attrs.InterleaveCount > 0) { Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"), ConstantAsMetadata::get(ConstantInt::get( Type::getInt32Ty(Ctx), Attrs.InterleaveCount))}; Args.push_back(MDNode::get(Ctx, Vals)); } // Setting interleave.count if (Attrs.UnrollCount > 0) { Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"), ConstantAsMetadata::get(ConstantInt::get( Type::getInt32Ty(Ctx), Attrs.UnrollCount))}; Args.push_back(MDNode::get(Ctx, Vals)); } // Setting vectorize.enable if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) { Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"), ConstantAsMetadata::get(ConstantInt::get( Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable == LoopAttributes::Enable)))}; Args.push_back(MDNode::get(Ctx, Vals)); } // Setting unroll.full or unroll.disable if (Attrs.UnrollEnable != LoopAttributes::Unspecified) { std::string Name; if (Attrs.UnrollEnable == LoopAttributes::Enable) Name = "llvm.loop.unroll.enable"; else if (Attrs.UnrollEnable == LoopAttributes::Full) Name = "llvm.loop.unroll.full"; else Name = "llvm.loop.unroll.disable"; Metadata *Vals[] = {MDString::get(Ctx, Name)}; Args.push_back(MDNode::get(Ctx, Vals)); } if (Attrs.DistributeEnable != LoopAttributes::Unspecified) { Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"), ConstantAsMetadata::get(ConstantInt::get( Type::getInt1Ty(Ctx), (Attrs.DistributeEnable == LoopAttributes::Enable)))}; Args.push_back(MDNode::get(Ctx, Vals)); } // Set the first operand to itself. MDNode *LoopID = MDNode::get(Ctx, Args); LoopID->replaceOperandWith(0, LoopID); return LoopID; }
/// Create a clone of the blocks in a loop and connect them together. /// If CreateRemainderLoop is false, loop structure will not be cloned, /// otherwise a new loop will be created including all cloned blocks, and the /// iterator of it switches to count NewIter down to 0. /// The cloned blocks should be inserted between InsertTop and InsertBot. /// If loop structure is cloned InsertTop should be new preheader, InsertBot /// new loop exit. /// Return the new cloned loop that is created when CreateRemainderLoop is true. static Loop * CloneLoopBlocks(Loop *L, Value *NewIter, const bool CreateRemainderLoop, const bool UseEpilogRemainder, const bool UnrollRemainder, BasicBlock *InsertTop, BasicBlock *InsertBot, BasicBlock *Preheader, std::vector<BasicBlock *> &NewBlocks, LoopBlocksDFS &LoopBlocks, ValueToValueMapTy &VMap, DominatorTree *DT, LoopInfo *LI) { StringRef suffix = UseEpilogRemainder ? "epil" : "prol"; BasicBlock *Header = L->getHeader(); BasicBlock *Latch = L->getLoopLatch(); Function *F = Header->getParent(); LoopBlocksDFS::RPOIterator BlockBegin = LoopBlocks.beginRPO(); LoopBlocksDFS::RPOIterator BlockEnd = LoopBlocks.endRPO(); Loop *ParentLoop = L->getParentLoop(); NewLoopsMap NewLoops; NewLoops[ParentLoop] = ParentLoop; if (!CreateRemainderLoop) NewLoops[L] = ParentLoop; // For each block in the original loop, create a new copy, // and update the value map with the newly created values. for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) { BasicBlock *NewBB = CloneBasicBlock(*BB, VMap, "." + suffix, F); NewBlocks.push_back(NewBB); // If we're unrolling the outermost loop, there's no remainder loop, // and this block isn't in a nested loop, then the new block is not // in any loop. Otherwise, add it to loopinfo. if (CreateRemainderLoop || LI->getLoopFor(*BB) != L || ParentLoop) addClonedBlockToLoopInfo(*BB, NewBB, LI, NewLoops); VMap[*BB] = NewBB; if (Header == *BB) { // For the first block, add a CFG connection to this newly // created block. InsertTop->getTerminator()->setSuccessor(0, NewBB); } if (DT) { if (Header == *BB) { // The header is dominated by the preheader. DT->addNewBlock(NewBB, InsertTop); } else { // Copy information from original loop to unrolled loop. BasicBlock *IDomBB = DT->getNode(*BB)->getIDom()->getBlock(); DT->addNewBlock(NewBB, cast<BasicBlock>(VMap[IDomBB])); } } if (Latch == *BB) { // For the last block, if CreateRemainderLoop is false, create a direct // jump to InsertBot. If not, create a loop back to cloned head. VMap.erase((*BB)->getTerminator()); BasicBlock *FirstLoopBB = cast<BasicBlock>(VMap[Header]); BranchInst *LatchBR = cast<BranchInst>(NewBB->getTerminator()); IRBuilder<> Builder(LatchBR); if (!CreateRemainderLoop) { Builder.CreateBr(InsertBot); } else { PHINode *NewIdx = PHINode::Create(NewIter->getType(), 2, suffix + ".iter", FirstLoopBB->getFirstNonPHI()); Value *IdxSub = Builder.CreateSub(NewIdx, ConstantInt::get(NewIdx->getType(), 1), NewIdx->getName() + ".sub"); Value *IdxCmp = Builder.CreateIsNotNull(IdxSub, NewIdx->getName() + ".cmp"); Builder.CreateCondBr(IdxCmp, FirstLoopBB, InsertBot); NewIdx->addIncoming(NewIter, InsertTop); NewIdx->addIncoming(IdxSub, NewBB); } LatchBR->eraseFromParent(); } } // Change the incoming values to the ones defined in the preheader or // cloned loop. for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) { PHINode *NewPHI = cast<PHINode>(VMap[&*I]); if (!CreateRemainderLoop) { if (UseEpilogRemainder) { unsigned idx = NewPHI->getBasicBlockIndex(Preheader); NewPHI->setIncomingBlock(idx, InsertTop); NewPHI->removeIncomingValue(Latch, false); } else { VMap[&*I] = NewPHI->getIncomingValueForBlock(Preheader); cast<BasicBlock>(VMap[Header])->getInstList().erase(NewPHI); } } else { unsigned idx = NewPHI->getBasicBlockIndex(Preheader); NewPHI->setIncomingBlock(idx, InsertTop); BasicBlock *NewLatch = cast<BasicBlock>(VMap[Latch]); idx = NewPHI->getBasicBlockIndex(Latch); Value *InVal = NewPHI->getIncomingValue(idx); NewPHI->setIncomingBlock(idx, NewLatch); if (Value *V = VMap.lookup(InVal)) NewPHI->setIncomingValue(idx, V); } } if (CreateRemainderLoop) { Loop *NewLoop = NewLoops[L]; assert(NewLoop && "L should have been cloned"); // Only add loop metadata if the loop is not going to be completely // unrolled. if (UnrollRemainder) return NewLoop; // Add unroll disable metadata to disable future unrolling for this loop. SmallVector<Metadata *, 4> MDs; // Reserve first location for self reference to the LoopID metadata node. MDs.push_back(nullptr); MDNode *LoopID = NewLoop->getLoopID(); if (LoopID) { // First remove any existing loop unrolling metadata. for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) { bool IsUnrollMetadata = false; MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i)); if (MD) { const MDString *S = dyn_cast<MDString>(MD->getOperand(0)); IsUnrollMetadata = S && S->getString().startswith("llvm.loop.unroll."); } if (!IsUnrollMetadata) MDs.push_back(LoopID->getOperand(i)); } } LLVMContext &Context = NewLoop->getHeader()->getContext(); SmallVector<Metadata *, 1> DisableOperands; DisableOperands.push_back(MDString::get(Context, "llvm.loop.unroll.disable")); MDNode *DisableNode = MDNode::get(Context, DisableOperands); MDs.push_back(DisableNode); MDNode *NewLoopID = MDNode::get(Context, MDs); // Set operand 0 to refer to the loop id itself. NewLoopID->replaceOperandWith(0, NewLoopID); NewLoop->setLoopID(NewLoopID); return NewLoop; } else return nullptr; }