/// buildCFICheck - emits __cfi_check for the current module. void CrossDSOCFI::buildCFICheck() { // FIXME: verify that __cfi_check ends up near the end of the code section, // but before the jump slots created in LowerBitSets. llvm::DenseSet<uint64_t> BitSetIds; NamedMDNode *BitSetNM = M->getNamedMetadata("llvm.bitsets"); if (BitSetNM) for (unsigned I = 0, E = BitSetNM->getNumOperands(); I != E; ++I) if (ConstantInt *TypeId = extractBitSetTypeId(BitSetNM->getOperand(I))) BitSetIds.insert(TypeId->getZExtValue()); LLVMContext &Ctx = M->getContext(); Constant *C = M->getOrInsertFunction( "__cfi_check", FunctionType::get( Type::getVoidTy(Ctx), {Type::getInt64Ty(Ctx), PointerType::getUnqual(Type::getInt8Ty(Ctx))}, false)); Function *F = dyn_cast<Function>(C); F->setAlignment(4096); auto args = F->arg_begin(); Argument &CallSiteTypeId = *(args++); CallSiteTypeId.setName("CallSiteTypeId"); Argument &Addr = *(args++); Addr.setName("Addr"); assert(args == F->arg_end()); BasicBlock *BB = BasicBlock::Create(Ctx, "entry", F); BasicBlock *TrapBB = BasicBlock::Create(Ctx, "trap", F); IRBuilder<> IRBTrap(TrapBB); Function *TrapFn = Intrinsic::getDeclaration(M, Intrinsic::trap); llvm::CallInst *TrapCall = IRBTrap.CreateCall(TrapFn); TrapCall->setDoesNotReturn(); TrapCall->setDoesNotThrow(); IRBTrap.CreateUnreachable(); BasicBlock *ExitBB = BasicBlock::Create(Ctx, "exit", F); IRBuilder<> IRBExit(ExitBB); IRBExit.CreateRetVoid(); IRBuilder<> IRB(BB); SwitchInst *SI = IRB.CreateSwitch(&CallSiteTypeId, TrapBB, BitSetIds.size()); for (uint64_t TypeId : BitSetIds) { ConstantInt *CaseTypeId = ConstantInt::get(Type::getInt64Ty(Ctx), TypeId); BasicBlock *TestBB = BasicBlock::Create(Ctx, "test", F); IRBuilder<> IRBTest(TestBB); Function *BitsetTestFn = Intrinsic::getDeclaration(M, Intrinsic::bitset_test); Value *Test = IRBTest.CreateCall( BitsetTestFn, {&Addr, MetadataAsValue::get( Ctx, ConstantAsMetadata::get(CaseTypeId))}); BranchInst *BI = IRBTest.CreateCondBr(Test, ExitBB, TrapBB); BI->setMetadata(LLVMContext::MD_prof, VeryLikelyWeights); SI->addCase(CaseTypeId, TestBB); ++TypeIds; } }
/// SplitBlockAndInsertIfThenElse is similar to SplitBlockAndInsertIfThen, /// but also creates the ElseBlock. /// Before: /// Head /// SplitBefore /// Tail /// After: /// Head /// if (Cond) /// ThenBlock /// else /// ElseBlock /// SplitBefore /// Tail void llvm::SplitBlockAndInsertIfThenElse(Value *Cond, Instruction *SplitBefore, TerminatorInst **ThenTerm, TerminatorInst **ElseTerm, MDNode *BranchWeights) { BasicBlock *Head = SplitBefore->getParent(); BasicBlock *Tail = Head->splitBasicBlock(SplitBefore); TerminatorInst *HeadOldTerm = Head->getTerminator(); LLVMContext &C = Head->getContext(); BasicBlock *ThenBlock = BasicBlock::Create(C, "", Head->getParent(), Tail); BasicBlock *ElseBlock = BasicBlock::Create(C, "", Head->getParent(), Tail); *ThenTerm = BranchInst::Create(Tail, ThenBlock); *ElseTerm = BranchInst::Create(Tail, ElseBlock); BranchInst *HeadNewTerm = BranchInst::Create(/*ifTrue*/ThenBlock, /*ifFalse*/ElseBlock, Cond); HeadNewTerm->setMetadata(LLVMContext::MD_prof, BranchWeights); ReplaceInstWithInst(HeadOldTerm, HeadNewTerm); }
TerminatorInst *llvm::SplitBlockAndInsertIfThen(Instruction *Cmp, bool Unreachable, MDNode *BranchWeights) { Instruction *SplitBefore = Cmp->getNextNode(); BasicBlock *Head = SplitBefore->getParent(); BasicBlock *Tail = Head->splitBasicBlock(SplitBefore); TerminatorInst *HeadOldTerm = Head->getTerminator(); LLVMContext &C = Head->getContext(); BasicBlock *ThenBlock = BasicBlock::Create(C, "", Head->getParent(), Tail); TerminatorInst *CheckTerm; if (Unreachable) CheckTerm = new UnreachableInst(C, ThenBlock); else CheckTerm = BranchInst::Create(Tail, ThenBlock); BranchInst *HeadNewTerm = BranchInst::Create(/*ifTrue*/ThenBlock, /*ifFalse*/Tail, Cmp); HeadNewTerm->setMetadata(LLVMContext::MD_prof, BranchWeights); ReplaceInstWithInst(HeadOldTerm, HeadNewTerm); return CheckTerm; }
TerminatorInst * llvm::SplitBlockAndInsertIfThen(Value *Cond, Instruction *SplitBefore, bool Unreachable, MDNode *BranchWeights, DominatorTree *DT, LoopInfo *LI) { BasicBlock *Head = SplitBefore->getParent(); BasicBlock *Tail = Head->splitBasicBlock(SplitBefore->getIterator()); TerminatorInst *HeadOldTerm = Head->getTerminator(); LLVMContext &C = Head->getContext(); BasicBlock *ThenBlock = BasicBlock::Create(C, "", Head->getParent(), Tail); TerminatorInst *CheckTerm; if (Unreachable) CheckTerm = new UnreachableInst(C, ThenBlock); else CheckTerm = BranchInst::Create(Tail, ThenBlock); CheckTerm->setDebugLoc(SplitBefore->getDebugLoc()); BranchInst *HeadNewTerm = BranchInst::Create(/*ifTrue*/ThenBlock, /*ifFalse*/Tail, Cond); HeadNewTerm->setMetadata(LLVMContext::MD_prof, BranchWeights); ReplaceInstWithInst(HeadOldTerm, HeadNewTerm); if (DT) { if (DomTreeNode *OldNode = DT->getNode(Head)) { std::vector<DomTreeNode *> Children(OldNode->begin(), OldNode->end()); DomTreeNode *NewNode = DT->addNewBlock(Tail, Head); for (DomTreeNode *Child : Children) DT->changeImmediateDominator(Child, NewNode); // Head dominates ThenBlock. DT->addNewBlock(ThenBlock, Head); } } if (LI) { if (Loop *L = LI->getLoopFor(Head)) { L->addBasicBlockToLoop(ThenBlock, *LI); L->addBasicBlockToLoop(Tail, *LI); } } return CheckTerm; }
/// \brief Peel off the first \p PeelCount iterations of loop \p L. /// /// Note that this does not peel them off as a single straight-line block. /// Rather, each iteration is peeled off separately, and needs to check the /// exit condition. /// For loops that dynamically execute \p PeelCount iterations or less /// this provides a benefit, since the peeled off iterations, which account /// for the bulk of dynamic execution, can be further simplified by scalar /// optimizations. bool llvm::peelLoop(Loop *L, unsigned PeelCount, LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, bool PreserveLCSSA) { if (!canPeel(L)) return false; LoopBlocksDFS LoopBlocks(L); LoopBlocks.perform(LI); BasicBlock *Header = L->getHeader(); BasicBlock *PreHeader = L->getLoopPreheader(); BasicBlock *Latch = L->getLoopLatch(); BasicBlock *Exit = L->getUniqueExitBlock(); Function *F = Header->getParent(); // Set up all the necessary basic blocks. It is convenient to split the // preheader into 3 parts - two blocks to anchor the peeled copy of the loop // body, and a new preheader for the "real" loop. // Peeling the first iteration transforms. // // PreHeader: // ... // Header: // LoopBody // If (cond) goto Header // Exit: // // into // // InsertTop: // LoopBody // If (!cond) goto Exit // InsertBot: // NewPreHeader: // ... // Header: // LoopBody // If (cond) goto Header // Exit: // // Each following iteration will split the current bottom anchor in two, // and put the new copy of the loop body between these two blocks. That is, // after peeling another iteration from the example above, we'll split // InsertBot, and get: // // InsertTop: // LoopBody // If (!cond) goto Exit // InsertBot: // LoopBody // If (!cond) goto Exit // InsertBot.next: // NewPreHeader: // ... // Header: // LoopBody // If (cond) goto Header // Exit: BasicBlock *InsertTop = SplitEdge(PreHeader, Header, DT, LI); BasicBlock *InsertBot = SplitBlock(InsertTop, InsertTop->getTerminator(), DT, LI); BasicBlock *NewPreHeader = SplitBlock(InsertBot, InsertBot->getTerminator(), DT, LI); InsertTop->setName(Header->getName() + ".peel.begin"); InsertBot->setName(Header->getName() + ".peel.next"); NewPreHeader->setName(PreHeader->getName() + ".peel.newph"); ValueToValueMapTy LVMap; // If we have branch weight information, we'll want to update it for the // newly created branches. BranchInst *LatchBR = cast<BranchInst>(cast<BasicBlock>(Latch)->getTerminator()); unsigned HeaderIdx = (LatchBR->getSuccessor(0) == Header ? 0 : 1); uint64_t TrueWeight, FalseWeight; uint64_t ExitWeight = 0, BackEdgeWeight = 0; if (LatchBR->extractProfMetadata(TrueWeight, FalseWeight)) { ExitWeight = HeaderIdx ? TrueWeight : FalseWeight; BackEdgeWeight = HeaderIdx ? FalseWeight : TrueWeight; } // For each peeled-off iteration, make a copy of the loop. for (unsigned Iter = 0; Iter < PeelCount; ++Iter) { SmallVector<BasicBlock *, 8> NewBlocks; ValueToValueMapTy VMap; // The exit weight of the previous iteration is the header entry weight // of the current iteration. So this is exactly how many dynamic iterations // the current peeled-off static iteration uses up. // FIXME: due to the way the distribution is constructed, we need a // guard here to make sure we don't end up with non-positive weights. if (ExitWeight < BackEdgeWeight) BackEdgeWeight -= ExitWeight; else BackEdgeWeight = 1; cloneLoopBlocks(L, Iter, InsertTop, InsertBot, Exit, NewBlocks, LoopBlocks, VMap, LVMap, LI); updateBranchWeights(InsertBot, cast<BranchInst>(VMap[LatchBR]), Iter, PeelCount, ExitWeight); InsertTop = InsertBot; InsertBot = SplitBlock(InsertBot, InsertBot->getTerminator(), DT, LI); InsertBot->setName(Header->getName() + ".peel.next"); F->getBasicBlockList().splice(InsertTop->getIterator(), F->getBasicBlockList(), NewBlocks[0]->getIterator(), F->end()); // Remap to use values from the current iteration instead of the // previous one. remapInstructionsInBlocks(NewBlocks, VMap); } // Now adjust the phi nodes in the loop header to get their initial values // from the last peeled-off iteration instead of the preheader. for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) { PHINode *PHI = cast<PHINode>(I); Value *NewVal = PHI->getIncomingValueForBlock(Latch); Instruction *LatchInst = dyn_cast<Instruction>(NewVal); if (LatchInst && L->contains(LatchInst)) NewVal = LVMap[LatchInst]; PHI->setIncomingValue(PHI->getBasicBlockIndex(NewPreHeader), NewVal); } // Adjust the branch weights on the loop exit. if (ExitWeight) { MDBuilder MDB(LatchBR->getContext()); MDNode *WeightNode = HeaderIdx ? MDB.createBranchWeights(ExitWeight, BackEdgeWeight) : MDB.createBranchWeights(BackEdgeWeight, ExitWeight); LatchBR->setMetadata(LLVMContext::MD_prof, WeightNode); } // If the loop is nested, we changed the parent loop, update SE. if (Loop *ParentLoop = L->getParentLoop()) SE->forgetLoop(ParentLoop); NumPeeled++; return true; }
/// buildCFICheck - emits __cfi_check for the current module. void CrossDSOCFI::buildCFICheck(Module &M) { // FIXME: verify that __cfi_check ends up near the end of the code section, // but before the jump slots created in LowerTypeTests. llvm::DenseSet<uint64_t> TypeIds; SmallVector<MDNode *, 2> Types; for (GlobalObject &GO : M.global_objects()) { Types.clear(); GO.getMetadata(LLVMContext::MD_type, Types); for (MDNode *Type : Types) { // Sanity check. GO must not be a function declaration. assert(!isa<Function>(&GO) || !cast<Function>(&GO)->isDeclaration()); if (ConstantInt *TypeId = extractNumericTypeId(Type)) TypeIds.insert(TypeId->getZExtValue()); } } LLVMContext &Ctx = M.getContext(); Constant *C = M.getOrInsertFunction( "__cfi_check", Type::getVoidTy(Ctx), Type::getInt64Ty(Ctx), Type::getInt8PtrTy(Ctx), Type::getInt8PtrTy(Ctx), nullptr); Function *F = dyn_cast<Function>(C); F->setAlignment(4096); auto args = F->arg_begin(); Value &CallSiteTypeId = *(args++); CallSiteTypeId.setName("CallSiteTypeId"); Value &Addr = *(args++); Addr.setName("Addr"); Value &CFICheckFailData = *(args++); CFICheckFailData.setName("CFICheckFailData"); assert(args == F->arg_end()); BasicBlock *BB = BasicBlock::Create(Ctx, "entry", F); BasicBlock *ExitBB = BasicBlock::Create(Ctx, "exit", F); BasicBlock *TrapBB = BasicBlock::Create(Ctx, "fail", F); IRBuilder<> IRBFail(TrapBB); Constant *CFICheckFailFn = M.getOrInsertFunction( "__cfi_check_fail", Type::getVoidTy(Ctx), Type::getInt8PtrTy(Ctx), Type::getInt8PtrTy(Ctx), nullptr); IRBFail.CreateCall(CFICheckFailFn, {&CFICheckFailData, &Addr}); IRBFail.CreateBr(ExitBB); IRBuilder<> IRBExit(ExitBB); IRBExit.CreateRetVoid(); IRBuilder<> IRB(BB); SwitchInst *SI = IRB.CreateSwitch(&CallSiteTypeId, TrapBB, TypeIds.size()); for (uint64_t TypeId : TypeIds) { ConstantInt *CaseTypeId = ConstantInt::get(Type::getInt64Ty(Ctx), TypeId); BasicBlock *TestBB = BasicBlock::Create(Ctx, "test", F); IRBuilder<> IRBTest(TestBB); Function *BitsetTestFn = Intrinsic::getDeclaration(&M, Intrinsic::type_test); Value *Test = IRBTest.CreateCall( BitsetTestFn, {&Addr, MetadataAsValue::get( Ctx, ConstantAsMetadata::get(CaseTypeId))}); BranchInst *BI = IRBTest.CreateCondBr(Test, ExitBB, TrapBB); BI->setMetadata(LLVMContext::MD_prof, VeryLikelyWeights); SI->addCase(CaseTypeId, TestBB); ++NumTypeIds; } }
/// Peel off the first \p PeelCount iterations of loop \p L. /// /// Note that this does not peel them off as a single straight-line block. /// Rather, each iteration is peeled off separately, and needs to check the /// exit condition. /// For loops that dynamically execute \p PeelCount iterations or less /// this provides a benefit, since the peeled off iterations, which account /// for the bulk of dynamic execution, can be further simplified by scalar /// optimizations. bool llvm::peelLoop(Loop *L, unsigned PeelCount, LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC, bool PreserveLCSSA) { assert(PeelCount > 0 && "Attempt to peel out zero iterations?"); assert(canPeel(L) && "Attempt to peel a loop which is not peelable?"); LoopBlocksDFS LoopBlocks(L); LoopBlocks.perform(LI); BasicBlock *Header = L->getHeader(); BasicBlock *PreHeader = L->getLoopPreheader(); BasicBlock *Latch = L->getLoopLatch(); BasicBlock *Exit = L->getUniqueExitBlock(); Function *F = Header->getParent(); // Set up all the necessary basic blocks. It is convenient to split the // preheader into 3 parts - two blocks to anchor the peeled copy of the loop // body, and a new preheader for the "real" loop. // Peeling the first iteration transforms. // // PreHeader: // ... // Header: // LoopBody // If (cond) goto Header // Exit: // // into // // InsertTop: // LoopBody // If (!cond) goto Exit // InsertBot: // NewPreHeader: // ... // Header: // LoopBody // If (cond) goto Header // Exit: // // Each following iteration will split the current bottom anchor in two, // and put the new copy of the loop body between these two blocks. That is, // after peeling another iteration from the example above, we'll split // InsertBot, and get: // // InsertTop: // LoopBody // If (!cond) goto Exit // InsertBot: // LoopBody // If (!cond) goto Exit // InsertBot.next: // NewPreHeader: // ... // Header: // LoopBody // If (cond) goto Header // Exit: BasicBlock *InsertTop = SplitEdge(PreHeader, Header, DT, LI); BasicBlock *InsertBot = SplitBlock(InsertTop, InsertTop->getTerminator(), DT, LI); BasicBlock *NewPreHeader = SplitBlock(InsertBot, InsertBot->getTerminator(), DT, LI); InsertTop->setName(Header->getName() + ".peel.begin"); InsertBot->setName(Header->getName() + ".peel.next"); NewPreHeader->setName(PreHeader->getName() + ".peel.newph"); ValueToValueMapTy LVMap; // If we have branch weight information, we'll want to update it for the // newly created branches. BranchInst *LatchBR = cast<BranchInst>(cast<BasicBlock>(Latch)->getTerminator()); unsigned HeaderIdx = (LatchBR->getSuccessor(0) == Header ? 0 : 1); uint64_t TrueWeight, FalseWeight; uint64_t ExitWeight = 0, CurHeaderWeight = 0; if (LatchBR->extractProfMetadata(TrueWeight, FalseWeight)) { ExitWeight = HeaderIdx ? TrueWeight : FalseWeight; // The # of times the loop body executes is the sum of the exit block // weight and the # of times the backedges are taken. CurHeaderWeight = TrueWeight + FalseWeight; } // For each peeled-off iteration, make a copy of the loop. for (unsigned Iter = 0; Iter < PeelCount; ++Iter) { SmallVector<BasicBlock *, 8> NewBlocks; ValueToValueMapTy VMap; // Subtract the exit weight from the current header weight -- the exit // weight is exactly the weight of the previous iteration's header. // FIXME: due to the way the distribution is constructed, we need a // guard here to make sure we don't end up with non-positive weights. if (ExitWeight < CurHeaderWeight) CurHeaderWeight -= ExitWeight; else CurHeaderWeight = 1; cloneLoopBlocks(L, Iter, InsertTop, InsertBot, Exit, NewBlocks, LoopBlocks, VMap, LVMap, DT, LI); // Remap to use values from the current iteration instead of the // previous one. remapInstructionsInBlocks(NewBlocks, VMap); if (DT) { // Latches of the cloned loops dominate over the loop exit, so idom of the // latter is the first cloned loop body, as original PreHeader dominates // the original loop body. if (Iter == 0) DT->changeImmediateDominator(Exit, cast<BasicBlock>(LVMap[Latch])); #ifdef EXPENSIVE_CHECKS assert(DT->verify(DominatorTree::VerificationLevel::Fast)); #endif } auto *LatchBRCopy = cast<BranchInst>(VMap[LatchBR]); updateBranchWeights(InsertBot, LatchBRCopy, Iter, PeelCount, ExitWeight); // Remove Loop metadata from the latch branch instruction // because it is not the Loop's latch branch anymore. LatchBRCopy->setMetadata(LLVMContext::MD_loop, nullptr); InsertTop = InsertBot; InsertBot = SplitBlock(InsertBot, InsertBot->getTerminator(), DT, LI); InsertBot->setName(Header->getName() + ".peel.next"); F->getBasicBlockList().splice(InsertTop->getIterator(), F->getBasicBlockList(), NewBlocks[0]->getIterator(), F->end()); } // Now adjust the phi nodes in the loop header to get their initial values // from the last peeled-off iteration instead of the preheader. for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) { PHINode *PHI = cast<PHINode>(I); Value *NewVal = PHI->getIncomingValueForBlock(Latch); Instruction *LatchInst = dyn_cast<Instruction>(NewVal); if (LatchInst && L->contains(LatchInst)) NewVal = LVMap[LatchInst]; PHI->setIncomingValue(PHI->getBasicBlockIndex(NewPreHeader), NewVal); } // Adjust the branch weights on the loop exit. if (ExitWeight) { // The backedge count is the difference of current header weight and // current loop exit weight. If the current header weight is smaller than // the current loop exit weight, we mark the loop backedge weight as 1. uint64_t BackEdgeWeight = 0; if (ExitWeight < CurHeaderWeight) BackEdgeWeight = CurHeaderWeight - ExitWeight; else BackEdgeWeight = 1; MDBuilder MDB(LatchBR->getContext()); MDNode *WeightNode = HeaderIdx ? MDB.createBranchWeights(ExitWeight, BackEdgeWeight) : MDB.createBranchWeights(BackEdgeWeight, ExitWeight); LatchBR->setMetadata(LLVMContext::MD_prof, WeightNode); } if (Loop *ParentLoop = L->getParentLoop()) L = ParentLoop; // We modified the loop, update SE. SE->forgetTopmostLoop(L); // FIXME: Incrementally update loop-simplify simplifyLoop(L, DT, LI, SE, AC, PreserveLCSSA); NumPeeled++; return true; }